Loki Metrics Dashboard
Loki Metrics Dashboard
A metrics dashboard for the metrics provided by Grafana Loki.

Metrics are expected to be collected by Prometheus from Loki. Supports multiple Prometheus jobs for querying metrics from multiple Grafana Loki instances.

Where applicable the graphs are able to work with the tenant feature in Grafana Loki.

Initially created with Grafana v9.3.1 and Grafana Loki v2.7.0

See also: [The Grafana Loki metrics dashboard on Githubb](https://github.com/frank-fegert/grafana-dashboards/blob/main/Loki_Metrics_Dashboard.json)

Loki Metrics Dashboard
{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__elements": {},
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "9.3.2"
    },
    {
      "type": "panel",
      "id": "heatmap",
      "name": "Heatmap",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "stat",
      "name": "Stat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "timeseries",
      "name": "Time series",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "$$hashKey": "object:7",
        "builtIn": 1,
        "datasource": {
          "type": "datasource",
          "uid": "grafana"
        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      }
    ]
  },
  "description": "Loki Metrics Dashboard",
  "editable": false,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 102,
      "panels": [],
      "title": "Loki General Info",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 0,
        "y": 1
      },
      "id": 59,
      "interval": "$interval",
      "maxDataPoints": 1,
      "options": {
        "colorMode": "background",
        "graphMode": "none",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "/.*/",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "exemplar": false,
          "expr": "loki_build_info{job=\"$jobs\",instance=\"$host:$instance\"}",
          "format": "table",
          "instant": false,
          "interval": "$interval",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Loki Version",
      "transformations": [
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true,
              "Value": true,
              "__name__": true,
              "goversion": true,
              "instance": true,
              "job": true,
              "version": true
            },
            "indexByName": {
              "Time": 0,
              "Value": 8,
              "__name__": 1,
              "branch": 2,
              "goversion": 4,
              "instance": 5,
              "job": 6,
              "revision": 3,
              "version": 7
            },
            "renameByName": {
              "branch": "Loki Branch",
              "goversion": "",
              "instance": "",
              "revision": "Loki Revision",
              "version": ""
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "decimals": 0,
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 3,
        "y": 1
      },
      "id": 10,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "none",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(loki_internal_log_messages_total{job=\"$jobs\",instance=\"$host:$instance\"})",
          "instant": false,
          "interval": "$interval",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Internal Log Messages",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 6,
        "y": 1
      },
      "id": 80,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "none",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "deprecated_flags_inuse_total{job=\"$jobs\",instance=\"$host:$instance\"}",
          "instant": false,
          "interval": "$interval",
          "legendFormat": "Deprecated",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "loki_experimental_features_in_use_total{job=\"$jobs\",instance=\"$host:$instance\"}",
          "hide": false,
          "instant": false,
          "interval": "$interval",
          "legendFormat": "Experimental",
          "refId": "B"
        }
      ],
      "title": "Features Currently Set",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "semi-dark-red",
                "value": 1
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 9,
        "y": 1
      },
      "id": 24,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "loki_panic_total{job=\"$jobs\",instance=\"$host:$instance\"}",
          "instant": false,
          "interval": "$interval",
          "legendFormat": "The total number of panics triggered",
          "refId": "A"
        }
      ],
      "title": "Panics",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "semi-dark-red",
                "value": 0
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 12,
        "y": 1
      },
      "id": 112,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "loki_distributor_replication_factor{job=\"$jobs\",instance=\"$host:$instance\"}",
          "instant": false,
          "interval": "$interval",
          "legendFormat": "The configured replication factor",
          "refId": "A"
        }
      ],
      "title": "Replication Faktor",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 15,
        "y": 1
      },
      "id": 81,
      "interval": "$interval",
      "maxDataPoints": 1,
      "options": {
        "colorMode": "background",
        "graphMode": "none",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "/.*/",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "exemplar": false,
          "expr": "go_info{job=\"$jobs\",instance=\"$host:$instance\"}",
          "format": "table",
          "instant": false,
          "interval": "$interval",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Go Version",
      "transformations": [
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true,
              "Value": true,
              "__name__": true,
              "branch": true,
              "instance": true,
              "job": true,
              "revision": true,
              "version": false
            },
            "indexByName": {
              "Time": 0,
              "Value": 8,
              "__name__": 1,
              "branch": 2,
              "goversion": 4,
              "instance": 5,
              "job": 6,
              "revision": 3,
              "version": 7
            },
            "renameByName": {
              "branch": "",
              "goversion": "Go Version",
              "instance": "",
              "revision": "",
              "version": "Go Version"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "semi-dark-red",
                "value": 500
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 18,
        "y": 1
      },
      "id": 86,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "exemplar": false,
          "expr": "sum(go_goroutines{job=\"$jobs\",instance=\"$host:$instance\"})",
          "format": "table",
          "instant": false,
          "interval": "$interval",
          "legendFormat": "Number of goroutines that currently exist",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Total Goroutines",
      "transformations": [],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "semi-dark-red",
                "value": 25
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 21,
        "y": 1
      },
      "id": 87,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "exemplar": false,
          "expr": "sum(go_threads{job=\"$jobs\",instance=\"$host:$instance\"})",
          "format": "table",
          "instant": false,
          "interval": "$interval",
          "legendFormat": "Number of OS threads created",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Total Threads",
      "transformations": [],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "cps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 0,
        "y": 4
      },
      "id": 235,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(loki_distributor_lines_received_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(tenant)",
          "hide": false,
          "instant": false,
          "interval": "$interval",
          "legendFormat": "{{ tenant }}",
          "refId": "A"
        }
      ],
      "title": "Distributor Lines",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "fake": "Default Tenant"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "binBps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 6,
        "y": 4
      },
      "id": 234,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(loki_distributor_bytes_received_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(tenant)",
          "hide": false,
          "instant": false,
          "interval": "$interval",
          "legendFormat": "{{ tenant }}",
          "refId": "A"
        }
      ],
      "title": "Distributor Volume",
      "transformations": [
        {
          "id": "calculateField",
          "options": {
            "mode": "reduceRow",
            "reduce": {
              "include": [],
              "reducer": "sum"
            }
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Received / fake": "Received / Default Tenant",
              "fake": "Default Tenant"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "ops"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 12,
        "y": 4
      },
      "id": 230,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "rate(loki_ingester_chunks_created_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
          "hide": false,
          "instant": false,
          "interval": "$interval",
          "legendFormat": "Create",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(loki_ingester_chunks_stored_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant) ",
          "hide": false,
          "instant": false,
          "interval": "$interval",
          "legendFormat": "Store / {{ tenant }}",
          "refId": "B"
        }
      ],
      "title": "Ingester Chunk Operations",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Store / fake": "Store / Default Tenant"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "binBps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 18,
        "y": 4
      },
      "id": 188,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(loki_ingester_chunk_stored_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)",
          "instant": false,
          "interval": "$interval",
          "legendFormat": "{{ tenant }}",
          "refId": "A"
        }
      ],
      "title": "Ingester Chunks Volume",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "fake": "Default Tenant"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 0,
        "y": 10
      },
      "id": 23,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(loki_store_series_total{job=\"$jobs\",instance=\"$host:$instance\"}) by(status)",
          "instant": false,
          "interval": "$interval",
          "legendFormat": "{{ status }}",
          "refId": "A"
        }
      ],
      "title": "Store Series",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "discarded": "Discarded",
              "matched": "Matched"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 3,
        "y": 10
      },
      "id": 131,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(loki_store_chunks_downloaded_total{job=\"$jobs\",instance=\"$host:$instance\"}) by(status)",
          "instant": false,
          "interval": "$interval",
          "legendFormat": "{{ status }}",
          "refId": "A"
        }
      ],
      "title": "Store Chunks Downloaded",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "discarded": "Discarded",
              "matched": "Matched"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 6,
        "y": 10
      },
      "id": 137,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(loki_index_chunk_refs_total{job=\"$jobs\",instance=\"$host:$instance\"}) by(status)",
          "instant": false,
          "interval": "$interval",
          "legendFormat": "{{ status }}",
          "refId": "A"
        }
      ],
      "title": "Index Chunks Downloaded",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "discarded": "Discarded",
              "matched": "Matched"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 12,
        "y": 10
      },
      "id": 191,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "loki_ingester_memory_chunks{job=\"$jobs\",instance=\"$host:$instance\"}",
          "hide": false,
          "instant": false,
          "interval": "$interval",
          "legendFormat": "Chunks in memory",
          "refId": "B"
        }
      ],
      "title": "Ingester Chunks In Memory",
      "transformations": [],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 15,
        "y": 10
      },
      "id": 192,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(loki_ingester_memory_streams{job=\"$jobs\",instance=\"$host:$instance\"}) by(tenant)",
          "instant": false,
          "interval": "$interval",
          "legendFormat": "{{ tenant }}",
          "refId": "A"
        }
      ],
      "title": "Ingester Streams in Memory",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {
              "Time": 0,
              "Total": 2,
              "fake": 1
            },
            "renameByName": {
              "discarded": "Discarded",
              "fake": "Default Tenant",
              "matched": "Matched"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "ops"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 18,
        "y": 10
      },
      "id": 233,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(loki_ingester_streams_created_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)",
          "hide": false,
          "instant": false,
          "interval": "$interval",
          "legendFormat": "Create / {{ tenant }}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(loki_ingester_streams_removed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)",
          "hide": false,
          "instant": false,
          "interval": "$interval",
          "legendFormat": "Delete / {{ tenant }}",
          "refId": "B"
        }
      ],
      "title": "Ingester Stream Operations",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Create / fake": "Create / Default Tenant",
              "Delete / fake": "Delete / Default Tenant",
              "Store / fake": "Store / Default Tenant"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "ops"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 6,
        "x": 0,
        "y": 13
      },
      "id": 236,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(loki_chunk_store_fetched_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (user)",
          "hide": false,
          "instant": false,
          "interval": "$interval",
          "legendFormat": "{{ user}}",
          "refId": "A"
        }
      ],
      "title": "Store Operations Fetched",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Create / fake": "Create / Default Tenant",
              "Delete / fake": "Delete / Default Tenant",
              "Fetch / fake": "Fetch / Default Tenant",
              "Store / fake": "Store / Default Tenant",
              "fake": "Default Tenant"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "ops"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 6,
        "x": 6,
        "y": 13
      },
      "id": 237,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(loki_chunk_store_stored_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (user)",
          "hide": false,
          "instant": false,
          "interval": "$interval",
          "legendFormat": "{{ user}}",
          "refId": "A"
        }
      ],
      "title": "Store Operations Stored",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Create / fake": "Create / Default Tenant",
              "Delete / fake": "Delete / Default Tenant",
              "Fetch / fake": "Default Tenant",
              "Store / fake": "Store / Default Tenant",
              "fake": "Default Tenant"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 12,
        "y": 13
      },
      "id": 231,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_size_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
          "hide": false,
          "instant": false,
          "interval": "$interval",
          "legendFormat": "Chunk Size",
          "refId": "A"
        }
      ],
      "title": "Ingester Chunk Size",
      "transformations": [],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "percent"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 15,
        "y": 13
      },
      "id": 232,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_compression_ratio_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
          "hide": false,
          "instant": false,
          "interval": "$interval",
          "legendFormat": "Compression Ratio",
          "refId": "A"
        }
      ],
      "title": "Ingester Chunk Compression",
      "transformations": [],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "binBps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 6,
        "x": 0,
        "y": 16
      },
      "id": 238,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "rate(loki_chunk_store_fetched_chunk_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
          "hide": false,
          "instant": false,
          "interval": "$interval",
          "legendFormat": "{{ user}}",
          "refId": "A"
        }
      ],
      "title": "Store Volume Fetched",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Create / fake": "Create / Default Tenant",
              "Delete / fake": "Delete / Default Tenant",
              "Fetch / fake": "Fetch / Default Tenant",
              "Store / fake": "Store / Default Tenant",
              "fake": "Default Tenant"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [
            {
              "options": {
                "null": {
                  "index": 0,
                  "text": "N/A"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "binBps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 6,
        "x": 6,
        "y": 16
      },
      "id": 239,
      "interval": "$interval",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "rate(loki_chunk_store_stored_chunk_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
          "hide": false,
          "instant": false,
          "interval": "$interval",
          "legendFormat": "{{ user}}",
          "refId": "A"
        }
      ],
      "title": "Store Volume Stored",
      "transformations": [
        {
          "id": "calculateField",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Create / fake": "Create / Default Tenant",
              "Delete / fake": "Delete / Default Tenant",
              "Fetch / fake": "Fetch / Default Tenant",
              "Store / fake": "Store / Default Tenant",
              "fake": "Default Tenant"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 19
      },
      "id": 143,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 2
          },
          "id": 151,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,method,route,status_code))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ method }} / {{ route }} / {{ status_code }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki API - Request Duration",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Requests",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 2
          },
          "id": 128,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (method, route)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ method }} / {{ route }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki API - Request Current",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Size",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 10
          },
          "id": 153,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_response_message_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,method,route))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ method }} / {{ route }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki API - Response Size",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Size",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 10
          },
          "id": 152,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_request_message_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,method,route))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ method }} / {{ route }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki API - Request Size",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Connections",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 18
          },
          "id": 96,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_tcp_connections{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ protocol }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki - TCP Connections",
          "transformations": [
            {
              "id": "calculateField",
              "options": {
                "mode": "reduceRow",
                "reduce": {
                  "include": [
                    "grpc",
                    "http"
                  ],
                  "reducer": "sum"
                }
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {
                  "Time": 1,
                  "Total": 0,
                  "grpc": 2,
                  "http": 3
                },
                "renameByName": {
                  "grpc": "gRPC",
                  "http": "HTTP"
                }
              }
            }
          ],
          "type": "timeseries"
        }
      ],
      "title": "Loki API / Connections Metrics",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 20
      },
      "id": 114,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Cache Hits",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 3
          },
          "id": 115,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_cache_hits{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(name)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "__auto",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Cache - Hits",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Keys",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 3
          },
          "id": 116,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_cache_fetched_keys{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(name)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "__auto",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Cache - Fetched Keys",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 11
          },
          "id": 118,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_cache_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,name,method))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ name }} / {{ method }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Cache - Request Duration",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*) / (.*)\\.(.*)",
                "renamePattern": "$1 / $3"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Size",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 11
          },
          "id": 119,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_cache_value_size_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,name,method))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ name }} / {{ method }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Cache - Value Size",
          "transformations": [],
          "type": "timeseries"
        }
      ],
      "title": "Loki Cache Metrics",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 21
      },
      "id": 108,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Bytes",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "binBps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 4
          },
          "id": 109,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_distributor_bytes_received_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(tenant)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "__auto",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Distributor - Received Bytes",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {},
                "renameByName": {
                  "fake": "Default Tenant"
                }
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Lines",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 4
          },
          "id": 110,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_distributor_lines_received_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(tenant)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "__auto",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Distributor - Received Lines",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {},
                "renameByName": {
                  "fake": "Default Tenant"
                }
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Appends",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 12
          },
          "id": 111,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_distributor_ingester_appends_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(ingester)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "__auto",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Distributor - Batch Appents Sent To Ingester",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {},
                "renameByName": {
                  "fake": "Default Tenant"
                }
              }
            }
          ],
          "type": "timeseries"
        }
      ],
      "title": "Loki Distributor Metrics",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 22
      },
      "id": 127,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 5
          },
          "id": 130,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_index_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,component,operation,status_code))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ component }} / {{ operation }} / {{ status_code }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Index - Request Duration",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*) / (.*)\\.(.*)",
                "renamePattern": "$1 / $3"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Chunks",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 5
          },
          "id": 138,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_index_chunk_refs_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (status)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ status }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Index - Chunks Downloaded",
          "transformations": [
            {
              "id": "calculateField",
              "options": {
                "mode": "reduceRow",
                "reduce": {
                  "include": [
                    "debug",
                    "error",
                    "info",
                    "warn"
                  ],
                  "reducer": "sum"
                }
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {
                  "Time": 1,
                  "Total": 0,
                  "grpc": 2,
                  "http": 3
                },
                "renameByName": {
                  "debug": "Debug",
                  "discarded": "Discarded",
                  "error": "Error",
                  "grpc": "gRPC",
                  "http": "HTTP",
                  "info": "Info",
                  "matched": "Matched",
                  "warn": "Warn"
                }
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 13
          },
          "id": 201,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_kv_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,kv_name,operation,role,status_code,type))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ kv_name }} / {{ type }} / {{ operation }} / {{ status_code }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki KV - Request Duration",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 13
          },
          "id": 202,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(cortex_kv_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,kv_name,operation,role,status_code,type))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ kv_name }} / {{ type }} / {{ operation }} / {{ status_code }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Cortex KV - Request Duration",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 21
          },
          "id": 228,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(cortex_consul_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,kv_name,operation,status_code))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ kv_name }} / {{ operation }} / {{ status_code }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Cortex KV - Request Duration",
          "transformations": [],
          "type": "timeseries"
        }
      ],
      "title": "Loki Index Metrics",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 23
      },
      "id": 174,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 6
          },
          "id": 189,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_client_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,operation,status_code))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ operation }} / {{ status_code }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Ingester - Client Request Duration",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Operations",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 6
          },
          "id": 185,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_ingester_chunks_created_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Chunk creations ",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_ingester_chunks_flushed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (reason)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Chunk flushes / {{ reason }}",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_ingester_chunks_stored_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant) ",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Chunks stored / {{ tenant }}",
              "range": true,
              "refId": "C"
            }
          ],
          "title": "Loki Ingester - Chunk Operations",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Checkpoints",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 14
          },
          "id": 175,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_ingester_checkpoint_creations_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Checkpoint creations ",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_ingester_checkpoint_creations_failed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Checkpoint creation failures",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_ingester_checkpoint_deletions_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Checkpoint deletions",
              "range": true,
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_ingester_checkpoint_deletions_failed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Checkpoint deletion failures",
              "range": true,
              "refId": "D"
            }
          ],
          "title": "Loki Ingester - Checkpoints",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Bytes",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "binBps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 14
          },
          "id": 177,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_ingester_checkpoint_logged_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Bytes written to disk for checkpointing",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Ingester - Checkpoints Volume",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 22
          },
          "id": 183,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_ingester_checkpoint_duration_seconds{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.99\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Time taken to create a checkpoint (0.99 quantile)",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_ingester_checkpoint_duration_seconds{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.9\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Time taken to create a checkpoint (0.9 quantile)",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_ingester_checkpoint_duration_seconds{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.5\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Time taken to create a checkpoint (0.5 quantile)",
              "range": true,
              "refId": "C"
            }
          ],
          "title": "Loki Ingester - Checkpoint Duration",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Blocks",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 22
          },
          "id": 176,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_blocks_per_chunk_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Blocks per Chunk",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Ingester - Blocks Per Chunk",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "%",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "percent"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 30
          },
          "id": 180,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_compression_ratio_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Compression ratio of chunks (when stored)",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Ingester - Chunk Compression Ratio",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Count",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 30
          },
          "id": 190,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_ingester_memory_chunks{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Chunks in memory",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Ingester - Chunks In Memory",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Bytes",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 38
          },
          "id": 184,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_size_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Distribution of stored chunk sizes (when stored)",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Ingester - Chunk Size",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 38
          },
          "id": 178,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_age_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Distribution of chunk ages (when stored)",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Ingester - Chunk Age",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "%",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "percentunit"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 46
          },
          "id": 186,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_utilization_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Distribution of stored chunk utilization (when stored)",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Ingester - Chunk Usage",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Bytes",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 46
          },
          "id": 187,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_ingester_chunk_stored_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Bytes stored in chunks / {{ tenant }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Ingester - Chunk Stored Volume",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Samples",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 54
          },
          "id": 194,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_samples_per_chunk_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Samples per chunk",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Ingester - Samples Per Chunk",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "h"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 54
          },
          "id": 179,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_bounds_hours_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Distribution of chunk end-start durations",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Ingester - Chunk Bounds (Hours)",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 62
          },
          "id": 181,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_encode_time_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Distribution of chunk encode times",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Ingester - Chunk Encoding Time",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Lines",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 62
          },
          "id": 182,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_entries_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Distribution of stored lines per chunk (when stored)",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Ingester - Lines Per Chunk",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Operations",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 70
          },
          "id": 196,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_ingester_streams_created_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Stream creations / {{ tenant }}",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_ingester_streams_removed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Stream deletetions / {{ tenant }}",
              "range": true,
              "refId": "B"
            }
          ],
          "title": "Loki Ingester - Stream Operations",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Streams",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 70
          },
          "id": 193,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(loki_ingester_memory_streams{job=\"$jobs\",instance=\"$host:$instance\"}) by (tenant)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Streams in memory / {{ tenant }}",
              "range": true,
              "refId": "B"
            }
          ],
          "title": "Loki Ingester - Streams In Memory",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Records",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 78
          },
          "id": 197,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_ingester_wal_records_logged_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "WAL records logged",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_ingester_wal_duplicate_entries_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Discarded entries during WAL replay due to existing in checkpoints",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_ingester_wal_disk_full_failures_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "WAL write failures due to full disk",
              "range": true,
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_ingester_wal_discarded_samples_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (reason)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "WAL segment entries discarded during replay / {{ reason }}",
              "range": true,
              "refId": "D"
            }
          ],
          "title": "Loki Ingester - WAL Records",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Bytes",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 78
          },
          "id": 200,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_ingester_wal_bytes_in_use{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "legendFormat": "Currently in use by the WAL recovery process",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_ingester_wal_logged_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Written to disk for WAL records",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_ingester_wal_discarded_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (reason)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "WAL segment bytes discarded during replay / {{ reason }}",
              "range": true,
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_ingester_wal_discarded_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (reason)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Recovered from the WAL",
              "range": true,
              "refId": "D"
            }
          ],
          "title": "Loki Ingester - WAL Volume",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Operations",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 86
          },
          "id": 198,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_ingester_wal_recovered_entries_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Entries recovered from the WAL",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_ingester_wal_recovered_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Chunks recovered from the WAL checkpoints",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_ingester_wal_recovered_streams_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Streams recovered from the WAL",
              "range": true,
              "refId": "C"
            }
          ],
          "title": "Loki Ingester - WAL Recovery Operations",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Count",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 86
          },
          "id": 199,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_ingester_wal_replay_active{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "WAL is replaying",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_ingester_wal_replay_flushing{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "WAL replay in flushing phase due to backpressure",
              "range": true,
              "refId": "B"
            }
          ],
          "title": "Loki Ingester - WAL Replay Status",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Length",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 94
          },
          "id": 218,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "cortex_ingester_flush_queue_length{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Series pending in the flush queue",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Cortex Ingester - Flush Queue",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Clients",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 94
          },
          "id": 229,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "cortex_distributor_ingester_clients{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Ingester clients",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Cortex Ingester - Clients",
          "transformations": [],
          "type": "timeseries"
        }
      ],
      "title": "Loki Ingester Metrics",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 24
      },
      "id": 121,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Messages",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 7
          },
          "id": 124,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_internal_log_messages_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (level)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ protocol }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki - Internal Log Messages",
          "transformations": [
            {
              "id": "calculateField",
              "options": {
                "mode": "reduceRow",
                "reduce": {
                  "include": [
                    "debug",
                    "error",
                    "info",
                    "warn"
                  ],
                  "reducer": "sum"
                }
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {
                  "Time": 1,
                  "Total": 0,
                  "grpc": 2,
                  "http": 3
                },
                "renameByName": {
                  "debug": "Debug",
                  "error": "Error",
                  "grpc": "gRPC",
                  "http": "HTTP",
                  "info": "Info",
                  "warn": "Warn"
                }
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Flushes",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 7
          },
          "id": 217,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_log_flushes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Log flushes - line-buffered logger",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Line - Log Flushes",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "custom": {
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "scaleDistribution": {
                  "type": "linear"
                }
              },
              "links": []
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 15
          },
          "id": 122,
          "maxDataPoints": 50,
          "options": {
            "calculate": false,
            "cellGap": 2,
            "cellValues": {
              "unit": "bytes"
            },
            "color": {
              "exponent": 0.5,
              "fill": "red",
              "mode": "scheme",
              "reverse": false,
              "scale": "exponential",
              "scheme": "BrBG",
              "steps": 64
            },
            "exemplars": {
              "color": "rgba(255,0,255,0.7)"
            },
            "filterValues": {
              "le": 1e-9
            },
            "legend": {
              "show": true
            },
            "rowsFrame": {
              "layout": "auto"
            },
            "tooltip": {
              "show": true,
              "yHistogram": false
            },
            "yAxis": {
              "axisLabel": "Bucket Size",
              "axisPlacement": "left",
              "reverse": false,
              "unit": "bytes"
            }
          },
          "pluginVersion": "9.3.2",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "exemplar": false,
              "expr": "rate(loki_bytes_per_line_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
              "format": "heatmap",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Line - Line Size Distribution",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {},
                "renameByName": {
                  "+Inf": "Unlimited",
                  "0.00010239999999999998": "10e-5",
                  "0.0010485759999999998": "10e-4",
                  "0.010485759999999998": "10e-3",
                  "0.10485759999999998": "10e-2",
                  "1.0239999999999999e-05": "10e-6",
                  "1.0239999999999999e-06": "10e-7",
                  "9.999999999999998e-08": "10e-8",
                  "9.999999999999999e-09": "10e-9",
                  "9.999999999999999e-10": "10e-10"
                }
              }
            }
          ],
          "type": "heatmap"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Size",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 15
          },
          "id": 123,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_bytes_per_line_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Number of bytes per line",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Line - Line Size",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*) / (.*)\\.(.*)",
                "renamePattern": "$1 / $3"
              }
            }
          ],
          "type": "timeseries"
        }
      ],
      "title": "Loki Line Metrics",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 25
      },
      "id": 140,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Lines",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 8
          },
          "id": 148,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_logql_querystats_ingester_sent_lines_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Lines",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki LogQL - Lines Received From Ingester",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Duplicates",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 8
          },
          "id": 150,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_logql_querystats_duplicates_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Lines",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki LogQL - Duplicates Found",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 16
          },
          "id": 125,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(logql_query_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,query_type))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ query_type }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki LogQL - Query Duration",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Zime",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 16
          },
          "id": 147,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_latency_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,type,status_code))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ type }} / {{ status_code }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki LogQL - Query Latency",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Size",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "binBps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 24
          },
          "id": 141,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_bytes_processed_per_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,type,status_code))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ type }} / {{ status_code }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki LogQL - Query Volume Processed",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Chunks",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 24
          },
          "id": 149,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_logql_querystats_downloaded_chunk_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(type,status_code)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ type }} / {{ status_code }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki LogQL - Downloaded Chunks",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Zime",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 32
          },
          "id": 146,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_chunk_download_latency_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,type,status_code))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ type }} / {{ status_code }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki LogQL - Chunk Download Latency",
          "transformations": [],
          "type": "timeseries"
        }
      ],
      "title": "Loki LogQL Metrics",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 26
      },
      "id": 204,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 9
          },
          "id": 206,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_querier_query_frontend_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,operation,status_code))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ operation }} / {{ status_code }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Querier - Frontend Request Duration",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Count",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 9
          },
          "id": 210,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_query_frontend_log_result_cache_hit_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Cache hits for the frontend cache",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_query_frontend_log_result_cache_miss_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Cache misses for the frontend cache",
              "range": true,
              "refId": "B"
            }
          ],
          "title": "Loki Querier - Frontend Cache",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 17
          },
          "id": 211,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_query_frontend_partitions_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Time-based partitions (sub-requests) per request",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Querier - Frontend Partitions",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Count",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 17
          },
          "id": 212,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_query_frontend_shard_factor_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,mapper))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Downstream queries per request / {{ mapper }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Querier - Frontend Shard Factor",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Count",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 25
          },
          "id": 213,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_query_frontend_sharding_parsed_queries_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,mapper,type)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Parsed queries by evaluation type / {{ mapper }} / {{ type }}",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_query_frontend_shards_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,mapper,type))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Downstream queries by expression type / {{ mapper }} / {{ type }}",
              "range": true,
              "refId": "B"
            }
          ],
          "title": "Loki Querier - Frontend Shards",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Count",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 25
          },
          "id": 209,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_querier_query_frontend_clients{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Clients connected to query-frontend",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_querier_tail_active{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Active tailers",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_querier_tail_active_streams{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Active streams being tailed",
              "range": true,
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_querier_worker_concurrency{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Concurrent querier workers",
              "range": true,
              "refId": "D"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_querier_worker_inflight_queries{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Queries being processed by the querier workers",
              "range": true,
              "refId": "E"
            }
          ],
          "title": "Loki Querier - Index Cache Clients / Workers",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Operations",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 33
          },
          "id": 208,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_querier_index_cache_gets_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Gets for the index cache",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_querier_index_cache_puts_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Puts for the index cache",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_querier_index_cache_hits_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Cache hits for the index cache",
              "range": true,
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_querier_index_cache_corruptions_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Cache corruptions for the index cache",
              "range": true,
              "refId": "D"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_querier_index_cache_encode_errors_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Errors for the index cache while encoding the body",
              "range": true,
              "refId": "E"
            }
          ],
          "title": "Loki Querier - Index Cache Operations",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Connections",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 33
          },
          "id": 219,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "cortex_query_frontend_connected_schedulers{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Schedulers this frontend is connected to",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "cortex_query_scheduler_connected_frontend_clients{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Query-frontend worker clients",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "cortex_query_scheduler_connected_querier_clients{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Querier worker clients",
              "range": true,
              "refId": "C"
            }
          ],
          "title": "Cortex Query - Frontend Connections",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Operations",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 41
          },
          "id": 215,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(querier_cache_added_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Entries added to the cache / {{ cache }}",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(querier_cache_added_new_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "New entries added to the cache / {{ cache }}",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(querier_cache_evicted_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache,reason)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Entries evicted from the cache / {{ cache }} / {{ reason }}",
              "range": true,
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(querier_cache_gets_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Get calls to the cache / {{ cache }}",
              "range": true,
              "refId": "D"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(querier_cache_misses_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Missed calls to the cache / {{ cache }}",
              "range": true,
              "refId": "E"
            }
          ],
          "title": "Loki Query Cache - Throughput",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Count",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 41
          },
          "id": 214,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(querier_cache_entries{job=\"$jobs\",instance=\"$host:$instance\"}) by (cache)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Cache Entries / {{ cache }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Query Cache - Entries",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Bytes",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 49
          },
          "id": 216,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(querier_cache_memory_bytes{job=\"$jobs\",instance=\"$host:$instance\"}) by (cache)",
              "hide": false,
              "legendFormat": "Current cache size / {{ cache }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Query Cache - Size",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Count",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 49
          },
          "id": 207,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(cortex_query_frontend_retries_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Request retries",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Cortex Query - Frontend Retries",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 57
          },
          "id": 205,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(cortex_frontend_query_range_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,method,status_code))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ method }} / {{ status_code }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Cortex Query - Frontend Range Request Duration",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Connections",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 57
          },
          "id": 220,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "cortex_query_frontend_queries_in_progress{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Queries in progress",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Cortex Query - Frontend Operations",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Requests",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 65
          },
          "id": 221,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.99\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Number of inflight requests (0.99 quantile)",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.95\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Number of inflight requests (0.95 quantile)",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.9\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Number of inflight requests (0.9 quantile)",
              "range": true,
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.8\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Number of inflight requests (0.8 quantile)",
              "range": true,
              "refId": "D"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.75\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Number of inflight requests (0.75 quantile)",
              "range": true,
              "refId": "E"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.5\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Number of inflight requests (0.5 quantile)",
              "range": true,
              "refId": "F"
            }
          ],
          "title": "Cortex Query - Scheduler Inflight Requests",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 65
          },
          "id": 222,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Time spend by requests in queue before getting picked up by a querier",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Cortex Query - Scheduler Queue Duration",
          "transformations": [],
          "type": "timeseries"
        }
      ],
      "title": "Loki Querier",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 27
      },
      "id": 224,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Heartbeats",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 10
          },
          "id": 225,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(ring_member_heartbeats_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (name)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Heartbeats sent / {{ name }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Ring - Heathbeats",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Tokens",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 10
          },
          "id": 226,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(ring_member_tokens_owned{job=\"$jobs\",instance=\"$host:$instance\"}) by (name)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Tokens owned / {{ name }}",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(ring_member_tokens_to_own{job=\"$jobs\",instance=\"$host:$instance\"}) by (name)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Tokens to be owned / {{ name }}",
              "range": true,
              "refId": "B"
            }
          ],
          "title": "Loki Ring - Tokens",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Heartbeats",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 18
          },
          "id": 227,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(cortex_member_consul_heartbeats_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (name)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Heartbeats sent / {{ name }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Cortex Ring Member - Heathbeats",
          "transformations": [],
          "type": "timeseries"
        }
      ],
      "title": "Loki Ring Metrics",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 28
      },
      "id": 133,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Chunks",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 11
          },
          "id": 135,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_store_series_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (status)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ status }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Store - Series Referenced",
          "transformations": [
            {
              "id": "calculateField",
              "options": {
                "mode": "reduceRow",
                "reduce": {
                  "include": [
                    "debug",
                    "error",
                    "info",
                    "warn"
                  ],
                  "reducer": "sum"
                }
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {
                  "Time": 1,
                  "Total": 0,
                  "grpc": 2,
                  "http": 3
                },
                "renameByName": {
                  "debug": "Debug",
                  "discarded": "Discarded",
                  "error": "Error",
                  "grpc": "gRPC",
                  "http": "HTTP",
                  "info": "Info",
                  "matched": "Matched",
                  "warn": "Warn"
                }
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Chunks",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 11
          },
          "id": 134,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_store_chunks_downloaded_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (status)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ status }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Store - Chunks Downloaded",
          "transformations": [
            {
              "id": "calculateField",
              "options": {
                "mode": "reduceRow",
                "reduce": {
                  "include": [
                    "debug",
                    "error",
                    "info",
                    "warn"
                  ],
                  "reducer": "sum"
                }
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {
                  "Time": 1,
                  "Total": 0,
                  "grpc": 2,
                  "http": 3
                },
                "renameByName": {
                  "debug": "Debug",
                  "discarded": "Discarded",
                  "error": "Error",
                  "grpc": "gRPC",
                  "http": "HTTP",
                  "info": "Info",
                  "matched": "Matched",
                  "warn": "Warn"
                }
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Batch Size",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 19
          },
          "id": 136,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_store_chunks_per_batch_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,status))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ status }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Store - Chunk Batch Size",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Count",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 19
          },
          "id": 165,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_chunk_fetcher_cache_dequeued_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Dequeue from buffer",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_chunk_fetcher_cache_enqueued_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Enqueue to buffer",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_chunk_fetcher_cache_skipped_buffer_full_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Skipped due to buffer full",
              "range": true,
              "refId": "C"
            }
          ],
          "title": "Loki Store - Chunk Fetcher Cache",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Chunks",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 27
          },
          "id": 166,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_chunk_store_fetched_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (user)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Fetched / {{ user }}",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "sum(rate(loki_chunk_store_stored_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (user)",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Stored / {{ user }}",
              "range": true,
              "refId": "B"
            }
          ],
          "title": "Loki Store - Fetched / Stored Chunks",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Size",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "binBps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 27
          },
          "id": 167,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_chunk_store_fetched_chunk_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Fetched / {{ user }}",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(loki_chunk_store_stored_chunk_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Stored / {{ user }}",
              "range": true,
              "refId": "B"
            }
          ],
          "title": "Loki Store - Fetched / Stored Chunk Volume",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "custom": {
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "scaleDistribution": {
                  "type": "linear"
                }
              },
              "links": []
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 35
          },
          "id": 163,
          "maxDataPoints": 50,
          "options": {
            "calculate": false,
            "cellGap": 2,
            "cellValues": {
              "unit": "cps"
            },
            "color": {
              "exponent": 0.5,
              "fill": "red",
              "mode": "scheme",
              "reverse": false,
              "scale": "exponential",
              "scheme": "RdBu",
              "steps": 64
            },
            "exemplars": {
              "color": "rgba(255,0,255,0.7)"
            },
            "filterValues": {
              "le": 0
            },
            "legend": {
              "show": true
            },
            "rowsFrame": {
              "layout": "auto"
            },
            "tooltip": {
              "show": true,
              "yHistogram": false
            },
            "yAxis": {
              "axisLabel": "Bucket Entries",
              "axisPlacement": "left",
              "reverse": false,
              "unit": "none"
            }
          },
          "pluginVersion": "9.3.2",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "exemplar": false,
              "expr": "rate(loki_chunk_store_chunks_per_query_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
              "format": "heatmap",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Store - Chunks Per Query Distribution",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {},
                "renameByName": {
                  "+Inf": "Unlimited",
                  "0.00010239999999999998": "10e-5",
                  "0.0010485759999999998": "10e-4",
                  "0.010485759999999998": "10e-3",
                  "0.10485759999999998": "10e-2",
                  "1.0239999999999999e-05": "10e-6",
                  "1.0239999999999999e-06": "10e-7",
                  "13568.999999999998": "13569",
                  "144.99999999999997": "145",
                  "1536.9999999999998": "1537",
                  "24.999999999999996": "25",
                  "27264.999999999996": "27265",
                  "320.99999999999994": "321",
                  "3200.9999999999995": "3201",
                  "64.99999999999999": "65",
                  "6528.999999999999": "6529",
                  "704.9999999999999": "705",
                  "8.999999999999998": "9",
                  "9.999999999999998e-08": "10e-8",
                  "9.999999999999999e-09": "10e-9",
                  "9.999999999999999e-10": "10e-10"
                }
              }
            }
          ],
          "type": "heatmap"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Bucket Entries",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 35
          },
          "id": 156,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_chunks_per_query_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Store - Chunks Per Query",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "custom": {
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "scaleDistribution": {
                  "type": "linear"
                }
              },
              "links": []
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 43
          },
          "id": 154,
          "maxDataPoints": 50,
          "options": {
            "calculate": false,
            "cellGap": 2,
            "cellValues": {
              "unit": "cps"
            },
            "color": {
              "exponent": 0.5,
              "fill": "red",
              "mode": "scheme",
              "reverse": false,
              "scale": "exponential",
              "scheme": "RdBu",
              "steps": 64
            },
            "exemplars": {
              "color": "rgba(255,0,255,0.7)"
            },
            "filterValues": {
              "le": 0
            },
            "legend": {
              "show": true
            },
            "rowsFrame": {
              "layout": "auto"
            },
            "tooltip": {
              "show": true,
              "yHistogram": false
            },
            "yAxis": {
              "axisLabel": "Bucket Entries",
              "axisPlacement": "left",
              "reverse": false,
              "unit": "none"
            }
          },
          "pluginVersion": "9.3.2",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "exemplar": false,
              "expr": "rate(loki_chunk_store_index_entries_per_chunk_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
              "format": "heatmap",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Store - Index Entries Per Chunk Distribution",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {},
                "renameByName": {
                  "+Inf": "Unlimited",
                  "0.00010239999999999998": "10e-5",
                  "0.0010485759999999998": "10e-4",
                  "0.010485759999999998": "10e-3",
                  "0.10485759999999998": "10e-2",
                  "1.0239999999999999e-05": "10e-6",
                  "1.0239999999999999e-06": "10e-7",
                  "13568.999999999998": "13569",
                  "144.99999999999997": "145",
                  "1536.9999999999998": "1537",
                  "24.999999999999996": "25",
                  "27264.999999999996": "27265",
                  "320.99999999999994": "321",
                  "3200.9999999999995": "3201",
                  "64.99999999999999": "65",
                  "6528.999999999999": "6529",
                  "704.9999999999999": "705",
                  "8.999999999999998": "9",
                  "9.999999999999998e-08": "10e-8",
                  "9.999999999999999e-09": "10e-9",
                  "9.999999999999999e-10": "10e-10"
                }
              }
            }
          ],
          "type": "heatmap"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Bucket Entries",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 43
          },
          "id": 164,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_index_entries_per_chunk_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Store - Index Entries Per Chunk",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "custom": {
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "scaleDistribution": {
                  "type": "linear"
                }
              },
              "links": []
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 51
          },
          "id": 159,
          "maxDataPoints": 50,
          "options": {
            "calculate": false,
            "cellGap": 2,
            "cellValues": {
              "unit": "cps"
            },
            "color": {
              "exponent": 0.5,
              "fill": "red",
              "mode": "scheme",
              "reverse": false,
              "scale": "exponential",
              "scheme": "RdBu",
              "steps": 64
            },
            "exemplars": {
              "color": "rgba(255,0,255,0.7)"
            },
            "filterValues": {
              "le": 0
            },
            "legend": {
              "show": true
            },
            "rowsFrame": {
              "layout": "auto"
            },
            "tooltip": {
              "show": true,
              "yHistogram": false
            },
            "yAxis": {
              "axisLabel": "Bucket Entries",
              "axisPlacement": "left",
              "reverse": false,
              "unit": "none"
            }
          },
          "pluginVersion": "9.3.2",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "exemplar": false,
              "expr": "rate(loki_chunk_store_index_lookups_per_query_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
              "format": "heatmap",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Store - Index Lookups Per Query Distribution",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {},
                "renameByName": {
                  "+Inf": "Unlimited",
                  "0.00010239999999999998": "10e-5",
                  "0.0010485759999999998": "10e-4",
                  "0.010485759999999998": "10e-3",
                  "0.10485759999999998": "10e-2",
                  "1.0239999999999999e-05": "10e-6",
                  "1.0239999999999999e-06": "10e-7",
                  "13568.999999999998": "13569",
                  "144.99999999999997": "145",
                  "1536.9999999999998": "1537",
                  "24.999999999999996": "25",
                  "27264.999999999996": "27265",
                  "320.99999999999994": "321",
                  "3200.9999999999995": "3201",
                  "64.99999999999999": "65",
                  "6528.999999999999": "6529",
                  "704.9999999999999": "705",
                  "8.999999999999998": "9",
                  "9.999999999999998e-08": "10e-8",
                  "9.999999999999999e-09": "10e-9",
                  "9.999999999999999e-10": "10e-10"
                }
              }
            }
          ],
          "type": "heatmap"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Bucket Entries",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 51
          },
          "id": 160,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_index_lookups_per_query_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Store - Index Lookups Per Query",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "custom": {
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "scaleDistribution": {
                  "type": "linear"
                }
              },
              "links": []
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 59
          },
          "id": 157,
          "maxDataPoints": 50,
          "options": {
            "calculate": false,
            "cellGap": 2,
            "cellValues": {
              "unit": "cps"
            },
            "color": {
              "exponent": 0.5,
              "fill": "red",
              "mode": "scheme",
              "reverse": false,
              "scale": "exponential",
              "scheme": "RdBu",
              "steps": 64
            },
            "exemplars": {
              "color": "rgba(255,0,255,0.7)"
            },
            "filterValues": {
              "le": 0
            },
            "legend": {
              "show": true
            },
            "rowsFrame": {
              "layout": "auto"
            },
            "tooltip": {
              "show": true,
              "yHistogram": false
            },
            "yAxis": {
              "axisLabel": "Bucket Entries",
              "axisPlacement": "left",
              "reverse": false,
              "unit": "none"
            }
          },
          "pluginVersion": "9.3.2",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "exemplar": false,
              "expr": "rate(loki_chunk_store_series_pre_intersection_per_query_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
              "format": "heatmap",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Store - Series (Pre Intersection) Per Chunk Distribution",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {},
                "renameByName": {
                  "+Inf": "Unlimited",
                  "0.00010239999999999998": "10e-5",
                  "0.0010485759999999998": "10e-4",
                  "0.010485759999999998": "10e-3",
                  "0.10485759999999998": "10e-2",
                  "1.0239999999999999e-05": "10e-6",
                  "1.0239999999999999e-06": "10e-7",
                  "13568.999999999998": "13569",
                  "144.99999999999997": "145",
                  "1536.9999999999998": "1537",
                  "24.999999999999996": "25",
                  "27264.999999999996": "27265",
                  "320.99999999999994": "321",
                  "3200.9999999999995": "3201",
                  "64.99999999999999": "65",
                  "6528.999999999999": "6529",
                  "704.9999999999999": "705",
                  "8.999999999999998": "9",
                  "9.999999999999998e-08": "10e-8",
                  "9.999999999999999e-09": "10e-9",
                  "9.999999999999999e-10": "10e-10"
                }
              }
            }
          ],
          "type": "heatmap"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Bucket Entries",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 59
          },
          "id": 158,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_series_pre_intersection_per_query_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Store - Series (Pre Intersection) Per Chunk",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "custom": {
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "scaleDistribution": {
                  "type": "linear"
                }
              },
              "links": []
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 67
          },
          "id": 161,
          "maxDataPoints": 50,
          "options": {
            "calculate": false,
            "cellGap": 2,
            "cellValues": {
              "unit": "cps"
            },
            "color": {
              "exponent": 0.5,
              "fill": "red",
              "mode": "scheme",
              "reverse": false,
              "scale": "exponential",
              "scheme": "RdBu",
              "steps": 64
            },
            "exemplars": {
              "color": "rgba(255,0,255,0.7)"
            },
            "filterValues": {
              "le": 0
            },
            "legend": {
              "show": true
            },
            "rowsFrame": {
              "layout": "auto"
            },
            "tooltip": {
              "show": true,
              "yHistogram": false
            },
            "yAxis": {
              "axisLabel": "Bucket Entries",
              "axisPlacement": "left",
              "reverse": false,
              "unit": "none"
            }
          },
          "pluginVersion": "9.3.2",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "exemplar": false,
              "expr": "rate(loki_chunk_store_series_post_intersection_per_query_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
              "format": "heatmap",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Store - Series (Post Intersection) Per Chunk Distribution",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {},
                "renameByName": {
                  "+Inf": "Unlimited",
                  "0.00010239999999999998": "10e-5",
                  "0.0010485759999999998": "10e-4",
                  "0.010485759999999998": "10e-3",
                  "0.10485759999999998": "10e-2",
                  "1.0239999999999999e-05": "10e-6",
                  "1.0239999999999999e-06": "10e-7",
                  "13568.999999999998": "13569",
                  "144.99999999999997": "145",
                  "1536.9999999999998": "1537",
                  "24.999999999999996": "25",
                  "27264.999999999996": "27265",
                  "320.99999999999994": "321",
                  "3200.9999999999995": "3201",
                  "64.99999999999999": "65",
                  "6528.999999999999": "6529",
                  "704.9999999999999": "705",
                  "8.999999999999998": "9",
                  "9.999999999999998e-08": "10e-8",
                  "9.999999999999999e-09": "10e-9",
                  "9.999999999999999e-10": "10e-10"
                }
              }
            }
          ],
          "type": "heatmap"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Bucket Entries",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 67
          },
          "id": 162,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_series_post_intersection_per_query_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Store - Series (Post Intersection) Per Chunk",
          "transformations": [],
          "type": "timeseries"
        }
      ],
      "title": "Loki Store Metrics",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 29
      },
      "id": 170,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Errors",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 12
          },
          "id": 171,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_table_manager_create_failures{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Table create failures",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "loki_table_manager_delete_failures{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Table delete failures",
              "range": true,
              "refId": "B"
            }
          ],
          "title": "Loki Table Manager - Errors",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Time",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 12
          },
          "id": 172,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "histogram_quantile(0.95, sum(rate(loki_table_manager_sync_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,operation,status_code))",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "{{ operation }} / {{ status_code }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Loki Table Manager - Sync Duration",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)\\.(.*)",
                "renamePattern": "$1 $2"
              }
            }
          ],
          "type": "timeseries"
        }
      ],
      "title": "Loki Table Manager Metrics",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 30
      },
      "id": 83,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Count",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 13
          },
          "id": 168,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_goroutines{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Goroutines",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_threads{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Threads",
              "range": true,
              "refId": "B"
            }
          ],
          "title": "Go - Goroutines / Threads",
          "transformations": [
            {
              "id": "renameByRegex",
              "options": {
                "regex": "(.*)fake$",
                "renamePattern": "$1Default Tenant"
              }
            }
          ],
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Avg. Latency",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 13
          },
          "id": 91,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "exemplar": false,
              "expr": "rate(go_sched_latencies_seconds_sum{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_sched_latencies_seconds_count{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Average Goroutine latency",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Go - Goroutines Latency",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "custom": {
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "scaleDistribution": {
                  "type": "linear"
                }
              },
              "links": []
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 21
          },
          "id": 94,
          "maxDataPoints": 50,
          "options": {
            "calculate": false,
            "cellGap": 2,
            "cellValues": {
              "unit": "cps"
            },
            "color": {
              "exponent": 0.5,
              "fill": "red",
              "mode": "scheme",
              "reverse": false,
              "scale": "exponential",
              "scheme": "RdBu",
              "steps": 64
            },
            "exemplars": {
              "color": "rgba(255,0,255,0.7)"
            },
            "filterValues": {
              "le": 1e-9
            },
            "legend": {
              "show": true
            },
            "rowsFrame": {
              "layout": "auto"
            },
            "tooltip": {
              "show": true,
              "yHistogram": false
            },
            "yAxis": {
              "axisLabel": "Bucket Latency",
              "axisPlacement": "left",
              "reverse": false,
              "unit": "s"
            }
          },
          "pluginVersion": "9.3.2",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "exemplar": false,
              "expr": "rate(go_sched_latencies_seconds_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
              "format": "heatmap",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Go - Goroutines Latency Distribution",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {},
                "renameByName": {
                  "+Inf": "Unlimited",
                  "0.00010239999999999998": "10e-5",
                  "0.0010485759999999998": "10e-4",
                  "0.010485759999999998": "10e-3",
                  "0.10485759999999998": "10e-2",
                  "1.0239999999999999e-05": "10e-6",
                  "1.0239999999999999e-06": "10e-7",
                  "13568.999999999998": "13569",
                  "144.99999999999997": "145",
                  "1536.9999999999998": "1537",
                  "24.999999999999996": "25",
                  "27264.999999999996": "27265",
                  "320.99999999999994": "321",
                  "3200.9999999999995": "3201",
                  "64.99999999999999": "65",
                  "6528.999999999999": "6529",
                  "704.9999999999999": "705",
                  "8.999999999999998": "9",
                  "9.999999999999998e-08": "10e-8",
                  "9.999999999999999e-09": "10e-9",
                  "9.999999999999999e-10": "10e-10"
                }
              }
            }
          ],
          "type": "heatmap"
        }
      ],
      "title": "Go General Info",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 31
      },
      "id": 104,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Bytes",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 14
          },
          "id": 92,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_memory_classes_heap_free_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "interval": "$interval",
              "legendFormat": "Free memory",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_memory_classes_heap_objects_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Allocated objects memory",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_memory_classes_heap_released_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Released memory",
              "range": true,
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_memory_classes_heap_stacks_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Stack reserved memory",
              "range": true,
              "refId": "D"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_memory_classes_heap_unused_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Reserved unused memory",
              "range": true,
              "refId": "E"
            }
          ],
          "title": "Go - Memory Heap",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Bytes",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 14
          },
          "id": 93,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_memory_classes_metadata_mcache_free_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "interval": "$interval",
              "legendFormat": "Free mcache memory",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_memory_classes_metadata_mcache_inuse_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Used mcache memory",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_memory_classes_metadata_mspan_free_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Free mspan memory",
              "range": true,
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_memory_classes_metadata_mspan_inuse_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Used mspan memory",
              "range": true,
              "refId": "D"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_memory_classes_metadata_other_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Used other metadata memory",
              "range": true,
              "refId": "E"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_memory_classes_other_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Used other memory",
              "range": true,
              "refId": "F"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_memory_classes_profiling_buckets_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Used profiling bucket memory",
              "range": true,
              "refId": "G"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "go_memory_classes_total_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": true,
              "interval": "$interval",
              "legendFormat": "Used total memory",
              "range": true,
              "refId": "H"
            }
          ],
          "title": "Go - Memory Metadata",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Avg. Object Size",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 22
          },
          "id": 95,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "exemplar": false,
              "expr": "rate(go_gc_heap_allocs_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_gc_heap_allocs_objects_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Average object size allocated to the heap by the application",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "exemplar": false,
              "expr": "rate(go_gc_heap_frees_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_gc_heap_frees_objects_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "instant": false,
              "interval": "$interval",
              "legendFormat": "Average object size freed from the heap by the GC",
              "range": true,
              "refId": "B"
            }
          ],
          "title": "Go - Heap Object Allocation / Free Sizes",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "custom": {
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "scaleDistribution": {
                  "type": "linear"
                }
              },
              "links": []
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 22
          },
          "id": 89,
          "maxDataPoints": 50,
          "options": {
            "calculate": false,
            "cellGap": 2,
            "cellValues": {
              "unit": "cps"
            },
            "color": {
              "exponent": 0.5,
              "fill": "red",
              "mode": "scheme",
              "reverse": false,
              "scale": "exponential",
              "scheme": "PuBuGn",
              "steps": 64
            },
            "exemplars": {
              "color": "rgba(255,0,255,0.7)"
            },
            "filterValues": {
              "le": 1e-9
            },
            "legend": {
              "show": true
            },
            "rowsFrame": {
              "layout": "auto"
            },
            "tooltip": {
              "show": true,
              "yHistogram": false
            },
            "yAxis": {
              "axisLabel": "Bucket Size",
              "axisPlacement": "left",
              "reverse": false,
              "unit": "bytes"
            }
          },
          "pluginVersion": "9.3.2",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "exemplar": false,
              "expr": "rate(go_gc_heap_frees_by_size_bytes_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
              "format": "heatmap",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Go - Heap Free Distribution",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {},
                "renameByName": {
                  "+Inf": "Unlimited",
                  "13568.999999999998": "13569",
                  "144.99999999999997": "145",
                  "1536.9999999999998": "1537",
                  "24.999999999999996": "25",
                  "27264.999999999996": "27265",
                  "320.99999999999994": "321",
                  "3200.9999999999995": "3201",
                  "64.99999999999999": "65",
                  "6528.999999999999": "6529",
                  "704.9999999999999": "705",
                  "8.999999999999998": "9"
                }
              }
            }
          ],
          "type": "heatmap"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "custom": {
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "scaleDistribution": {
                  "type": "linear"
                }
              },
              "links": []
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 30
          },
          "id": 88,
          "maxDataPoints": 50,
          "options": {
            "calculate": false,
            "cellGap": 2,
            "cellValues": {
              "unit": "cps"
            },
            "color": {
              "exponent": 0.5,
              "fill": "red",
              "mode": "scheme",
              "reverse": false,
              "scale": "exponential",
              "scheme": "RdBu",
              "steps": 64
            },
            "exemplars": {
              "color": "rgba(255,0,255,0.7)"
            },
            "filterValues": {
              "le": 1e-9
            },
            "legend": {
              "show": true
            },
            "rowsFrame": {
              "layout": "auto"
            },
            "tooltip": {
              "show": true,
              "yHistogram": false
            },
            "yAxis": {
              "axisLabel": "Bucket Size",
              "axisPlacement": "left",
              "reverse": false,
              "unit": "bytes"
            }
          },
          "pluginVersion": "9.3.2",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "exemplar": false,
              "expr": "rate(go_gc_heap_allocs_by_size_bytes_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
              "format": "heatmap",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Go - Heap Allocation Distribution",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {},
                "renameByName": {
                  "+Inf": "Unlimited",
                  "13568.999999999998": "13569",
                  "144.99999999999997": "145",
                  "1536.9999999999998": "1537",
                  "24.999999999999996": "25",
                  "27264.999999999996": "27265",
                  "320.99999999999994": "321",
                  "3200.9999999999995": "3201",
                  "64.99999999999999": "65",
                  "6528.999999999999": "6529",
                  "704.9999999999999": "705",
                  "8.999999999999998": "9"
                }
              }
            }
          ],
          "type": "heatmap"
        }
      ],
      "title": "Go Memory Metrics",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 32
      },
      "id": 106,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Cycles",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "cps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 15
          },
          "id": 84,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(go_gc_cycles_total_gc_cycles_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Count of all completed GC cycles",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(go_gc_cycles_automatic_gc_cycles_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Count of completed GC cycles generated by the Go runtime",
              "range": true,
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(go_gc_cycles_forced_gc_cycles_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "interval": "$interval",
              "legendFormat": "Count of completed GC cycles forced by the application.",
              "range": true,
              "refId": "C"
            }
          ],
          "title": "Go - Garbage Collection Cycles",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Avg. Pause Duration",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 15
          },
          "id": 85,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(go_gc_duration_seconds_sum{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_gc_duration_seconds_count{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Average GC cycle pause duration",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(go_gc_pauses_seconds_sum{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_gc_pauses_seconds_count{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Average GC cycle halt duration",
              "range": true,
              "refId": "B"
            }
          ],
          "title": "Go - Garbage Collection Pause Duration",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "custom": {
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "scaleDistribution": {
                  "type": "linear"
                }
              },
              "links": []
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 23
          },
          "id": 90,
          "maxDataPoints": 50,
          "options": {
            "calculate": false,
            "cellGap": 2,
            "cellValues": {
              "unit": "cps"
            },
            "color": {
              "exponent": 0.5,
              "fill": "red",
              "mode": "scheme",
              "reverse": false,
              "scale": "exponential",
              "scheme": "BrBG",
              "steps": 64
            },
            "exemplars": {
              "color": "rgba(255,0,255,0.7)"
            },
            "filterValues": {
              "le": 1e-9
            },
            "legend": {
              "show": true
            },
            "rowsFrame": {
              "layout": "auto"
            },
            "tooltip": {
              "show": true,
              "yHistogram": false
            },
            "yAxis": {
              "axisLabel": "Bucket Duration",
              "axisPlacement": "left",
              "reverse": false,
              "unit": "s"
            }
          },
          "pluginVersion": "9.3.2",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "exemplar": false,
              "expr": "rate(go_gc_pauses_seconds_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
              "format": "heatmap",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ le }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Go - Garbage Collection Pauses Distribution",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {},
                "renameByName": {
                  "+Inf": "Unlimited",
                  "0.00010239999999999998": "10e-5",
                  "0.0010485759999999998": "10e-4",
                  "0.010485759999999998": "10e-3",
                  "0.10485759999999998": "10e-2",
                  "1.0239999999999999e-05": "10e-6",
                  "1.0239999999999999e-06": "10e-7",
                  "9.999999999999998e-08": "10e-8",
                  "9.999999999999999e-09": "10e-9",
                  "9.999999999999999e-10": "10e-10"
                }
              }
            }
          ],
          "type": "heatmap"
        }
      ],
      "title": "Go GC Metrics",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 33
      },
      "id": 75,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Seconds",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 16
          },
          "id": 78,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "rate(process_cpu_seconds_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Total user and system CPU time spent in seconds",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Process - CPU seconds",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Bytes",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 16
          },
          "id": 79,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "process_virtual_memory_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "interval": "$interval",
              "legendFormat": "Virtual memory size",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "process_resident_memory_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Resident memory size",
              "range": true,
              "refId": "B"
            }
          ],
          "title": "Process - Memory",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "Descriptors",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 24
          },
          "id": 77,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean",
                "last"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "process_max_fds{job=\"$jobs\",instance=\"$host:$instance\"}",
              "hide": false,
              "interval": "$interval",
              "legendFormat": "Maximum number of open file descriptors",
              "range": true,
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "editorMode": "code",
              "expr": "process_open_fds{job=\"$jobs\",instance=\"$host:$instance\"}",
              "interval": "$interval",
              "legendFormat": "Number of open file descriptors",
              "range": true,
              "refId": "B"
            }
          ],
          "title": "Process - File descriptors",
          "type": "timeseries"
        }
      ],
      "title": "Process Info",
      "type": "row"
    }
  ],
  "refresh": "1m",
  "schemaVersion": 37,
  "style": "dark",
  "tags": [
    "loki"
  ],
  "templating": {
    "list": [
      {
        "current": {
          "selected": false,
          "text": "default",
          "value": "default"
        },
        "hide": 2,
        "includeAll": false,
        "label": "Datasource",
        "multi": false,
        "name": "DS_PROMETHEUS",
        "options": [],
        "query": "prometheus",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "type": "datasource"
      },
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "label_values(loki_build_info,job)",
        "hide": 0,
        "includeAll": false,
        "label": "Job",
        "multi": false,
        "name": "jobs",
        "options": [],
        "query": {
          "query": "label_values(loki_build_info,job)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query",
        "useTags": false
      },
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "up{job=~\"$jobs\"}",
        "hide": 0,
        "includeAll": false,
        "label": "Host",
        "multi": false,
        "name": "host",
        "options": [],
        "query": {
          "query": "up{job=~\"$jobs\"}",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": ".*instance=\"(.*?):.*\".*",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query"
      },
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "up{job=~\"$jobs\"}",
        "hide": 0,
        "includeAll": false,
        "label": "Instance",
        "multi": false,
        "name": "instance",
        "options": [],
        "query": {
          "query": "up{job=~\"$jobs\"}",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": ".*instance=\".*:(.*?)\".*",
        "skipUrlSync": false,
        "sort": 3,
        "type": "query"
      },
      {
        "auto": true,
        "auto_count": 30,
        "auto_min": "10s",
        "current": {
          "selected": false,
          "text": "30s",
          "value": "30s"
        },
        "hide": 0,
        "label": "Interval",
        "name": "interval",
        "options": [
          {
            "selected": false,
            "text": "auto",
            "value": "$__auto_interval_interval"
          },
          {
            "selected": true,
            "text": "30s",
            "value": "30s"
          },
          {
            "selected": false,
            "text": "1m",
            "value": "1m"
          },
          {
            "selected": false,
            "text": "5m",
            "value": "5m"
          },
          {
            "selected": false,
            "text": "1h",
            "value": "1h"
          },
          {
            "selected": false,
            "text": "6h",
            "value": "6h"
          },
          {
            "selected": false,
            "text": "1d",
            "value": "1d"
          }
        ],
        "query": "30s,1m,5m,1h,6h,1d",
        "refresh": 2,
        "skipUrlSync": false,
        "type": "interval"
      }
    ]
  },
  "time": {
    "from": "now-24h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "browser",
  "title": "Loki Metrics Dashboard",
  "uid": "",
  "version": 13,
  "weekStart": "",
  "gnetId": 17781
}