Elasticsearch - History Stats
Grafana Dashboards for Elasticsearch using Prometheus Datasource. Dashboard for History Stats.
# grafana-dashboards-elasticsearch

Grafana Dashboards for Elasticsearch using Prometheus Datasource.

Please visit my GitHub at [grafana-dashboards-elasticsearch](https://github.com/huynhsamha/grafana-dashboards-elasticsearch) for all dashboards related to this dashboard.

## Requirements

- **Datasource**: [Prometheus](https://prometheus.io/)
- **Exporter**: [Elasticsearch Exporter](https://github.com/justwatchcom/elasticsearch_exporter)

## Quickstart

My dashboards require **Prometheus** Datasource for _mananging metrics_, and **Elasticsearch Exporter** for _crawling metrics_ from Elasticsearch Cluster and _exposing_ these metrics. Prometheus is scheduled for _pulling metrics_ from Elasticsearch Exporter.

I published my dashboards to [Grafana Dashboards](https://grafana.com/grafana/dashboards) so you can easily import these dashboards to your Grafana.

- `13071`: [Dashboard for Elasticsearch Cluster Stats](https://grafana.com/grafana/dashboards/13071)
- `13073`: [Dashboard for Elasticsearch Node Stats](https://grafana.com/grafana/dashboards/13073)
- `13072`: [Dashboard for Elasticsearch Index Stats](https://grafana.com/grafana/dashboards/13072)
- `13074`: [Dashboard for Elasticsearch History Stats](https://grafana.com/grafana/dashboards/13074)

Many thanks to https://grafana.com/grafana/dashboards/6483 and https://grafana.com/grafana/dashboards/2322


## Screenshots

This is a screenshot of this dashboard. To view all screenshots, you can visit my [GitHub](https://github.com/huynhsamha/grafana-dashboards-elasticsearch) for detail.

![Cluster Stats Dashboard Screenshot](https://raw.githubusercontent.com/huynhsamha/grafana-dashboards-elasticsearch/master/screenshots/es-history-stats-dashboard.jpg)


## Changelogs

- **30/09/2020**: first release with 4 dashboards, including [Cluster Level Stats](https://grafana.com/grafana/dashboards/13071), [Node Stats](https://grafana.com/grafana/dashboards/13073), [Index Stats](https://grafana.com/grafana/dashboards/13072) and [History Stats](https://grafana.com/grafana/dashboards/13074)

{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "3.1"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "singlestat",
      "name": "Singlestat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "description": "Grafana Dashboards for Elasticsearch using Prometheus Datasource. Dashboard for History Stats.",
  "editable": true,
  "gnetId": 13074,
  "graphTooltip": 1,
  "id": 79,
  "iteration": 1601026115630,
  "links": [
    {
      "asDropdown": true,
      "icon": "external link",
      "tags": [
        "elasticsearch"
      ],
      "title": "ES Dashboards",
      "type": "dashboards"
    }
  ],
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 231,
      "panels": [],
      "title": "Cluster Level - $cluster",
      "type": "row"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "description": "",
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 1
      },
      "id": 213,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": false,
        "max": true,
        "min": true,
        "rightSide": true,
        "show": true,
        "sort": "avg",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "alias": "ProcRate - Today",
          "bars": true,
          "color": "rgb(19, 81, 164)",
          "legend": false,
          "lines": false,
          "yaxis": 2,
          "zindex": -3
        },
        {
          "alias": "Today",
          "color": "#FFEE52"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "sum (rate(elasticsearch_indices_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\"}[$interval]))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Today",
          "refId": "A"
        },
        {
          "expr": "1 / (\nsum (rate(elasticsearch_indices_indexing_index_time_seconds_total{instance=~\"$instance\",cluster=\"$cluster\"}[$interval])) / \nsum (rate(elasticsearch_indices_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\"}[$interval]))\n)",
          "hide": false,
          "legendFormat": "ProcRate - Today",
          "refId": "B"
        },
        {
          "expr": "label_replace(sum (rate(elasticsearch_indices_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\"}[$interval] offset $day)), \"time\", \"$day\", \"\", \".*\") or\nlabel_replace(sum (rate(elasticsearch_indices_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\"}[$interval] offset $day1)), \"time\", \"$day1\", \"\", \".*\") or\nlabel_replace(sum (rate(elasticsearch_indices_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\"}[$interval] offset $day2)), \"time\", \"$day2\", \"\", \".*\")",
          "legendFormat": "Last {{ time }}",
          "refId": "C"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Cluster Indexing - $cluster",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "decimals": null,
          "format": "reqps",
          "label": "ReqRate",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "format": "reqps",
          "label": "ProcRate",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "description": "",
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 8
      },
      "id": 234,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": false,
        "max": true,
        "min": true,
        "rightSide": true,
        "show": true,
        "sort": "avg",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "maxPerRow": 3,
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "repeatDirection": "h",
      "seriesOverrides": [
        {
          "alias": "ProcRate - Today",
          "bars": true,
          "color": "rgb(17, 79, 185)",
          "legend": false,
          "lines": false,
          "yaxis": 2,
          "zindex": -3
        },
        {
          "alias": "Today",
          "color": "#FFEE52"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "sum (rate(elasticsearch_indices_search_query_total{instance=~\"$instance\",cluster=\"$cluster\"}[$interval]))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Today",
          "refId": "B"
        },
        {
          "expr": "1 / (\n(sum(rate(elasticsearch_indices_search_query_time_seconds{instance=~\"$instance\",cluster=\"$cluster\"}[$interval]))\n+ sum(rate(elasticsearch_indices_search_fetch_time_seconds{instance=~\"$instance\",cluster=\"$cluster\"}[$interval]))\n+ sum(rate(elasticsearch_indices_search_scroll_time_seconds{instance=~\"$instance\",cluster=\"$cluster\"}[$interval]))\n+ sum(rate(elasticsearch_indices_search_suggest_time_seconds{instance=~\"$instance\",cluster=\"$cluster\"}[$interval]))\n) /\nsum(rate(elasticsearch_indices_search_query_total{instance=~\"$instance\",cluster=\"$cluster\"}[$interval]))\n)",
          "legendFormat": "ProcRate - Today",
          "refId": "A"
        },
        {
          "expr": "label_replace(sum (rate(elasticsearch_indices_search_query_total{instance=~\"$instance\",cluster=\"$cluster\"}[$interval] offset $day)), \"time\", \"$day\", \"\", \".*\") or\nlabel_replace(sum (rate(elasticsearch_indices_search_query_total{instance=~\"$instance\",cluster=\"$cluster\"}[$interval] offset $day1)), \"time\", \"$day1\", \"\", \".*\") or\nlabel_replace(sum (rate(elasticsearch_indices_search_query_total{instance=~\"$instance\",cluster=\"$cluster\"}[$interval] offset $day2)), \"time\", \"$day2\", \"\", \".*\")",
          "legendFormat": "Last {{ time }}",
          "refId": "C"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Cluster Search Query - $cluster",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "decimals": null,
          "format": "reqps",
          "label": "ReqRate",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "format": "reqps",
          "label": "ProcRate",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 15
      },
      "id": 236,
      "panels": [],
      "repeat": null,
      "title": "Node Level - $name",
      "type": "row"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "description": "",
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 16
      },
      "id": 242,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": false,
        "max": true,
        "min": true,
        "rightSide": true,
        "show": true,
        "sort": "avg",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "alias": "ProcRate - Today",
          "bars": true,
          "color": "rgb(19, 81, 164)",
          "legend": false,
          "lines": false,
          "yaxis": 2,
          "zindex": -3
        },
        {
          "alias": "Today",
          "color": "#FFEE52"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "rate(elasticsearch_indices_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Today",
          "refId": "A"
        },
        {
          "expr": "1 / (\nrate(elasticsearch_indices_indexing_index_time_seconds_total{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval]) / rate(elasticsearch_indices_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval])\n)",
          "legendFormat": "ProcRate - Today",
          "refId": "B"
        },
        {
          "expr": "label_replace(rate(elasticsearch_indices_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval] offset $day), \"time\", \"$day\", \"\", \".*\") or\nlabel_replace(rate(elasticsearch_indices_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval] offset $day1), \"time\", \"$day1\", \"\", \".*\") or\nlabel_replace(rate(elasticsearch_indices_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval] offset $day2), \"time\", \"$day2\", \"\", \".*\")",
          "legendFormat": "Last {{ time }}",
          "refId": "C"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Node Indexing - $name",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "decimals": null,
          "format": "reqps",
          "label": "ReqRate",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "format": "reqps",
          "label": "ProcRate",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "description": "",
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 23
      },
      "id": 244,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": false,
        "max": true,
        "min": true,
        "rightSide": true,
        "show": true,
        "sort": "avg",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "maxPerRow": 3,
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "repeatDirection": "h",
      "seriesOverrides": [
        {
          "alias": "ProcRate - Today",
          "bars": true,
          "color": "rgb(17, 79, 185)",
          "legend": false,
          "lines": false,
          "yaxis": 2,
          "zindex": -3
        },
        {
          "alias": "Today",
          "color": "#FFEE52"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "rate(elasticsearch_indices_search_query_total{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Today",
          "refId": "B"
        },
        {
          "expr": "1 / (\n(rate(elasticsearch_indices_search_query_time_seconds{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval])\n+ rate(elasticsearch_indices_search_fetch_time_seconds{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval])\n+ rate(elasticsearch_indices_search_scroll_time_seconds{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval])\n+ rate(elasticsearch_indices_search_suggest_time_seconds{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval])\n) /\nrate(elasticsearch_indices_search_query_total{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval])\n)",
          "legendFormat": "ProcRate - Today",
          "refId": "A"
        },
        {
          "expr": "label_replace(rate(elasticsearch_indices_search_query_total{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval] offset $day), \"time\", \"$day\", \"\", \".*\") or\nlabel_replace(rate(elasticsearch_indices_search_query_total{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval] offset $day1), \"time\", \"$day1\", \"\", \".*\") or\nlabel_replace(rate(elasticsearch_indices_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\",name=~\"$name\"}[$interval] offset $day2), \"time\", \"$day2\", \"\", \".*\")",
          "legendFormat": "Last {{ time }}",
          "refId": "C"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Node Search Query - $name",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "decimals": null,
          "format": "reqps",
          "label": "ReqRate",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "format": "reqps",
          "label": "ProcRate",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 30
      },
      "id": 241,
      "panels": [],
      "title": "Index Level - $index",
      "type": "row"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "description": "",
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 31
      },
      "id": 243,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": false,
        "max": true,
        "min": true,
        "rightSide": true,
        "show": true,
        "sort": "avg",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "alias": "ProcRate - Today",
          "bars": true,
          "color": "rgb(19, 81, 164)",
          "legend": false,
          "lines": false,
          "yaxis": 2,
          "zindex": -3
        },
        {
          "alias": "Today",
          "color": "#FFEE52"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "rate(elasticsearch_index_stats_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Today",
          "refId": "A"
        },
        {
          "expr": "1 / (\nrate(elasticsearch_index_stats_indexing_index_time_seconds_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval]) / rate(elasticsearch_index_stats_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval])\n)",
          "legendFormat": "ProcRate - Today",
          "refId": "B"
        },
        {
          "expr": "label_replace(rate(elasticsearch_index_stats_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval] offset $day), \"time\", \"$day\", \"\", \".*\") or\nlabel_replace(rate(elasticsearch_index_stats_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval] offset $day1), \"time\", \"$day1\", \"\", \".*\") or\nlabel_replace(rate(elasticsearch_index_stats_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval] offset $day2), \"time\", \"$day2\", \"\", \".*\")",
          "legendFormat": "Last {{ time }}",
          "refId": "C"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Indexing - $index",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "decimals": null,
          "format": "reqps",
          "label": "ReqRate",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "format": "reqps",
          "label": "ProcRate",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "description": "",
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 38
      },
      "id": 245,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": false,
        "max": true,
        "min": true,
        "rightSide": true,
        "show": true,
        "sort": "avg",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "maxPerRow": 3,
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "repeatDirection": "h",
      "seriesOverrides": [
        {
          "alias": "ProcRate - Today",
          "bars": true,
          "color": "rgb(17, 79, 185)",
          "legend": false,
          "lines": false,
          "yaxis": 2,
          "zindex": -3
        },
        {
          "alias": "Today",
          "color": "#FFEE52"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "rate(elasticsearch_index_stats_search_query_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Today",
          "refId": "B"
        },
        {
          "expr": "1 / (\n(rate(elasticsearch_index_stats_search_query_time_seconds_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval])\n+ rate(elasticsearch_index_stats_search_fetch_time_seconds_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval])\n+ rate(elasticsearch_index_stats_search_scroll_time_seconds_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval])\n+ rate(elasticsearch_index_stats_search_suggest_time_seconds_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval])\n) / rate(elasticsearch_index_stats_search_query_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval])\n)",
          "legendFormat": "ProcRate - Today",
          "refId": "A"
        },
        {
          "expr": "label_replace(rate(elasticsearch_index_stats_search_query_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval] offset $day), \"time\", \"$day\", \"\", \".*\") or\nlabel_replace(rate(elasticsearch_index_stats_search_query_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval] offset $day1), \"time\", \"$day1\", \"\", \".*\") or\nlabel_replace(rate(elasticsearch_index_stats_indexing_index_total{instance=~\"$instance\",cluster=\"$cluster\",index=~\"$index\"}[$interval] offset $day2), \"time\", \"$day2\", \"\", \".*\")",
          "legendFormat": "Last {{ time }}",
          "refId": "C"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Search Query - $index",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "decimals": null,
          "format": "reqps",
          "label": "ReqRate",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "format": "reqps",
          "label": "ProcRate",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    }
  ],
  "refresh": "5m",
  "schemaVersion": 20,
  "style": "dark",
  "tags": [
    "elasticsearch",
    "es-cluster-hist"
  ],
  "templating": {
    "list": [
      {
        "allValue": null,
        "current": {
          "text": "",
          "value": ""
        },
        "datasource": "${DS_PROMETHEUS}",
        "definition": "",
        "hide": 0,
        "includeAll": false,
        "label": "Cluster",
        "multi": false,
        "name": "cluster",
        "options": [],
        "query": "label_values(elasticsearch_indices_docs,cluster)",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "tagValuesQuery": null,
        "tags": [],
        "tagsQuery": null,
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {
          "text": "localhost:9115",
          "value": "localhost:9115"
        },
        "datasource": "${DS_PROMETHEUS}",
        "definition": "",
        "hide": 2,
        "includeAll": false,
        "label": "Source of metrics",
        "multi": false,
        "name": "instance",
        "options": [],
        "query": "label_values(elasticsearch_indices_docs{cluster=\"$cluster\", name!=\"\"},instance)",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "tagValuesQuery": null,
        "tags": [],
        "tagsQuery": null,
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {
          "text": "",
          "value": ""
        },
        "datasource": "${DS_PROMETHEUS}",
        "definition": "label_values(elasticsearch_indices_docs{instance=\"$instance\",cluster=\"$cluster\", name!=\"\"},name)",
        "hide": 0,
        "includeAll": false,
        "label": "Node name",
        "multi": false,
        "name": "name",
        "options": [],
        "query": "label_values(elasticsearch_indices_docs{instance=\"$instance\",cluster=\"$cluster\", name!=\"\"},name)",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 2,
        "tagValuesQuery": null,
        "tags": [],
        "tagsQuery": null,
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {
          "text": "",
          "value": ""
        },
        "datasource": "${DS_PROMETHEUS}",
        "definition": "label_values(elasticsearch_index_stats_get_total{instance=\"$instance\",cluster=\"$cluster\"},index)",
        "hide": 0,
        "includeAll": false,
        "label": "Index",
        "multi": false,
        "name": "index",
        "options": [],
        "query": "label_values(elasticsearch_index_stats_get_total{instance=\"$instance\",cluster=\"$cluster\"},index)",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 2,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "auto": true,
        "auto_count": 30,
        "auto_min": "10s",
        "current": {
          "text": "5m",
          "value": "5m"
        },
        "hide": 2,
        "label": "Rate Interval",
        "name": "interval",
        "options": [
          {
            "selected": false,
            "text": "auto",
            "value": "$__auto_interval_interval"
          },
          {
            "selected": true,
            "text": "5m",
            "value": "5m"
          },
          {
            "selected": false,
            "text": "10m",
            "value": "10m"
          },
          {
            "selected": false,
            "text": "30m",
            "value": "30m"
          },
          {
            "selected": false,
            "text": "1h",
            "value": "1h"
          },
          {
            "selected": false,
            "text": "6h",
            "value": "6h"
          },
          {
            "selected": false,
            "text": "12h",
            "value": "12h"
          },
          {
            "selected": false,
            "text": "1d",
            "value": "1d"
          },
          {
            "selected": false,
            "text": "7d",
            "value": "7d"
          },
          {
            "selected": false,
            "text": "14d",
            "value": "14d"
          },
          {
            "selected": false,
            "text": "30d",
            "value": "30d"
          }
        ],
        "query": "5m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
        "refresh": 2,
        "skipUrlSync": false,
        "type": "interval"
      },
      {
        "allValue": null,
        "current": {
          "tags": [],
          "text": "7d",
          "value": "7d"
        },
        "hide": 0,
        "includeAll": false,
        "label": "Day",
        "multi": false,
        "name": "day",
        "options": [
          {
            "selected": false,
            "text": "1d",
            "value": "1d"
          },
          {
            "selected": false,
            "text": "2d",
            "value": "2d"
          },
          {
            "selected": false,
            "text": "3d",
            "value": "3d"
          },
          {
            "selected": false,
            "text": "4d",
            "value": "4d"
          },
          {
            "selected": false,
            "text": "5d",
            "value": "5d"
          },
          {
            "selected": false,
            "text": "6d",
            "value": "6d"
          },
          {
            "selected": true,
            "text": "7d",
            "value": "7d"
          },
          {
            "selected": false,
            "text": "8d",
            "value": "8d"
          },
          {
            "selected": false,
            "text": "9d",
            "value": "9d"
          },
          {
            "selected": false,
            "text": "10d",
            "value": "10d"
          },
          {
            "selected": false,
            "text": "11d",
            "value": "11d"
          },
          {
            "selected": false,
            "text": "12d",
            "value": "12d"
          },
          {
            "selected": false,
            "text": "13d",
            "value": "13d"
          },
          {
            "selected": false,
            "text": "14d",
            "value": "14d"
          },
          {
            "selected": false,
            "text": "15d",
            "value": "15d"
          },
          {
            "selected": false,
            "text": "16d",
            "value": "16d"
          },
          {
            "selected": false,
            "text": "17d",
            "value": "17d"
          },
          {
            "selected": false,
            "text": "18d",
            "value": "18d"
          },
          {
            "selected": false,
            "text": "19d",
            "value": "19d"
          },
          {
            "selected": false,
            "text": "20d",
            "value": "20d"
          },
          {
            "selected": false,
            "text": "21d",
            "value": "21d"
          },
          {
            "selected": false,
            "text": "22d",
            "value": "22d"
          },
          {
            "selected": false,
            "text": "23d",
            "value": "23d"
          },
          {
            "selected": false,
            "text": "24d",
            "value": "24d"
          },
          {
            "selected": false,
            "text": "25d",
            "value": "25d"
          },
          {
            "selected": false,
            "text": "26d",
            "value": "26d"
          },
          {
            "selected": false,
            "text": "27d",
            "value": "27d"
          },
          {
            "selected": false,
            "text": "28d",
            "value": "28d"
          },
          {
            "selected": false,
            "text": "29d",
            "value": "29d"
          },
          {
            "selected": false,
            "text": "30d",
            "value": "30d"
          }
        ],
        "query": "1d,2d,3d,4d,5d,6d,7d,8d,9d,10d,11d,12d,13d,14d,15d,16d,17d,18d,19d,20d,21d,22d,23d,24d,25d,26d,27d,28d,29d,30d",
        "skipUrlSync": false,
        "type": "custom"
      },
      {
        "allValue": null,
        "current": {
          "tags": [],
          "text": "7d",
          "value": "7d"
        },
        "hide": 0,
        "includeAll": false,
        "label": "Day",
        "multi": false,
        "name": "day1",
        "options": [
          {
            "selected": false,
            "text": "1d",
            "value": "1d"
          },
          {
            "selected": false,
            "text": "2d",
            "value": "2d"
          },
          {
            "selected": false,
            "text": "3d",
            "value": "3d"
          },
          {
            "selected": false,
            "text": "4d",
            "value": "4d"
          },
          {
            "selected": false,
            "text": "5d",
            "value": "5d"
          },
          {
            "selected": false,
            "text": "6d",
            "value": "6d"
          },
          {
            "selected": true,
            "text": "7d",
            "value": "7d"
          },
          {
            "selected": false,
            "text": "8d",
            "value": "8d"
          },
          {
            "selected": false,
            "text": "9d",
            "value": "9d"
          },
          {
            "selected": false,
            "text": "10d",
            "value": "10d"
          },
          {
            "selected": false,
            "text": "11d",
            "value": "11d"
          },
          {
            "selected": false,
            "text": "12d",
            "value": "12d"
          },
          {
            "selected": false,
            "text": "13d",
            "value": "13d"
          },
          {
            "selected": false,
            "text": "14d",
            "value": "14d"
          },
          {
            "selected": false,
            "text": "15d",
            "value": "15d"
          },
          {
            "selected": false,
            "text": "16d",
            "value": "16d"
          },
          {
            "selected": false,
            "text": "17d",
            "value": "17d"
          },
          {
            "selected": false,
            "text": "18d",
            "value": "18d"
          },
          {
            "selected": false,
            "text": "19d",
            "value": "19d"
          },
          {
            "selected": false,
            "text": "20d",
            "value": "20d"
          },
          {
            "selected": false,
            "text": "21d",
            "value": "21d"
          },
          {
            "selected": false,
            "text": "22d",
            "value": "22d"
          },
          {
            "selected": false,
            "text": "23d",
            "value": "23d"
          },
          {
            "selected": false,
            "text": "24d",
            "value": "24d"
          },
          {
            "selected": false,
            "text": "25d",
            "value": "25d"
          },
          {
            "selected": false,
            "text": "26d",
            "value": "26d"
          },
          {
            "selected": false,
            "text": "27d",
            "value": "27d"
          },
          {
            "selected": false,
            "text": "28d",
            "value": "28d"
          },
          {
            "selected": false,
            "text": "29d",
            "value": "29d"
          },
          {
            "selected": false,
            "text": "30d",
            "value": "30d"
          }
        ],
        "query": "1d,2d,3d,4d,5d,6d,7d,8d,9d,10d,11d,12d,13d,14d,15d,16d,17d,18d,19d,20d,21d,22d,23d,24d,25d,26d,27d,28d,29d,30d",
        "skipUrlSync": false,
        "type": "custom"
      },
      {
        "allValue": null,
        "current": {
          "tags": [],
          "text": "7d",
          "value": "7d"
        },
        "hide": 0,
        "includeAll": false,
        "label": "Day",
        "multi": false,
        "name": "day2",
        "options": [
          {
            "selected": false,
            "text": "1d",
            "value": "1d"
          },
          {
            "selected": false,
            "text": "2d",
            "value": "2d"
          },
          {
            "selected": false,
            "text": "3d",
            "value": "3d"
          },
          {
            "selected": false,
            "text": "4d",
            "value": "4d"
          },
          {
            "selected": false,
            "text": "5d",
            "value": "5d"
          },
          {
            "selected": false,
            "text": "6d",
            "value": "6d"
          },
          {
            "selected": true,
            "text": "7d",
            "value": "7d"
          },
          {
            "selected": false,
            "text": "8d",
            "value": "8d"
          },
          {
            "selected": false,
            "text": "9d",
            "value": "9d"
          },
          {
            "selected": false,
            "text": "10d",
            "value": "10d"
          },
          {
            "selected": false,
            "text": "11d",
            "value": "11d"
          },
          {
            "selected": false,
            "text": "12d",
            "value": "12d"
          },
          {
            "selected": false,
            "text": "13d",
            "value": "13d"
          },
          {
            "selected": false,
            "text": "14d",
            "value": "14d"
          },
          {
            "selected": false,
            "text": "15d",
            "value": "15d"
          },
          {
            "selected": false,
            "text": "16d",
            "value": "16d"
          },
          {
            "selected": false,
            "text": "17d",
            "value": "17d"
          },
          {
            "selected": false,
            "text": "18d",
            "value": "18d"
          },
          {
            "selected": false,
            "text": "19d",
            "value": "19d"
          },
          {
            "selected": false,
            "text": "20d",
            "value": "20d"
          },
          {
            "selected": false,
            "text": "21d",
            "value": "21d"
          },
          {
            "selected": false,
            "text": "22d",
            "value": "22d"
          },
          {
            "selected": false,
            "text": "23d",
            "value": "23d"
          },
          {
            "selected": false,
            "text": "24d",
            "value": "24d"
          },
          {
            "selected": false,
            "text": "25d",
            "value": "25d"
          },
          {
            "selected": false,
            "text": "26d",
            "value": "26d"
          },
          {
            "selected": false,
            "text": "27d",
            "value": "27d"
          },
          {
            "selected": false,
            "text": "28d",
            "value": "28d"
          },
          {
            "selected": false,
            "text": "29d",
            "value": "29d"
          },
          {
            "selected": false,
            "text": "30d",
            "value": "30d"
          }
        ],
        "query": "1d,2d,3d,4d,5d,6d,7d,8d,9d,10d,11d,12d,13d,14d,15d,16d,17d,18d,19d,20d,21d,22d,23d,24d,25d,26d,27d,28d,29d,30d",
        "skipUrlSync": false,
        "type": "custom"
      }
    ]
  },
  "time": {
    "from": "now/d",
    "to": "now/d"
  },
  "timepicker": {
    "refresh_intervals": [
      "1s",
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "",
  "title": "Elasticsearch - History Stats",
  "uid": "7820e5c4d6ae456aa21390618b6515dd",
  "version": 1
}