Basic Infra Alert
Alerts for Linux Nodes using prometheus, node_exporter and blackbox. You can have alerts for Disk space, CPU and Memory.
Alerts for Linux Nodes using prometheus and node_exporter. You can have alerts for Disk space, CPU and Memory. Also added a log of alerts and alert status.
{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS-CHANNEL",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    },
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "7.2.0"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      }
    ]
  },
  "description": "Alerts for Linux Nodes using prometheus, node_exporter and blackbox. You can have alerts for Disk space, CPU and Memory.",
  "editable": true,
  "gnetId": 16819,
  "graphTooltip": 0,
  "id": null,
  "links": [],
  "panels": [
    {
      "collapsed": false,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 18,
      "panels": [],
      "title": "服务器UP/DOWN",
      "type": "row"
    },
    {
      "cacheTimeout": null,
      "datasource": "${DS_PROMETHEUS-CHANNEL}",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": "left",
            "displayMode": "color-background",
            "filterable": false
          },
          "mappings": [
            {
              "from": "",
              "id": 0,
              "text": "DOWN",
              "to": "",
              "type": 1,
              "value": "0"
            },
            {
              "from": "",
              "id": 1,
              "text": "UP",
              "to": "",
              "type": 1,
              "value": "1"
            },
            {
              "from": "",
              "id": 2,
              "text": "DOWN",
              "to": "",
              "type": 1,
              "value": "null"
            }
          ],
          "max": 1,
          "min": 0,
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              },
              {
                "color": "red",
                "value": 0
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          },
          "unit": "none"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Host"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 313
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 7,
        "x": 0,
        "y": 1
      },
      "id": 20,
      "interval": null,
      "links": [],
      "options": {
        "showHeader": true,
        "sortBy": [
          {
            "desc": false,
            "displayName": "Status"
          }
        ]
      },
      "pluginVersion": "7.2.0",
      "targets": [
        {
          "expr": "probe_success{job=~\"host up/down\"}",
          "format": "table",
          "instant": true,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "服务器UP/DOWN",
      "transformations": [
        {
          "id": "filterFieldsByName",
          "options": {
            "include": {
              "names": [
                "instance",
                "Value"
              ]
            }
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Value": "Status",
              "instance": "Host"
            }
          }
        }
      ],
      "type": "table"
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                1
              ],
              "type": "lt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "1m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "last"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "alerting",
        "for": "20s",
        "frequency": "10s",
        "handler": 1,
        "message": "有服务器挂了！",
        "name": "服务器UP/DOWN预警",
        "noDataState": "alerting",
        "notifications": [
          {
            "uid": "fnQuJ0Cnk"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "cacheTimeout": null,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS-CHANNEL}",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": "left",
            "displayMode": "color-background",
            "filterable": false
          },
          "mappings": [
            {
              "from": "",
              "id": 0,
              "text": "DOWN",
              "to": "",
              "type": 1,
              "value": "0"
            },
            {
              "from": "",
              "id": 1,
              "text": "UP",
              "to": "",
              "type": 1,
              "value": "1"
            },
            {
              "from": "",
              "id": 2,
              "text": "DOWN",
              "to": "",
              "type": 1,
              "value": "null"
            }
          ],
          "max": 1,
          "min": 0,
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              },
              {
                "color": "red",
                "value": 0
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 17,
        "x": 7,
        "y": 1
      },
      "hiddenSeries": false,
      "id": 21,
      "interval": null,
      "legend": {
        "alignAsTable": false,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null as zero",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.2.0",
      "pointradius": 2,
      "points": true,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "probe_success{job=~\"host up/down\"}",
          "format": "time_series",
          "instant": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "lt",
          "value": 1
        }
      ],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "服务器UP/DOWN预警",
      "tooltip": {
        "shared": true,
        "sort": 1,
        "value_type": "individual"
      },
      "transformations": [],
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:8525",
          "decimals": -1,
          "format": "none",
          "label": null,
          "logBase": 1,
          "max": "1",
          "min": "0",
          "show": true
        },
        {
          "$$hashKey": "object:8526",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "collapsed": false,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 9
      },
      "id": 23,
      "panels": [],
      "title": "数据库UP/DOWN",
      "type": "row"
    },
    {
      "cacheTimeout": null,
      "datasource": "${DS_PROMETHEUS-CHANNEL}",
      "description": "Coordinator ([pc, sc]) \nWorker ([pw1, sw1], [pw2, sw2], [pw3, sw3])",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": "left",
            "displayMode": "color-background",
            "filterable": false
          },
          "mappings": [
            {
              "from": "",
              "id": 0,
              "text": "DOWN",
              "to": "",
              "type": 1,
              "value": "0"
            },
            {
              "from": "",
              "id": 1,
              "text": "UP",
              "to": "",
              "type": 1,
              "value": "1"
            },
            {
              "from": "",
              "id": 2,
              "text": "DOWN",
              "to": "",
              "type": 1,
              "value": "null"
            }
          ],
          "max": 1,
          "min": 0,
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              },
              {
                "color": "red",
                "value": 0
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          },
          "unit": "none"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Coordinator ([pc, sc]) - Worker ([pw1, sw1], [pw2, sw2], [pw3, sw3])"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 241
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Host"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 222
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "pg_role"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 96
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 10,
        "w": 7,
        "x": 0,
        "y": 10
      },
      "id": 24,
      "interval": null,
      "links": [],
      "options": {
        "showHeader": true,
        "sortBy": [
          {
            "desc": false,
            "displayName": "Status"
          }
        ]
      },
      "pluginVersion": "7.2.0",
      "targets": [
        {
          "expr": "probe_success{job=~\"pg up/down\", env=\"uat\"}",
          "format": "table",
          "instant": true,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "UAT PG UP/DOWN",
      "transformations": [
        {
          "id": "filterFieldsByName",
          "options": {
            "include": {
              "names": [
                "instance",
                "pg_role",
                "Value"
              ]
            }
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Value": "Status",
              "instance": "Host",
              "pg_role": ""
            }
          }
        }
      ],
      "type": "table"
    },
    {
      "cacheTimeout": null,
      "datasource": "${DS_PROMETHEUS-CHANNEL}",
      "description": "Coordinator ([pc, sc]) \nWorker ([pw1, sw1], [pw2, sw2], [pw3, sw3])",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": "left",
            "displayMode": "color-background",
            "filterable": false
          },
          "mappings": [
            {
              "from": "",
              "id": 0,
              "text": "DOWN",
              "to": "",
              "type": 1,
              "value": "0"
            },
            {
              "from": "",
              "id": 1,
              "text": "UP",
              "to": "",
              "type": 1,
              "value": "1"
            },
            {
              "from": "",
              "id": 2,
              "text": "DOWN",
              "to": "",
              "type": 1,
              "value": "null"
            }
          ],
          "max": 1,
          "min": 0,
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              },
              {
                "color": "red",
                "value": 0
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          },
          "unit": "none"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Coordinator ([pc, sc]) - Worker ([pw1, sw1], [pw2, sw2], [pw3, sw3])"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 241
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Host"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 222
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "pg_role"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 96
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 10,
        "w": 7,
        "x": 7,
        "y": 10
      },
      "id": 25,
      "interval": null,
      "links": [],
      "options": {
        "showHeader": true,
        "sortBy": [
          {
            "desc": false,
            "displayName": "Status"
          }
        ]
      },
      "pluginVersion": "7.2.0",
      "targets": [
        {
          "expr": "probe_success{job=~\"pg up/down\", env=\"prd\"}",
          "format": "table",
          "instant": true,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "PRD PG UP/DOWN",
      "transformations": [
        {
          "id": "filterFieldsByName",
          "options": {
            "include": {
              "names": [
                "instance",
                "pg_role",
                "Value"
              ]
            }
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Value": "Status",
              "instance": "Host",
              "pg_role": ""
            }
          }
        }
      ],
      "type": "table"
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                1
              ],
              "type": "lt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "1m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "last"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "alerting",
        "for": "20s",
        "frequency": "10s",
        "handler": 1,
        "message": "有PG实例挂了！",
        "name": "PG UP/DOWN预警",
        "noDataState": "alerting",
        "notifications": [
          {
            "uid": "Vd9e6XW4k"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "cacheTimeout": null,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS-CHANNEL}",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": "left",
            "displayMode": "color-background",
            "filterable": false
          },
          "mappings": [
            {
              "from": "",
              "id": 0,
              "text": "DOWN",
              "to": "",
              "type": 1,
              "value": "0"
            },
            {
              "from": "",
              "id": 1,
              "text": "UP",
              "to": "",
              "type": 1,
              "value": "1"
            },
            {
              "from": "",
              "id": 2,
              "text": "DOWN",
              "to": "",
              "type": 1,
              "value": "null"
            }
          ],
          "max": 1,
          "min": 0,
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              },
              {
                "color": "red",
                "value": 0
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 10,
        "w": 10,
        "x": 14,
        "y": 10
      },
      "hiddenSeries": false,
      "id": 26,
      "interval": null,
      "legend": {
        "alignAsTable": false,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null as zero",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.2.0",
      "pointradius": 2,
      "points": true,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "probe_success{job=~\"pg up/down\"}",
          "format": "time_series",
          "instant": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "lt",
          "value": 1
        }
      ],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "PG UP/DOWN预警",
      "tooltip": {
        "shared": true,
        "sort": 1,
        "value_type": "individual"
      },
      "transformations": [],
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:8525",
          "decimals": -1,
          "format": "none",
          "label": null,
          "logBase": 1,
          "max": "1",
          "min": "0",
          "show": true
        },
        {
          "$$hashKey": "object:8526",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 20
      },
      "id": 10,
      "title": "硬盘",
      "type": "row"
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                90
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "1m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "min"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "alerting",
        "for": "10m",
        "frequency": "1m",
        "handler": 1,
        "message": "硬盘使用率超过90%，请清理硬盘！",
        "name": "硬盘使用率预警",
        "noDataState": "alerting",
        "notifications": [
          {
            "uid": "3ZUw9kO7z"
          },
          {
            "uid": "fnQuJ0Cnk"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS-CHANNEL}",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "links": [],
          "unit": "percent"
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 21
      },
      "hiddenSeries": false,
      "id": 1,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "hideEmpty": false,
        "hideZero": false,
        "max": true,
        "min": true,
        "rightSide": true,
        "show": true,
        "sideWidth": null,
        "sort": "max",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.2.0",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "100.0 - 100 * ((node_filesystem_avail_bytes{fstype=~\"ext.*|xfs\", mountpoint !~\".*(?:pod|boot|run|ramdisk|s3fs|obsfs|ossfs).*\"} / 1000 / 1000 ) / (node_filesystem_size_bytes{fstype=~\"ext.*|xfs\", mountpoint !~\".*(?:pod|boot|run|ramdisk|s3fs|obsfs|ossfs).*\"}  / 1024 / 1024))",
          "format": "time_series",
          "instant": false,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "{{instance}} - {{mountpoint}}",
          "refId": "A"
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 90
        }
      ],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "硬盘使用率",
      "tooltip": {
        "shared": false,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:151",
          "format": "percent",
          "label": null,
          "logBase": 1,
          "max": "100",
          "min": "0",
          "show": true
        },
        {
          "$$hashKey": "object:152",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "collapsed": false,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 28
      },
      "id": 16,
      "panels": [],
      "title": "内存",
      "type": "row"
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                98
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "alerting",
        "for": "20m",
        "frequency": "1m",
        "handler": 1,
        "message": "内存使用率超过98%，请注意观察服务器内存使用情况！",
        "name": "内存使用率预警-256GB内存机器",
        "noDataState": "no_data",
        "notifications": [
          {
            "uid": "3ZUw9kO7z"
          },
          {
            "uid": "fnQuJ0Cnk"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS-CHANNEL}",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 29
      },
      "hiddenSeries": false,
      "id": 5,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "max": true,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "paceLength": 10,
      "percentage": false,
      "pluginVersion": "7.2.0",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "exemplar": true,
          "expr": "100*(node_memory_MemTotal_bytes{instance=~\".*(?:bare|es).*\"} - node_memory_MemFree_bytes{instance=~\".*(?:bare|es).*\"} - node_memory_Buffers_bytes{instance=~\".*(?:bare|es).*\"} - node_memory_Cached_bytes{instance=~\".*(?:bare|es).*\"}) / node_memory_MemTotal_bytes{instance=~\".*(?:bare|es).*\"}",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 98
        }
      ],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "内存使用率-256GB内存机器",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:261",
          "decimals": null,
          "format": "percent",
          "label": null,
          "logBase": 1,
          "max": "100",
          "min": "0",
          "show": true
        },
        {
          "$$hashKey": "object:262",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                95
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "alerting",
        "for": "20m",
        "frequency": "1m",
        "handler": 1,
        "message": "内存使用率超过95%，请注意观察服务器内存使用情况！",
        "name": "内存使用率预警-小于32GB内存机器",
        "noDataState": "no_data",
        "notifications": [
          {
            "uid": "fnQuJ0Cnk"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS-CHANNEL}",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 29
      },
      "hiddenSeries": false,
      "id": 12,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "max": true,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "paceLength": 10,
      "percentage": false,
      "pluginVersion": "7.2.0",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "exemplar": true,
          "expr": "100*(node_memory_MemTotal_bytes{instance!~\".*(?:bare|es).*\"} - node_memory_MemFree_bytes{instance!~\".*(?:bare|es).*\"} - node_memory_Buffers_bytes{instance!~\".*(?:bare|es).*\"} - node_memory_Cached_bytes{instance!~\".*(?:bare|es).*\"}) / node_memory_MemTotal_bytes{instance!~\".*(?:bare|es).*\"}",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 95
        }
      ],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "内存使用率-小于32GB内存机器",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:261",
          "decimals": null,
          "format": "percent",
          "label": null,
          "logBase": 1,
          "max": "100",
          "min": "0",
          "show": true
        },
        {
          "$$hashKey": "object:262",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "collapsed": false,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 38
      },
      "id": 8,
      "panels": [],
      "title": "系统负载",
      "type": "row"
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                96
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "alerting",
        "for": "60m",
        "frequency": "1m",
        "handler": 1,
        "message": "机器持续60分钟负载过高，请注意观察服务器负载情况！",
        "name": "系统负载预警-48线程",
        "noDataState": "no_data",
        "notifications": [
          {
            "uid": "3ZUw9kO7z"
          },
          {
            "uid": "fnQuJ0Cnk"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS-CHANNEL}",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 39
      },
      "hiddenSeries": false,
      "id": 6,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "max": true,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "paceLength": 10,
      "percentage": false,
      "pluginVersion": "7.2.0",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "exemplar": true,
          "expr": "node_load5{instance=~\"fs-channel-bare-hdp.*\"}",
          "format": "time_series",
          "instant": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 96
        }
      ],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "系统负载-48线程",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:261",
          "decimals": null,
          "format": "none",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "$$hashKey": "object:262",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                200
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "alerting",
        "for": "60m",
        "frequency": "1m",
        "handler": 1,
        "message": "机器持续60分钟负载过高，请注意观察服务器负载情况！",
        "name": "系统负载预警-128线程",
        "noDataState": "no_data",
        "notifications": [
          {
            "uid": "fnQuJ0Cnk"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS-CHANNEL}",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 39
      },
      "hiddenSeries": false,
      "id": 13,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "max": true,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "paceLength": 10,
      "percentage": false,
      "pluginVersion": "7.2.0",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "exemplar": true,
          "expr": "node_load5{instance=~\"fs-channel-bare-abhoc.*\"}",
          "format": "time_series",
          "instant": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 200
        }
      ],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "系统负载-128线程",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:261",
          "decimals": null,
          "format": "none",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "$$hashKey": "object:262",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                20
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "alerting",
        "for": "60m",
        "frequency": "1m",
        "handler": 1,
        "name": "系统负载预警-虚拟机",
        "noDataState": "no_data",
        "notifications": [
          {
            "uid": "fnQuJ0Cnk"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS-CHANNEL}",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 47
      },
      "hiddenSeries": false,
      "id": 14,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "max": true,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "paceLength": 10,
      "percentage": false,
      "pluginVersion": "7.2.0",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "exemplar": true,
          "expr": "node_load5{instance=~\".*vm.*\"}",
          "format": "time_series",
          "instant": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 20
        }
      ],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "系统负载-虚拟机",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:261",
          "decimals": null,
          "format": "none",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "$$hashKey": "object:262",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    }
  ],
  "refresh": "10s",
  "schemaVersion": 26,
  "style": "dark",
  "tags": [
    "alerts",
    "linux"
  ],
  "templating": {
    "list": []
  },
  "time": {
    "from": "now-12h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "",
  "title": "Basic Infra Alert",
  "uid": "000000012",
  "version": 54
}