{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__requires": [ { "type": "panel", "id": "alertlist", "name": "Alert list", "version": "" }, { "type": "panel", "id": "gauge", "name": "Gauge", "version": "" }, { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "7.5.2" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "table", "name": "Table", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "id": null, "iteration": 1619790416969, "links": [], "panels": [ { "collapsed": false, "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 8, "panels": [], "title": "DRBD", "type": "row" }, { "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 18, "x": 0, "y": 1 }, "id": 6, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "text": {} }, "pluginVersion": "7.5.2", "targets": [ { "exemplar": true, "expr": "drbd_resource_resources{instance=~\"$instance\"}", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "title": "Number of DRBD Resources", "type": "gauge" }, { "dashboardFilter": "", "dashboardTags": [], "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": {}, "overrides": [] }, "folderId": null, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 1 }, "id": 18, "limit": 10, "nameFilter": "", "onlyAlertsOnDashboard": false, "pluginVersion": "7.5.2", "show": "current", "sortOrder": 1, "stateFilter": [], "timeFrom": null, "timeShift": null, "title": "Alerts", "type": "alertlist" }, { "alert": { "alertRuleTags": {}, "conditions": [ { "evaluator": { "params": [ 0.1 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "B", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "for": "5m", "frequency": "1m", "handler": 1, "message": "Scraping from ${instance} is slow (took >100ms).", "name": "Slow Scrape", "noDataState": "no_data", "notifications": [ { "uid": "Aoev7clMk" } ] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, "hiddenSeries": false, "id": 27, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "avg_over_time(scrape_duration_seconds{instance=~\"$instance\"}[5m])", "interval": "", "legendFormat": "{{instance}}", "refId": "A" }, { "exemplar": true, "expr": "avg_over_time(scrape_duration_seconds{instance=~\".*:9942\"}[5m])", "hide": true, "interval": "", "legendFormat": "", "refId": "B" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.1, "visible": true } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Scrape Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": null, "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "DRBD Resource" }, "properties": [ { "id": "custom.width", "value": 279 } ] }, { "matcher": { "id": "byName", "options": "Host" }, "properties": [ { "id": "custom.width", "value": 238 } ] }, { "matcher": { "id": "byName", "options": "Out of Sync" }, "properties": [ { "id": "custom.width", "value": 93 } ] } ] }, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 9 }, "id": 16, "options": { "frameIndex": 0, "showHeader": true, "sortBy": [] }, "pluginVersion": "7.5.2", "targets": [ { "exemplar": false, "expr": "sum(drbd_peerdevice_outofsync_bytes{instance=~\"$instance\"}) by(name, instance) > 0", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "DRBD Resources Out of Sync", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": { "Time": 0, "Value": 3, "instance": 2, "name": 1 }, "renameByName": { "Time": "", "Value": "Out of Sync", "instance": "Host", "name": "DRBD Resource" } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "Out of Sync" } ] } } ], "type": "table" }, { "alert": { "alertRuleTags": {}, "conditions": [ { "evaluator": { "params": [ 0 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "B", "5m", "now" ] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "executionErrorState": "alerting", "for": "5m", "frequency": "1m", "handler": 1, "message": "Data is out of sync!!!", "name": "DRBD out-of-sync", "noDataState": "no_data", "notifications": [ { "uid": "Aoev7clMk" } ] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "DRBD data out of sync with a peer", "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 9 }, "hiddenSeries": false, "id": 12, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "drbd_peerdevice_outofsync_bytes{instance=~\"$instance\"} > 0", "interval": "", "legendFormat": "{{name}} on {{instance}}", "refId": "A" }, { "exemplar": true, "expr": "drbd_peerdevice_outofsync_bytes > 0", "hide": true, "interval": "", "legendFormat": "", "refId": "B" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0, "visible": true } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "DRBD out-of-sync", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 17 }, "hiddenSeries": false, "id": 14, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "drbd_connection_state{drbd_connection_state=~\"!UpToDate\", instance=~\"$instance\"} == 1", "interval": "", "legendFormat": "{{name}} {{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Standalone DRBD resources", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 17 }, "hiddenSeries": false, "id": 4, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "drbd_device_unintentionaldiskless == 1", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of unintentional diskless resources", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": null, "displayMode": "auto", "filterable": false }, "mappings": [ { "from": "", "id": 1, "text": "No", "to": "", "type": 1, "value": "1" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 17 }, "id": 22, "maxDataPoints": null, "options": { "frameIndex": 0, "showHeader": true }, "pluginVersion": "7.5.2", "targets": [ { "exemplar": true, "expr": "count(drbd_device_quorum{instance=~\"$instance\"} == 0) by(instance, name)", "format": "table", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "DRBD Resources Without Quorum", "transformations": [ { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": { "Value #A": "Quorum?", "instance": "Instance", "name": "DRBD Resource" } } } ], "type": "table" }, { "alert": { "alertRuleTags": {}, "conditions": [ { "evaluator": { "params": [ 0.99 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "B", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "for": "5m", "frequency": "1m", "handler": 1, "message": "Resource ${name} does not have quorum", "name": "Resources Without Quorum", "noDataState": "ok", "notifications": [ { "uid": "Aoev7clMk" } ] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 17 }, "hiddenSeries": false, "id": 20, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "count(drbd_device_quorum{instance=~\"$instance\"} == 0)", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}} on {{instance}}", "refId": "A" }, { "exemplar": true, "expr": "count(drbd_device_quorum == 0)", "hide": true, "interval": "", "legendFormat": "", "refId": "B" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.99, "visible": true } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of Resources Without Quorum", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "alertRuleTags": {}, "conditions": [ { "evaluator": { "params": [ 10485760 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "B", "10m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "for": "10m", "frequency": "1m", "handler": 1, "message": "Excessive write rate on volume ${name} on ${instance}.\n\nDRBD write rate has exceeded 10MiB/s for more than 10 minutes.", "name": "Excessive Write Rate", "noDataState": "no_data", "notifications": [ { "uid": "Aoev7clMk" } ] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "unit": "binBps" }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 }, "hiddenSeries": false, "id": 24, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "rate(drbd_device_written_bytes_total{instance=~\"$instance\"}[5m]) and topk(5, avg_over_time(drbd_device_written_bytes_total{instance=~\"$instance\"}[6h]) > 0)", "instant": false, "interval": "", "legendFormat": "{{name}} on {{instance}}", "refId": "A" }, { "exemplar": true, "expr": "rate(drbd_device_written_bytes_total[5m]) and topk(5, avg_over_time(drbd_device_written_bytes_total[6h]) > 0)", "hide": true, "interval": "", "legendFormat": "", "refId": "B" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 10485760, "visible": true } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Write Rate (5 Most Active Volumes)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "alertRuleTags": {}, "conditions": [ { "evaluator": { "params": [ 10485760 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "B", "10m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "for": "10m", "frequency": "1m", "handler": 1, "message": "Excessive Read Rate on volume ${name} on ${instance}.\n\nDRBD read rate has exceeded 10MiB/s for more than 10 minutes.", "name": "Excessive Read Rate", "noDataState": "no_data", "notifications": [ { "uid": "Aoev7clMk" } ] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "unit": "binBps" }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 }, "hiddenSeries": false, "id": 25, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "rate(drbd_device_read_bytes_total{instance=~\"$instance\"}[5m]) and topk(5, avg_over_time(drbd_device_read_bytes_total{instance=~\"$instance\"}[6h]) > 0)", "instant": false, "interval": "", "legendFormat": "{{name}} on {{instance}}", "refId": "A" }, { "exemplar": true, "expr": "rate(drbd_device_read_bytes_total[5m]) and topk(5, avg_over_time(drbd_device_read_bytes_total[6h]) > 0)", "hide": true, "interval": "", "legendFormat": "", "refId": "B" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 10485760, "visible": true } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Read Rate (5 Most Active Volumes)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": "30s", "schemaVersion": 27, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", "definition": "label_values(drbdreactor_up, instance)", "description": null, "error": null, "hide": 0, "includeAll": true, "label": null, "multi": true, "name": "instance", "options": [], "query": { "query": "label_values(drbdreactor_up, instance)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "DRBD", "uid": "f_tZtVlMz", "version": 44 }