{ "__inputs": [], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "6.1.0" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "singlestat", "name": "Singlestat", "version": "" }, { "type": "panel", "id": "table", "name": "Table", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" }, { "datasource": "$datasource", "enable": true, "expr": "changes(process_start_time_seconds{ instance=~\"$instance\"}[2m]) > 0", "hide": false, "iconColor": "#bf1b00", "name": "Restarts", "showIn": 0, "step": "1m", "tagKeys": "instance", "titleFormat": "Restart" } ] }, "description": "Dashboard showing Prometheus Alertmanager metrics for observing status of the cluster and possible debbuging.", "editable": true, "gnetId": null, "graphTooltip": 1, "id": null, "iteration": 1555334727804, "links": [ { "icon": "doc", "tags": [], "targetBlank": true, "title": "Docs", "tooltip": "Official documentation of Alertmanager", "type": "link", "url": "https://prometheus.io/docs/alerting/alertmanager/" }, { "icon": "info", "tags": [], "targetBlank": true, "title": "GitHub", "tooltip": "Alertmanager sources on GitHub", "type": "link", "url": "https://github.com/prometheus/alertmanager" }, { "icon": "info", "tags": [], "targetBlank": true, "title": "Twitter", "tooltip": "Twitter account with prometheus related info", "type": "link", "url": "https://twitter.com/PrometheusIO" }, { "icon": "question", "tags": [], "targetBlank": true, "title": "Mailing list", "tooltip": "Prometheus users mailing list", "type": "link", "url": "https://groups.google.com/forum/#!forum/prometheus-users" }, { "icon": "question", "tags": [], "targetBlank": true, "title": "IRC", "tooltip": "Join IRC using Riot", "type": "link", "url": "https://riot.im/app/#/room/#prometheus:matrix.org" } ], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 36, "panels": [], "title": "General info", "type": "row" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a" ], "datasource": "$datasource", "decimals": 0, "format": "short", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 5, "w": 3, "x": 0, "y": 1 }, "id": 4, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(244, 213, 152, 0.12)", "full": false, "lineColor": "#f4d598", "show": true }, "tableColumn": "", "targets": [ { "expr": "count(alertmanager_build_info{instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "thresholds": "", "title": "Number of instances", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "avg" }, { "columns": [], "datasource": "$datasource", "description": "Table containing list of Alertmanager instances showing it's version, up time, last reload time and if it was successful.", "fontSize": "90%", "gridPos": { "h": 5, "w": 9, "x": 3, "y": 1 }, "id": 26, "links": [], "pageSize": null, "scroll": true, "showHeader": true, "sort": { "col": 13, "desc": false }, "styles": [ { "alias": "Time", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "Time", "type": "hidden" }, { "alias": "Instance", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "instance", "thresholds": [], "type": "string", "unit": "short" }, { "alias": "Version", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "version", "thresholds": [], "type": "string", "unit": "short" }, { "alias": "Up time", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 0, "mappingType": 1, "pattern": "Value #A", "thresholds": [], "type": "number", "unit": "s" }, { "alias": "Last reload", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 0, "mappingType": 1, "pattern": "Value #B", "thresholds": [], "type": "number", "unit": "s" }, { "alias": "Last reload sucessfull", "colorMode": "cell", "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "Value #C", "thresholds": [ "0", "1" ], "type": "number", "unit": "short" }, { "alias": "", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "/.*/", "thresholds": [], "type": "hidden", "unit": "short" } ], "targets": [ { "expr": "time() - (alertmanager_build_info{instance=~\"$instance\"} * on (instance, cluster, job) group_left process_start_time_seconds{instance=~\"$instance\"})", "format": "table", "instant": true, "intervalFactor": 1, "refId": "A" }, { "expr": "time() - alertmanager_config_last_reload_success_timestamp_seconds{instance=~\"$instance\"}", "format": "table", "instant": true, "intervalFactor": 1, "refId": "B" }, { "expr": "alertmanager_config_last_reload_successful{instance=~\"$instance\"}", "format": "table", "instant": true, "intervalFactor": 1, "refId": "C" } ], "title": "Instance versions and up time", "transform": "table", "type": "table" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a" ], "datasource": "$datasource", "decimals": 0, "description": "Number of peers in the Alertmanager cluster.", "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 5, "w": 3, "x": 12, "y": 1 }, "id": 207, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(191, 27, 0, 0.08)", "full": false, "lineColor": "#e5ac0e", "show": true }, "tableColumn": "", "targets": [ { "expr": "max(alertmanager_cluster_members{instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "thresholds": "", "title": "Cluster size", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "avg" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a" ], "datasource": "$datasource", "description": "Current number of active alerts.", "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 5, "w": 3, "x": 15, "y": 1 }, "id": 2, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(191, 27, 0, 0.08)", "full": false, "lineColor": "#bf1b00", "show": true }, "tableColumn": "", "targets": [ { "expr": "max(alertmanager_alerts{state=\"active\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "thresholds": "", "title": "Number of active alerts", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "avg" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a" ], "datasource": "$datasource", "description": "Current number of suppressed alerts.", "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 5, "w": 3, "x": 18, "y": 1 }, "id": 3, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(249, 226, 210, 0.18)", "full": false, "lineColor": "#f9e2d2", "show": true }, "tableColumn": "", "targets": [ { "expr": "max(alertmanager_alerts{state=\"suppressed\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "thresholds": "", "title": "Number of suppressed alerts", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "avg" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a" ], "datasource": "$datasource", "description": "Current number of active silences.", "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 5, "w": 3, "x": 21, "y": 1 }, "id": 121, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(249, 226, 210, 0.18)", "full": false, "lineColor": "#f9e2d2", "show": true }, "tableColumn": "", "targets": [ { "expr": "max(alertmanager_silences{state=\"active\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "thresholds": "", "title": "Number of active silences", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "avg" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "id": 113, "panels": [], "title": "Notifications", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Number of sent notifications to distinct integrations such as PagerDuty, Slack and so on. On negative axis are displayed failed notifications.", "fill": 1, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 7 }, "id": 118, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "total", "sortDesc": true, "total": true, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "seriesOverrides": [ { "alias": "/Failed.*/", "color": "#99440a", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(increase(alertmanager_notifications_total{instance=~\"$instance\"}[$__interval])) by (integration)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ integration}}", "refId": "B" }, { "expr": "sum(increase(alertmanager_notifications_failed_total{instance=~\"$instance\"}[$__interval])) by (integration)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "Failed {{ integration }}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Notifications sent from $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Duration of notification sends in 0.99 and 0.9 quantiles per integration.", "fill": 0, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 12 }, "id": 115, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "seriesOverrides": [ { "alias": "/0.99.*/", "linewidth": 1 }, { "alias": "/0.5 .*/", "linewidth": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(histogram_quantile(0.9,rate(alertmanager_notification_latency_seconds_bucket{instance=~\"$instance\"}[$__interval]))) by (integration)", "format": "time_series", "intervalFactor": 1, "legendFormat": "0.9q {{ integration }}", "refId": "B" }, { "expr": "sum(histogram_quantile(0.99,rate(alertmanager_notification_latency_seconds_bucket{instance=~\"$instance\"}[$__interval]))) by (integration)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "0.99q {{ integration }}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Notification durations per integration on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, "id": 18, "panels": [], "title": "Alerts", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Number of alerts by state such as `active`, `suppressed` etc.", "fill": 4, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 19 }, "id": 6, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "seriesOverrides": [ { "alias": "active", "color": "#bf1b00" }, { "alias": "suppressed", "color": "#2f575e" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(alertmanager_alerts{instance=~\"$instance\"}) by (state)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{state}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Active alerts in $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Number of received alerts from Prometheus by status `firing` on positive axis and `resolved` on negative axis.", "fill": 1, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 24 }, "id": 8, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "seriesOverrides": [ { "alias": "resolved", "color": "#7eb26d", "transform": "negative-Y" }, { "alias": "firing", "color": "#99440a" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(increase(alertmanager_alerts_received_total{instance=~\"$instance\"}[$__interval])) by (status)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{ status }}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Received alerts by status for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 29 }, "id": 34, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows cluster score representing cluster health. From Hashicorps official documentation: \n> This metric describes a node's perception of its own health based on how well it is meeting the soft real-time requirements of the protocol. This metric ranges from 0 to 8, where 0 indicates \"totally healthy\".\n\nFor more info see https://www.consul.io/docs/agent/telemetry.html#cluster-health", "fill": 1, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 20 }, "id": 57, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Number of failed peers", "color": "#bf1b00", "fill": 7 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_cluster_health_score{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster health score", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Clusterhealth score for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "Shows cluster score representing cluster health. From Hashicorps official documentation: \n> This metric describes a node's perception of its own health based on how well it is meeting the soft real-time requirements of the protocol. This metric ranges from 0 to 8, where 0 indicates \"totally healthy\".\n\nFor more info see https://www.consul.io/docs/agent/telemetry.html#cluster-health", "fill": 1, "gridPos": { "h": 4, "w": 6, "x": 6, "y": 20 }, "id": 408, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 57, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Number of failed peers", "color": "#bf1b00", "fill": 7 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_cluster_health_score{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster health score", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Clusterhealth score for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "Shows cluster score representing cluster health. From Hashicorps official documentation: \n> This metric describes a node's perception of its own health based on how well it is meeting the soft real-time requirements of the protocol. This metric ranges from 0 to 8, where 0 indicates \"totally healthy\".\n\nFor more info see https://www.consul.io/docs/agent/telemetry.html#cluster-health", "fill": 1, "gridPos": { "h": 4, "w": 6, "x": 12, "y": 20 }, "id": 409, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 57, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Number of failed peers", "color": "#bf1b00", "fill": 7 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_cluster_health_score{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster health score", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Clusterhealth score for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "Shows cluster score representing cluster health. From Hashicorps official documentation: \n> This metric describes a node's perception of its own health based on how well it is meeting the soft real-time requirements of the protocol. This metric ranges from 0 to 8, where 0 indicates \"totally healthy\".\n\nFor more info see https://www.consul.io/docs/agent/telemetry.html#cluster-health", "fill": 1, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 20 }, "id": 410, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 57, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Number of failed peers", "color": "#bf1b00", "fill": 7 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_cluster_health_score{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster health score", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Clusterhealth score for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows gossip cluster members count in time and failing peers in case of any in red color.", "fill": 1, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 24 }, "id": 38, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Number of failed peers", "color": "#bf1b00", "fill": 7 } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "alertmanager_cluster_members{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Number of cluster members", "refId": "A" }, { "expr": "alertmanager_cluster_failed_peers{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Number of failed peers", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster members count on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "Shows gossip cluster members count in time and failing peers in case of any in red color.", "fill": 1, "gridPos": { "h": 4, "w": 6, "x": 6, "y": 24 }, "id": 411, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 38, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Number of failed peers", "color": "#bf1b00", "fill": 7 } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "alertmanager_cluster_members{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Number of cluster members", "refId": "A" }, { "expr": "alertmanager_cluster_failed_peers{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Number of failed peers", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster members count on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "Shows gossip cluster members count in time and failing peers in case of any in red color.", "fill": 1, "gridPos": { "h": 4, "w": 6, "x": 12, "y": 24 }, "id": 412, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 38, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Number of failed peers", "color": "#bf1b00", "fill": 7 } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "alertmanager_cluster_members{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Number of cluster members", "refId": "A" }, { "expr": "alertmanager_cluster_failed_peers{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Number of failed peers", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster members count on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "Shows gossip cluster members count in time and failing peers in case of any in red color.", "fill": 1, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 24 }, "id": 413, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 38, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Number of failed peers", "color": "#bf1b00", "fill": 7 } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "alertmanager_cluster_members{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Number of cluster members", "refId": "A" }, { "expr": "alertmanager_cluster_failed_peers{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Number of failed peers", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster members count on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "On positive axis shows number of peers that joined the cluster and on negative axis number of peers that left the cluster.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 28 }, "id": 75, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Cluster left peers", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_cluster_peers_left_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster joined peers", "refId": "A" }, { "expr": "increase(alertmanager_cluster_peers_joined_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster left peers", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster peers left/joined on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis shows number of peers that joined the cluster and on negative axis number of peers that left the cluster.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 28 }, "id": 414, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 75, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Cluster left peers", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_cluster_peers_left_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster joined peers", "refId": "A" }, { "expr": "increase(alertmanager_cluster_peers_joined_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster left peers", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster peers left/joined on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis shows number of peers that joined the cluster and on negative axis number of peers that left the cluster.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 28 }, "id": 415, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 75, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Cluster left peers", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_cluster_peers_left_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster joined peers", "refId": "A" }, { "expr": "increase(alertmanager_cluster_peers_joined_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster left peers", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster peers left/joined on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis shows number of peers that joined the cluster and on negative axis number of peers that left the cluster.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 28 }, "id": 416, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 75, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Cluster left peers", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_cluster_peers_left_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster joined peers", "refId": "A" }, { "expr": "increase(alertmanager_cluster_peers_joined_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster left peers", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster peers left/joined on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "On positive axis is number of attempts to reconnect the cluster. On negative axis if number of failed attempts.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 34 }, "id": 68, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Failed reconnections", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_cluster_reconnections_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Sucessful reconnections", "refId": "A" }, { "expr": "increase(alertmanager_cluster_reconnections_failed_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Failed reconnections", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster reconnections on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis is number of attempts to reconnect the cluster. On negative axis if number of failed attempts.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 34 }, "id": 417, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 68, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Failed reconnections", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_cluster_reconnections_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Sucessful reconnections", "refId": "A" }, { "expr": "increase(alertmanager_cluster_reconnections_failed_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Failed reconnections", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster reconnections on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis is number of attempts to reconnect the cluster. On negative axis if number of failed attempts.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 34 }, "id": 418, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 68, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Failed reconnections", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_cluster_reconnections_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Sucessful reconnections", "refId": "A" }, { "expr": "increase(alertmanager_cluster_reconnections_failed_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Failed reconnections", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster reconnections on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis is number of attempts to reconnect the cluster. On negative axis if number of failed attempts.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 34 }, "id": 419, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 68, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Failed reconnections", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_cluster_reconnections_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Sucessful reconnections", "refId": "A" }, { "expr": "increase(alertmanager_cluster_reconnections_failed_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Failed reconnections", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster reconnections on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "On positive axis is number of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 40 }, "id": 48, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "/received.*/", "transform": "negative-Y" }, { "alias": "/.*full_state.*/", "color": "#629e51" }, { "alias": "/.*update.*/", "color": "#f4d598" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(increase(alertmanager_cluster_messages_sent_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "sent {{msg_type}}", "refId": "A" }, { "expr": "sum(increase(alertmanager_cluster_messages_received_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "received {{msg_type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster messages count on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis is number of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 40 }, "id": 420, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 48, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "/received.*/", "transform": "negative-Y" }, { "alias": "/.*full_state.*/", "color": "#629e51" }, { "alias": "/.*update.*/", "color": "#f4d598" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(increase(alertmanager_cluster_messages_sent_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "sent {{msg_type}}", "refId": "A" }, { "expr": "sum(increase(alertmanager_cluster_messages_received_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "received {{msg_type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster messages count on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis is number of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 40 }, "id": 421, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 48, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "/received.*/", "transform": "negative-Y" }, { "alias": "/.*full_state.*/", "color": "#629e51" }, { "alias": "/.*update.*/", "color": "#f4d598" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(increase(alertmanager_cluster_messages_sent_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "sent {{msg_type}}", "refId": "A" }, { "expr": "sum(increase(alertmanager_cluster_messages_received_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "received {{msg_type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster messages count on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis is number of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 40 }, "id": 422, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 48, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "/received.*/", "transform": "negative-Y" }, { "alias": "/.*full_state.*/", "color": "#629e51" }, { "alias": "/.*update.*/", "color": "#f4d598" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(increase(alertmanager_cluster_messages_sent_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "sent {{msg_type}}", "refId": "A" }, { "expr": "sum(increase(alertmanager_cluster_messages_received_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "received {{msg_type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster messages count on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "On positive axis is size of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 46 }, "id": 53, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "/received.*/", "transform": "negative-Y" }, { "alias": "/.*full_state.*/", "color": "#629e51" }, { "alias": "/.*update.*/", "color": "#f4d598" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(increase(alertmanager_cluster_messages_sent_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "sent {{msg_type}}", "refId": "A" }, { "expr": "sum(increase(alertmanager_cluster_messages_received_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "received {{msg_type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster messages size on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis is size of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 46 }, "id": 423, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 53, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "/received.*/", "transform": "negative-Y" }, { "alias": "/.*full_state.*/", "color": "#629e51" }, { "alias": "/.*update.*/", "color": "#f4d598" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(increase(alertmanager_cluster_messages_sent_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "sent {{msg_type}}", "refId": "A" }, { "expr": "sum(increase(alertmanager_cluster_messages_received_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "received {{msg_type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster messages size on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis is size of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 46 }, "id": 424, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 53, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "/received.*/", "transform": "negative-Y" }, { "alias": "/.*full_state.*/", "color": "#629e51" }, { "alias": "/.*update.*/", "color": "#f4d598" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(increase(alertmanager_cluster_messages_sent_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "sent {{msg_type}}", "refId": "A" }, { "expr": "sum(increase(alertmanager_cluster_messages_received_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "received {{msg_type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster messages size on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis is size of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 46 }, "id": 425, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 53, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "/received.*/", "transform": "negative-Y" }, { "alias": "/.*full_state.*/", "color": "#629e51" }, { "alias": "/.*update.*/", "color": "#f4d598" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(increase(alertmanager_cluster_messages_sent_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "sent {{msg_type}}", "refId": "A" }, { "expr": "sum(increase(alertmanager_cluster_messages_received_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "received {{msg_type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster messages size on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "On positive axis is number of queued cluster messages and on negative axis number of pruned messages.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 52 }, "id": 62, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Pruned messaged", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_cluster_messages_pruned_total{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Pruned messaged", "refId": "A" }, { "expr": "alertmanager_cluster_messages_queued{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Queued messages", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster messages queue on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis is number of queued cluster messages and on negative axis number of pruned messages.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 52 }, "id": 426, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 62, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Pruned messaged", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_cluster_messages_pruned_total{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Pruned messaged", "refId": "A" }, { "expr": "alertmanager_cluster_messages_queued{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Queued messages", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster messages queue on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis is number of queued cluster messages and on negative axis number of pruned messages.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 52 }, "id": 427, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 62, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Pruned messaged", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_cluster_messages_pruned_total{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Pruned messaged", "refId": "A" }, { "expr": "alertmanager_cluster_messages_queued{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Queued messages", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster messages queue on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "On positive axis is number of queued cluster messages and on negative axis number of pruned messages.", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 52 }, "id": 428, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 62, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Pruned messaged", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_cluster_messages_pruned_total{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Pruned messaged", "refId": "A" }, { "expr": "alertmanager_cluster_messages_queued{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Queued messages", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster messages queue on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Cluster members", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, "id": 284, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 10 }, "id": 314, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "/dropped/", "color": "#cca300", "transform": "negative-Y" }, { "alias": "/failed/", "color": "#bf1b00", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_oversized_gossip_message_sent_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{key}}", "refId": "A" }, { "expr": "increase(alertmanager_oversized_gossip_message_dropped_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "dropped {{key}}", "refId": "B" }, { "expr": "increase(alertmanager_oversized_gossip_message_failure_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "failed {{key}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Count of oversized gossip messages on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 10 }, "id": 441, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 314, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "/dropped/", "color": "#cca300", "transform": "negative-Y" }, { "alias": "/failed/", "color": "#bf1b00", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_oversized_gossip_message_sent_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{key}}", "refId": "A" }, { "expr": "increase(alertmanager_oversized_gossip_message_dropped_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "dropped {{key}}", "refId": "B" }, { "expr": "increase(alertmanager_oversized_gossip_message_failure_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "failed {{key}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Count of oversized gossip messages on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 10 }, "id": 442, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 314, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "/dropped/", "color": "#cca300", "transform": "negative-Y" }, { "alias": "/failed/", "color": "#bf1b00", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_oversized_gossip_message_sent_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{key}}", "refId": "A" }, { "expr": "increase(alertmanager_oversized_gossip_message_dropped_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "dropped {{key}}", "refId": "B" }, { "expr": "increase(alertmanager_oversized_gossip_message_failure_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "failed {{key}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Count of oversized gossip messages on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 10 }, "id": 443, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 314, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "/dropped/", "color": "#cca300", "transform": "negative-Y" }, { "alias": "/failed/", "color": "#bf1b00", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_oversized_gossip_message_sent_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{key}}", "refId": "A" }, { "expr": "increase(alertmanager_oversized_gossip_message_dropped_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "dropped {{key}}", "refId": "B" }, { "expr": "increase(alertmanager_oversized_gossip_message_failure_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "failed {{key}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Count of oversized gossip messages on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 16 }, "id": 307, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1,rate(alertmanager_oversize_gossip_message_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{key}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Duration of oversized gossip messages on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 16 }, "id": 444, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 307, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1,rate(alertmanager_oversize_gossip_message_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{key}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Duration of oversized gossip messages on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 16 }, "id": 445, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 307, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1,rate(alertmanager_oversize_gossip_message_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{key}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Duration of oversized gossip messages on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 16 }, "id": 446, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 307, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1,rate(alertmanager_oversize_gossip_message_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{key}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Duration of oversized gossip messages on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 22 }, "id": 303, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_silences_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "silences", "refId": "A" }, { "expr": "increase(alertmanager_nflog_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "nf_log", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of propagated gossip messages on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 22 }, "id": 447, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 303, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_silences_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "silences", "refId": "A" }, { "expr": "increase(alertmanager_nflog_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "nf_log", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of propagated gossip messages on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 22 }, "id": 448, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 303, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_silences_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "silences", "refId": "A" }, { "expr": "increase(alertmanager_nflog_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "nf_log", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of propagated gossip messages on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 22 }, "id": 449, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 303, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_silences_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "silences", "refId": "A" }, { "expr": "increase(alertmanager_nflog_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "nf_log", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of propagated gossip messages on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Gossip messages", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, "id": 84, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 11 }, "id": 94, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_nflog_queries_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log query count", "refId": "A" }, { "expr": "increase(alertmanager_nflog_query_errors_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log query errors", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log queries count for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 11 }, "id": 450, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 94, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_nflog_queries_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log query count", "refId": "A" }, { "expr": "increase(alertmanager_nflog_query_errors_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log query errors", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log queries count for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 11 }, "id": 451, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 94, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_nflog_queries_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log query count", "refId": "A" }, { "expr": "increase(alertmanager_nflog_query_errors_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log query errors", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log queries count for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 11 }, "id": 452, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 94, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_nflog_queries_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log query count", "refId": "A" }, { "expr": "increase(alertmanager_nflog_query_errors_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log query errors", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log queries count for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 17 }, "id": 106, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1,rate(alertmanager_nflog_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log query duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log query duration for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 17 }, "id": 453, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 106, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1,rate(alertmanager_nflog_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log query duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log query duration for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 17 }, "id": 454, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 106, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1,rate(alertmanager_nflog_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log query duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log query duration for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 17 }, "id": 455, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 106, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1,rate(alertmanager_nflog_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log query duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log query duration for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 23 }, "id": 97, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_nflog_snapshot_size_bytes{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log snapshot size", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log snapshot size for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 23 }, "id": 456, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 97, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_nflog_snapshot_size_bytes{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log snapshot size", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log snapshot size for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 23 }, "id": 457, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 97, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_nflog_snapshot_size_bytes{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log snapshot size", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log snapshot size for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 23 }, "id": 458, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 97, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_nflog_snapshot_size_bytes{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nf log snapshot size", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log snapshot size for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 29 }, "id": 101, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval]) / rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Nf log snapshot size", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log snapshot duration for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 29 }, "id": 459, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 101, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval]) / rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Nf log snapshot size", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log snapshot duration for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 29 }, "id": 460, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 101, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval]) / rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Nf log snapshot size", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log snapshot duration for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 29 }, "id": 461, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 101, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval]) / rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Nf log snapshot size", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log snapshot duration for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 35 }, "id": 92, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Cluster left peers", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_nflog_gc_duration_seconds{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster joined peers", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log Go GC time for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 35 }, "id": 462, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 92, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Cluster left peers", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_nflog_gc_duration_seconds{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster joined peers", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log Go GC time for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 35 }, "id": 463, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 92, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Cluster left peers", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_nflog_gc_duration_seconds{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster joined peers", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log Go GC time for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 35 }, "id": 464, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 92, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Cluster left peers", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_nflog_gc_duration_seconds{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "Cluster joined peers", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nf log Go GC time for $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Nflog", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "id": 123, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 12 }, "id": 129, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences{instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{state}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences count by state on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 12 }, "id": 465, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 129, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences{instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{state}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences count by state on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 12 }, "id": 466, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 129, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences{instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{state}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences count by state on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 12 }, "id": 467, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 129, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences{instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{state}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences count by state on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 18 }, "id": 134, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Silecnces query fails", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_silences_queries_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces query count", "refId": "A" }, { "expr": "increase(alertmanager_silences_query_errors_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Silecnces query fails", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences query count on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 18 }, "id": 468, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 134, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Silecnces query fails", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_silences_queries_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces query count", "refId": "A" }, { "expr": "increase(alertmanager_silences_query_errors_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Silecnces query fails", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences query count on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 18 }, "id": 469, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 134, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Silecnces query fails", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_silences_queries_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces query count", "refId": "A" }, { "expr": "increase(alertmanager_silences_query_errors_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Silecnces query fails", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences query count on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 18 }, "id": 470, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 134, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Silecnces query fails", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "increase(alertmanager_silences_queries_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces query count", "refId": "A" }, { "expr": "increase(alertmanager_silences_query_errors_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "Silecnces query fails", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences query count on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 24 }, "id": 138, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1,rate(alertmanager_silences_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces query duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences query duration on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 24 }, "id": 471, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 138, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1,rate(alertmanager_silences_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces query duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences query duration on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 24 }, "id": 472, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 138, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1,rate(alertmanager_silences_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces query duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences query duration on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 24 }, "id": 473, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 138, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1,rate(alertmanager_silences_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces query duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences query duration on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 30 }, "id": 149, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences_snapshot_size_bytes{instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces snapshot size", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences snapshot size on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 30 }, "id": 474, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 149, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences_snapshot_size_bytes{instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces snapshot size", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences snapshot size on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 30 }, "id": 475, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 149, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences_snapshot_size_bytes{instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces snapshot size", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences snapshot size on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 30 }, "id": 476, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 149, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences_snapshot_size_bytes{instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces snapshot size", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences snapshot size on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 36 }, "id": 143, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences_snapshot_duration_seconds{instance=~\"$instance\", quantile=\"0.99\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces snapshot duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences snapshot duration on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 36 }, "id": 477, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 143, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences_snapshot_duration_seconds{instance=~\"$instance\", quantile=\"0.99\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces snapshot duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences snapshot duration on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 36 }, "id": 478, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 143, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences_snapshot_duration_seconds{instance=~\"$instance\", quantile=\"0.99\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces snapshot duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences snapshot duration on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 36 }, "id": 479, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 143, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences_snapshot_duration_seconds{instance=~\"$instance\", quantile=\"0.99\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces snapshot duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences snapshot duration on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 42 }, "id": 131, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences_gc_duration_seconds{instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces GC duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences GC duraton on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 42 }, "id": 480, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 131, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences_gc_duration_seconds{instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces GC duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences GC duraton on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 42 }, "id": 481, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 131, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences_gc_duration_seconds{instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces GC duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences GC duraton on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 42 }, "id": 482, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 131, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "Nf log query errors", "color": "#890f02", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "alertmanager_silences_gc_duration_seconds{instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "Silecnces GC duration", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Silences GC duraton on $instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Silences", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 33 }, "id": 173, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "decimals": 2, "fill": 1, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 13 }, "id": 175, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "/Limit .*/", "color": "#C15C17", "dashes": true, "fill": 0 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(process_cpu_seconds_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "groupBy": [ { "params": [ "$__interval" ], "type": "time" }, { "params": [ "null" ], "type": "fill" } ], "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ [ { "params": [ "value" ], "type": "field" }, { "params": [], "type": "mean" } ] ], "tags": [] }, { "expr": "max(kube_pod_container_resource_limits_cpu_cores{pod=~\"$instance\"}) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Limit {{pod}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU usage/s for $instance", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "decimals": 2, "fill": 1, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 13 }, "id": 483, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 175, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "/Limit .*/", "color": "#C15C17", "dashes": true, "fill": 0 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(process_cpu_seconds_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "groupBy": [ { "params": [ "$__interval" ], "type": "time" }, { "params": [ "null" ], "type": "fill" } ], "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ [ { "params": [ "value" ], "type": "field" }, { "params": [], "type": "mean" } ] ], "tags": [] }, { "expr": "max(kube_pod_container_resource_limits_cpu_cores{pod=~\"$instance\"}) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Limit {{pod}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU usage/s for $instance", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "decimals": 2, "fill": 1, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 13 }, "id": 484, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 175, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "/Limit .*/", "color": "#C15C17", "dashes": true, "fill": 0 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(process_cpu_seconds_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "groupBy": [ { "params": [ "$__interval" ], "type": "time" }, { "params": [ "null" ], "type": "fill" } ], "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ [ { "params": [ "value" ], "type": "field" }, { "params": [], "type": "mean" } ] ], "tags": [] }, { "expr": "max(kube_pod_container_resource_limits_cpu_cores{pod=~\"$instance\"}) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Limit {{pod}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU usage/s for $instance", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "decimals": 2, "fill": 1, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 13 }, "id": 485, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 175, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "/Limit .*/", "color": "#C15C17", "dashes": true, "fill": 0 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(process_cpu_seconds_total{instance=~\"$instance\"}[$__interval])", "format": "time_series", "groupBy": [ { "params": [ "$__interval" ], "type": "time" }, { "params": [ "null" ], "type": "fill" } ], "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ [ { "params": [ "value" ], "type": "field" }, { "params": [], "type": "mean" } ] ], "tags": [] }, { "expr": "max(kube_pod_container_resource_limits_cpu_cores{pod=~\"$instance\"}) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Limit {{pod}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU usage/s for $instance", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 20 }, "id": 177, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "instance", "repeatDirection": "h", "scopedVars": { "instance": { "selected": false, "text": "alertmanager-54b9fdf9bf-gqxs4", "value": "alertmanager-54b9fdf9bf-gqxs4" } }, "seriesOverrides": [ { "alias": "/Limit .*/", "color": "#C15C17", "dashes": true, "fill": 0 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "process_resident_memory_bytes{instance=~\"$instance\"}", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "refId": "E" }, { "expr": "max(kube_pod_container_resource_limits_memory_bytes{pod=~\"$instance\"}) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Limit {{ pod }}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory usage for $instance", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 20 }, "id": 486, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 177, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-7f5c977766-kwr2s", "value": "alertmanager-7f5c977766-kwr2s" } }, "seriesOverrides": [ { "alias": "/Limit .*/", "color": "#C15C17", "dashes": true, "fill": 0 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "process_resident_memory_bytes{instance=~\"$instance\"}", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "refId": "E" }, { "expr": "max(kube_pod_container_resource_limits_memory_bytes{pod=~\"$instance\"}) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Limit {{ pod }}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory usage for $instance", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 20 }, "id": 487, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 177, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-5b9858bf54-r8skx", "value": "alertmanager-sre-5b9858bf54-r8skx" } }, "seriesOverrides": [ { "alias": "/Limit .*/", "color": "#C15C17", "dashes": true, "fill": 0 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "process_resident_memory_bytes{instance=~\"$instance\"}", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "refId": "E" }, { "expr": "max(kube_pod_container_resource_limits_memory_bytes{pod=~\"$instance\"}) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Limit {{ pod }}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory usage for $instance", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 20 }, "id": 488, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "h", "repeatIteration": 1555332813958, "repeatPanelId": 177, "scopedVars": { "instance": { "selected": false, "text": "alertmanager-sre-8455c65f47-mmwlq", "value": "alertmanager-sre-8455c65f47-mmwlq" } }, "seriesOverrides": [ { "alias": "/Limit .*/", "color": "#C15C17", "dashes": true, "fill": 0 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "process_resident_memory_bytes{instance=~\"$instance\"}", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "refId": "E" }, { "expr": "max(kube_pod_container_resource_limits_memory_bytes{pod=~\"$instance\"}) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Limit {{ pod }}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory usage for $instance", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Resources", "type": "row" } ], "refresh": "5m", "schemaVersion": 18, "style": "dark", "tags": [ "alertmanager", "prometheus", "alerting" ], "templating": { "list": [ { "current": { "tags": [], "text": "default", "value": "default" }, "hide": 0, "includeAll": false, "label": "Prometheus datasource", "multi": false, "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": "", "current": {}, "datasource": "$datasource", "definition": "query_result(alertmanager_build_info)", "hide": 0, "includeAll": true, "label": "Instance", "multi": true, "name": "instance", "options": [], "query": "query_result(alertmanager_build_info)", "refresh": 2, "regex": "/.*instance=\"([^\"]+)\".*/", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "label_values(alertmanager_build_info, instance)", "tags": [], "tagsQuery": "label_values(alertmanager_build_info, namespace)", "type": "query", "useTags": false } ] }, "time": { "from": "now-12h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "", "title": "Alertmanager", "uid": "eea-9_sik", "version": 27 }