{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": 1, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 22, "panels": [], "title": "header", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 0, "y": 1 }, "id": 19, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.1.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(node_cpu_info{instance=~\".*master.*\"}) by (instance)", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "CPU Count By node (Control Planes)", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 4, "y": 1 }, "id": 18, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.1.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(machine_memory_bytes{node=~\".*master.*\"}) by (node) ", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Mem Total By node (Control Planes)", "transparent": true, "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "id": 21, "panels": [], "title": "Cluster", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 5 }, "id": 9, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true, "width": 0 }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(resource_verb:apiserver_request_duration_seconds_bucket:rate:5m{apiserver=\"kube-apiserver\"}) by (verb, le))", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "api-kas-request-duration-p99", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 5 }, "id": 10, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, operation:etcd_request_duration_seconds_bucket:rate5m)", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "etcd-request-duration-p99", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 5 }, "id": 1, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"etcd\", pod=~\"etcd-$instance\"}[5m])) by (instance, le))", "instant": false, "legendFormat": "{{instance}}-WAL", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"etcd\", pod=~\"etcd-$instance\"}[5m])) by (instance, le))", "hide": false, "instant": false, "legendFormat": "{{instance}}-DB", "range": true, "refId": "B" } ], "title": "etcd disk fsync (WAL | DB) p99", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 0, "y": 11 }, "id": 11, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "(instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"\", instance=~\"$instance\"} / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"\", instance=~\"$instance\"}))) != 0", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(node_cpu_info{instance=~\"$instance\"}) by (instance)", "hide": false, "instant": false, "interval": "", "legendFormat": "{{instance}}_Total", "range": true, "refId": "B" } ], "title": "node-cpu-saturation-load1", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 5, "y": 11 }, "id": 12, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": false }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "instance:node_vmstat_pgmajfault:rate1m{job=\"node-exporter\", cluster=\"\", instance=~\"$instance\"}", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "node-memory-saturation", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 10, "y": 11 }, "id": 8, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", id!=\"\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"\", namespace!=\"\", namespace=~\"$namespace\"}[5m]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", id!=\"\", cluster=\"\", namespace!=\"\", namespace=~\"$namespace\"}[5m]))", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "cluster-storage-throughput (by namespace)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 15, "y": 11 }, "id": 14, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "( instance_device:node_disk_io_time_weighted_seconds:rate1m{job=\"node-exporter\", cluster=\"\", instance=~\"$instance\"} / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate1m{job=\"node-exporter\", cluster=\"\", instance=~\"$instance\"}))) != 0", "instant": false, "legendFormat": "{{instance}}_{{device}}", "range": true, "refId": "A" } ], "title": "node-disk-saturation", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 20, "y": 11 }, "id": 3, "options": { "legend": { "calcs": [ "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "timezone": [ "utc" ], "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"etcd\", pod=~\"etcd-$instance\"}[5m])))", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "etcd-peer-round-trip-time", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 0, "y": 18 }, "id": 5, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"\", namespace=~\"$namespace\"}) by (namespace)", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"\", namespace=~\"$namespace\"})", "hide": false, "instant": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "cluster-cpu-usage (by namespace)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 5, "y": 18 }, "id": 6, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"\", container!=\"\", namespace=~\"$namespace\", node=~\".*master.*\"}) by (namespace)", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"\", container!=\"\", namespace=~\"$namespace\", node=~\".*master.*\"}) by (node)", "hide": false, "instant": false, "legendFormat": "Total-Node_{{node}}", "range": true, "refId": "B" } ], "title": "cluster-memory-usage-wo-cache (by namespace and Node Total)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 10, "y": 18 }, "id": 7, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", id!=\"\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"\", namespace!=\"\", namespace=~\"$namespace\"}[5m]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", id!=\"\", cluster=\"\", namespace!=\"\", namespace=~\"$namespace\"}[5m])))", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "cluster-storage-iops", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 15, "y": 18 }, "id": 13, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "instance:node_network_transmit_drop_excluding_lo:rate1m{job=\"node-exporter\", cluster=\"\", instance=~\"$instance\"} != 0", "instant": false, "legendFormat": "{{instance}}_tx", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "instance:node_network_receive_drop_excluding_lo:rate1m{job=\"node-exporter\", cluster=\"\", instance=~\"$instance\"} != 0", "hide": false, "instant": false, "legendFormat": "{{instance}}_rx", "range": true, "refId": "B" } ], "title": "node-network-saturation RX/TX", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 20, "y": 18 }, "id": 2, "options": { "legend": { "calcs": [ "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "changes(etcd_server_leader_changes_seen_total{job=\"etcd\"}[1d])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "etcd-total-leader-elections-day", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 0, "y": 25 }, "id": 28, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "changes(kube_endpoint_address_available{endpoint=~\"api.*\"}[5m])", "hide": false, "instant": false, "legendFormat": "{{namespace}}_{{ endpoint }}", "range": true, "refId": "A" } ], "title": "Endpoint Availability", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 10, "y": 25 }, "id": 23, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(rate(node_network_transmit_bytes_total{instance=~\"$instance\"}[5m])) by (instance, device)", "hide": false, "instant": false, "legendFormat": "{{instance}}_{{device}}_tx", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(rate(node_network_receive_bytes_total{instance=~\"$instance\"}[5m])) by (instance, device)", "hide": false, "instant": false, "legendFormat": "{{instance}}_{{device}}_rx", "range": true, "refId": "B" } ], "title": "node network bytes total (RX/TX)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 15, "y": 25 }, "id": 24, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(rate(node_network_transmit_packets_total{instance=~\"$instance\"}[5m])) by (instance, device)", "hide": false, "instant": false, "legendFormat": "{{instance}}_{{device}}_tx", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(rate(node_network_receive_packets_total{instance=~\"$instance\"}[5m])) by (instance, device)", "hide": false, "instant": false, "legendFormat": "{{instance}}_{{device}}_rx", "range": true, "refId": "B" } ], "title": "node network packets total (RX/TX)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 20, "y": 25 }, "id": 4, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"etcd\", pod=~\"etcd-$instance\"}[5m])) by (instance, le))", "instant": false, "legendFormat": "{{instance}}-WAL", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"etcd\", pod=~\"etcd-$instance\"}[5m])) by (instance, le))", "hide": false, "instant": false, "legendFormat": "{{instance}}-DB", "range": true, "refId": "B" } ], "title": "etcd disk fsync (WAL | DB) p50", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 10, "y": 32 }, "id": 25, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(kube_namespace_status_phase) by (phase)", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Total Namespaces by Phase", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 15, "y": 32 }, "id": 26, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(kube_pod_status_phase) by (phase)", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Total Pods (by Phase)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 20, "y": 32 }, "id": 27, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "timezone": [ "utc" ], "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(kube_pod_container_status_ready)", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "editorMode": "code", "expr": "sum(kube_pod_container_status_waiting)", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "B" } ], "title": "Total Containers", "type": "timeseries" } ], "refresh": false, "schemaVersion": 38, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": ".*", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "definition": "label_values(node_cpu_info,instance)", "hide": 0, "includeAll": true, "label": "Node Name", "multi": true, "name": "instance", "options": [], "query": { "query": "label_values(node_cpu_info,instance)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" }, { "allValue": ".*", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "${Datasource}" }, "definition": "label_values(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate,namespace)", "hide": 0, "includeAll": true, "multi": true, "name": "namespace", "options": [], "query": { "query": "label_values(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate,namespace)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 2, "type": "query" }, { "current": { "selected": false, "text": "CI 1714274463920427008", "value": "e7b59dc8-01b7-4fed-876f-5e13e41f8102" }, "hide": 0, "includeAll": false, "label": "Datasource", "multi": false, "name": "Datasource", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": ".*", "current": { "selected": false, "text": [ "All" ], "value": [ "$__all" ] }, "hide": 0, "includeAll": true, "multi": true, "name": "node_group", "options": [ { "selected": true, "text": "All", "value": "$__all" }, { "selected": false, "text": ".*master.*", "value": ".*master.*" }, { "selected": false, "text": ".*worker.*", "value": ".*worker.*" } ], "query": ".*master.*, .*worker.*", "queryValue": "", "skipUrlSync": false, "type": "custom" } ] }, "time": { "from": "2023-10-17T14:20:27.696Z", "to": "2023-10-17T15:44:13.208Z" }, "timepicker": {}, "timezone": "utc", "title": "OpenShift", "uid": "a0efd871-4fc5-4fe8-8037-a1d7ab1e4ccd", "version": 58, "weekStart": "" }