{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__elements": { "d476ac28-999a-4363-890e-170933aa51c7": { "name": "Packets dropped due to PPS (Packets Per Second) limits to link-local proxy services", "uid": "d476ac28-999a-4363-890e-170933aa51c7", "kind": 1, "model": { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "The linklocal_allowance_exceeded is number of packets dropped because the PPS of the traffic to local proxy services exceeded the maximum for the network interface. This impacts traffic to the DNS service, the Instance Metadata Service, and the Amazon Time Sync Service.\nhttps://docs.aws.amazon.com/eks/latest/best-practices/monitoring_eks_workloads_for_network_performance_issues.html#:~:text=The%20linklocal_allowance_exceeded%20is%20number%20of%20packets%20dropped%20because%20the%20PPS%20of%20the%20traffic%20to%20local%20proxy%20services%20exceeded%20the%20maximum%20for%20the%20network%20interface.%20This%20impacts%20traffic%20to%20the%20DNS%20service%2C%20the%20Instance%20Metadata%20Service%2C%20and%20the%20Amazon%20Time%20Sync%20Service.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "min", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum by (instance) (\n rate(\n node_ethtool_linklocal_allowance_exceeded{job=\"node-exporter\"}[1m]\n )\n)", "legendFormat": "{{instance}} ", "range": true, "refId": "A" } ], "title": "Packets dropped due to PPS (Packets Per Second) limits to link-local proxy services", "type": "timeseries" } }, "bd060192-bb67-413c-ae00-f568c01c5c25": { "name": "AWS instance-level available connections", "uid": "bd060192-bb67-413c-ae00-f568c01c5c25", "kind": 1, "model": { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "conntrack_allowance_available is the number of tracked connections that can be established by the instance before hitting the Connections Tracked allowance of that instance type (supported for nitro-based instance only).\nhttps://docs.aws.amazon.com/eks/latest/best-practices/monitoring_eks_workloads_for_network_performance_issues.html#:~:text=conntrack_allowance_exceeded%20is%20the%20number%20of%20packets%20dropped%20because%20connection%20tracking%20exceeded%20the%20maximum%20for%20the%20instance%20and%20new%20connections%20could%20not%20be%20established.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "min", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Min", "sortDesc": false }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "node_ethtool_conntrack_allowance_available{job=\"node-exporter\"}", "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "AWS instance-level available connections", "type": "timeseries" } }, "f70e0915-c07a-4d7c-8cc4-b3427bb506be": { "name": "Packets dropped per second due to inbound bandwidth limits", "uid": "f70e0915-c07a-4d7c-8cc4-b3427bb506be", "kind": 1, "model": { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Packets that were queued or dropped due to inbound bandwidth exceeding the instance's network performance limitshttps://docs.aws.amazon.com/eks/latest/best-practices/monitoring_eks_workloads_for_network_performance_issues.html#:~:text=conntrack_allowance_exceeded%20is%20the%20number%20of%20packets%20dropped%20because%20connection%20tracking%20exceeded%20the%20maximum%20for%20the%20instance%20and%20new%20connections%20could%20not%20be%20established.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "min", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum by (instance) (\n rate(node_ethtool_bw_in_allowance_exceeded{job=\"node-exporter\"}[1m])\n )", "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Packets dropped per second due to inbound bandwidth limits", "type": "timeseries" } }, "bb9df1df-0e04-447f-85e3-12f96a0d0abf": { "name": "Packets Dropped Per Second due to connection tracking exceeded", "uid": "bb9df1df-0e04-447f-85e3-12f96a0d0abf", "kind": 1, "model": { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "conntrack_allowance_exceeded is the number of packets dropped because connection tracking exceeded the maximum for the instance and new connections could not be established.\nType: This is a cumulative counter metric, not a rate metric. It represents the total number of packets that have been dropped since the metric collection started.\nhttps://docs.aws.amazon.com/eks/latest/best-practices/monitoring_eks_workloads_for_network_performance_issues.html#:~:text=conntrack_allowance_exceeded%20is%20the%20number%20of%20packets%20dropped%20because%20connection%20tracking%20exceeded%20the%20maximum%20for%20the%20instance%20and%20new%20connections%20could%20not%20be%20established.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "min", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum by (instance) (\n rate(node_ethtool_conntrack_allowance_exceeded{job=\"node-exporter\"}[1m])\n )", "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Packets Dropped Per Second due to connection tracking exceeded", "type": "timeseries" } }, "b4eae9c7-2b00-4cb8-8c20-04df57ce6d6a": { "name": "Packets dropped per second due to outbound bandwidth limits", "uid": "b4eae9c7-2b00-4cb8-8c20-04df57ce6d6a", "kind": 1, "model": { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Packets that were queued or dropped due to outbound bandwidth exceeding the instance's network performance limitshttps://docs.aws.amazon.com/eks/latest/best-practices/monitoring_eks_workloads_for_network_performance_issues.html#:~:text=conntrack_allowance_exceeded%20is%20the%20number%20of%20packets%20dropped%20because%20connection%20tracking%20exceeded%20the%20maximum%20for%20the%20instance%20and%20new%20connections%20could%20not%20be%20established.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "min", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum by (instance) (\n rate(node_ethtool_bw_out_allowance_exceeded{job=\"node-exporter\"}[1m])\n )", "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Packets dropped per second due to outbound bandwidth limits", "type": "timeseries" } }, "b253c185-47ee-4b64-9bce-a19ecdcd344a": { "name": "Packets dropped per second due to bidirectional PPS limits", "uid": "b253c185-47ee-4b64-9bce-a19ecdcd344a", "kind": 1, "model": { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Packets that were queued or dropped due to bidirectional PPS exceeding the instance's network performance limitshttps://docs.aws.amazon.com/eks/latest/best-practices/monitoring_eks_workloads_for_network_performance_issues.html#:~:text=conntrack_allowance_exceeded%20is%20the%20number%20of%20packets%20dropped%20because%20connection%20tracking%20exceeded%20the%20maximum%20for%20the%20instance%20and%20new%20connections%20could%20not%20be%20established.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "min", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum by (instance) (\n rate(node_ethtool_pps_allowance_exceeded{job=\"node-exporter\"}[1m])\n )", "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Packets dropped per second due to bidirectional PPS limits", "type": "timeseries" } } }, "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.0.4" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 2, "id": null, "links": [], "liveNow": false, "panels": [ { "gridPos": { "h": 11, "w": 12, "x": 0, "y": 0 }, "id": 2, "libraryPanel": { "uid": "d476ac28-999a-4363-890e-170933aa51c7", "name": "Packets dropped due to PPS (Packets Per Second) limits to link-local proxy services" } }, { "gridPos": { "h": 11, "w": 12, "x": 12, "y": 0 }, "id": 4, "libraryPanel": { "uid": "bd060192-bb67-413c-ae00-f568c01c5c25", "name": "AWS instance-level available connections" } }, { "gridPos": { "h": 9, "w": 12, "x": 0, "y": 11 }, "id": 8, "libraryPanel": { "uid": "f70e0915-c07a-4d7c-8cc4-b3427bb506be", "name": "Packets dropped per second due to inbound bandwidth limits" } }, { "gridPos": { "h": 9, "w": 12, "x": 12, "y": 11 }, "id": 6, "libraryPanel": { "uid": "bb9df1df-0e04-447f-85e3-12f96a0d0abf", "name": "Packets Dropped Per Second due to connection tracking exceeded" } }, { "gridPos": { "h": 11, "w": 12, "x": 0, "y": 20 }, "id": 12, "libraryPanel": { "uid": "b4eae9c7-2b00-4cb8-8c20-04df57ce6d6a", "name": "Packets dropped per second due to outbound bandwidth limits" } }, { "gridPos": { "h": 11, "w": 12, "x": 12, "y": 20 }, "id": 10, "libraryPanel": { "uid": "b253c185-47ee-4b64-9bce-a19ecdcd344a", "name": "Packets dropped per second due to bidirectional PPS limits" } } ], "refresh": "", "schemaVersion": 38, "style": "dark", "tags": [], "templating": { "list": [] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Network limits", "uid": "ed8d7485-c51a-4993-a4da-fca3f1947728", "version": 5, "weekStart": "" }