{ "title": "HAMi Ascend hard slicing - lab core metrics", "uid": "hami-ascend-lab-core", "tags": ["hami", "ascend", "lab"], "timezone": "browser", "schemaVersion": 39, "version": 1, "refresh": "30s", "time": { "from": "now-1h", "to": "now" }, "templating": { "list": [ { "name": "datasource", "label": "Data source", "type": "datasource", "query": "prometheus", "current": {}, "hide": 0 }, { "name": "node", "label": "Node", "type": "query", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "query": "label_values(hami_gpu_memory_limit_bytes, node)", "includeAll": true, "multi": true, "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, "refresh": 1, "hide": 0 }, { "name": "namespace", "label": "Namespace", "type": "query", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "query": "label_values(hami_vgpu_memory_allocated_bytes, namespace)", "includeAll": true, "multi": true, "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, "refresh": 1, "hide": 0 } ] }, "panels": [ { "type": "stat", "title": "Shared containers (\u5207\u7247\u5171\u5361\u6570)", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 0, "w": 4, "h": 5 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(hami_gpu_shared_count{node=~\"$node\"})", "instant": true, "refId": "A", "format": "time_series" } ], "options": { "graphMode": "none", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto", "justifyMode": "auto" }, "fieldConfig": { "defaults": { "unit": null, "decimals": null, "color": { "mode": "fixed", "fixedColor": "#73BF69" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] } }, { "type": "stat", "title": "Allocated GPU memory (\u5df2\u5206\u914d\u663e\u5b58)", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 4, "y": 0, "w": 4, "h": 5 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(hami_gpu_memory_allocated_bytes{node=~\"$node\"})", "instant": true, "refId": "A", "format": "time_series" } ], "options": { "graphMode": "none", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto", "justifyMode": "auto" }, "fieldConfig": { "defaults": { "unit": "bytes", "decimals": null, "color": { "mode": "fixed", "fixedColor": "#FADE2A" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] } }, { "type": "stat", "title": "GPU memory limit (\u5361\u603b\u663e\u5b58)", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 8, "y": 0, "w": 4, "h": 5 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(hami_gpu_memory_limit_bytes{node=~\"$node\"})", "instant": true, "refId": "A", "format": "time_series" } ], "options": { "graphMode": "none", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto", "justifyMode": "auto" }, "fieldConfig": { "defaults": { "unit": "bytes", "decimals": null, "color": { "mode": "fixed", "fixedColor": "text" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] } }, { "type": "gauge", "title": "GPU memory allocated % (\u663e\u5b58\u5206\u914d\u7387)", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 0, "w": 6, "h": 5 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "100 * sum(hami_gpu_memory_allocated_bytes{node=~\"$node\"}) / clamp_min(sum(hami_gpu_memory_limit_bytes{node=~\"$node\"}), 1)", "instant": true, "refId": "A" } ], "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] } }, { "type": "stat", "title": "Namespace quota used (huawei.com/Ascend310P-memory)", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 18, "y": 0, "w": 6, "h": 5 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "hami_resource_quota_used{namespace=~\"$namespace\", quota_name=\"huawei.com/Ascend310P-memory\"}", "instant": true, "refId": "A", "format": "time_series" } ], "options": { "graphMode": "none", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto", "justifyMode": "auto" }, "fieldConfig": { "defaults": { "unit": null, "decimals": null, "color": { "mode": "fixed", "fixedColor": "#FF9830" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] } }, { "type": "table", "title": "Per-Pod allocated memory (\u6bcf\u4e2a\u5207\u7247 Pod \u7684\u5206\u914d\u91cf)", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 5, "w": 14, "h": 9 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "hami_vgpu_memory_allocated_bytes{namespace=~\"$namespace\"}", "instant": true, "refId": "A", "format": "table" } ], "options": { "showHeader": true, "footer": { "show": false, "reducer": ["sum"], "countRows": false, "fields": "" } }, "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "unit": "bytes", "decimals": 2 }, "overrides": [] }, "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "instance": true, "container_index": true, "device_uuid": true, "zone": true, "node": false }, "renameByName": { "pod": "Pod", "namespace": "Namespace", "Value": "\u5df2\u5206\u914d", "node": "Node", "quota_name": "\u914d\u989d\u9879", "lim": "\u4e0a\u9650" } } } ] }, { "type": "bargauge", "title": "Per-Pod allocated memory (bar)", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 14, "y": 5, "w": 10, "h": 9 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "hami_vgpu_memory_allocated_bytes{namespace=~\"$namespace\"}", "instant": true, "refId": "A", "legendFormat": "{{pod}}" } ], "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showUnfilled": true, "minVizHeight": 16, "minVizWidth": 8 }, "fieldConfig": { "defaults": { "unit": "bytes", "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] } }, { "type": "table", "title": "Device inventory (\u8bbe\u5907\u6e05\u5355)", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 14, "w": 12, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "hami_node_gpu_overview{node=~\"$node\"}", "instant": true, "refId": "A", "format": "table" } ], "options": { "showHeader": true, "footer": { "show": false, "reducer": ["sum"], "countRows": false, "fields": "" } }, "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "unit": "none", "decimals": 0 }, "overrides": [] }, "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "instance": true, "container_index": true, "device_uuid": true, "zone": true, "node": false }, "renameByName": { "pod": "Pod", "namespace": "Namespace", "Value": "\u5df2\u5206\u914d", "node": "Node", "quota_name": "\u914d\u989d\u9879", "lim": "\u4e0a\u9650" } } } ] }, { "type": "table", "title": "Namespace quota (\u547d\u540d\u7a7a\u95f4\u914d\u989d)", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 14, "w": 12, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "hami_resource_quota_used{namespace=~\"$namespace\"}", "instant": true, "refId": "A", "format": "table" } ], "options": { "showHeader": true, "footer": { "show": false, "reducer": ["sum"], "countRows": false, "fields": "" } }, "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "unit": "none", "decimals": 0 }, "overrides": [] }, "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "instance": true, "container_index": true, "device_uuid": true, "zone": true, "node": false }, "renameByName": { "pod": "Pod", "namespace": "Namespace", "Value": "\u5df2\u5206\u914d", "node": "Node", "quota_name": "\u914d\u989d\u9879", "lim": "\u4e0a\u9650" } } } ] }, { "type": "timeseries", "title": "Allocated vs limit over time", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 22, "w": 8, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (node) (hami_gpu_memory_allocated_bytes{node=~\"$node\"})", "legendFormat": "allocated {{node}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (node) (hami_gpu_memory_limit_bytes{node=~\"$node\"})", "legendFormat": "limit {{node}}", "refId": "B" } ], "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 12, "showPoints": "never", "spanNulls": true } }, "overrides": [] } }, { "type": "timeseries", "title": "Shared containers over time", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 8, "y": 22, "w": 8, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(hami_gpu_shared_count{node=~\"$node\"})", "legendFormat": "shared containers", "refId": "A" } ], "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "fieldConfig": { "defaults": { "unit": null, "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 12, "showPoints": "never", "spanNulls": true } }, "overrides": [] } }, { "type": "timeseries", "title": "Node GPU memory allocated ratio", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 16, "y": 22, "w": 8, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "hami_node_gpu_memory_allocated_ratio{node=~\"$node\"}", "legendFormat": "{{node}} idx{{device_index}}", "refId": "A" } ], "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "fieldConfig": { "defaults": { "unit": "percentunit", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 12, "showPoints": "never", "spanNulls": true } }, "overrides": [] } } ], "annotations": { "list": [] } }