{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" }, { "name": "DS_EXPRESSION", "label": "Expression", "description": "", "type": "datasource", "pluginId": "__expr__" }, { "name": "VAR_INSTANCE_LABEL", "type": "constant", "label": "Instance Label", "value": "job", "description": "" } ], "__elements": {}, "__requires": [ { "type": "datasource", "id": "__expr__", "version": "1.0.0" }, { "type": "panel", "id": "bargauge", "name": "Bar gauge", "version": "" }, { "type": "panel", "id": "gauge", "name": "Gauge", "version": "" }, { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "12.2.1" }, { "type": "panel", "id": "heatmap", "name": "Heatmap", "version": "" }, { "type": "panel", "id": "piechart", "name": "Pie chart", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, { "type": "panel", "id": "table", "name": "Table", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "links": [], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 96, "panels": [], "repeat": "instance", "title": "Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 0, "y": 1 }, "id": 22, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": { "valueSize": 20 }, "textMode": "name", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "exemplar": false, "expr": "reth_info{$instance_label=\"$instance\"}", "instant": true, "legendFormat": "{{version}}", "range": false, "refId": "A" } ], "title": "Version", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 4, "y": 1 }, "id": 192, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": { "valueSize": 20 }, "textMode": "name", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "exemplar": false, "expr": "reth_info{$instance_label=\"$instance\"}", "instant": true, "legendFormat": "{{build_timestamp}}", "range": false, "refId": "A" } ], "title": "Build Timestamp", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 8, "y": 1 }, "id": 193, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": { "valueSize": 20 }, "textMode": "name", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "exemplar": false, "expr": "reth_info{$instance_label=\"$instance\"}", "instant": true, "legendFormat": "{{git_sha}}", "range": false, "refId": "A" } ], "title": "Git SHA", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 12, "y": 1 }, "id": 195, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": { "valueSize": 20 }, "textMode": "name", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "exemplar": false, "expr": "reth_info{$instance_label=\"$instance\"}", "instant": true, "legendFormat": "{{build_profile}}", "range": false, "refId": "A" } ], "title": "Build Profile", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 16, "y": 1 }, "id": 196, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": { "valueSize": 20 }, "textMode": "name", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "exemplar": false, "expr": "reth_info{$instance_label=\"$instance\"}", "instant": true, "legendFormat": "{{target_triple}}", "range": false, "refId": "A" } ], "title": "Target Triple", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 20, "y": 1 }, "id": 197, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": { "valueSize": 20 }, "textMode": "name", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "exemplar": false, "expr": "reth_info{$instance_label=\"$instance\"}", "instant": true, "legendFormat": "{{cargo_features}}", "range": false, "refId": "A" } ], "title": "Cargo Features", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "dark-red", "value": 0 }, { "color": "semi-dark-orange", "value": 10 }, { "color": "semi-dark-yellow", "value": 20 }, { "color": "semi-dark-green", "value": 30 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 8, "x": 0, "y": 4 }, "id": 194, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "exemplar": false, "expr": "reth_network_connected_peers{$instance_label=\"$instance\"}", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" } ], "title": "Connected peers", "transparent": true, "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The checkpoints mark the last block a stage can recover from in the case of a crash or shutdown of the node", "fieldConfig": { "defaults": { "color": { "mode": "continuous-RdYlGr" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 8, "x": 8, "y": 4 }, "id": 20, "options": { "displayMode": "lcd", "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "maxVizHeight": 300, "minVizHeight": 16, "minVizWidth": 8, "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "sizing": "auto", "valueMode": "color" }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "exemplar": false, "expr": "max by (stage) (reth_sync_checkpoint{$instance_label=\"$instance\"})", "instant": true, "legendFormat": "{{stage}}", "range": false, "refId": "A" } ], "title": "Stage checkpoints", "transformations": [ { "id": "joinByField", "options": { "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": {}, "includeByName": {}, "indexByName": { "AccountHashing": 6, "Bodies": 1, "Execution": 3, "Finish": 14, "Headers": 0, "IndexAccountHistory": 12, "IndexStorageHistory": 11, "MerkleChangeSets": 9, "MerkleExecute": 8, "MerkleUnwind": 5, "Prune": 13, "PruneSenderRecovery": 4, "SenderRecovery": 2, "StorageHashing": 7, "Time": 15, "TransactionLookup": 10 }, "renameByName": {} } } ], "transparent": true, "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 9, "w": 8, "x": 16, "y": 4 }, "id": 218, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(reth_db_table_size{$instance_label=\"$instance\"})", "legendFormat": "Database", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(reth_db_freelist{$instance_label=\"$instance\"} * reth_db_page_size{$instance_label=\"$instance\"})", "hide": false, "instant": false, "legendFormat": "Freelist", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(reth_static_files_segment_size{$instance_label=\"$instance\"})", "hide": false, "instant": false, "legendFormat": "Static Files", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "(sum(reth_rocksdb_table_size{$instance_label=\"$instance\"}) or vector(0)) + (sum(reth_rocksdb_wal_size{$instance_label=\"$instance\"}) or vector(0))", "hide": false, "instant": false, "legendFormat": "RocksDB", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(reth_db_table_size{$instance_label=\"$instance\"}) + sum(reth_db_freelist{$instance_label=\"$instance\"} * reth_db_page_size{$instance_label=\"$instance\"}) + sum(reth_static_files_segment_size{$instance_label=\"$instance\"}) + (sum(reth_rocksdb_table_size{$instance_label=\"$instance\"}) or vector(0)) + (sum(reth_rocksdb_wal_size{$instance_label=\"$instance\"}) or vector(0))", "hide": false, "instant": false, "legendFormat": "Total", "range": true, "refId": "E" } ], "title": "Storage size", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, "id": 69, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "avg by (stage) (reth_sync_entities_processed{$instance_label=\"$instance\"} / reth_sync_entities_total{$instance_label=\"$instance\"})", "legendFormat": "{{stage}}", "range": true, "refId": "A" } ], "title": "Sync progress (stage progress in %)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, "id": 12, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "max by (stage) (reth_sync_checkpoint{$instance_label=\"$instance\"})", "legendFormat": "{{stage}}", "range": true, "refId": "A" } ], "title": "Sync progress (stage progress as highest block number reached)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Latency histogram for the engine_forkchoiceUpdated RPC API", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [ { "matcher": { "id": "byValue", "options": { "op": "gte", "reducer": "allIsZero", "value": 0 } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": true, "tooltip": true, "viz": true } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, "id": 211, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_consensus_engine_beacon_forkchoice_updated_latency{$instance_label=\"$instance\", quantile=\"0\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "min", "range": true, "refId": "K", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_consensus_engine_beacon_forkchoice_updated_latency{$instance_label=\"$instance\", quantile=\"0.5\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "p50", "range": true, "refId": "L", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_consensus_engine_beacon_forkchoice_updated_latency{$instance_label=\"$instance\", quantile=\"0.9\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "p90", "range": true, "refId": "M", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_consensus_engine_beacon_forkchoice_updated_latency{$instance_label=\"$instance\", quantile=\"0.95\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "p95", "range": true, "refId": "N", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_consensus_engine_beacon_forkchoice_updated_latency{$instance_label=\"$instance\", quantile=\"0.99\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "p99", "range": true, "refId": "O", "useBackend": false } ], "title": "Engine API forkchoiceUpdated Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Latency histogram for the engine_newPayload RPC API", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [ { "matcher": { "id": "byValue", "options": { "op": "gte", "reducer": "allIsZero", "value": 0 } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": true, "tooltip": true, "viz": true } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 }, "id": 210, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_consensus_engine_beacon_new_payload_latency{$instance_label=\"$instance\", quantile=\"0\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "min", "range": true, "refId": "P", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_consensus_engine_beacon_new_payload_latency{$instance_label=\"$instance\", quantile=\"0.5\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "p50", "range": true, "refId": "Q", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_consensus_engine_beacon_new_payload_latency{$instance_label=\"$instance\", quantile=\"0.9\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "p90", "range": true, "refId": "R", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_consensus_engine_beacon_new_payload_latency{$instance_label=\"$instance\", quantile=\"0.95\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "p95", "range": true, "refId": "S", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_consensus_engine_beacon_new_payload_latency{$instance_label=\"$instance\", quantile=\"0.99\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "p99", "range": true, "refId": "T", "useBackend": false } ], "title": "Engine API newPayload Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The metric is the amount of gas processed in a block", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "sishort" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 }, "id": 1004, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_consensus_engine_beacon_new_payload_total_gas{$instance_label=\"$instance\", quantile=\"0.5\"}", "legendFormat": "p50", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_consensus_engine_beacon_new_payload_total_gas{$instance_label=\"$instance\", quantile=\"0.9\"}", "hide": false, "legendFormat": "p90", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_consensus_engine_beacon_new_payload_total_gas{$instance_label=\"$instance\", quantile=\"0.95\"}", "hide": false, "legendFormat": "p95", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_consensus_engine_beacon_new_payload_total_gas{$instance_label=\"$instance\", quantile=\"0.99\"}", "hide": false, "legendFormat": "p99", "range": true, "refId": "D" } ], "title": "Engine API newPayload Total Gas", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The throughput of the Engine API newPayload method. The metric is the amount of gas processed in a block, divided by the time it took to process the newPayload request.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "si: gas/s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 }, "id": 1003, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_consensus_engine_beacon_new_payload_gas_per_second{$instance_label=\"$instance\", quantile=\"0.5\"}", "legendFormat": "p50", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_consensus_engine_beacon_new_payload_gas_per_second{$instance_label=\"$instance\", quantile=\"0.9\"}", "hide": false, "legendFormat": "p90", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_consensus_engine_beacon_new_payload_gas_per_second{$instance_label=\"$instance\", quantile=\"0.95\"}", "hide": false, "legendFormat": "p95", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_consensus_engine_beacon_new_payload_gas_per_second{$instance_label=\"$instance\", quantile=\"0.99\"}", "hide": false, "legendFormat": "p99", "range": true, "refId": "D" } ], "title": "Engine API newPayload Throughput", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The throughput of the node's executor. The metric is the amount of gas processed in a block, divided by the time it took to process the block.\n\nNote: For mainnet, the block range 2,383,397-2,620,384 will be slow because of the 2016 DoS attack.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "si: gas/s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 37 }, "id": 56, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_sync_execution_gas_per_second{$instance_label=\"$instance\"}", "legendFormat": "Gas/s", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "avg_over_time(reth_sync_execution_gas_per_second{$instance_label=\"$instance\"}[1m])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "Avg Gas/s (1m)", "range": true, "refId": "B", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "avg_over_time(reth_sync_execution_gas_per_second{$instance_label=\"$instance\"}[5m])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "Avg Gas/s (5m)", "range": true, "refId": "C", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "avg_over_time(reth_sync_execution_gas_per_second{$instance_label=\"$instance\"}[10m])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "Avg Gas/s (10m)", "range": true, "refId": "D", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "avg_over_time(reth_sync_execution_gas_per_second{$instance_label=\"$instance\"}[30m])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "Avg Gas/s (30m)", "range": true, "refId": "E", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "avg_over_time(reth_sync_execution_gas_per_second{$instance_label=\"$instance\"}[1h])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "Avg Gas/s (1h)", "range": true, "refId": "F", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "avg_over_time(reth_sync_execution_gas_per_second{$instance_label=\"$instance\"}[24h])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "Avg Gas/s (24h)", "range": true, "refId": "G", "useBackend": false } ], "title": "Execution throughput", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 37 }, "id": 240, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_sync_block_validation_state_root_duration{$instance_label=\"$instance\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "State Root Duration", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_sync_execution_execution_duration{$instance_label=\"$instance\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Execution Duration", "range": true, "refId": "B", "useBackend": false } ], "title": "Block Processing Latency", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 }, "id": 87, "panels": [], "repeat": "instance", "title": "Engine API", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Engine API messages received by the CL, either engine_newPayload or engine_forkchoiceUpdated", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 46 }, "id": 84, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "rate(reth_consensus_engine_beacon_forkchoice_updated_messages{$instance_label=\"$instance\"}[$__rate_interval])", "legendFormat": "forkchoiceUpdated", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "rate(reth_consensus_engine_beacon_new_payload_messages{$instance_label=\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "newPayload", "range": true, "refId": "B" } ], "title": "Engine API messages", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Counts the number of failed response deliveries due to client request termination.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 46 }, "id": 249, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "rate(reth_consensus_engine_beacon_failed_new_payload_response_deliveries{$instance_label=\"$instance\"}[$__rate_interval])", "legendFormat": "newPayload", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "rate(reth_consensus_engine_beacon_failed_forkchoice_updated_response_deliveries{$instance_label=\"$instance\"}[$__rate_interval])", "legendFormat": "forkchoiceUpdated", "range": true, "refId": "B" } ], "title": "Failed Engine API Response Deliveries", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Latency histogram for the engine_newPayload to engine_forkchoiceUpdated", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 54 }, "id": 213, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_consensus_engine_beacon_new_payload_forkchoice_updated_time_diff{$instance_label=\"$instance\"}", "legendFormat": "p{{quantile}}", "range": true, "refId": "A" } ], "title": "Engine API latency between forkchoiceUpdated and newPayload", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Latency histograms for the engine_getPayloadBodiesByHashV1 and engine_getPayloadBodiesByRangeV1 RPC APIs", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 54 }, "id": 212, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{$instance_label=\"$instance\", quantile=\"0\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getPayloadBodiesByHashV1 min", "range": true, "refId": "O", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{$instance_label=\"$instance\", quantile=\"0.5\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getPayloadBodiesByHashV1 p50", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{$instance_label=\"$instance\", quantile=\"0.9\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getPayloadBodiesByHashV1 p90", "range": true, "refId": "B", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{$instance_label=\"$instance\", quantile=\"0.95\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getPayloadBodiesByHashV1 p95", "range": true, "refId": "C", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{$instance_label=\"$instance\", quantile=\"0.99\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getPayloadBodiesByHashV1 p99", "range": true, "refId": "D", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{$instance_label=\"$instance\", quantile=\"0\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getPayloadBodiesByRangeV1 min", "range": true, "refId": "E", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{$instance_label=\"$instance\", quantile=\"0.5\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getPayloadBodiesByRangeV1 p50", "range": true, "refId": "F", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{$instance_label=\"$instance\", quantile=\"0.9\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getPayloadBodiesByRangeV1 p90", "range": true, "refId": "G", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{$instance_label=\"$instance\", quantile=\"0.95\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getPayloadBodiesByRangeV1 p95", "range": true, "refId": "H", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{$instance_label=\"$instance\", quantile=\"0.99\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getPayloadBodiesByRangeV1 p99", "range": true, "refId": "I", "useBackend": false } ], "title": "Engine API getPayloadBodies Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 62 }, "id": 1000, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "rate(reth_engine_rpc_blobs_blob_count{$instance_label=\"$instance\"}[$__rate_interval])", "legendFormat": "Found", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "rate(reth_engine_rpc_blobs_blob_misses{$instance_label=\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Missed", "range": true, "refId": "B" } ], "title": "Blob Count and Misses", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 62 }, "id": 258, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_blobs_v2{$instance_label=\"$instance\", quantile=\"0.5\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getBlobsV2 p50", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_blobs_v2{$instance_label=\"$instance\", quantile=\"0.95\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getBlobsV2 p95", "range": true, "refId": "B", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_blobs_v2{$instance_label=\"$instance\", quantile=\"0.99\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getBlobsV2 p99", "range": true, "refId": "C", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_blobs_v2{$instance_label=\"$instance\", quantile=\"0\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getBlobsV2 min", "range": true, "refId": "D", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_engine_rpc_get_blobs_v2{$instance_label=\"$instance\", quantile=\"1\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "engine_getBlobsV2 max", "range": true, "refId": "E", "useBackend": false } ], "title": "Engine API getBlobs Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Total pipeline runs triggered by the sync controller", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 70 }, "id": 85, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_consensus_engine_beacon_pipeline_runs{$instance_label=\"$instance\"}", "legendFormat": "Pipeline runs", "range": true, "refId": "A" } ], "title": "Pipeline runs", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 70 }, "id": 83, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_consensus_engine_beacon_active_block_downloads{$instance_label=\"$instance\"}", "legendFormat": "Active block downloads", "range": true, "refId": "A" } ], "title": "Active block downloads", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 78 }, "id": 46, "panels": [], "repeat": "instance", "title": "Execution", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 24, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "percent" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 79 }, "id": 1001, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_sync_block_validation_state_root_duration{$instance_label=\"$instance\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "State Root Duration", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_sync_execution_execution_duration{$instance_label=\"$instance\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Execution Duration", "range": true, "refId": "B", "useBackend": false } ], "title": "Block Processing Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 79 }, "id": 251, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_sync_caching_account_cache_hits{$instance_label=\"$instance\"} / (reth_sync_caching_account_cache_hits{$instance_label=\"$instance\"} + reth_sync_caching_account_cache_misses{$instance_label=\"$instance\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Account cache hits", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_sync_caching_storage_cache_hits{$instance_label=\"$instance\"} / (reth_sync_caching_storage_cache_hits{$instance_label=\"$instance\"} + reth_sync_caching_storage_cache_misses{$instance_label=\"$instance\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Storage cache hits", "range": true, "refId": "B", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_sync_caching_code_cache_hits{$instance_label=\"$instance\"} / (reth_sync_caching_code_cache_hits{$instance_label=\"$instance\"} + reth_sync_caching_code_cache_misses{$instance_label=\"$instance\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Code cache hits", "range": true, "refId": "C", "useBackend": false } ], "title": "Execution cache hitrate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The time it takes for operations that are part of block validation, but not execution or state root, to complete.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 87 }, "id": 252, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_sync_block_validation_trie_input_duration{$instance_label=\"$instance\", quantile=~\"(0|0.5|0.9|0.95|1)\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Trie input creation duration p{{quantile}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Block validation overhead", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "Precompile cache hits" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": true, "viz": true } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 87 }, "id": 1005, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "reth_sync_caching_precompile_cache_hits{$instance_label=\"$instance\"} / (reth_sync_caching_precompile_cache_hits{$instance_label=\"$instance\"} + reth_sync_caching_precompile_cache_misses{$instance_label=\"$instance\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Precompile cache hits", "range": true, "refId": "A", "useBackend": false } ], "title": "Precompile cache hitrate", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 95 }, "id": 214, "panels": [], "title": "State Root Task", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 96 }, "id": 255, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_proofs_processed_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", "instant": false, "legendFormat": "{{quantile}} percentile", "range": true, "refId": "Branch Nodes" } ], "title": "Proofs Processed", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 96 }, "id": 254, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_proof_calculation_duration_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", "instant": false, "legendFormat": "{{quantile}} percentile", "range": true, "refId": "Branch Nodes" } ], "title": "Proof calculation duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 104 }, "id": 257, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_pending_account_multiproofs_histogram{$instance_label=\"$instance\", quantile=\"0.5\"}", "instant": false, "legendFormat": "accounts p50", "range": true, "refId": "Branch Nodes" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_pending_account_multiproofs_histogram{$instance_label=\"$instance\", quantile=\"0.9\"}", "hide": false, "instant": false, "legendFormat": "accounts p90", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_pending_account_multiproofs_histogram{$instance_label=\"$instance\", quantile=\"0.95\"}", "hide": false, "instant": false, "legendFormat": "accounts p95", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_pending_account_multiproofs_histogram{$instance_label=\"$instance\", quantile=\"0.99\"}", "hide": false, "instant": false, "legendFormat": "accounts p99", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_pending_storage_multiproofs_histogram{$instance_label=\"$instance\", quantile=\"0.5\"}", "hide": false, "instant": false, "legendFormat": "storage p50", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_pending_storage_multiproofs_histogram{$instance_label=\"$instance\", quantile=\"0.9\"}", "hide": false, "instant": false, "legendFormat": "storage p90", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_pending_storage_multiproofs_histogram{$instance_label=\"$instance\", quantile=\"0.95\"}", "hide": false, "instant": false, "legendFormat": "storage p95", "range": true, "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_pending_storage_multiproofs_histogram{$instance_label=\"$instance\", quantile=\"0.99\"}", "hide": false, "instant": false, "legendFormat": "storage p99", "range": true, "refId": "G" } ], "title": "Pending MultiProof requests", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 104 }, "id": 256, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_active_account_workers_histogram{$instance_label=\"$instance\",quantile=\"0.5\"}", "instant": false, "legendFormat": "accounts p50", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_active_account_workers_histogram{$instance_label=\"$instance\",quantile=\"0.9\"}", "hide": false, "instant": false, "legendFormat": "accounts p90", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_active_account_workers_histogram{$instance_label=\"$instance\",quantile=\"0.95\"}", "hide": false, "instant": false, "legendFormat": "accounts p95", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_active_account_workers_histogram{$instance_label=\"$instance\",quantile=\"0.99\"}", "hide": false, "instant": false, "legendFormat": "accounts p99", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_active_storage_workers_histogram{$instance_label=\"$instance\",quantile=\"0.5\"}", "instant": false, "legendFormat": "storages p50", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_active_storage_workers_histogram{$instance_label=\"$instance\",quantile=\"0.9\"}", "hide": false, "instant": false, "legendFormat": "storages p90", "range": true, "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_active_storage_workers_histogram{$instance_label=\"$instance\",quantile=\"0.95\"}", "hide": false, "instant": false, "legendFormat": "storages p95", "range": true, "refId": "G" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_active_storage_workers_histogram{$instance_label=\"$instance\",quantile=\"0.99\"}", "hide": false, "instant": false, "legendFormat": "storages p99", "range": true, "refId": "H" } ], "title": "Active multiproof workers", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 112 }, "id": 260, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_sparse_state_trie_multiproof_total_account_nodes{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", "instant": false, "legendFormat": "Account {{quantile}} percentile", "range": true, "refId": "Branch Nodes" } ], "title": "Total multiproof account nodes", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 112 }, "id": 259, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_sparse_state_trie_multiproof_total_storage_nodes{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", "instant": false, "legendFormat": "Storage {{quantile}} percentile", "range": true, "refId": "Branch Nodes" } ], "title": "Total multiproof storage nodes", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 120 }, "id": 262, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_sparse_state_trie_multiproof_skipped_account_nodes{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", "hide": false, "instant": false, "legendFormat": "Account {{quantile}} percentile", "range": true, "refId": "A" } ], "title": "Redundant multiproof account nodes", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 120 }, "id": 261, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_sparse_state_trie_multiproof_skipped_storage_nodes{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", "instant": false, "legendFormat": "Storage {{quantile}} percentile", "range": true, "refId": "Branch Nodes" } ], "title": "Redundant multiproof storage nodes", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "How much time is spent in the multiproof task", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 128 }, "id": 263, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_tree_root_multiproof_task_total_duration_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", "hide": false, "instant": false, "legendFormat": "Task duration {{quantile}} percentile", "range": true, "refId": "A" } ], "title": "Proof fetching total duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Histogram for state root latency, the time spent blocked waiting for the state root.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 128 }, "id": 1006, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_sync_block_validation_state_root_histogram{$instance_label=\"$instance\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "State Root Duration p{{quantile}}", "range": true, "refId": "A", "useBackend": false } ], "title": "State root latency", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 136 }, "id": 38, "panels": [], "repeat": "instance", "title": "Database", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The average commit time for database transactions. Generally, this should not be a limiting factor in syncing.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic", "seriesBy": "last" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 137 }, "id": 40, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "avg(rate(reth_database_transaction_close_duration_seconds_sum{$instance_label=\"$instance\", outcome=\"commit\"}[$__rate_interval]) / rate(reth_database_transaction_close_duration_seconds_count{$instance_label=\"$instance\", outcome=\"commit\"}[$__rate_interval]) >= 0)", "format": "time_series", "instant": false, "legendFormat": "Commit time", "range": true, "refId": "A" } ], "title": "Average commit time", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 137 }, "id": 42, "maxDataPoints": 25, "options": { "calculate": false, "cellGap": 1, "cellValues": { "unit": "s" }, "color": { "exponent": 0.2, "fill": "dark-orange", "min": 0, "mode": "opacity", "reverse": false, "scale": "exponential", "scheme": "Oranges", "steps": 128 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1E-9 }, "legend": { "show": true }, "rowsFrame": { "layout": "auto", "value": "Commit time" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisLabel": "Quantile", "axisPlacement": "left", "reverse": false, "unit": "percentunit" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "avg(max_over_time(reth_database_transaction_close_duration_seconds{$instance_label=\"$instance\", outcome=\"commit\"}[$__rate_interval])) by (quantile)", "format": "time_series", "instant": false, "legendFormat": "{{quantile}}", "range": true, "refId": "A" } ], "title": "Commit time heatmap", "type": "heatmap" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The average time a database transaction was open.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic", "seriesBy": "last" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 145 }, "id": 117, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "sum(rate(reth_database_transaction_open_duration_seconds_sum{$instance_label=\"$instance\", outcome!=\"\"}[$__rate_interval]) / rate(reth_database_transaction_open_duration_seconds_count{$instance_label=\"$instance\", outcome!=\"\"}[$__rate_interval])) by (outcome, mode)", "format": "time_series", "instant": false, "legendFormat": "{{mode}}, {{outcome}}", "range": true, "refId": "A" } ], "title": "Average transaction open time", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The maximum time the database transaction was open.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 145 }, "id": 116, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "max(max_over_time(reth_database_transaction_open_duration_seconds{$instance_label=\"$instance\", outcome!=\"\", quantile=\"1\"}[$__interval])) by (outcome, mode)", "format": "time_series", "instant": false, "legendFormat": "{{mode}}, {{outcome}}", "range": true, "refId": "A" } ], "title": "Max transaction open time", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "txs", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "Diff(opened-closed)" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" }, { "id": "custom.lineStyle", "value": { "dash": [ 0, 10 ], "fill": "dot" } }, { "id": "custom.axisLabel", "value": "diff" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 153 }, "id": 119, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "sum(reth_database_transaction_opened_total{$instance_label=\"$instance\", mode=\"read-write\"})", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Opened", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "sum(reth_database_transaction_closed_total{$instance_label=\"$instance\", mode=\"read-write\"})", "format": "time_series", "instant": false, "legendFormat": "Closed {{mode}}", "range": true, "refId": "B" }, { "datasource": { "type": "__expr__", "uid": "${DS_EXPRESSION}" }, "expression": "${A} - ${B}", "hide": false, "refId": "Diff(opened-closed)", "type": "math" } ], "title": "Number of read-write transactions", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "txs", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "Diff(opened, closed)" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" }, { "id": "custom.lineStyle", "value": { "dash": [ 0, 10 ], "fill": "dot" } }, { "id": "custom.axisLabel", "value": "diff" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 153 }, "id": 250, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "exemplar": false, "expr": "reth_database_transaction_opened_total{$instance_label=\"$instance\", mode=\"read-only\"}", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Opened", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "exemplar": false, "expr": "sum(reth_database_transaction_closed_total{$instance_label=\"$instance\", mode=\"read-only\"})", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Closed {{mode}}", "range": true, "refId": "B", "useBackend": false }, { "datasource": { "type": "__expr__", "uid": "${DS_EXPRESSION}" }, "expression": "${A} - ${B}", "hide": false, "refId": "Diff(opened, closed)", "type": "math" } ], "title": "Number of read-only transactions", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The size of tables in the database", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 161 }, "id": 48, "options": { "displayLabels": [ "name" ], "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "sort": "desc", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_db_table_size{$instance_label=\"$instance\"}", "interval": "", "legendFormat": "{{table}}", "range": true, "refId": "A" } ], "title": "Database tables", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The maximum time the database transaction operation which inserts a large value took.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 161 }, "id": 118, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "max(max_over_time(reth_database_operation_large_value_duration_seconds{$instance_label=\"$instance\", quantile=\"1\"}[$__interval]) > 0) by (table)", "format": "time_series", "instant": false, "legendFormat": "{{table}}", "range": true, "refId": "A" } ], "title": "Max insertion operation time", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The type of the pages in the database:\n\n- **Leaf** pages contain KV pairs.\n- **Branch** pages contain information about keys in the leaf pages\n- **Overflow** pages store large values and should generally be avoided if possible", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 169 }, "id": 50, "options": { "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "sort": "desc", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "sum by (type) ( reth_db_table_pages{$instance_label=\"$instance\"} )", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Database pages", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The size of the database over time", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 4, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 169 }, "id": 52, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (job) ( reth_db_table_size{$instance_label=\"$instance\"} )", "legendFormat": "Size ({{job}})", "range": true, "refId": "A" } ], "title": "Database growth", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The number of pages on the MDBX freelist", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 177 }, "id": 113, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(reth_db_freelist{$instance_label=\"$instance\"}) by (job)", "legendFormat": "Pages ({{job}})", "range": true, "refId": "A" } ], "title": "Freelist", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "left", "cellOptions": { "type": "auto" }, "footer": { "reducers": [] }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Time" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] }, { "matcher": { "id": "byName", "options": "__name__" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] }, { "matcher": { "id": "byName", "options": "instance" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] }, { "matcher": { "id": "byName", "options": "job" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] }, { "matcher": { "id": "byName", "options": "type" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] }, { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "unit", "value": "locale" }, { "id": "displayName", "value": "Overflow pages" } ] }, { "matcher": { "id": "byName", "options": "table" }, "properties": [ { "id": "displayName", "value": "Table" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 177 }, "id": 58, "options": { "cellHeight": "sm", "showHeader": true }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "sort_desc(reth_db_table_pages{$instance_label=\"$instance\", type=\"overflow\"} != 0)", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" } ], "title": "Overflow pages by table", "type": "table" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 185 }, "id": 203, "panels": [], "title": "Static Files", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The size of segments in the static files", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 186 }, "id": 202, "options": { "displayLabels": [ "name" ], "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "sort": "desc", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_static_files_segment_size{$instance_label=\"$instance\"}", "interval": "", "legendFormat": "{{segment}}", "range": true, "refId": "A" } ], "title": "Segments size", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "left", "cellOptions": { "type": "auto" }, "footer": { "reducers": [] }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "unit", "value": "locale" }, { "id": "displayName", "value": "Entries" } ] }, { "matcher": { "id": "byName", "options": "segment" }, "properties": [ { "id": "displayName", "value": "Segment" } ] }, { "matcher": { "id": "byName", "options": "Time" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] }, { "matcher": { "id": "byName", "options": "instance" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] }, { "matcher": { "id": "byName", "options": "job" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] }, { "matcher": { "id": "byName", "options": "__name__" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] } ] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 186 }, "id": 204, "options": { "cellHeight": "sm", "showHeader": true }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "reth_static_files_segment_entries{$instance_label=\"$instance\"}", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" } ], "title": "Entries per segment", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "left", "cellOptions": { "type": "auto" }, "footer": { "reducers": [] }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "unit", "value": "locale" }, { "id": "displayName", "value": "Files" } ] }, { "matcher": { "id": "byName", "options": "segment" }, "properties": [ { "id": "displayName", "value": "Segment" } ] }, { "matcher": { "id": "byName", "options": "Time" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] }, { "matcher": { "id": "byName", "options": "instance" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] }, { "matcher": { "id": "byName", "options": "job" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] }, { "matcher": { "id": "byName", "options": "__name__" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] } ] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 186 }, "id": 205, "options": { "cellHeight": "sm", "showHeader": true }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "reth_static_files_segment_files{$instance_label=\"$instance\"}", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" } ], "title": "Files per segment", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The size of the static files over time", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 194 }, "id": 206, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (job) ( reth_static_files_segment_size{$instance_label=\"$instance\"} )", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Static Files growth", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The maximum time the static files operation which commits a writer took.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 194 }, "id": 207, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "max(max_over_time(reth_static_files_jar_provider_write_duration_seconds{$instance_label=\"$instance\", operation=\"commit-writer\", quantile=\"1\"}[$__interval]) > 0) by (segment)", "legendFormat": "{{segment}}", "range": true, "refId": "A" } ], "title": "Max writer commit time", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 202 }, "id": 1008, "panels": [], "title": "RocksDB", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Size of each RocksDB table (SST files + memtables)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 203 }, "id": 1009, "options": { "displayLabels": [ "name" ], "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "sort": "desc", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_rocksdb_table_size{$instance_label=\"$instance\"}", "legendFormat": "{{table}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_rocksdb_wal_size{$instance_label=\"$instance\"}", "legendFormat": "WAL", "range": true, "refId": "B" } ], "title": "RocksDB Tables Size", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "left", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 203 }, "id": 1010, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_rocksdb_table_entries{$instance_label=\"$instance\"}", "format": "table", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "RocksDB Entries per Table", "transformations": [ { "id": "groupBy", "options": { "fields": { "Value": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "table": { "aggregations": [], "operation": "groupby" } } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "Value (lastNotNull)" } ] } }, { "id": "organize", "options": { "renameByName": { "table": "Table", "Value (lastNotNull)": "Entries" } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Bytes pending compaction per table", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "left", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 203 }, "id": 1011, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_rocksdb_pending_compaction_bytes{$instance_label=\"$instance\"}", "format": "table", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "RocksDB Pending Compaction", "transformations": [ { "id": "groupBy", "options": { "fields": { "Value": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "table": { "aggregations": [], "operation": "groupby" } } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "Value (lastNotNull)" } ] } }, { "id": "organize", "options": { "renameByName": { "table": "Table", "Value (lastNotNull)": "Pending Bytes" } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Total RocksDB size over time", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 211 }, "id": 1012, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (job) ( reth_rocksdb_table_size{$instance_label=\"$instance\"} ) + (sum by (job) ( reth_rocksdb_wal_size{$instance_label=\"$instance\"} ) or vector(0))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "RocksDB Growth", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Pending compaction bytes over time", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 211 }, "id": 1013, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_rocksdb_pending_compaction_bytes{$instance_label=\"$instance\"}", "legendFormat": "{{table}}", "range": true, "refId": "A" } ], "title": "RocksDB Pending Compaction Over Time", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 219 }, "id": 79, "panels": [], "repeat": "instance", "title": "Blockchain Tree", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The block number of the tip of the canonical chain from the blockchain tree.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 220 }, "id": 74, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_blockchain_tree_canonical_chain_height{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Canonical chain height", "range": true, "refId": "B" } ], "title": "Canonical chain height", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Total number of blocks in the tree's block buffer", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 220 }, "id": 80, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_blockchain_tree_block_buffer_blocks{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Buffered blocks", "range": true, "refId": "B" } ], "title": "Block buffer blocks", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 228 }, "id": 1002, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "increase(reth_blockchain_tree_reorgs{$instance_label=\"$instance\"}[$__rate_interval])", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Reorgs", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 228 }, "id": 190, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_blockchain_tree_latest_reorg_depth{$instance_label=\"$instance\"}", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Latest Reorg Depth", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 236 }, "id": 108, "panels": [], "title": "RPC server", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "http" }, "properties": [ { "id": "displayName", "value": "HTTP" } ] }, { "matcher": { "id": "byName", "options": "ws" }, "properties": [ { "id": "displayName", "value": "WebSocket" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 237 }, "id": 109, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "sum(reth_rpc_server_connections_connections_opened_total{$instance_label=\"$instance\"} - reth_rpc_server_connections_connections_closed_total{$instance_label=\"$instance\"}) by (transport)", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{transport}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Active Connections", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 237 }, "id": 111, "maxDataPoints": 25, "options": { "calculate": false, "cellGap": 1, "cellValues": { "unit": "s" }, "color": { "exponent": 0.2, "fill": "dark-orange", "min": 0, "mode": "opacity", "reverse": false, "scale": "exponential", "scheme": "Oranges", "steps": 128 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1E-9 }, "legend": { "show": true }, "rowsFrame": { "layout": "auto", "value": "Latency time" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisLabel": "Quantile", "axisPlacement": "left", "reverse": false, "unit": "percentunit" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "avg(max_over_time(reth_rpc_server_connections_request_time_seconds{$instance_label=\"$instance\"}[$__rate_interval]) > 0) by (quantile)", "format": "time_series", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Request Latency time", "type": "heatmap" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 245 }, "id": 120, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "max(max_over_time(reth_rpc_server_calls_time_seconds{$instance_label=\"$instance\"}[$__rate_interval])) by (method) > 0", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Maximum call latency per method", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 245 }, "id": 112, "maxDataPoints": 25, "options": { "calculate": false, "cellGap": 1, "cellValues": { "unit": "s" }, "color": { "exponent": 0.2, "fill": "dark-orange", "min": 0, "mode": "opacity", "reverse": false, "scale": "exponential", "scheme": "Oranges", "steps": 128 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1E-9 }, "legend": { "show": true }, "rowsFrame": { "layout": "auto", "value": "Latency time" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisLabel": "Quantile", "axisPlacement": "left", "reverse": false, "unit": "percentunit" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "avg(max_over_time(reth_rpc_server_calls_time_seconds{$instance_label=\"$instance\"}[$__rate_interval]) > 0) by (quantile)", "format": "time_series", "instant": false, "legendFormat": "{{quantile}}", "range": true, "refId": "A" } ], "title": "Call Latency time", "type": "heatmap" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/.*cached items.*/" }, "properties": [ { "id": "custom.axisLabel", "value": "Items" } ] }, { "matcher": { "id": "byRegexp", "options": "/.*consumers.*/" }, "properties": [ { "id": "custom.axisLabel", "value": "Queued consumers" } ] }, { "matcher": { "id": "byRegexp", "options": "/.memory usage*/" }, "properties": [ { "id": "unit", "value": "decbytes" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 253 }, "id": 198, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_rpc_eth_cache_cached_count{$instance_label=\"$instance\", cache=\"headers\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Headers cache cached items", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_rpc_eth_cache_queued_consumers_count{$instance_label=\"$instance\", cache=\"receipts\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Receipts cache queued consumers", "range": true, "refId": "B", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_rpc_eth_cache_queued_consumers_count{$instance_label=\"$instance\", cache=\"headers\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Headers cache queued consumers", "range": true, "refId": "C", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_rpc_eth_cache_queued_consumers_count{$instance_label=\"$instance\", cache=\"blocks\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Block cache queued consumers", "range": true, "refId": "D", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_rpc_eth_cache_memory_usage{$instance_label=\"$instance\", cache=\"blocks\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Blocks cache memory usage", "range": true, "refId": "E", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_rpc_eth_cache_cached_count{$instance_label=\"$instance\", cache=\"receipts\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Receipts cache cached items", "range": true, "refId": "F", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_rpc_eth_cache_memory_usage{$instance_label=\"$instance\", cache=\"receipts\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Receipts cache memory usage", "range": true, "refId": "G", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "reth_rpc_eth_cache_cached_count{$instance_label=\"$instance\", cache=\"blocks\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Block cache cached items", "range": true, "refId": "H", "useBackend": false } ], "title": "RPC Cache Metrics", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 253 }, "id": 246, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(rate(reth_rpc_server_calls_successful_total{$instance_label=\"$instance\"}[$__rate_interval])) by (method) > 0", "instant": false, "legendFormat": "{{method}}", "range": true, "refId": "A" } ], "title": "RPC Throughput", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 261 }, "id": 24, "panels": [], "repeat": "instance", "title": "Downloader: Headers", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] }, { "matcher": { "id": "byFrameRefID", "options": "D" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 262 }, "id": 26, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_downloaders_headers_total_downloaded{$instance_label=\"$instance\"}", "legendFormat": "Downloaded", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_downloaders_headers_total_flushed{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Flushed", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "rate(reth_downloaders_headers_total_downloaded{$instance_label=\"$instance\"}[$__rate_interval])", "hide": false, "instant": false, "legendFormat": "Downloaded/s", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "rate(reth_downloaders_headers_total_flushed{$instance_label=\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Flushed/s", "range": true, "refId": "D" } ], "title": "I/O", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Internal errors in the header downloader. These are expected to happen from time to time.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 262 }, "id": 33, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "rate(reth_downloaders_headers_timeout_errors{$instance_label=\"$instance\"}[$__rate_interval])", "legendFormat": "Request timed out", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "rate(reth_downloaders_headers_unexpected_errors{$instance_label=\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Unexpected error", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "rate(reth_downloaders_headers_validation_errors{$instance_label=\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Invalid response", "range": true, "refId": "C" } ], "title": "Errors", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The number of connected peers and in-progress requests for headers.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 270 }, "id": 36, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_downloaders_headers_in_flight_requests{$instance_label=\"$instance\"}", "legendFormat": "In flight requests", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_network_connected_peers{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Connected peers", "range": true, "refId": "B" } ], "title": "Requests", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 278 }, "id": 32, "panels": [], "repeat": "instance", "title": "Downloader: Bodies", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The internal state of the headers downloader: the number of downloaded headers, and the number of headers sent to the header stage.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "locale" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" }, { "id": "unit", "value": "ops" } ] }, { "matcher": { "id": "byFrameRefID", "options": "D" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" }, { "id": "unit", "value": "ops" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 279 }, "id": 30, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_downloaders_bodies_total_downloaded{$instance_label=\"$instance\"}", "legendFormat": "Downloaded", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_downloaders_bodies_total_flushed{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Flushed", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "rate(reth_downloaders_bodies_total_flushed{$instance_label=\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Flushed/s", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "rate(reth_downloaders_bodies_total_downloaded{$instance_label=\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Downloaded/s", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_downloaders_bodies_buffered_responses{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Buffered responses", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_downloaders_bodies_buffered_blocks{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Buffered blocks", "range": true, "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_downloaders_bodies_queued_blocks{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Queued blocks", "range": true, "refId": "G" } ], "title": "I/O", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Internal errors in the bodies downloader. These are expected to happen from time to time.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 279 }, "id": 28, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "rate(reth_downloaders_bodies_timeout_errors{$instance_label=\"$instance\"}[$__rate_interval])", "legendFormat": "Request timed out", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "rate(reth_downloaders_bodies_unexpected_errors{$instance_label=\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Unexpected error", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "rate(reth_downloaders_bodies_validation_errors{$instance_label=\"$instance\"}[$__rate_interval])", "hide": false, "legendFormat": "Invalid response", "range": true, "refId": "C" } ], "title": "Errors", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The number of connected peers and in-progress requests for bodies.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 287 }, "id": 35, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_downloaders_bodies_in_flight_requests{$instance_label=\"$instance\"}", "legendFormat": "In flight requests", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_network_connected_peers{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Connected peers", "range": true, "refId": "B" } ], "title": "Requests", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The number of blocks and size in bytes of those blocks", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" }, { "id": "unit", "value": "blocks" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 287 }, "id": 73, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_downloaders_bodies_buffered_blocks_size_bytes{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Buffered blocks size ", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_downloaders_bodies_buffered_blocks{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Buffered blocks", "range": true, "refId": "B" } ], "title": "Downloader buffer", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The number of blocks in a request and size in bytes of those block responses", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" }, { "id": "unit", "value": "blocks" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 295 }, "id": 102, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_downloaders_bodies_response_response_size_bytes{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Response size", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_downloaders_bodies_response_response_length{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Individual response length (number of bodies in response)", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_downloaders_bodies_response_response_size_bytes / reth_downloaders_bodies_response_response_length{$instance_label=\"$instance\"}", "hide": false, "instant": false, "legendFormat": "Mean body size in response", "range": true, "refId": "C" } ], "title": "Block body response sizes", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 303 }, "id": 226, "panels": [], "title": "Eth Requests", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "http" }, "properties": [ { "id": "displayName", "value": "HTTP" } ] }, { "matcher": { "id": "byName", "options": "ws" }, "properties": [ { "id": "displayName", "value": "WebSocket" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 304 }, "id": 225, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "maxHeight": 600, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "rate(reth_network_eth_headers_requests_received_total{$instance_label=\"$instance\"}[$__rate_interval])", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "Headers Requests/s", "range": true, "refId": "A", "useBackend": false } ], "title": "Headers Requests Received", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "http" }, "properties": [ { "id": "displayName", "value": "HTTP" } ] }, { "matcher": { "id": "byName", "options": "ws" }, "properties": [ { "id": "displayName", "value": "WebSocket" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 304 }, "id": 227, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "maxHeight": 600, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "rate(reth_network_eth_receipts_requests_received_total{$instance_label=\"$instance\"}[$__rate_interval])", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "Receipts Requests/s", "range": true, "refId": "A", "useBackend": false } ], "title": "Receipts Requests Received", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "http" }, "properties": [ { "id": "displayName", "value": "HTTP" } ] }, { "matcher": { "id": "byName", "options": "ws" }, "properties": [ { "id": "displayName", "value": "WebSocket" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 312 }, "id": 235, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "maxHeight": 600, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "rate(reth_network_eth_bodies_requests_received_total{$instance_label=\"$instance\"}[$__rate_interval])", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "Bodies Requests/s", "range": true, "refId": "A", "useBackend": false } ], "title": "Bodies Requests Received", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "http" }, "properties": [ { "id": "displayName", "value": "HTTP" } ] }, { "matcher": { "id": "byName", "options": "ws" }, "properties": [ { "id": "displayName", "value": "WebSocket" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 312 }, "id": 234, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "maxHeight": 600, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "rate(reth_network_eth_node_data_requests_received_total{$instance_label=\"$instance\"}[$__rate_interval])", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "Node Data Requests/s", "range": true, "refId": "A", "useBackend": false } ], "title": "Node Data Requests Received", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 320 }, "id": 68, "panels": [], "repeat": "instance", "title": "Payload Builder", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Number of active jobs", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 321 }, "id": 60, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_payloads_active_jobs{$instance_label=\"$instance\"}", "legendFormat": "Active Jobs", "range": true, "refId": "A" } ], "title": "Active Jobs", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Total number of initiated jobs", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 321 }, "id": 62, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_payloads_initiated_jobs{$instance_label=\"$instance\"}", "legendFormat": "Initiated Jobs", "range": true, "refId": "A" } ], "title": "Initiated Jobs", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Total number of failed jobs", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 329 }, "id": 64, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_payloads_failed_jobs{$instance_label=\"$instance\"}", "legendFormat": "Failed Jobs", "range": true, "refId": "A" } ], "title": "Failed Jobs", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 337 }, "id": 105, "panels": [], "title": "Pruning", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 338 }, "id": 106, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "rate(reth_pruner_duration_seconds_sum{$instance_label=\"$instance\"}[$__rate_interval]) / rate(reth_pruner_duration_seconds_count{$instance_label=\"$instance\"}[$__rate_interval])", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Pruner duration, total", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 338 }, "id": 107, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "rate(reth_pruner_segments_duration_seconds_sum{$instance_label=\"$instance\"}[$__rate_interval]) / rate(reth_pruner_segments_duration_seconds_count{$instance_label=\"$instance\"}[$__rate_interval])", "instant": false, "legendFormat": "{{segment}}", "range": true, "refId": "A" } ], "title": "Pruner duration, per segment", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 346 }, "id": 217, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Archive and full nodes prune headers, transactions and receipts in MDBX (hot db) after they have been written to static files (cold db). Full nodes additionally prune history indices.", "editorMode": "code", "expr": "reth_pruner_segments_highest_pruned_block{$instance_label=\"$instance\"}", "instant": false, "legendFormat": "{{segment}}", "range": true, "refId": "A" } ], "title": "Highest pruned block, per segment", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 354 }, "id": 97, "panels": [], "title": "Process", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Retained" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 355 }, "id": 98, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_jemalloc_active{$instance_label=\"$instance\"}", "instant": false, "legendFormat": "Active", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_jemalloc_allocated{$instance_label=\"$instance\"}", "hide": false, "instant": false, "legendFormat": "Allocated", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_jemalloc_mapped{$instance_label=\"$instance\"}", "hide": false, "instant": false, "legendFormat": "Mapped", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_jemalloc_metadata{$instance_label=\"$instance\"}", "hide": false, "instant": false, "legendFormat": "Metadata", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_jemalloc_resident{$instance_label=\"$instance\"}", "hide": false, "instant": false, "legendFormat": "Resident", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_jemalloc_retained{$instance_label=\"$instance\"}", "hide": false, "instant": false, "legendFormat": "Retained", "range": true, "refId": "F" } ], "title": "Jemalloc Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 355 }, "id": 101, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_process_resident_memory_bytes{$instance_label=\"$instance\"}", "instant": false, "legendFormat": "Resident", "range": true, "refId": "A" } ], "title": "Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "100% = 1 core", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 363 }, "id": 99, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "avg(rate(reth_process_cpu_seconds_total{$instance_label=\"$instance\"}[1m]))", "instant": false, "legendFormat": "Process", "range": true, "refId": "A" } ], "title": "CPU", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 363 }, "id": 100, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_process_open_fds{$instance_label=\"$instance\"}", "instant": false, "legendFormat": "Open", "range": true, "refId": "A" } ], "title": "File Descriptors", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Tracks the number of critical tasks currently ran by the executor.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "semi-dark-red", "value": 0 } ] }, "unit": "tasks" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 371 }, "id": 248, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_executor_spawn_critical_tasks_total{$instance_label=\"$instance\"}- reth_executor_spawn_finished_critical_tasks_total{$instance_label=\"$instance\"}", "hide": false, "instant": false, "legendFormat": "Tasks running", "range": true, "refId": "C" } ], "title": "Task Executor critical tasks", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Tracks the number of regular tasks currently ran by the executor.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "semi-dark-red", "value": 80 } ] }, "unit": "tasks/s" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "unit", "value": "tasks" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 371 }, "id": 247, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "rate(reth_executor_spawn_regular_tasks_total{$instance_label=\"$instance\"}[$__rate_interval])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, "instant": false, "legendFormat": "Tasks started", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_executor_spawn_regular_tasks_total{$instance_label=\"$instance\"} - reth_executor_spawn_finished_regular_tasks_total{$instance_label=\"$instance\"}", "hide": false, "instant": false, "legendFormat": "Tasks running", "range": true, "refId": "C" } ], "title": "Task Executor regular tasks", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Tracks the number of regular blocking tasks currently ran by the executor.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "semi-dark-red", "value": 80 } ] }, "unit": "tasks/s" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "unit", "value": "tasks" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 379 }, "id": 1007, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "rate(reth_executor_spawn_regular_blocking_tasks_total{$instance_label=\"$instance\"}[$__rate_interval])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, "instant": false, "legendFormat": "Tasks started", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_executor_spawn_regular_blocking_tasks_total{$instance_label=\"$instance\"} - reth_executor_spawn_finished_regular_blocking_tasks_total{$instance_label=\"$instance\"}", "hide": false, "instant": false, "legendFormat": "Tasks running", "range": true, "refId": "C" } ], "title": "Task Executor regular blocking tasks", "type": "timeseries" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 387 }, "id": 236, "panels": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The total number of canonical state notifications sent to ExExes.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 387 }, "id": 237, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_exex_notifications_sent_total{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Total Notifications Sent", "range": true, "refId": "B" } ], "title": "Total Notifications Sent", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "The total number of events ExExes have sent to the manager.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 387 }, "id": 238, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_exex_events_sent_total{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Total Events Sent", "range": true, "refId": "B" } ], "title": "Total Events Sent", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Current and Maximum capacity of the internal state notifications buffer.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 395 }, "id": 239, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_exex_manager_current_capacity{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Current size", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "max_over_time(reth_exex_manager_max_capacity{$instance_label=\"$instance\"}[1h])", "hide": false, "legendFormat": "Max size", "range": true, "refId": "C" } ], "title": "Current and Max Capacity", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Current size of the internal state notifications buffer.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 395 }, "id": 219, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_exex_manager_buffer_size{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Max size", "range": true, "refId": "B" } ], "title": "Buffer Size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Total number of ExExes installed in the node", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 403 }, "id": 220, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "builder", "expr": "reth_exex_manager_num_exexs{$instance_label=\"$instance\"}", "hide": false, "legendFormat": "Number of ExExs", "range": true, "refId": "A" } ], "title": "Number of ExExes", "type": "stat" } ], "title": "Execution Extensions", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 388 }, "id": 241, "panels": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 388 }, "id": 243, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_exex_wal_lowest_committed_block_height{$instance_label=\"$instance\"}", "hide": false, "instant": false, "legendFormat": "Lowest Block", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_exex_wal_highest_committed_block_height{$instance_label=\"$instance\"}", "hide": false, "instant": false, "legendFormat": "Highest Block", "range": true, "refId": "C" } ], "title": "Current Committed Block Heights", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 388 }, "id": 244, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_exex_wal_committed_blocks_count{$instance_label=\"$instance\"}", "hide": false, "instant": false, "legendFormat": "Committed Blocks", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_exex_wal_notifications_count{$instance_label=\"$instance\"}", "hide": false, "instant": false, "legendFormat": "Notifications", "range": true, "refId": "B" } ], "title": "Number of entities", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 396 }, "id": 245, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_exex_wal_size_bytes{$instance_label=\"$instance\"}", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "C" } ], "title": "Total size of all notifications", "type": "timeseries" } ], "title": "Execution Extensions Write-Ahead Log", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "SST file size and memtable size per RocksDB table", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "left", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 211 }, "id": 1017, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_rocksdb_sst_size{$instance_label=\"$instance\"}", "format": "table", "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "reth_rocksdb_memtable_size{$instance_label=\"$instance\"}", "format": "table", "legendFormat": "__auto", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "label_replace(reth_rocksdb_wal_size{$instance_label=\"$instance\"}, \"table\", \"WAL\", \"\", \"\")", "format": "table", "legendFormat": "__auto", "range": true, "refId": "C" } ], "transformations": [ { "id": "merge", "options": {} }, { "id": "groupBy", "options": { "fields": { "Value #A": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "Value #B": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "Value #C": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "table": { "aggregations": [], "operation": "groupby" } } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "Value #A (lastNotNull)" } ] } }, { "id": "organize", "options": { "renameByName": { "table": "Table", "Value #A (lastNotNull)": "SST Size", "Value #B (lastNotNull)": "Memtable Size", "Value #C (lastNotNull)": "WAL Size" } } } ], "title": "RocksDB Storage Breakdown", "type": "table" } ], "refresh": "5s", "schemaVersion": 42, "tags": [], "templating": { "list": [ { "current": {}, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(reth_info,$instance_label)", "includeAll": false, "label": "Instance", "name": "instance", "options": [], "query": { "qryType": 1, "query": "label_values(reth_info,$instance_label)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "sort": 1, "type": "query" }, { "current": {}, "includeAll": false, "label": "Datasource", "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "type": "datasource" }, { "hide": 2, "label": "Instance Label", "name": "instance_label", "query": "${VAR_INSTANCE_LABEL}", "skipUrlSync": true, "type": "constant", "current": { "value": "${VAR_INSTANCE_LABEL}", "text": "${VAR_INSTANCE_LABEL}", "selected": false }, "options": [ { "value": "${VAR_INSTANCE_LABEL}", "text": "${VAR_INSTANCE_LABEL}", "selected": false } ] } ] }, "time": { "from": "now-12h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Reth", "uid": "2k8BXz24x", "version": 4, "weekStart": "" }