{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "Weaver Usenet downloader: throughput, pipeline backpressure, per-server health, and post-processing.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "panels": [], "title": "Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Bytes per second pulled from Usenet, measured as the rate of the download counter rather than the smoothed speed gauge.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 5, "w": 5, "x": 0, "y": 1 }, "id": 2, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(rate(weaver_pipeline_bytes_downloaded_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "download", "range": true, "refId": "A" } ], "title": "Download speed", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Bytes downloaded in the last 24 hours. Compare against your download quota.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 5, "w": 5, "x": 5, "y": 1 }, "id": 3, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(increase(weaver_pipeline_bytes_downloaded_total{job=\"$job\",instance=~\"$instance\"}[24h]))", "legendFormat": "24h", "range": true, "refId": "A" } ], "title": "Downloaded / 24h", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Article segments waiting to be fetched. Recovery work is included; subtract weaver_pipeline_recovery_queue_depth for required-only depth.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 7, "x": 10, "y": 1 }, "id": 4, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_pipeline_download_queue_depth{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "queue depth", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_pipeline_active_downloads{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "active downloads", "range": true, "refId": "B" } ], "title": "Queue depth", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Jobs by status. Statuses with no jobs report 0, so the series set is stable.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 40, "lineWidth": 1, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" } }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 7, "x": 17, "y": 1 }, "id": 5, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (status) (weaver_pipeline_jobs{job=\"$job\",instance=~\"$instance\"}) > 0", "legendFormat": "{{status}}", "range": true, "refId": "A" } ], "title": "Jobs by status", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "id": 10, "panels": [], "title": "Pipeline", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "What is currently limiting the downloader. Exactly one limiter is 1 at any moment.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 60, "lineInterpolation": "stepAfter", "lineWidth": 0, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" } }, "mappings": [], "max": 1, "min": 0, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 7 }, "id": 11, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_pipeline_download_observed_limiter{job=\"$job\",instance=~\"$instance\"} == 1", "legendFormat": "{{limiter}}", "range": true, "refId": "A" } ], "title": "Observed limiter", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Backpressure state and the gate reason blocking downloads, if any.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 60, "lineInterpolation": "stepAfter", "lineWidth": 0, "showPoints": "never", "spanNulls": true }, "mappings": [], "max": 1, "min": 0, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 7 }, "id": 12, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_pipeline_download_pressure_state{job=\"$job\",instance=~\"$instance\",state!=\"clear\"} == 1", "legendFormat": "pressure {{state}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_pipeline_download_gate{job=\"$job\",instance=~\"$instance\",reason!=\"none\"} == 1", "legendFormat": "gate {{reason}}", "range": true, "refId": "B" } ], "title": "Pressure and gate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Bytes held in the decode and write stages against their soft pressure limits.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*soft limit.*" }, "properties": [ { "id": "custom.lineStyle", "value": { "dash": [10, 10], "fill": "dash" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 13 }, "id": 13, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_pipeline_decode_pending_bytes{job=\"$job\",instance=~\"$instance\"} + weaver_pipeline_decode_active_bytes{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "decode backlog", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_pipeline_decode_pressure_soft_limit_bytes{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "decode soft limit", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_pipeline_write_buffered_bytes{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "write buffered", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_pipeline_write_pressure_soft_limit_bytes{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "write soft limit", "range": true, "refId": "D" } ], "title": "Decode and write backlog", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Download lanes by scheduler state. Lanes stuck in awaiting_work or parking mean the scheduler cannot feed them.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 40, "lineWidth": 1, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" } }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 13 }, "id": 14, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_pipeline_download_lane_states_active{job=\"$job\",instance=~\"$instance\"} > 0", "legendFormat": "{{state}}", "range": true, "refId": "A" } ], "title": "Download lanes by state", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 19 }, "id": 20, "panels": [], "title": "Servers", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Per-server health state. Anything other than healthy is worth a look; the reason is on weaver_server_state_reason.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 60, "lineInterpolation": "stepAfter", "lineWidth": 0, "showPoints": "never", "spanNulls": true }, "mappings": [], "max": 1, "min": 0, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 20 }, "id": 21, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_server_state{job=\"$job\",instance=~\"$instance\"} == 1", "legendFormat": "{{server}} — {{state}}", "range": true, "refId": "A" } ], "title": "Server state", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Connection permits still free versus the runtime maximum. A flat zero means the servers, not the pipeline, are the bottleneck.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "short" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*max.*" }, "properties": [ { "id": "custom.lineStyle", "value": { "dash": [10, 10], "fill": "dash" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 20 }, "id": 22, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_server_connections_available{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "{{server}} available", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_server_connections_max{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "{{server}} max", "range": true, "refId": "B" } ], "title": "Connections available vs max", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "EWMA request latency per server.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "s" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 26 }, "id": 23, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_server_latency_seconds{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "{{server}}", "range": true, "refId": "A" } ], "title": "Server latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "BODY bytes per second from each server. Shows which provider is actually carrying the job.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 40, "lineWidth": 1, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" } }, "mappings": [], "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 26 }, "id": 24, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (server) (rate(weaver_server_download_bytes_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{server}}", "range": true, "refId": "A" } ], "title": "Per-server throughput", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Fraction of each server's configured download quota consumed in the current window.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true, "thresholdsStyle": { "mode": "dashed" } }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.8 }, { "color": "red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 26 }, "id": 25, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_server_download_quota_used_bytes{job=\"$job\",instance=~\"$instance\"} / clamp_min(weaver_server_download_quota_limit_bytes{job=\"$job\",instance=~\"$instance\"}, 1)", "legendFormat": "{{server}}", "range": true, "refId": "A" } ], "title": "Server quota used", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Share of article fetches each server answered with 'article not found'. The clearest provider-retention signal there is: a climbing miss ratio means that server is losing articles, even while it still looks healthy.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true, "thresholdsStyle": { "mode": "dashed" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 0.05 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 32 }, "id": 26, "options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (server) (rate(weaver_server_article_attempts_total{job=\"$job\",instance=~\"$instance\",outcome=\"not_found\"}[5m])) / clamp_min(sum by (server) (rate(weaver_server_article_attempts_total{job=\"$job\",instance=~\"$instance\"}[5m])), 0.001)", "legendFormat": "{{server}}", "range": true, "refId": "A" } ], "title": "Article miss ratio per server", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "95th percentile article fetch latency per server.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "s" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 32 }, "id": 27, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, server) (rate(weaver_server_article_latency_seconds_bucket{job=\"$job\",instance=~\"$instance\"}[5m])))", "legendFormat": "{{server}}", "range": true, "refId": "A" } ], "title": "Article latency p95 per server", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 38 }, "id": 40, "panels": [], "title": "Jobs", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Jobs reaching a terminal state, by result. A rising 'failed' share is the headline reliability number.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 40, "lineWidth": 1, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" } }, "mappings": [], "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 39 }, "id": 41, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (result) (rate(weaver_jobs_finished_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{result}}", "range": true, "refId": "A" } ], "title": "Jobs finished by result", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "95th percentile wall-clock time jobs spent in each pipeline stage. Shows where the time actually goes.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "s" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 39 }, "id": 42, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, stage) (rate(weaver_job_stage_duration_seconds_bucket{job=\"$job\",instance=~\"$instance\"}[1h])))", "legendFormat": "{{stage}}", "range": true, "refId": "A" } ], "title": "Stage duration p95", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 }, "id": 30, "panels": [], "title": "Storage & post-processing", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Post-processing scripts waiting and running.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 46 }, "id": 31, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_post_processing_queue_depth{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "queued", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_post_processing_active_attempts{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "running", "range": true, "refId": "B" } ], "title": "Post-processing queue", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Completed post-processing attempts per second, by outcome.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 40, "lineWidth": 1, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" } }, "mappings": [], "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 46 }, "id": 32, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(weaver_post_processing_attempts_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "{{result}}", "range": true, "refId": "A" } ], "title": "Post-processing results", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Archive entries refused by the extraction guardrails. A sustained non-zero rate usually means a malformed or hostile archive, not a Weaver bug.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 40, "lineWidth": 1, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" } }, "mappings": [], "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 46 }, "id": 33, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(weaver_extraction_rejections_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]) > 0", "legendFormat": "{{reason}}", "range": true, "refId": "A" } ], "title": "Extraction rejections", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Verify, repair, and extract workers busy right now, plus disk write latency.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "disk write latency" }, "properties": [ { "id": "unit", "value": "s" }, { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 52 }, "id": 34, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_pipeline_verify_active{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "verify", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_pipeline_repair_active{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "repair", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_pipeline_extract_active{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "extract", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "weaver_pipeline_disk_write_latency_seconds{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "disk write latency", "range": true, "refId": "D" } ], "title": "Post-download workers", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 58 }, "id": 50, "panels": [], "title": "Process", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Resident and virtual memory of the Weaver process. Some platforms cannot report these cheaply, in which case the series are absent rather than zero.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 59 }, "id": 51, "options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "process_resident_memory_bytes{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "resident", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "process_virtual_memory_bytes{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "virtual", "range": true, "refId": "B" } ], "title": "Process memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "CPU cores consumed by the Weaver process, plus thread and file-descriptor counts.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "cpu cores" }, "properties": [{ "id": "custom.fillOpacity", "value": 20 }] }, { "matcher": { "id": "byRegexp", "options": "threads|open fds" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 59 }, "id": 52, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(process_cpu_seconds_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "cpu cores", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "process_threads{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "threads", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "process_open_fds{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "open fds", "range": true, "refId": "C" } ], "title": "Process CPU, threads and fds", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 65 }, "id": 60, "panels": [], "title": "Scripts", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Each script's last run: status, duration and age, with its run rate and failure ratio over the last hour. Failures count failed, timed_out, not_started and interrupted runs.", "fieldConfig": { "defaults": { "custom": { "align": "auto" }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byName", "options": "Last duration" }, "properties": [ { "id": "unit", "value": "s" } ] }, { "matcher": { "id": "byName", "options": "Since last run" }, "properties": [ { "id": "unit", "value": "s" } ] }, { "matcher": { "id": "byName", "options": "Failure ratio" }, "properties": [ { "id": "unit", "value": "percentunit" } ] } ] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 66 }, "id": 61, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max by (script, status) (weaver_post_processing_script_last_run_status{job=\"$job\",instance=~\"$instance\"} == 1)", "legendFormat": "", "refId": "A", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max by (script) (weaver_post_processing_script_last_duration_seconds{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "B", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "time() - max by (script) (weaver_post_processing_script_last_run_timestamp_seconds{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "C", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (script) (increase(weaver_post_processing_script_runs_total{job=\"$job\",instance=~\"$instance\"}[1h]))", "legendFormat": "", "refId": "D", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (script) (increase(weaver_post_processing_script_runs_total{job=\"$job\",instance=~\"$instance\",status=~\"failed|timed_out|not_started|interrupted\"}[1h])) / clamp_min(sum by (script) (increase(weaver_post_processing_script_runs_total{job=\"$job\",instance=~\"$instance\"}[1h])), 1)", "legendFormat": "", "refId": "E", "instant": true, "range": false, "format": "table" } ], "title": "Script status", "type": "table", "transformations": [ { "id": "joinByField", "options": { "byField": "script", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": { "Time": true, "Value #A": true }, "renameByName": { "script": "Script", "status": "Last status", "Value #A": "", "Value #B": "Last duration", "Value #C": "Since last run", "Value #D": "Runs / h", "Value #E": "Failure ratio" } } } ] }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Run duration quantiles per script across every run kind.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "s" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 74 }, "id": 62, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.5, sum by (script, le) (rate(weaver_post_processing_attempt_duration_seconds_bucket{job=\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "{{script}} p50", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (script, le) (rate(weaver_post_processing_attempt_duration_seconds_bucket{job=\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "{{script}} p95", "refId": "B", "range": true } ], "title": "Script duration p50 / p95", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Finished script runs per second, stacked by status.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 40, "lineWidth": 1, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" } }, "mappings": [], "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 74 }, "id": 63, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (status) (rate(weaver_post_processing_script_runs_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{status}}", "refId": "A", "range": true } ], "title": "Script runs by status", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Script runs holding a slot, by run kind, against the configured concurrency limit.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 80 }, "id": 64, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (kind) (weaver_post_processing_runs_running{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "{{kind}}", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max(weaver_post_processing_concurrency_limit{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "limit", "refId": "B", "range": true } ], "title": "Running vs limit", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Script runs waiting for a slot now, and the rate runs enter a wait, by run kind.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 80 }, "id": 65, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (kind) (weaver_post_processing_waiting_for_slot{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "{{kind}} waiting", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (kind) (rate(weaver_post_processing_slot_waits_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{kind}} waits/s", "refId": "B", "range": true } ], "title": "Waiting for slot", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Queue events recorded for scripts and not yet started.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 80 }, "id": 66, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max(weaver_post_processing_queue_event_backlog{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "backlog", "refId": "A", "range": true } ], "title": "Queue-event backlog", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Jobs whose post-processing pass finished, by summary.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 40, "lineWidth": 1, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" } }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 86 }, "id": 67, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (summary) (increase(weaver_post_processing_job_summaries_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{summary}}", "refId": "A", "range": true } ], "title": "Job summaries", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Script runs whose captured output was truncated, per script.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 86 }, "id": 68, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (script) (increase(weaver_post_processing_script_output_truncations_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{script}}", "refId": "A", "range": true } ], "title": "Output truncations", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 92 }, "id": 69, "panels": [], "title": "Schedules", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Each configured rule: whether it is enabled, its last outcome, time since it last fired and its fires over the last day.", "fieldConfig": { "defaults": { "custom": { "align": "auto" }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byName", "options": "Since last fire" }, "properties": [ { "id": "unit", "value": "s" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 93 }, "id": 70, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max by (rule_id, action) (weaver_schedule_rule_enabled{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "A", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max by (rule_id, outcome) (weaver_schedule_rule_last_outcome{job=\"$job\",instance=~\"$instance\"} == 1)", "legendFormat": "", "refId": "B", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "time() - max by (rule_id) (weaver_schedule_rule_last_fire_timestamp_seconds{job=\"$job\",instance=~\"$instance\"} > 0)", "legendFormat": "", "refId": "C", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (rule_id) (increase(weaver_schedule_rule_fires_total{job=\"$job\",instance=~\"$instance\"}[24h]))", "legendFormat": "", "refId": "D", "instant": true, "range": false, "format": "table" } ], "title": "Schedule rules", "type": "table", "transformations": [ { "id": "joinByField", "options": { "byField": "rule_id", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": { "Time": true, "Value #B": true }, "renameByName": { "rule_id": "Rule", "action": "Action", "Value #A": "Enabled", "outcome": "Last outcome", "Value #C": "Since last fire", "Value #D": "Fires / 24h" } } } ] }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Schedule actions applied per second, by action kind and outcome.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 93 }, "id": 71, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (action, outcome) (rate(weaver_schedule_actions_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{action}} {{outcome}}", "refId": "A", "range": true } ], "title": "Schedule actions by kind", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Why the schedule holds download admission; none means no hold.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 101 }, "id": 72, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max by (reason) (weaver_schedule_admission_hold{job=\"$job\",instance=~\"$instance\"}) == 1", "legendFormat": "{{reason}}", "refId": "A", "range": true } ], "title": "Admission hold", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Schedule actions replayed, by reason, and wall-clock jumps the evaluators detected.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 101 }, "id": 73, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (reason) (increase(weaver_schedule_replays_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "replay {{reason}}", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (evaluator) (increase(weaver_schedule_clock_jumps_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "jump {{evaluator}}", "refId": "B", "range": true } ], "title": "Replays and clock jumps", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 107 }, "id": 74, "panels": [], "title": "Networking", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Each route leg: state, time since a scrape first saw it, connections, throughput and the ladder rung its last connection used.", "fieldConfig": { "defaults": { "custom": { "align": "auto" }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byName", "options": "In state" }, "properties": [ { "id": "unit", "value": "s" } ] }, { "matcher": { "id": "byName", "options": "Throughput" }, "properties": [ { "id": "unit", "value": "Bps" } ] } ] }, "gridPos": { "h": 8, "w": 14, "x": 0, "y": 108 }, "id": 75, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max by (consumer, position, egress_id, state) (weaver_network_leg_state{job=\"$job\",instance=~\"$instance\"} == 1)", "legendFormat": "", "refId": "A", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "time() - max by (consumer, position, egress_id) (weaver_network_leg_state_since_timestamp_seconds{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "B", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max by (consumer, position, egress_id) (weaver_network_leg_open_connections{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "C", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max by (consumer, position, egress_id) (weaver_network_leg_target_connections{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "D", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max by (consumer, position, egress_id) (weaver_network_leg_throughput_bytes_per_second{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "E", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max by (consumer, position, egress_id) (weaver_network_leg_rung{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "F", "instant": true, "range": false, "format": "table" } ], "title": "Route legs", "type": "table", "transformations": [ { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": { "Time": true, "Value #A": true }, "renameByName": { "consumer": "Consumer", "position": "Leg", "egress_id": "Egress", "state": "State", "Value #B": "In state", "Value #C": "Open", "Value #D": "Target", "Value #E": "Throughput", "Value #F": "Rung" } } } ] }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Each egress: health, time since a scrape first saw it, leg dial failures and cooldowns over the last hour, and download rate.", "fieldConfig": { "defaults": { "custom": { "align": "auto" }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byName", "options": "In state" }, "properties": [ { "id": "unit", "value": "s" } ] }, { "matcher": { "id": "byName", "options": "Download" }, "properties": [ { "id": "unit", "value": "Bps" } ] } ] }, "gridPos": { "h": 8, "w": 10, "x": 14, "y": 108 }, "id": 76, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max by (egress_id, health) (weaver_network_egress_health{job=\"$job\",instance=~\"$instance\"} == 1)", "legendFormat": "", "refId": "A", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "time() - max by (egress_id) (weaver_network_egress_health_since_timestamp_seconds{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "B", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (egress_id) (increase(weaver_network_egress_dials_total{job=\"$job\",instance=~\"$instance\",result=~\"failed|blocked\"}[1h]))", "legendFormat": "", "refId": "C", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (egress_id) (increase(weaver_network_egress_cooldowns_total{job=\"$job\",instance=~\"$instance\"}[1h]))", "legendFormat": "", "refId": "D", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (egress_id) (rate(weaver_egress_download_bytes_total{job=\"$job\",instance=~\"$instance\"}[5m]))", "legendFormat": "", "refId": "E", "instant": true, "range": false, "format": "table" } ], "title": "Egresses", "type": "table", "transformations": [ { "id": "joinByField", "options": { "byField": "egress_id", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": { "Time": true, "Value #A": true }, "renameByName": { "egress_id": "Egress", "health": "Health", "Value #B": "In state", "Value #C": "Failures / h", "Value #D": "Cooldowns / h", "Value #E": "Download" } } } ] }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Session preparations per second on session-based proxy hops, and the share that failed, by proxy kind.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 116 }, "id": 77, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (kind) (rate(weaver_tunnel_session_prepares_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{kind}} /s", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (kind) (rate(weaver_tunnel_session_prepares_total{job=\"$job\",instance=~\"$instance\",result=\"failed\"}[$__rate_interval])) / clamp_min(sum by (kind) (rate(weaver_tunnel_session_prepares_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval])), 1e-9)", "legendFormat": "{{kind}} failed ratio", "refId": "B", "range": true } ], "title": "Proxy session handshakes", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Streams opened through proxy hops per second, and the share that failed, by proxy kind.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 116 }, "id": 78, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (kind) (rate(weaver_tunnel_streams_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{kind}} /s", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (kind) (rate(weaver_tunnel_streams_total{job=\"$job\",instance=~\"$instance\",result!~\"success|skipped|at_capacity\"}[$__rate_interval])) / clamp_min(sum by (kind) (rate(weaver_tunnel_streams_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval])), 1e-9)", "legendFormat": "{{kind}} failed ratio", "refId": "B", "range": true } ], "title": "Proxy streams", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Members of each proxy pool stage: total, warmed and blocked.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 116 }, "id": 79, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max by (pool_id, egress_id, stage) (weaver_network_pool_members{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "pool {{pool_id}} egress {{egress_id}} {{stage}}", "refId": "A", "range": true } ], "title": "Pool stage sizes", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Share of host name lookups for routed connections that failed, by resolver.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "percentunit", "min": 0, "max": 1 }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 122 }, "id": 80, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (resolver) (rate(weaver_network_dns_resolutions_total{job=\"$job\",instance=~\"$instance\",result=\"failed\"}[$__rate_interval])) / clamp_min(sum by (resolver) (rate(weaver_network_dns_resolutions_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval])), 1e-9)", "legendFormat": "{{resolver}}", "refId": "A", "range": true } ], "title": "DNS failure ratio", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Route stages revoked, and connections made past a ladder's first rung, by rung.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 122 }, "id": 81, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(increase(weaver_network_leg_revocations_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "revocations", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (rung) (increase(weaver_network_rung_fallbacks_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "fallback rung {{rung}}", "refId": "B", "range": true }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (rung) (increase(weaver_network_rung_cooldowns_total{job=\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "cooldown rung {{rung}}", "refId": "C", "range": true } ], "title": "Revocations and ladder fallbacks", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Bytes per second each route leg carried.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never", "spanNulls": true }, "mappings": [], "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 122 }, "id": 82, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (consumer, position) (weaver_network_leg_throughput_bytes_per_second{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "{{consumer}} leg {{position}}", "refId": "A", "range": true } ], "title": "Leg throughput", "type": "timeseries" } ], "preload": false, "refresh": "30s", "schemaVersion": 39, "tags": ["weaver", "usenet"], "templating": { "list": [ { "current": {}, "hide": 0, "includeAll": false, "label": "Prometheus", "multi": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(weaver_build_info, job)", "hide": 0, "includeAll": false, "label": "Job", "multi": false, "name": "job", "options": [], "query": { "qryType": 1, "query": "label_values(weaver_build_info, job)", "refId": "job" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(weaver_build_info{job=\"$job\"}, instance)", "hide": 0, "includeAll": true, "label": "Instance", "multi": true, "name": "instance", "options": [], "query": { "qryType": 1, "query": "label_values(weaver_build_info{job=\"$job\"}, instance)", "refId": "instance" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Weaver — Overview", "uid": "weaver-overview", "version": 1, "weekStart": "" }