{ "apiVersion": "dashboard.grafana.app/v2", "kind": "Dashboard", "metadata": { "name": "docker-fclones-scheduler" }, "spec": { "annotations": [ { "kind": "AnnotationQuery", "spec": { "query": { "kind": "DataQuery", "group": "grafana", "version": "v0", "datasource": { "name": "-- Grafana --" }, "spec": {} }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "builtIn": true, "legacyOptions": { "type": "dashboard" } } } ], "cursorSync": "Crosshair", "description": "How the latest fclones run ended, when the next success is due and how much space the runs freed, with what needs your attention. The attention tiles need the rules from alerts/promql.yaml loaded.", "editable": true, "elements": { "panel-1": { "kind": "Panel", "spec": { "id": 1, "title": "Run outcome", "description": "Shows while the FclonesRunFailed alert fires: the latest run failed, so no space was reclaimed. The Last run tile below says how it ended, and the container log has the detail. Needs the rules from alerts/promql.yaml in a ruler that stores its `ALERTS` series beside the metrics, as Prometheus and Mimir do.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "0 * max(ALERTS{alertname=\"FclonesRunFailed\", alertstate=\"firing\", instance=\"$instance\"})", "legendFormat": "", "instant": true, "range": false } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": { "maxDataPoints": 500 } } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "auto", "text": { "titleSize": 14, "valueSize": 28 } }, "fieldConfig": { "defaults": { "unit": "none", "noValue": "Nothing to report", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" } ] }, "decimals": 0, "mappings": [ { "type": "value", "options": { "0": { "index": 0, "text": "Failed", "color": "red" } } } ] }, "overrides": [] } } } } }, "panel-2": { "kind": "Panel", "spec": { "id": 2, "title": "Next success", "description": "Shows while the FclonesScanStalled alert fires: no run succeeded by the deadline, or, with no deadline, none finished in 30 hours. Check the Last run tile first, then that the schedule or your outside trigger still runs. Needs the rules from alerts/promql.yaml in a ruler that stores its `ALERTS` series beside the metrics, as Prometheus and Mimir do.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "0 * max(ALERTS{alertname=\"FclonesScanStalled\", alertstate=\"firing\", instance=\"$instance\"})", "legendFormat": "", "instant": true, "range": false } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": { "maxDataPoints": 500 } } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "auto", "text": { "titleSize": 14, "valueSize": 28 } }, "fieldConfig": { "defaults": { "unit": "none", "noValue": "Nothing to report", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" } ] }, "decimals": 0, "mappings": [ { "type": "value", "options": { "0": { "index": 0, "text": "Overdue", "color": "red" } } } ] }, "overrides": [] } } } } }, "panel-3": { "kind": "Panel", "spec": { "id": 3, "title": "Action summary", "description": "Shows while the FclonesFormatDrift alert fires: the container did not recognize the summary fclones printed after its action, so the space and files freed read 0 after real work. A --dry-run in FCLONES_ACTION_ARGS causes this on purpose. Otherwise, check the fclones version. Needs the rules from alerts/promql.yaml in a ruler that stores its `ALERTS` series beside the metrics, as Prometheus and Mimir do.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "0 * max(ALERTS{alertname=\"FclonesFormatDrift\", alertstate=\"firing\", instance=\"$instance\"})", "legendFormat": "", "instant": true, "range": false } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": { "maxDataPoints": 500 } } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "auto", "text": { "titleSize": 14, "valueSize": 28 } }, "fieldConfig": { "defaults": { "unit": "none", "noValue": "Nothing to report", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "yellow" } ] }, "decimals": 0, "mappings": [ { "type": "value", "options": { "0": { "index": 0, "text": "Not recognized", "color": "yellow" } } } ] }, "overrides": [] } } } } }, "panel-4": { "kind": "Panel", "spec": { "id": 4, "title": "fclones Scheduler scrape", "description": "Shows while the FclonesDown alert fires: no scrape has succeeded for 5 minutes, so every other panel is out of date. The tile counts the selected instance and any target that is gone, because the alert for a gone target carries no instance. Needs the rules from alerts/promql.yaml in a ruler that stores its `ALERTS` series beside the metrics, as Prometheus and Mimir do.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "0 * max((ALERTS{alertname=\"FclonesDown\", alertstate=\"firing\", instance=\"$instance\"} or ALERTS{alertname=\"FclonesDown\", alertstate=\"firing\", instance=\"\"}))", "legendFormat": "", "instant": true, "range": false } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": { "maxDataPoints": 500 } } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "auto", "text": { "titleSize": 14, "valueSize": 28 } }, "fieldConfig": { "defaults": { "unit": "none", "noValue": "Nothing to report", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "yellow" } ] }, "decimals": 0, "mappings": [ { "type": "value", "options": { "0": { "index": 0, "text": "Not scraped", "color": "yellow" } } } ] }, "overrides": [] } } } } }, "panel-5": { "kind": "Panel", "spec": { "id": 5, "title": "Space reclaimed", "description": "Shows while the FclonesSpaceReclaimed notice fires: runs in the last 30 hours linked, removed or deduplicated duplicate files. It needs no action, and the Space reclaimed row shows how much was freed. Needs the rules from alerts/promql.yaml in a ruler that stores its `ALERTS` series beside the metrics, as Prometheus and Mimir do.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "0 * max(ALERTS{alertname=\"FclonesSpaceReclaimed\", alertstate=\"firing\", instance=\"$instance\"})", "legendFormat": "", "instant": true, "range": false } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": { "maxDataPoints": 500 } } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "auto", "text": { "titleSize": 14, "valueSize": 28 } }, "fieldConfig": { "defaults": { "unit": "none", "noValue": "Nothing to report", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "blue" } ] }, "decimals": 0, "mappings": [ { "type": "value", "options": { "0": { "index": 0, "text": "Duplicates processed", "color": "blue" } } } ] }, "overrides": [] } } } } }, "panel-10": { "kind": "Panel", "spec": { "id": 10, "title": "Last run", "description": "How the latest run ended. A failure names its cause: fclones failed or timed out, its report could not be read, its action reclaimed nothing, the run could not start, or the scan lock failed. Right after a restart it reads only Succeeded or Failed until the next run. No value means no run has finished yet, or the container was not scraped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "(sum(fclones_scheduler_last_run_result{instance=\"$instance\", result=\"success\"}) + 2 * sum(fclones_scheduler_last_run_result{instance=\"$instance\", result=\"exec_error\"}) + 3 * sum(fclones_scheduler_last_run_result{instance=\"$instance\", result=\"timeout\"}) + 4 * sum(fclones_scheduler_last_run_result{instance=\"$instance\", result=\"decode_error\"}) + 5 * sum(fclones_scheduler_last_run_result{instance=\"$instance\", result=\"action_no_op\"}) + 6 * sum(fclones_scheduler_last_run_result{instance=\"$instance\", result=\"start_error\"}) + 7 * sum(fclones_scheduler_last_run_result{instance=\"$instance\", result=\"lock_error\"}) > 0) or on () (8 + fclones_scheduler_last_run_ok{instance=\"$instance\"} and on () fclones_scheduler_last_run_timestamp_seconds{instance=\"$instance\"} > 0)", "legendFormat": "", "instant": true, "range": false } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "auto", "text": { "titleSize": 14, "valueSize": 28 } }, "fieldConfig": { "defaults": { "unit": "none", "noValue": "No run yet", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "text" } ] }, "mappings": [ { "type": "value", "options": { "1": { "index": 0, "text": "Succeeded", "color": "green" }, "2": { "index": 1, "text": "fclones failed", "color": "red" }, "3": { "index": 2, "text": "Timed out", "color": "red" }, "4": { "index": 3, "text": "Report unreadable", "color": "red" }, "5": { "index": 4, "text": "Nothing reclaimed", "color": "red" }, "6": { "index": 5, "text": "Could not start", "color": "red" }, "7": { "index": 6, "text": "Lock failed", "color": "red" }, "8": { "index": 7, "text": "Failed", "color": "red" }, "9": { "index": 8, "text": "Succeeded", "color": "green" } } } ] }, "overrides": [] } } } } }, "panel-11": { "kind": "Panel", "spec": { "id": 11, "title": "Last run ended", "description": "When the latest run finished, kept across a restart. A time further back than SCAN_INTERVAL, 3 hours by default, plus one run's length means the schedule or your outside trigger has stopped. No value means no run has finished yet, or the container was not scraped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "(fclones_scheduler_last_run_timestamp_seconds{instance=\"$instance\"} > 0) * 1000", "legendFormat": "", "instant": true, "range": false } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "auto", "text": { "titleSize": 14, "valueSize": 28 } }, "fieldConfig": { "defaults": { "unit": "dateTimeFromNow", "noValue": "No run yet", "color": { "mode": "fixed", "fixedColor": "text" }, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "text" } ] } }, "overrides": [] } } } } }, "panel-12": { "kind": "Panel", "spec": { "id": 12, "title": "Last success", "description": "When a run last succeeded, kept across a restart. A time older than Last run ended means the runs since have failed. No value means no run has succeeded yet, the run before the last restart failed, or the container was not scraped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "(fclones_scheduler_last_success_timestamp_seconds{instance=\"$instance\"} > 0) * 1000", "legendFormat": "", "instant": true, "range": false } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "auto", "text": { "titleSize": 14, "valueSize": 28 } }, "fieldConfig": { "defaults": { "unit": "dateTimeFromNow", "noValue": "No success known", "color": { "mode": "fixed", "fixedColor": "text" }, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "text" } ] } }, "overrides": [] } } } } }, "panel-13": { "kind": "Panel", "spec": { "id": 13, "title": "Next success due", "description": "The time by which the next run must succeed: the last success, or the container's start when none is known, plus twice SCAN_INTERVAL and twice SCAN_TIMEOUT. FclonesScanStalled fires 30 minutes after it passes. No deadline means an outside scheduler, SCAN_INTERVAL=0 or SCAN_TIMEOUT=0, or that the container was not scraped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "(fclones_scheduler_run_deadline_timestamp_seconds{instance=\"$instance\"} > 0) * 1000", "legendFormat": "", "instant": true, "range": false } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "auto", "text": { "titleSize": 14, "valueSize": 28 } }, "fieldConfig": { "defaults": { "unit": "dateTimeFromNow", "noValue": "No deadline", "color": { "mode": "fixed", "fixedColor": "text" }, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "text" } ] } }, "overrides": [] } } } } }, "panel-20": { "kind": "Panel", "spec": { "id": 20, "title": "Space freed in ${span} ${spanunit}", "description": "The space the actions freed in the selected range, from fclones' own rounded summary, so for dedupe it is an upper bound. Counts reset when the container restarts, and the total still adds up across it. No value means the container was not scraped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "round(increase(fclones_scheduler_reclaimed_bytes_total{instance=\"$instance\"}[$__range]))", "legendFormat": "", "instant": true, "range": false } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "auto", "text": { "titleSize": 14, "valueSize": 28 } }, "fieldConfig": { "defaults": { "unit": "bytes", "noValue": "Not scraped", "color": { "mode": "fixed", "fixedColor": "text" }, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "text" } ] } }, "overrides": [] } } } } }, "panel-21": { "kind": "Panel", "spec": { "id": 21, "title": "Files processed in ${span} ${spanunit}", "description": "Duplicate files the actions linked, removed or deduplicated in the selected range. Counts reset when the container restarts, and the total still adds up across it. No value means the container was not scraped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "round(increase(fclones_scheduler_files_processed_total{instance=\"$instance\"}[$__range]))", "legendFormat": "", "instant": true, "range": false } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "auto", "text": { "titleSize": 14, "valueSize": 28 } }, "fieldConfig": { "defaults": { "unit": "none", "noValue": "Not scraped", "color": { "mode": "fixed", "fixedColor": "text" }, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "text" } ] } }, "overrides": [] } } } } }, "panel-22": { "kind": "Panel", "spec": { "id": 22, "title": "Duplicate files found", "description": "Duplicate files, keepers left out, in the latest scan report, before its action ran. A number that stays high run after run means the action is not freeing them, so check the configured action. A scan whose report was not read keeps the previous number, 0 after a restart, and Last run names why. No value means no scan has run since the container started, or the container was not scraped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "fclones_scheduler_duplicate_files{instance=\"$instance\"} and on (instance) fclones_scheduler_phase_duration_seconds{instance=\"$instance\", phase=\"scan\"} > 0", "legendFormat": "", "instant": true, "range": false } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "auto", "text": { "titleSize": 14, "valueSize": 28 } }, "fieldConfig": { "defaults": { "unit": "none", "noValue": "No scan since start", "color": { "mode": "fixed", "fixedColor": "text" }, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "text" } ] } }, "overrides": [] } } } } }, "panel-23": { "kind": "Panel", "spec": { "id": 23, "title": "Space duplicates take", "description": "The space the duplicates in the latest scan report take up, the most the next action can free. A scan whose report was not read keeps the previous number, 0 after a restart, and Last run names why. No value means no scan has run since the container started, or the container was not scraped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "fclones_scheduler_redundant_bytes{instance=\"$instance\"} and on (instance) fclones_scheduler_phase_duration_seconds{instance=\"$instance\", phase=\"scan\"} > 0", "legendFormat": "", "instant": true, "range": false } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "auto", "text": { "titleSize": 14, "valueSize": 28 } }, "fieldConfig": { "defaults": { "unit": "bytes", "noValue": "No scan since start", "color": { "mode": "fixed", "fixedColor": "text" }, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "text" } ] } }, "overrides": [] } } } } }, "panel-24": { "kind": "Panel", "spec": { "id": 24, "title": "Space freed", "description": "The space the actions freed in each time slot, as bars, with the total for the selected range in the legend. A run that freed nothing draws no bar, and a gap means the container was not scraped. A slot near a container restart, a failed scrape or a scrape outage can show fewer or more than happened.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "round(sum by (instance) (((last_over_time(fclones_scheduler_reclaimed_bytes_total{instance=\"$instance\"}[$__interval]) - last_over_time(fclones_scheduler_reclaimed_bytes_total{instance=\"$instance\"}[$__interval] offset $__interval)) unless on (instance) ((last_over_time(process_start_time_seconds{instance=\"$instance\"}[$__interval]) != last_over_time(process_start_time_seconds{instance=\"$instance\"}[$__interval] offset $__interval)) or (last_over_time(process_start_time_seconds{instance=\"$instance\"}[$__interval]) > time() - ${__interval_ms} / 1000))) or (last_over_time(fclones_scheduler_reclaimed_bytes_total{instance=\"$instance\"}[$__interval]) and on (instance) ((last_over_time(process_start_time_seconds{instance=\"$instance\"}[$__interval]) != last_over_time(process_start_time_seconds{instance=\"$instance\"}[$__interval] offset $__interval)) or (last_over_time(process_start_time_seconds{instance=\"$instance\"}[$__interval]) > time() - ${__interval_ms} / 1000))) or (last_over_time(fclones_scheduler_reclaimed_bytes_total{instance=\"$instance\"}[$__interval]) - min_over_time(fclones_scheduler_reclaimed_bytes_total{instance=\"$instance\"}[$__interval]))))", "legendFormat": "Freed" } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": { "interval": "1h" } } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": [ "sum" ] }, "tooltip": { "mode": "multi" } }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" } ] }, "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, "stacking": { "mode": "normal", "group": "A" } }, "noValue": "Not scraped", "min": 0 }, "overrides": [] } } } } }, "panel-25": { "kind": "Panel", "spec": { "id": 25, "title": "Duplicate space per scan", "description": "The space the duplicates in each scan report take up, over the selected range. A line that grows means duplicates arrive faster than the action frees them. A scan whose report was not read keeps the line flat, at 0 after a restart. A gap means no scan has run since the container started, or the container was not scraped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "fclones_scheduler_redundant_bytes{instance=\"$instance\"} and on (instance) fclones_scheduler_phase_duration_seconds{instance=\"$instance\", phase=\"scan\"} > 0", "legendFormat": "Duplicates" } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" } ] }, "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 20, "lineWidth": 2 }, "noValue": "No scan since start", "min": 0 }, "overrides": [] } } } } }, "panel-30": { "kind": "Panel", "spec": { "id": 30, "title": "Runs by result", "description": "Completed runs in each time slot by how they ended, as bars, with the total for the selected range in the legend. Any result but Succeeded is a failed run, and the container log names its cause. No value means no run finished in the range. A slot near a container restart, a failed scrape or a scrape outage can show fewer or more than happened.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "round(sum by (result) (((last_over_time(fclones_scheduler_runs_total{instance=\"$instance\"}[$__interval]) - last_over_time(fclones_scheduler_runs_total{instance=\"$instance\"}[$__interval] offset $__interval)) unless on (instance) ((last_over_time(process_start_time_seconds{instance=\"$instance\"}[$__interval]) != last_over_time(process_start_time_seconds{instance=\"$instance\"}[$__interval] offset $__interval)) or (last_over_time(process_start_time_seconds{instance=\"$instance\"}[$__interval]) > time() - ${__interval_ms} / 1000))) or (last_over_time(fclones_scheduler_runs_total{instance=\"$instance\"}[$__interval]) and on (instance) ((last_over_time(process_start_time_seconds{instance=\"$instance\"}[$__interval]) != last_over_time(process_start_time_seconds{instance=\"$instance\"}[$__interval] offset $__interval)) or (last_over_time(process_start_time_seconds{instance=\"$instance\"}[$__interval]) > time() - ${__interval_ms} / 1000))) or (last_over_time(fclones_scheduler_runs_total{instance=\"$instance\"}[$__interval]) - min_over_time(fclones_scheduler_runs_total{instance=\"$instance\"}[$__interval])))) > 0", "legendFormat": "{{result}}" } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": { "interval": "1h" } } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": [ "sum" ] }, "tooltip": { "mode": "multi" } }, "fieldConfig": { "defaults": { "unit": "none", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" } ] }, "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, "stacking": { "mode": "normal", "group": "A" } }, "noValue": "No run in the range", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "success" }, "properties": [ { "id": "displayName", "value": "Succeeded" }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] }, { "matcher": { "id": "byName", "options": "exec_error" }, "properties": [ { "id": "displayName", "value": "fclones failed" }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] }, { "matcher": { "id": "byName", "options": "timeout" }, "properties": [ { "id": "displayName", "value": "Timed out" }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] }, { "matcher": { "id": "byName", "options": "decode_error" }, "properties": [ { "id": "displayName", "value": "Report unreadable" }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] }, { "matcher": { "id": "byName", "options": "action_no_op" }, "properties": [ { "id": "displayName", "value": "Nothing reclaimed" }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] }, { "matcher": { "id": "byName", "options": "start_error" }, "properties": [ { "id": "displayName", "value": "Could not start" }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] }, { "matcher": { "id": "byName", "options": "lock_error" }, "properties": [ { "id": "displayName", "value": "Lock failed" }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] } ] } } } } }, "panel-31": { "kind": "Panel", "spec": { "id": 31, "title": "Phase time", "description": "How long the scan and the action took each time they ran, over the selected range. A line that climbs toward SCAN_TIMEOUT warns before a run times out. A gap means the phase has not run since the container started, or the container was not scraped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "expr": "fclones_scheduler_phase_duration_seconds{instance=\"$instance\"} > 0", "legendFormat": "{{phase}}" } }, "refId": "A", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "fieldConfig": { "defaults": { "unit": "s", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" } ] }, "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 20, "lineWidth": 2 }, "noValue": "No run since start", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "scan" }, "properties": [ { "id": "displayName", "value": "Scan" } ] }, { "matcher": { "id": "byName", "options": "action" }, "properties": [ { "id": "displayName", "value": "Action" } ] } ] } } } } } }, "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "title": "Needs attention", "collapse": false, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "visibility": "show", "condition": "and", "items": [ { "kind": "ConditionalRenderingVariable", "spec": { "variable": "attention", "operator": "matches", "value": "^[1-9]" } } ] } }, "layout": { "kind": "AutoGridLayout", "spec": { "maxColumnCount": 3, "columnWidthMode": "custom", "rowHeightMode": "custom", "rowHeight": 106, "fillScreen": false, "columnWidth": 272, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-1" }, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "visibility": "show", "condition": "or", "items": [ { "kind": "ConditionalRenderingData", "spec": { "value": true } }, { "kind": "ConditionalRenderingTimeRangeSize", "spec": { "value": "0s" } } ] } } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-2" }, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "visibility": "show", "condition": "or", "items": [ { "kind": "ConditionalRenderingData", "spec": { "value": true } }, { "kind": "ConditionalRenderingTimeRangeSize", "spec": { "value": "0s" } } ] } } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-3" }, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "visibility": "show", "condition": "or", "items": [ { "kind": "ConditionalRenderingData", "spec": { "value": true } }, { "kind": "ConditionalRenderingTimeRangeSize", "spec": { "value": "0s" } } ] } } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-4" }, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "visibility": "show", "condition": "or", "items": [ { "kind": "ConditionalRenderingData", "spec": { "value": true } }, { "kind": "ConditionalRenderingTimeRangeSize", "spec": { "value": "0s" } } ] } } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-5" }, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "visibility": "show", "condition": "or", "items": [ { "kind": "ConditionalRenderingData", "spec": { "value": true } }, { "kind": "ConditionalRenderingTimeRangeSize", "spec": { "value": "0s" } } ] } } } } ] } } } }, { "kind": "RowsLayoutRow", "spec": { "title": "Last run", "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "x": 0, "y": 0, "width": 6, "height": 3, "element": { "kind": "ElementReference", "name": "panel-10" } } }, { "kind": "GridLayoutItem", "spec": { "x": 6, "y": 0, "width": 6, "height": 3, "element": { "kind": "ElementReference", "name": "panel-11" } } }, { "kind": "GridLayoutItem", "spec": { "x": 12, "y": 0, "width": 6, "height": 3, "element": { "kind": "ElementReference", "name": "panel-12" } } }, { "kind": "GridLayoutItem", "spec": { "x": 18, "y": 0, "width": 6, "height": 3, "element": { "kind": "ElementReference", "name": "panel-13" } } } ] } } } }, { "kind": "RowsLayoutRow", "spec": { "title": "Space reclaimed", "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "x": 0, "y": 0, "width": 6, "height": 3, "element": { "kind": "ElementReference", "name": "panel-20" } } }, { "kind": "GridLayoutItem", "spec": { "x": 6, "y": 0, "width": 6, "height": 3, "element": { "kind": "ElementReference", "name": "panel-21" } } }, { "kind": "GridLayoutItem", "spec": { "x": 12, "y": 0, "width": 6, "height": 3, "element": { "kind": "ElementReference", "name": "panel-22" } } }, { "kind": "GridLayoutItem", "spec": { "x": 18, "y": 0, "width": 6, "height": 3, "element": { "kind": "ElementReference", "name": "panel-23" } } }, { "kind": "GridLayoutItem", "spec": { "x": 0, "y": 3, "width": 12, "height": 8, "element": { "kind": "ElementReference", "name": "panel-24" } } }, { "kind": "GridLayoutItem", "spec": { "x": 12, "y": 3, "width": 12, "height": 8, "element": { "kind": "ElementReference", "name": "panel-25" } } } ] } } } }, { "kind": "RowsLayoutRow", "spec": { "title": "Runs", "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "x": 0, "y": 0, "width": 12, "height": 8, "element": { "kind": "ElementReference", "name": "panel-30" } } }, { "kind": "GridLayoutItem", "spec": { "x": 12, "y": 0, "width": 12, "height": 8, "element": { "kind": "ElementReference", "name": "panel-31" } } } ] } } } } ] } }, "links": [], "liveNow": false, "preload": true, "tags": [ "duplicates", "storage" ], "timeSettings": { "timezone": "browser", "from": "now-7d", "to": "now", "autoRefresh": "5m", "autoRefreshIntervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "hideTimepicker": false, "fiscalYearStartMonth": 0 }, "title": "fclones Scheduler", "variables": [ { "kind": "DatasourceVariable", "spec": { "name": "datasource", "pluginId": "prometheus", "refresh": "never", "regex": "", "current": { "text": "", "value": "" }, "options": [], "multi": false, "includeAll": false, "label": "Data source", "hide": "dontHide", "skipUrlSync": false, "allowCustomValue": true } }, { "kind": "QueryVariable", "spec": { "name": "instance", "current": { "text": "", "value": "" }, "label": "fclones Scheduler", "hide": "dontHide", "refresh": "onTimeRangeChanged", "skipUrlSync": false, "description": "The fclones scheduler to show, by its scrape instance.", "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "query": "label_values(fclones_scheduler_build_info, instance)" } }, "regex": "", "sort": "alphabeticalAsc", "definition": "label_values(fclones_scheduler_build_info, instance)", "options": [], "multi": false, "includeAll": false, "allowCustomValue": false } }, { "kind": "QueryVariable", "spec": { "name": "spanread", "label": "Range length source", "description": "Reads the length of the time range for span, spanunit and spanthe, all their words in one answer.", "current": { "text": "", "value": "" }, "hide": "hideVariable", "refresh": "onTimeRangeChanged", "skipUrlSync": true, "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "query": "query_result(label_replace(label_replace(count_values by (u) (\"n\", label_replace(round(vector($__range_s / 86400), 0.1) and on () (vector($__range_s) >= 172800), \"u\", \"days\", \"\", \"\") or on () label_replace(round(vector($__range_s / 3600), 0.1) and on () (vector($__range_s) >= 7200), \"u\", \"hours\", \"\", \"\") or on () label_replace(round(vector($__range_s / 60)) and on () (vector($__range_s) >= 120), \"u\", \"minutes\", \"\", \"\") or on () label_replace(round(vector($__range_s)), \"u\", \"seconds\", \"\", \"\")), \"n\", \"$1 : \", \"n\", \"(.+)\"), \"u\", \"$1 : \", \"u\", \"(.+)\"))" } }, "regex": "/^\\{n=\"(?[^\"]+)\",u=\"(?[^\"]+)\"\\} /", "sort": "disabled", "options": [], "multi": false, "includeAll": false, "allowCustomValue": false, "definition": "query_result(label_replace(label_replace(count_values by (u) (\"n\", label_replace(round(vector($__range_s / 86400), 0.1) and on () (vector($__range_s) >= 172800), \"u\", \"days\", \"\", \"\") or on () label_replace(round(vector($__range_s / 3600), 0.1) and on () (vector($__range_s) >= 7200), \"u\", \"hours\", \"\", \"\") or on () label_replace(round(vector($__range_s / 60)) and on () (vector($__range_s) >= 120), \"u\", \"minutes\", \"\", \"\") or on () label_replace(round(vector($__range_s)), \"u\", \"seconds\", \"\", \"\")), \"n\", \"$1 : \", \"n\", \"(.+)\"), \"u\", \"$1 : \", \"u\", \"(.+)\"))" } }, { "kind": "CustomVariable", "spec": { "name": "span", "label": "Range length", "description": "The number of the dashboard time range's length, such as \"30\" in \"30 days\", from spanread. It reads \"the\" until spanread has read a length.", "query": "${spanread}the", "current": { "text": "the", "value": "the" }, "options": [], "hide": "hideVariable", "skipUrlSync": true, "multi": false, "includeAll": false, "allowCustomValue": false } }, { "kind": "CustomVariable", "spec": { "name": "spanunit", "label": "Range unit", "description": "The unit of the dashboard time range's length, such as \"days\" in \"30 days\", from spanread. It reads \"range\" until spanread has read a length.", "query": "${spanread:text}range", "current": { "text": "range", "value": "range" }, "options": [], "hide": "hideVariable", "skipUrlSync": true, "multi": false, "includeAll": false, "allowCustomValue": false } }, { "kind": "CustomVariable", "spec": { "name": "spanthe", "label": "Range length with its article", "description": "The word \"the\" and the number of the dashboard time range's length, such as \"the 30\" in \"the 30 days\", from spanread. It reads \"the\" until spanread has read a length.", "query": "the ${spanread}${spanread:text}", "current": { "text": "the", "value": "the" }, "options": [], "hide": "hideVariable", "skipUrlSync": true, "multi": false, "includeAll": false, "allowCustomValue": false } }, { "kind": "QueryVariable", "spec": { "name": "attention", "label": "Attention tiles shown", "description": "How many Needs attention tiles have data. The row shows only while one does.", "current": { "text": "0", "value": "0" }, "hide": "hideVariable", "refresh": "onTimeRangeChanged", "skipUrlSync": true, "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "datasource": { "name": "${datasource}" }, "spec": { "query": "query_result(count(label_replace(count(0 * max(ALERTS{alertname=\"FclonesRunFailed\", alertstate=\"firing\", instance=\"$instance\"})), \"tile\", \"1\", \"\", \"\") or label_replace(count(0 * max(ALERTS{alertname=\"FclonesScanStalled\", alertstate=\"firing\", instance=\"$instance\"})), \"tile\", \"2\", \"\", \"\") or label_replace(count(0 * max(ALERTS{alertname=\"FclonesFormatDrift\", alertstate=\"firing\", instance=\"$instance\"})), \"tile\", \"3\", \"\", \"\") or label_replace(count(0 * max((ALERTS{alertname=\"FclonesDown\", alertstate=\"firing\", instance=\"$instance\"} or ALERTS{alertname=\"FclonesDown\", alertstate=\"firing\", instance=\"\"}))), \"tile\", \"4\", \"\", \"\") or label_replace(count(0 * max(ALERTS{alertname=\"FclonesSpaceReclaimed\", alertstate=\"firing\", instance=\"$instance\"})), \"tile\", \"5\", \"\", \"\")) or vector(0))" } }, "definition": "query_result(count(label_replace(count(0 * max(ALERTS{alertname=\"FclonesRunFailed\", alertstate=\"firing\", instance=\"$instance\"})), \"tile\", \"1\", \"\", \"\") or label_replace(count(0 * max(ALERTS{alertname=\"FclonesScanStalled\", alertstate=\"firing\", instance=\"$instance\"})), \"tile\", \"2\", \"\", \"\") or label_replace(count(0 * max(ALERTS{alertname=\"FclonesFormatDrift\", alertstate=\"firing\", instance=\"$instance\"})), \"tile\", \"3\", \"\", \"\") or label_replace(count(0 * max((ALERTS{alertname=\"FclonesDown\", alertstate=\"firing\", instance=\"$instance\"} or ALERTS{alertname=\"FclonesDown\", alertstate=\"firing\", instance=\"\"}))), \"tile\", \"4\", \"\", \"\") or label_replace(count(0 * max(ALERTS{alertname=\"FclonesSpaceReclaimed\", alertstate=\"firing\", instance=\"$instance\"})), \"tile\", \"5\", \"\", \"\")) or vector(0))", "regex": "/^\\{\\} ([0-9]+) /", "sort": "disabled", "options": [], "multi": false, "includeAll": false, "allowCustomValue": false } } ] } }