{ "uid": "manticore-search-prometheus", "title": "Manticore Search (Prometheus)", "tags": [ "manticore", "manticore search", "prometheus" ], "timezone": "browser", "schemaVersion": 38, "version": 1, "refresh": "30s", "time": { "from": "now-6h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h" ] }, "templating": { "list": [ { "name": "datasource", "type": "datasource", "query": "prometheus", "current": { "selected": false, "text": "default", "value": "default" } }, { "name": "job", "type": "query", "datasource": { "type": "prometheus", "uid": "$datasource" }, "query": "label_values(up, job)", "refresh": 1, "current": { "selected": false, "text": "manticoresearch", "value": "manticoresearch" } }, { "name": "instance", "type": "query", "datasource": { "type": "prometheus", "uid": "$datasource" }, "query": "label_values(up{job=\"$job\"}, instance)", "refresh": 1, "includeAll": true, "allValue": ".*", "multi": true, "current": { "selected": false, "text": "All", "value": ".*" } }, { "name": "table", "type": "query", "datasource": { "type": "prometheus", "uid": "$datasource" }, "query": "label_values(manticore_indexed_documents_count{job=\"$job\",instance=~\"$instance\"}, table)", "refresh": 1, "includeAll": true, "allValue": ".*", "multi": true, "current": { "selected": false, "text": "All", "value": ".*" } }, { "name": "cluster", "type": "query", "datasource": { "type": "prometheus", "uid": "$datasource" }, "query": "label_values(manticore_cluster_numeric{job=\"$job\",instance=~\"$instance\"}, cluster)", "refresh": 1, "includeAll": true, "allValue": ".*", "multi": true, "current": { "selected": false, "text": "All", "value": ".*" } }, { "name": "slack_url", "type": "constant", "hide": 2, "query": "https://manticoresearch.com/slack/", "current": { "selected": false, "text": "https://manticoresearch.com/slack/", "value": "https://manticoresearch.com/slack/" } }, { "name": "forum_url", "type": "constant", "hide": 2, "query": "https://forum.manticoresearch.com/", "current": { "selected": false, "text": "https://forum.manticoresearch.com/", "value": "https://forum.manticoresearch.com/" } }, { "name": "support_url", "type": "constant", "hide": 2, "query": "https://manticoresearch.com/support/", "current": { "selected": false, "text": "https://manticoresearch.com/support/", "value": "https://manticoresearch.com/support/" } } ] }, "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "panels": [ { "id": 1, "type": "stat", "title": "Health / Uptime", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 4, "y": 5, "w": 4, "h": 4 }, "targets": [ { "expr": "max(manticore_uptime_seconds_gauge{job=\"$job\",instance=~\"$instance\"}) * on() (min(up{job=\"$job\",instance=~\"$instance\"}) == 1)", "legendFormat": "ManticoreSearch", "refId": "A", "instant": true } ], "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "blue" }, "unit": "s", "decimals": 0 }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "last" ], "values": false } }, "description": "Time since searchd start (seconds)." }, { "id": 2, "type": "table", "title": "Overview / Version", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 12, "y": 5, "w": 12, "h": 4 }, "targets": [ { "expr": "(label_replace(label_replace(label_replace(manticore_version{job=\"$job\",instance=~\"$instance\"}, \"component\", \"daemon\", \"version\", \".*\"), \"version_short\", \"$1\", \"version\", \"^([^(]+).*\"), \"order\", \"1\", \"component\", \".*\")) or (label_replace(label_replace(label_replace(manticore_version{job=\"$job\",instance=~\"$instance\"}, \"component\", \"columnar\", \"version\", \".*\"), \"version_short\", \"$1\", \"version\", \".*[(]columnar ([^)]*)[)].*\"), \"order\", \"2\", \"component\", \".*\")) or (label_replace(label_replace(label_replace(manticore_version{job=\"$job\",instance=~\"$instance\"}, \"component\", \"secondary\", \"version\", \".*\"), \"version_short\", \"$1\", \"version\", \".*[(]secondary ([^)]*)[)].*\"), \"order\", \"3\", \"component\", \".*\")) or (label_replace(label_replace(label_replace(manticore_version{job=\"$job\",instance=~\"$instance\"}, \"component\", \"buddy\", \"version\", \".*\"), \"version_short\", \"$1\", \"version\", \".*[(]buddy ([^)]*)[)].*\"), \"order\", \"4\", \"component\", \".*\"))", "refId": "A", "instant": true, "format": "table" } ], "fieldConfig": { "defaults": { "decimals": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "component" }, "properties": [ { "id": "custom.width", "value": 90 } ] }, { "matcher": { "id": "byName", "options": "version" }, "properties": [ { "id": "custom.width", "value": 520 } ] }, { "matcher": { "id": "byName", "options": "order" }, "properties": [ { "id": "custom.hidden", "value": true } ] } ] }, "options": { "showHeader": false, "sortBy": [ { "displayName": "order", "desc": false } ] }, "transformations": [ { "id": "labelsToFields", "options": { "mode": "columns" } }, { "id": "organize", "options": { "excludeByName": { "__name__": true, "Time": true, "Value": true, "job": true, "instance": true, "version": true }, "renameByName": { "version_short": "version" } } } ], "description": "Component versions parsed from manticore_version (daemon/columnar/secondary/buddy)." }, { "id": 3, "type": "stat", "title": "Health / Current connections", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 8, "y": 5, "w": 4, "h": 4 }, "targets": [ { "expr": "max(manticore_current_connections_count{job=\"$job\",instance=~\"$instance\"}) * on() (min(up{job=\"$job\",instance=~\"$instance\"}) == 1)", "legendFormat": "ManticoreSearch", "refId": "A", "instant": true } ], "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "last" ], "values": false } }, "description": "Current client connections (from SHOW STATUS: workers_clients)." }, { "id": 4, "type": "stat", "title": "Health / Connect latency", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 5, "w": 4, "h": 4 }, "targets": [ { "expr": "max(manticore_connect_time_seconds{job=\"$job\",instance=~\"$instance\"}) * on() (min(up{job=\"$job\",instance=~\"$instance\"}) == 1)", "legendFormat": "ManticoreSearch", "refId": "A", "instant": true } ], "fieldConfig": { "defaults": { "unit": "s", "decimals": 3, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "#56A64B", "value": null }, { "color": "yellow", "value": 0.05 }, { "color": "red", "value": 0.2 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "last" ], "values": false } }, "description": "Buddy \u2192 searchd connectivity latency (seconds)." }, { "id": 26, "type": "stat", "title": "Health / Up", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 1, "w": 4, "h": 4 }, "targets": [ { "expr": "min(up{job=\"$job\",instance=~\"$instance\"})", "refId": "A" } ], "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "type": "value", "options": { "0": { "text": "DOWN" }, "1": { "text": "UP" } } } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "#56A64B", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "last" ], "values": false } }, "description": "Connectivity health: 1=UP, 0=DOWN." }, { "id": 46, "type": "stat", "title": "Health / Crash indicator", "description": "1=OK (no crashes in range), 0=CRITICAL (crash detected). Based on increase(manticore_searchd_crashes_total[$__range]).", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 4, "y": 1, "w": 4, "h": 4 }, "targets": [ { "expr": "(1 - clamp_max((sum(increase(manticore_searchd_crashes_total{job=\"$job\",instance=~\"$instance\"}[$__range])) or vector(0)), 1)) * on() (min(up{job=\"$job\",instance=~\"$instance\"}) == 1)", "legendFormat": "crash_indicator", "refId": "A", "instant": true } ], "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "type": "value", "options": { "0": { "text": "CRITICAL" }, "1": { "text": "OK" } } } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "#56A64B", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "last" ], "values": false } } }, { "id": 5, "type": "timeseries", "title": "Performance / Load (All Queues)", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 17, "w": 12, "h": 6 }, "targets": [ { "expr": "manticore_load_1m_total{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "1m", "refId": "A" }, { "expr": "manticore_load_5m_total{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "5m", "refId": "B" }, { "expr": "manticore_load_15m_total{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "15m", "refId": "C" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 2 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Load averages from SHOW STATUS: load_{1m,5m,15m}." }, { "id": 6, "type": "timeseries", "title": "Performance / Work Queue Length", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 12, "y": 17, "w": 12, "h": 6 }, "targets": [ { "expr": "manticore_work_queue_length_count{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "queue", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Current work queue length (SHOW STATUS: work_queue_length)." }, { "id": 7, "type": "timeseries", "title": "Performance / Commands per Second", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 6, "y": 10, "w": 6, "h": 7 }, "targets": [ { "refId": "A", "expr": "sum(rate(manticore_command_search_count{job=\"$job\",instance=~\"$instance\"}[1m])) or vector(0)", "legendFormat": "search" }, { "refId": "B", "expr": "sum(rate(manticore_command_insert_count{job=\"$job\",instance=~\"$instance\"}[1m])) or vector(0)", "legendFormat": "insert" }, { "refId": "C", "expr": "sum(rate(manticore_command_update_count{job=\"$job\",instance=~\"$instance\"}[1m])) or vector(0)", "legendFormat": "update" }, { "refId": "D", "expr": "sum(rate(manticore_command_delete_count{job=\"$job\",instance=~\"$instance\"}[1m])) or vector(0)", "legendFormat": "delete" }, { "refId": "E", "expr": "sum(rate(manticore_command_replace_count{job=\"$job\",instance=~\"$instance\"}[1m])) or vector(0)", "legendFormat": "replace" } ], "fieldConfig": { "defaults": { "unit": "ops", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 2 }, "overrides": [ { "matcher": { "id": "byName", "options": "search" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] }, { "matcher": { "id": "byName", "options": "insert" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } } ] }, { "matcher": { "id": "byName", "options": "update" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] }, { "matcher": { "id": "byName", "options": "delete" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] }, { "matcher": { "id": "byName", "options": "replace" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "gray" } } ] } ] }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Per-command request rate (ops/s). Uses 1m rate; includes vector(0) so zeros still render." }, { "id": 8, "type": "timeseries", "title": "Performance / Search Latency (p95/p99)", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 12, "y": 10, "w": 6, "h": 7 }, "targets": [ { "expr": "manticore_search_stats_ms_pct95_5m_millisecond{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "p95", "refId": "A" }, { "expr": "manticore_search_stats_ms_pct99_5m_millisecond{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "p99", "refId": "B" } ], "fieldConfig": { "defaults": { "unit": "ms", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 100 }, { "color": "red", "value": 300 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "p95" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } } ] }, { "matcher": { "id": "byName", "options": "p99" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }, { "id": "custom.lineWidth", "value": 4 } ] } ] }, "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Search latency p95/p99 (ms) over 5m window." }, { "id": 9, "type": "timeseries", "title": "Performance / Query Cache Usage", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 23, "w": 12, "h": 7 }, "targets": [ { "expr": "manticore_qcache_used_bytes{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "used", "refId": "A" }, { "expr": "manticore_qcache_max_bytes{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "max", "refId": "B" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Query cache usage (bytes)." }, { "id": 10, "type": "timeseries", "title": "Performance / Query Cache Hit Rate", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 12, "y": 23, "w": 12, "h": 7 }, "targets": [ { "expr": "rate(manticore_qcache_hits_count{job=\"$job\",instance=~\"$instance\"}[5m])", "legendFormat": "hits/sec", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "ops", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 2 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Query cache hit rate (ops/s): rate(qcache_hits_count)." }, { "id": 11, "type": "timeseries", "title": "Tables / Disk Bytes (Top 10)", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 74, "w": 12, "h": 7 }, "targets": [ { "expr": "sum(manticore_disk_bytes{job=\"$job\",instance=~\"$instance\",table=~\"$table\"})", "legendFormat": "total", "refId": "A" }, { "expr": "topk(10, manticore_disk_bytes{job=\"$job\",instance=~\"$instance\",table=~\"$table\"})", "legendFormat": "{{table}}", "refId": "B" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Top 10 tables by total disk_bytes." }, { "id": 12, "type": "timeseries", "title": "Tables / Disk Mapped Cache Ratio (Worst 10)", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 12, "y": 74, "w": 12, "h": 7 }, "targets": [ { "expr": "min(manticore_disk_mapped_cached_ratio_percent{job=\"$job\",instance=~\"$instance\",table=~\"$table\"})", "legendFormat": "worst", "refId": "A" }, { "expr": "bottomk(10, manticore_disk_mapped_cached_ratio_percent{job=\"$job\",instance=~\"$instance\",table=~\"$table\"})", "legendFormat": "{{table}}", "refId": "B" } ], "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 2 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "description": "Worst 10 tables by disk_mapped_cached/disk_mapped ratio (lower is worse)." }, { "id": 30, "type": "timeseries", "title": "Tables / RAM Bytes (Top 10)", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 12, "y": 66, "w": 12, "h": 8 }, "targets": [ { "expr": "sum(manticore_ram_bytes{job=\"$job\",instance=~\"$instance\",table=~\"$table\"})", "legendFormat": "total", "refId": "A" }, { "expr": "topk(10, manticore_ram_bytes{job=\"$job\",instance=~\"$instance\",table=~\"$table\"})", "legendFormat": "{{table}}", "refId": "B" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 31, "type": "timeseries", "title": "Storage / Index & Binlog Size", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 12, "y": 30, "w": 12, "h": 7 }, "targets": [ { "expr": "manticore_index_files_bytes{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "index", "refId": "A" }, { "expr": "manticore_binlog_files_bytes{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "binlog", "refId": "B" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 13, "type": "timeseries", "title": "Performance / Slowest Thread", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 18, "y": 10, "w": 6, "h": 7 }, "targets": [ { "expr": "manticore_slowest_thread_seconds{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "seconds", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 3 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Longest currently running query time (seconds)." }, { "id": 14, "type": "timeseries", "title": "Performance / Connections by Type", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 30, "w": 12, "h": 7 }, "targets": [ { "expr": "manticore_current_connections_count{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "current", "refId": "A" }, { "expr": "manticore_workers_clients_buddy_count{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "buddy", "refId": "B" }, { "expr": "manticore_workers_clients_vip_count{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "vip", "refId": "C" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Connections by type (clients / VIP / Buddy)." }, { "id": 15, "type": "stat", "title": "Resources / FDs (searchd)", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 61, "w": 6, "h": 4 }, "targets": [ { "expr": "max(manticore_searchd_fd_count{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "ManticoreSearch", "refId": "A" } ], "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "blue" }, "unit": "short", "decimals": 0 }, "overrides": [] }, "description": "Open file descriptors in searchd process." }, { "id": 16, "type": "stat", "title": "Active Workers", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 6, "y": 61, "w": 6, "h": 4 }, "targets": [ { "expr": "max(manticore_workers_active_count{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "ManticoreSearch", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] }, "description": "Active worker threads (SHOW STATUS: workers_active)." }, { "id": 17, "type": "stat", "title": "Tables Count", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 12, "y": 61, "w": 6, "h": 4 }, "targets": [ { "expr": "max(manticore_tables_count{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "ManticoreSearch", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] }, "description": "Total number of tables known to searchd." }, { "id": 18, "type": "stat", "title": "Non-served Tables", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 18, "y": 61, "w": 6, "h": 4 }, "targets": [ { "expr": "max(manticore_non_served_tables_count{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "ManticoreSearch", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] }, "description": "Tables that are present but not served." }, { "id": 20, "type": "timeseries", "title": "RSS Bytes (Total)", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 57, "w": 24, "h": 4 }, "targets": [ { "expr": "manticore_searchd_rss_bytes{job=\"$job\",instance=~\"$instance\"} + manticore_buddy_rss_bytes{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "ManticoreSearch", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Process RSS (bytes) for searchd/buddy (if exported)." }, { "id": 22, "type": "stat", "title": "Cluster / Status", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 99, "w": 12, "h": 4 }, "targets": [ { "expr": "max by (cluster,value) (manticore_cluster_string{job=\"$job\",instance=~\"$instance\",cluster=~\"$cluster\", key=\"status\"})", "legendFormat": "{{cluster}} {{value}}", "refId": "A" } ], "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "gray" }, "unit": "short", "decimals": 0 }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "name" }, "description": "Cluster status (primary/non-primary) derived from SHOW STATUS cluster_* counters." }, { "id": 23, "type": "stat", "title": "Cluster / Node State", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 12, "y": 99, "w": 12, "h": 4 }, "targets": [ { "expr": "max by (cluster,value) (manticore_cluster_string{job=\"$job\",instance=~\"$instance\",cluster=~\"$cluster\", key=\"node_state\"})", "legendFormat": "{{cluster}} {{value}}", "refId": "A" } ], "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "gray" }, "unit": "short", "decimals": 0 }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "textMode": "name" }, "description": "Cluster node state (synced/joiner/etc)." }, { "id": 32, "type": "row", "title": "Tables", "gridPos": { "x": 0, "y": 65, "w": 24, "h": 1 }, "collapsed": false, "panels": [] }, { "id": 33, "type": "bargauge", "title": "Docs by Table", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 66, "w": 12, "h": 8 }, "targets": [ { "expr": "manticore_indexed_documents_count{job=\"$job\",instance=~\"$instance\",table=~\"$table\"}", "legendFormat": "{{table}}", "refId": "A", "instant": true } ], "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "color": { "mode": "palette-classic" } }, "overrides": [] }, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "showUnfilled": true }, "description": "Indexed documents per table (top by docs)." }, { "id": 36, "type": "stat", "title": "Load / Queue pressure", "description": "Backlog ratio: max(work_queue_length - workers_total, 0) / workers_total. 0% means no backlog.", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 16, "y": 1, "w": 4, "h": 4 }, "targets": [ { "expr": "(clamp_min(max(manticore_work_queue_length_count{job=\"$job\",instance=~\"$instance\"}) - max(manticore_workers_total_count{job=\"$job\",instance=~\"$instance\"}), 0) / clamp_min(max(manticore_workers_total_count{job=\"$job\",instance=~\"$instance\"}), 1)) * on() (min(up{job=\"$job\",instance=~\"$instance\"}) == 1)", "legendFormat": "queue/workers_total", "refId": "A", "instant": true } ], "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 0, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "#56A64B", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "last" ], "values": false } }, "transformations": [] }, { "id": 52, "type": "gauge", "title": "System Score", "description": "Composite health score (0-100): 100 - queue_backlog_ratio*20 - (workers_utilization>0.9)*10 - (maxed_out>0)*30 - (crashes>0)*40.", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 20, "y": 1, "w": 4, "h": 4 }, "targets": [ { "expr": "clamp_min(clamp_max(100 - ((clamp_min(max(manticore_work_queue_length_count{job=\"$job\",instance=~\"$instance\"}) - max(manticore_workers_total_count{job=\"$job\",instance=~\"$instance\"}), 0) / clamp_min(max(manticore_workers_total_count{job=\"$job\",instance=~\"$instance\"}), 1)) * 20) - ((max(manticore_workers_active_count{job=\"$job\",instance=~\"$instance\"} / clamp_min(manticore_workers_total_count{job=\"$job\",instance=~\"$instance\"}, 1)) > bool 0.9) * 10) - (((sum(increase(manticore_maxed_out_error_count{job=\"$job\",instance=~\"$instance\"}[$__range])) or vector(0)) > bool 0) * 30) - (((sum(increase(manticore_searchd_crashes_total{job=\"$job\",instance=~\"$instance\"}[$__range])) or vector(0)) > bool 0) * 40), 100), 0) * on() min(up{job=\"$job\",instance=~\"$instance\"})", "refId": "A", "instant": true } ], "fieldConfig": { "defaults": { "min": 0, "max": 100, "decimals": 0, "color": { "mode": "thresholds" }, "mappings": [ { "type": "range", "options": { "from": 90, "to": 100, "result": { "text": "Healthy" } } }, { "type": "range", "options": { "from": 70, "to": 89.999, "result": { "text": "Attention" } } }, { "type": "range", "options": { "from": 0, "to": 69.999, "result": { "text": "Degraded" } } } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 70 }, { "color": "#56A64B", "value": 90 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "last" ], "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true } }, { "id": 37, "type": "table", "title": "Tables / Health", "description": "Per-table instant snapshot (aggregated across instance): docs, RAM, disk, rt_mem_limit_rate%, locked, optimizing.", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 81, "w": 24, "h": 10 }, "targets": [ { "refId": "A", "expr": "sum by (table) (manticore_indexed_documents_count{job=\"$job\",instance=~\"$instance\",table=~\"$table\"})", "instant": true, "format": "table" }, { "refId": "B", "expr": "sum by (table) (manticore_ram_bytes{job=\"$job\",instance=~\"$instance\",table=~\"$table\"})", "instant": true, "format": "table" }, { "refId": "C", "expr": "sum by (table) (manticore_disk_bytes{job=\"$job\",instance=~\"$instance\",table=~\"$table\"})", "instant": true, "format": "table" }, { "refId": "D", "expr": "max by (table) (manticore_mem_limit_rate_percent{job=\"$job\",instance=~\"$instance\",table=~\"$table\"})", "instant": true, "format": "table" }, { "refId": "E", "expr": "max by (table) (manticore_locked{job=\"$job\",instance=~\"$instance\",table=~\"$table\"})", "instant": true, "format": "table" }, { "refId": "F", "expr": "max by (table) (manticore_optimizing{job=\"$job\",instance=~\"$instance\",table=~\"$table\"})", "instant": true, "format": "table" }, { "refId": "G", "expr": "100 * sum by (table) (manticore_ram_chunk_size{job=\"$job\",instance=~\"$instance\",table=~\"$table\"}) / clamp_min(max by (table) (manticore_mem_limit_bytes{job=\"$job\",instance=~\"$instance\",table=~\"$table\"}) * max by (table) (manticore_mem_limit_rate_percent{job=\"$job\",instance=~\"$instance\",table=~\"$table\"}) / 100, 1)", "instant": true, "format": "table" } ], "transformations": [ { "id": "labelsToFields", "options": { "mode": "columns" } }, { "id": "joinByField", "options": { "byField": "table", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": { "__name__": true, "__name__ 1": true, "__name__ 2": true, "__name__ 3": true, "__name__ 4": true, "__name__ 5": true, "Time": true, "Time 1": true, "Time 2": true, "Time 3": true, "Time 4": true, "Time 5": true, "job": true, "job 1": true, "job 2": true, "job 3": true, "job 4": true, "job 5": true, "instance": true, "instance 1": true, "instance 2": true, "instance 3": true, "instance 4": true, "instance 5": true, "table 1": true, "table 2": true, "table 3": true, "table 4": true, "table 5": true }, "renameByName": { "Value": "docs", "Value #A": "docs", "Value 1": "ram_bytes", "Value #B": "ram_bytes", "Value 2": "disk_bytes", "Value #C": "disk_bytes", "Value 3": "flush_threshold_percent", "Value #D": "flush_threshold_percent", "Value 4": "locked", "Value #E": "locked", "Value 5": "optimizing", "Value #F": "optimizing", "Value 6": "flush_threshold_fill_percent", "Value #G": "flush_threshold_fill_percent" }, "indexByName": { "table": 0, "docs": 1, "ram_bytes": 2, "disk_bytes": 3, "flush_threshold_percent": 4, "flush_threshold_fill_percent": 5, "locked": 6, "optimizing": 7 } } } ], "options": { "showHeader": true, "sortBy": [ { "displayName": "flush_threshold_percent", "desc": true } ] }, "fieldConfig": { "defaults": { "decimals": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "table" }, "properties": [ { "id": "custom.width", "value": 220 }, { "id": "links", "value": [ { "title": "Filter to table", "url": "?var-table=${__value.raw}", "targetBlank": false } ] } ] }, { "matcher": { "id": "byName", "options": "ram_bytes" }, "properties": [ { "id": "unit", "value": "bytes" }, { "id": "decimals", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "docs" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "disk_bytes" }, "properties": [ { "id": "unit", "value": "bytes" }, { "id": "decimals", "value": 0 } ] }, { "matcher": { "id": "byName", "options": "flush_threshold_percent" }, "properties": [ { "id": "unit", "value": "percent" }, { "id": "decimals", "value": 1 }, { "id": "color", "value": { "mode": "thresholds" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "orange", "value": 85 }, { "color": "red", "value": 95 } ] } } ] }, { "matcher": { "id": "byName", "options": "flush_threshold_fill_percent" }, "properties": [ { "id": "unit", "value": "percent" }, { "id": "decimals", "value": 1 }, { "id": "color", "value": { "mode": "thresholds" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "orange", "value": 85 }, { "color": "red", "value": 95 } ] } } ] }, { "matcher": { "id": "byName", "options": "locked" }, "properties": [ { "id": "mappings", "value": [ { "type": "value", "options": { "0": { "text": "no" }, "1": { "text": "LOCKED" } } } ] }, { "id": "color", "value": { "mode": "thresholds" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } } ] }, { "matcher": { "id": "byName", "options": "optimizing" }, "properties": [ { "id": "mappings", "value": [ { "type": "value", "options": { "0": { "text": "no" }, "1": { "text": "yes" } } } ] }, { "id": "color", "value": { "mode": "thresholds" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "gray", "value": null }, { "color": "yellow", "value": 1 } ] } } ] } ] } }, { "id": 38, "type": "stat", "title": "Health / Maxed out (range)", "description": "Count of maxed_out events over the selected range (server overload/rejections). Not SQL syntax errors.", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 8, "y": 1, "w": 4, "h": 4 }, "targets": [ { "expr": "increase(manticore_maxed_out_error_count{job=\"$job\",instance=~\"$instance\"}[$__range]) * on() (min(up{job=\"$job\",instance=~\"$instance\"}) == 1)", "refId": "A", "instant": true } ], "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "#56A64B", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "last" ], "values": false } } }, { "id": 39, "type": "bargauge", "title": "Workers Utilization %", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 12, "y": 1, "w": 4, "h": 4 }, "targets": [ { "expr": "(100 * max(manticore_workers_active_count{job=\"$job\",instance=~\"$instance\"}) / max(manticore_workers_total_count{job=\"$job\",instance=~\"$instance\"})) * on() (min(up{job=\"$job\",instance=~\"$instance\"}) == 1)", "refId": "A", "instant": true } ], "fieldConfig": { "defaults": { "unit": "percent", "decimals": 0, "color": { "mode": "palette-classic" }, "min": 0, "max": 100, "thresholds": { "mode": "absolute", "steps": [ { "color": "#56A64B", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 85 } ] } }, "overrides": [] }, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "values": false }, "showUnfilled": true }, "description": "100 * workers_active / workers_total." }, { "id": 40, "type": "timeseries", "title": "Performance / QPS Total", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 10, "w": 6, "h": 7 }, "targets": [ { "expr": "sum(rate(manticore_queries_count{job=\"$job\",instance=~\"$instance\"}[1m])) or vector(0)", "legendFormat": "qps", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "ops", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never", "spanNulls": true }, "decimals": 2 }, "overrides": [ { "matcher": { "id": "byName", "options": "qps" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] } ] }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Total QPS: rate(manticore_queries_count[1m]) (ops/s)." }, { "id": 41, "type": "row", "title": "Memory", "gridPos": { "x": 0, "y": 44, "w": 24, "h": 1 }, "collapsed": false, "panels": [] }, { "id": 42, "type": "timeseries", "title": "Memory / Searchd RSS", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 45, "w": 8, "h": 6 }, "targets": [ { "expr": "manticore_searchd_rss_bytes{job=\"$job\",instance=~\"$instance\"}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0, "color": { "mode": "fixed", "fixedColor": "blue" } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "searchd resident set size (bytes)." }, { "id": 43, "type": "timeseries", "title": "Memory / Buddy RSS", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 8, "y": 45, "w": 8, "h": 6 }, "targets": [ { "expr": "manticore_buddy_rss_bytes{job=\"$job\",instance=~\"$instance\"}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0, "color": { "mode": "fixed", "fixedColor": "purple" } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "buddy resident set size (bytes)." }, { "id": 44, "type": "timeseries", "title": "Memory / Searchd Virtual Memory", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 16, "y": 45, "w": 8, "h": 6 }, "targets": [ { "expr": "manticore_searchd_virt_bytes{job=\"$job\",instance=~\"$instance\"}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0, "color": { "mode": "fixed", "fixedColor": "gray" } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "searchd virtual memory size (bytes)." }, { "id": 53, "type": "timeseries", "title": "Memory / Searchd Anon RSS", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 51, "w": 8, "h": 6 }, "targets": [ { "expr": "manticore_searchd_anon_rss_bytes{job=\"$job\",instance=~\"$instance\"}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0, "color": { "mode": "fixed", "fixedColor": "dark-blue" } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "searchd anonymous RSS in bytes (heap-only, excludes file-backed pages)." }, { "id": 54, "type": "timeseries", "title": "Memory / Buddy Anon RSS", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 8, "y": 51, "w": 8, "h": 6 }, "targets": [ { "expr": "manticore_buddy_anon_rss_bytes{job=\"$job\",instance=~\"$instance\"}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0, "color": { "mode": "fixed", "fixedColor": "dark-purple" } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "buddy anonymous RSS in bytes (heap-only, excludes file-backed pages)." }, { "id": 55, "type": "timeseries", "title": "Memory / Anon RSS (Total)", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 16, "y": 51, "w": 8, "h": 6 }, "targets": [ { "expr": "manticore_searchd_anon_rss_bytes{job=\"$job\",instance=~\"$instance\"} + manticore_buddy_anon_rss_bytes{job=\"$job\",instance=~\"$instance\"}", "legendFormat": "ManticoreSearch", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0, "color": { "mode": "fixed", "fixedColor": "orange" } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Total anonymous RSS (searchd + buddy) in bytes." }, { "id": 45, "type": "timeseries", "title": "Load / Worker Saturation (History)", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 37, "w": 24, "h": 6 }, "targets": [ { "expr": "max(manticore_workers_active_count{job=\"$job\",instance=~\"$instance\"}) / clamp_min(max(manticore_workers_total_count{job=\"$job\",instance=~\"$instance\"}), 1)", "legendFormat": "saturation", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "percentunit", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never", "spanNulls": true }, "decimals": 0, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.6 }, { "color": "red", "value": 0.85 } ] } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "workers_active / workers_total over time (spikes indicate saturation)." }, { "id": 48, "type": "text", "title": "", "transparent": true, "gridPos": { "x": 0, "y": 0, "w": 24, "h": 1 }, "options": { "mode": "html", "content": "
\n
\n \"Manticore\"\n Manticore Search\n
\n
\n Slack\n Forum\n Support\n
\n
" } }, { "id": 49, "type": "row", "title": "Load", "collapsed": false, "gridPos": { "x": 0, "y": 9, "w": 24, "h": 1 }, "panels": [] }, { "id": 50, "type": "row", "title": "Cluster", "collapsed": false, "gridPos": { "x": 0, "y": 98, "w": 24, "h": 1 }, "panels": [] }, { "id": 51, "type": "row", "title": "History (collapsed)", "collapsed": true, "gridPos": { "x": 0, "y": 103, "w": 24, "h": 1 }, "panels": [ { "id": 29, "type": "timeseries", "title": "Overview / Version Changes", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 104, "w": 24, "h": 6 }, "targets": [ { "refId": "A", "expr": "max by (version) (manticore_version{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "{{version}}", "instant": false } ], "transformations": [], "fieldConfig": { "defaults": { "unit": "short", "min": 0, "max": 1.2, "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineWidth": 4, "fillOpacity": 40, "showPoints": "never", "spanNulls": false } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Version label changes over time (useful to spot upgrades)." }, { "id": 21, "type": "timeseries", "title": "Overview / Schema Hash", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 110, "w": 24, "h": 4 }, "targets": [ { "expr": "max by (hash8) (label_replace(manticore_schema_hash{job=\"$job\",instance=~\"$instance\"}, \"hash8\", \"$1\", \"hash\", \"^(.{8}).*\"))", "legendFormat": "{{hash8}}", "refId": "A", "instant": false } ], "transformations": [], "fieldConfig": { "defaults": { "unit": "short", "min": 0, "max": 1.2, "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineWidth": 4, "fillOpacity": 40, "showPoints": "never", "spanNulls": false } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Schema hash changes over time (useful to spot ALTER/DDL deployments)." }, { "id": 24, "type": "timeseries", "title": "Cluster / Size (Active Nodes)", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 121, "w": 24, "h": 7 }, "targets": [ { "expr": "manticore_cluster_numeric{job=\"$job\",instance=~\"$instance\",cluster=~\"$cluster\", key=\"size\"}", "legendFormat": "{{cluster}}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Active nodes in cluster (cluster size)." }, { "id": 27, "type": "timeseries", "title": "Load / Workers (History)", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 114, "w": 12, "h": 7 }, "targets": [ { "expr": "max by (instance) (manticore_workers_active_count{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "active", "refId": "A" }, { "expr": "max by (instance) (manticore_workers_total_count{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "total", "refId": "B" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Workers active/total history." }, { "id": 28, "type": "timeseries", "title": "Health / Uptime (History)", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 12, "y": 114, "w": 12, "h": 7 }, "targets": [ { "expr": "max by (instance) (manticore_uptime_seconds{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "ManticoreSearch", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true }, "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "Uptime history." }, { "id": 19, "type": "stat", "title": "Health / Crashes (range)", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 0, "y": 128, "w": 8, "h": 4 }, "targets": [ { "expr": "sum(increase(manticore_searchd_crashes_total{job=\"$job\",instance=~\"$instance\"}[$__range])) or vector(0)", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "#56A64B", "value": null }, { "color": "red", "value": 0.5 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false } }, "description": "searchd crashes in selected range (parsed from searchd.log)." }, { "id": 25, "type": "stat", "title": "Health / Restarts (range)", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 8, "y": 128, "w": 8, "h": 4 }, "targets": [ { "expr": "sum(resets(manticore_uptime_seconds{job=\"$job\",instance=~\"$instance\"}[$__range]))", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false } }, "description": "Restarts in selected range (based on uptime resets)." }, { "id": 35, "type": "stat", "title": "Load / Workers active/total", "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "x": 16, "y": 128, "w": 8, "h": 4 }, "targets": [ { "expr": "max(manticore_workers_active_count{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "active", "refId": "A" }, { "expr": "max(manticore_workers_total_count{job=\"$job\",instance=~\"$instance\"})", "legendFormat": "total", "refId": "B" } ], "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "values": false }, "orientation": "horizontal", "textMode": "value_and_name" }, "description": "Current active vs total workers (instant)." } ] } ] }