{ "description": "Metrics of bookkeeper cluster", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 25, "panels": [], "title": "Healthy", "type": "row" }, { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 3, "x": 0, "y": 1 }, "id": 27, "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "targets": [ { "expr": "count(bookie_SERVER_STATUS{job=~\"$job\"} == 1)\n ", "format": "time_series", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Writable Bookies", "type": "stat" }, { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "0" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 3, "x": 3, "y": 1 }, "id": 31, "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "targets": [ { "expr": "count(bookie_SERVER_STATUS{job=~\"$job\", kubernetes_pod_name=~\"$instance\"} == 0)\n", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "ReadOnly Bookies", "type": "stat" }, { "description": "success rate for different types of requests", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 9, "x": 6, "y": 1 }, "id": 1, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "targets": [ { "editorMode": "code", "expr": "sum(irate(bookkeeper_server_ADD_ENTRY_count{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}[1m]))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "Overall", "range": true, "refId": "A", "step": 2 }, { "editorMode": "code", "expr": "sum(irate(bookkeeper_server_ADD_ENTRY_count{success=\"false\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Failure", "range": true, "refId": "B" }, { "editorMode": "code", "expr": "sum(irate(bookkeeper_server_ADD_ENTRY_count{success=\"true\"}[1m]))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "Success", "range": true, "refId": "C", "step": 2 } ], "title": "Write Rates", "type": "timeseries" }, { "description": "success rate for different types of requests", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 9, "x": 15, "y": 1 }, "id": 61, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "targets": [ { "editorMode": "code", "expr": "sum(irate(bookkeeper_server_READ_ENTRY_count{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}[1m]))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "Overall", "range": true, "refId": "A", "step": 2 }, { "editorMode": "code", "expr": "sum(irate(bookkeeper_server_READ_ENTRY_count{success=\"false\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Failure", "range": true, "refId": "B" }, { "editorMode": "code", "expr": "sum(irate(bookkeeper_server_READ_ENTRY_count{success=\"true\"}[1m]))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "Success", "range": true, "refId": "C", "step": 2 } ], "title": "Read Rates", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#d44a3a", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 80 }, { "color": "#299c46", "value": 90 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 14, "w": 6, "x": 0, "y": 8 }, "id": 33, "maxDataPoints": 100, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": false, "sizing": "auto" }, "targets": [ { "expr": "100 \n* \ncount(bookie_SERVER_STATUS{job=~\"$job\", kubernetes_pod_name=~\"$instance\"} == 1)\n/\ncount(bookie_SERVER_STATUS{job=~\"$job\", kubernetes_pod_name=~\"$instance\"})\n ", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "writable bookies (percentage)", "type": "gauge" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 9, "x": 6, "y": 8 }, "id": 12, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "targets": [ { "expr": "sum(rate(bookie_WRITE_BYTES{job=~\"$job\"}[60s]))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "add entry", "refId": "A", "step": 2 }, { "expr": "sum(rate(bookie_READ_BYTES{job=~\"$job\"}[60s]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "read entry", "refId": "B" } ], "title": "Throughput (MB / s)", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 9, "x": 15, "y": 8 }, "id": 13, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "targets": [ { "expr": "avg(bookkeeper_server_ADD_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.99\"})", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "add (p99)", "refId": "A", "step": 2 }, { "expr": "avg(bookkeeper_server_ADD_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.999\"})", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "add (p999)", "refId": "B" }, { "expr": "avg(bookkeeper_server_ADD_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.5\"})", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "add (p50)", "refId": "C" }, { "expr": "avg(bookkeeper_server_READ_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.99\"})", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "read (p99)", "refId": "D" }, { "expr": "avg(bookkeeper_server_READ_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.999\"})", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "read (p999)", "refId": "E" }, { "expr": "avg(bookkeeper_server_READ_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.5\"})", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "read (p50)", "refId": "F" } ], "title": "Latency", "type": "timeseries" }, { "description": "success rate for different types of requests", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 9, "x": 6, "y": 15 }, "id": 60, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "targets": [ { "editorMode": "code", "expr": "sum(irate(bookkeeper_server_READ_ENTRY_count{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\"}[1m])) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{ kubernetes_pod_name }} read_entry", "range": true, "refId": "A", "step": 2 }, { "editorMode": "code", "expr": "sum(irate(bookkeeper_server_ADD_ENTRY_count{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\"}[1m])) by (kubernetes_pod_name)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ kubernetes_pod_name }} add_entry", "range": true, "refId": "B" } ], "title": "Success Rate By Bookie", "type": "timeseries" }, { "description": "success rate for different types of requests", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 9, "x": 15, "y": 15 }, "id": 59, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "targets": [ { "editorMode": "code", "expr": "sum(irate(bookkeeper_server_ADD_ENTRY_count{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"false\"}[1m])) by (kubernetes_pod_name)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ kubernetes_pod_name }} add_entry", "range": true, "refId": "B" }, { "editorMode": "code", "expr": "sum(irate(bookkeeper_server_READ_ENTRY_count{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"false\"}[1m])) by (kubernetes_pod_name)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{ kubernetes_pod_name }} read_entry", "range": true, "refId": "A" } ], "title": "Failure Rate by Bookie", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "id": 23, "panels": [], "title": "Server", "type": "row" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 23 }, "id": 3, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "targets": [ { "expr": "sum(irate(bookie_WRITE_BYTES{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}[60s])) by (kubernetes_pod_name) ", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{ instance }}", "refId": "A", "step": 2 } ], "title": "Write Bytes", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 23 }, "id": 51, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "targets": [ { "expr": "sum(irate(bookie_READ_BYTES{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}[60s])) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{ instance }}", "refId": "A", "step": 2 } ], "title": "Read Bytes (bytes/s)", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 30 }, "id": 52, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(irate(bookkeeper_server_ADD_ENTRY_count{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}[60s])) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{ instance }}", "refId": "A", "step": 2 } ], "title": "Add Entry Rate", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 30 }, "id": 53, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(irate(bookkeeper_server_READ_ENTRY_count{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}[60s])) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{ instance }}", "refId": "A", "step": 2 } ], "title": "Read Entry Rate", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 37 }, "id": 48, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookkeeper_server_ADD_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.5\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{ instance }}", "refId": "A", "step": 2 } ], "title": "Add entry latency (p50)", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 37 }, "id": 5, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookkeeper_server_ADD_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.99\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A", "step": 2 } ], "title": "Add entry latency (99pct)", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 37 }, "id": 49, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookkeeper_server_ADD_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.999\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{ instance }}", "refId": "A", "step": 2 } ], "title": "Add entry latency (p999)", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 44 }, "id": 54, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookkeeper_server_READ_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.5\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{ instance }}", "refId": "A", "step": 2 } ], "title": "Read entry latency (p50)", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 44 }, "id": 55, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookkeeper_server_READ_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.99\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{ instance }}", "refId": "A", "step": 2 } ], "title": "Read entry latency (p99)", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 44 }, "id": 56, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookkeeper_server_READ_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.999\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{ instance }}", "refId": "A", "step": 2 } ], "title": "Read entry latency (p999)", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, "id": 35, "panels": [], "title": "Journal", "type": "row" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 52 }, "id": 14, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(irate(bookie_journal_JOURNAL_SYNC_count{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\"}[60s])) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A", "step": 4 } ], "title": "Journal Sync Rate", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 52 }, "id": 40, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(irate(bookie_journal_JOURNAL_ADD_ENTRY_count{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\"}[60s])) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A", "step": 4 } ], "title": "Journal Add Rate", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 59 }, "id": 11, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookie_journal_JOURNAL_QUEUE_SIZE{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{kubernetes_pod_name}}", "refId": "A", "step": 2 } ], "title": "Journal Queue length", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 59 }, "id": 36, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookie_journal_JOURNAL_FORCE_WRITE_QUEUE_SIZE{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{kubernetes_pod_name}}", "refId": "A", "step": 2 } ], "title": "Journal Force Write Queue length", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 66 }, "id": 10, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookie_journal_JOURNAL_ADD_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.5\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{kubernetes_pod_name}}", "refId": "A", "step": 2 } ], "title": "Journal Add latency (p50)", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 66 }, "id": 38, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookie_journal_JOURNAL_ADD_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.99\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A", "step": 2 } ], "title": "Journal Add latency (p99)", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 66 }, "id": 39, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookie_journal_JOURNAL_ADD_ENTRY{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"1.0\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{kubernetes_pod_name}}", "refId": "A", "step": 2 } ], "title": "Journal Add latency (Max)", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 73 }, "id": 15, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookie_journal_JOURNAL_SYNC{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.5\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{kubernetes_pod_name}}", "refId": "A", "step": 4 } ], "title": "Journal Sync Latency (p50)", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 73 }, "id": 41, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookie_journal_JOURNAL_SYNC{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"0.99\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{kubernetes_pod_name}}", "refId": "A", "step": 4 } ], "title": "Journal Sync Latency (p99)", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 73 }, "id": 16, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookie_journal_JOURNAL_SYNC{job=~\"$job\", kubernetes_pod_name=~\"$instance\", success=\"true\", quantile=\"1.0\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{kubernetes_pod_name}}", "refId": "A", "step": 4 } ], "title": "Journal Sync Latency (Max)", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 80 }, "id": 43, "panels": [], "title": "Storage", "type": "row" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 81 }, "id": 8, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookie_ledgers_count{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{kubernetes_pod_name}}", "refId": "A", "step": 2 } ], "title": "Ledgers count", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 81 }, "id": 9, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookie_entries_count{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{kubernetes_pod_name}}", "refId": "A", "step": 2 } ], "title": "Entries count", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 88 }, "id": 18, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookie_write_cache_size{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{kubernetes_pod_name}}", "refId": "A", "step": 2 } ], "title": "Write Cache Size", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 88 }, "id": 19, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookie_read_cache_size{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{kubernetes_pod_name}}", "refId": "A", "step": 2 } ], "title": "Read Cache Size", "type": "timeseries" }, { "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 95 }, "id": 64, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "editorMode": "code", "expr": "sum(irate(bookie_read_cache_misses{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}[1m])) by (kubernetes_pod_name) / sum(irate(bookie_read_cache_hits{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}[1m])) by (kubernetes_pod_name)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "B" } ], "title": "Read Cache Miss-to-Hit Ratio", "type": "timeseries" }, { "description": "If >1 indicates read cache thrashing.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 95 }, "id": 65, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "editorMode": "code", "expr": "(sum(rate(bookie_readahead_batch_count_count[1m])) by (kubernetes_pod_name) + sum(rate(bookie_readahead_batch_count_sum[1m])) by (kubernetes_pod_name)) / sum(rate(bookie_read_entry_count[1m])) by (kubernetes_pod_name)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "B" } ], "title": "Entries Read from Disk per Read Op", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 102 }, "id": 45, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "editorMode": "code", "expr": "sum(bookie_DELETED_LEDGER_TOTAL{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{kubernetes_pod_name}}", "range": true, "refId": "A" } ], "title": "Deleted Ledger Count", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 102 }, "id": 46, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "targets": [ { "expr": "sum(bookie_ledger_writable_dirs{job=~\"$job\", kubernetes_pod_name=~\"$instance\"}) by (kubernetes_pod_name)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{kubernetes_pod_name}}", "refId": "A" } ], "title": "Writable Ledger Directory Count", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 111 }, "id": 63, "panels": [], "title": "Internal Queues", "type": "row" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent", "value": null }, { "color": "red", "value": 2400 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 112 }, "id": 66, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "targets": [ { "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "sum by(kubernetes_pod_name, thread) (bookkeeper_server_thread_executor_queue{job=~\"$job\", kubernetes_pod_name=~\"$instance\", thread=~\"BookieHighPriorityThread-.*\"})", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{kubernetes_pod_name}} {{thread}}", "range": true, "refId": "A", "step": 2, "useBackend": false } ], "title": "Write Thread Pool Queue by Thread", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent", "value": null }, { "color": "red", "value": 2400 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 112 }, "id": 58, "options": { "alertThreshold": true, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "targets": [ { "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "sum by(kubernetes_pod_name, thread) (bookkeeper_server_thread_executor_queue{job=~\"$job\", kubernetes_pod_name=~\"$instance\", thread=~\"BookieReadThreadPool-.*\"})", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{kubernetes_pod_name}} {{thread}}", "range": true, "refId": "A", "step": 2, "useBackend": false } ], "title": "Read Thread Pool Queue by Thread", "type": "timeseries" } ], "refresh": "30s", "schemaVersion": 40, "tags": [ "bookkeeper" ], "templating": { "list": [ { "current": { "text": "All", "value": "$__all" }, "definition": "label_values(bookie_SERVER_STATUS,job)", "includeAll": true, "label": "job", "name": "job", "options": [], "query": { "qryType": 1, "query": "label_values(bookie_SERVER_STATUS,job)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", "sort": 1, "type": "query" }, { "current": { "text": "All", "value": "$__all" }, "definition": "label_values(bookie_SERVER_STATUS{job=\"$job\"},kubernetes_pod_name)", "includeAll": true, "label": "Bookie", "multi": true, "name": "instance", "options": [], "query": { "qryType": 1, "query": "label_values(bookie_SERVER_STATUS{job=\"$job\"},kubernetes_pod_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", "sort": 1, "type": "query" } ] }, "time": { "from": "now-15m", "to": "now" }, "timezone": "utc", "title": "Pulsar / BookKeeper", "uid": "qAjftkhlA", "weekStart": "" }