{ "__inputs": [ { "name": "DS_LIGHTNING", "label": "lightning", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "4.6.3" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "singlestat", "name": "Singlestat", "version": "" }, { "type": "panel", "id": "table", "name": "Table", "version": "" }, { "type": "panel", "id": "bargauge", "name": "Bar gauge", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "hideControls": false, "id": null, "links": [], "refresh": false, "rows": [ { "collapse": false, "height": "250px", "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_LIGHTNING}", "fill": 1, "id": 1, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(tikv_import_write_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "write from lightning-importer-{{instance}}", "refId": "A" }, { "expr": "sum(rate(tikv_import_upload_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "upload to tikv-importer-{{instance}}", "refId": "B" }, { "expr": "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"imported\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "write to tikv-lightning-{{instance}}", "refId": "C" }, { "expr": "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"restored\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "read from source-lightning-{{instance}}", "refId": "D" }, { "expr": "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"written\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "write to local-lightning-{{instance}}", "refId": "E" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import speed", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_LIGHTNING}", "fill": 1, "id": 10, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "1/rate(lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"finished\"}[1m]) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Chunk process duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": false, "title": "Import Speed", "titleSize": "h6" }, { "collapse": false, "height": 250, "panels": [ { "datasource": "${DS_LIGHTNING}", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#d44a3a", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0 }, { "color": "#299c46", "value": 0 } ] }, "unit": "percentunit" }, "overrides": [] }, "id": 4, "links": [], "maxDataPoints": 100, "options": { "displayMode": "gradient", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "9.2.4", "targets": [ { "expr": "lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"finished\"} / ignoring(state) lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"estimated\"}", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A" } ], "title": "Import Progress", "type": "bargauge" }, { "datasource": "${DS_LIGHTNING}", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#d44a3a", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 0 }, { "color": "#299c46", "value": 0 } ] }, "unit": "percentunit" }, "overrides": [] }, "id": 12, "links": [], "maxDataPoints": 100, "options": { "displayMode": "gradient", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "9.2.4", "targets": [ { "expr": "lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"completed\"} / ignoring(state) lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"pending\"}", "format": "time_series", "instant": false, "intervalFactor": 1, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Checksum progress", "type": "bargauge" }, { "columns": [ { "text": "Max", "value": "max" } ], "datasource": "${DS_LIGHTNING}", "fontSize": "100%", "id": 8, "links": [], "pageSize": null, "scroll": true, "showHeader": true, "sort": { "col": 0, "desc": true }, "span": 6, "styles": [ { "alias": "Time", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "Time", "type": "date" }, { "alias": "Step", "colorMode": "cell", "colors": [ "rgba(50, 172, 45, 0.97)", "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], "decimals": 0, "pattern": "Metric", "thresholds": [ "1", "2" ], "type": "number", "unit": "none" }, { "alias": "Tables", "colorMode": "cell", "colors": [ "rgba(50, 172, 45, 0.97)", "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 0, "pattern": "Max", "thresholds": [ "0", "0" ], "type": "number", "unit": "none" } ], "targets": [ { "expr": "lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", result=\"failure\"}", "format": "time_series", "instant": false, "intervalFactor": 2, "legendFormat": "{{state}}", "refId": "A" } ], "title": "Failures", "transform": "timeseries_aggregations", "type": "table" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": false, "title": "Import Progress", "titleSize": "h6" }, { "collapse": false, "height": 250, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_LIGHTNING}", "fill": 1, "id": 7, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tikv-importer\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "importer RSS", "refId": "A" }, { "expr": "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb-lightning\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "lightning heap-{{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_LIGHTNING}", "fill": 1, "id": 9, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb-lightning\"}", "format": "time_series", "instant": false, "intervalFactor": 2, "legendFormat": "", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Number of Lightning Goroutines", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_LIGHTNING}", "fill": 1, "id": 3, "legend": { "alignAsTable": false, "avg": true, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb-lightning\"}[30s])*100", "format": "time_series", "intervalFactor": 2, "legendFormat": "Lightning-{{instance}}", "refId": "A" }, { "expr": "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tikv-importer\"}[30s])*100", "format": "time_series", "intervalFactor": 2, "legendFormat": "Importer-{{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU%", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percent", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": false, "title": "Resource usage", "titleSize": "h6" }, { "collapse": false, "height": 250, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_LIGHTNING}", "fill": 1, "id": 5, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 5, "stack": false, "steppedLine": true, "targets": [ { "expr": "lightning_idle_workers{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Idle workers", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_LIGHTNING}", "fill": 1, "id": 6, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 5, "stack": false, "steppedLine": true, "targets": [ { "expr": "lightning_kv_encoder{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"open\"} - ignoring(type) lightning_kv_encoder{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"closed\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "KV Encoder", "refId": "A" }, { "expr": "lightning_importer_engine{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"open\"} - ignoring(type) lightning_importer_engine{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"closed\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Importer Engines (via Lightning)", "refId": "B" }, { "expr": "tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", request=\"open_engine\",result=\"ok\"} - ignoring(request) tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", request=\"close_engine\",result=\"ok\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Importer Engines (via Importer)", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "External resources", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "columns": [ { "text": "Current", "value": "current" } ], "datasource": "${DS_LIGHTNING}", "fontSize": "100%", "id": 21, "links": [], "pageSize": null, "scroll": true, "showHeader": true, "sort": { "col": 0, "desc": true }, "span": 2, "styles": [ { "alias": "TiKV", "pattern": "Metric" }, { "alias": "", "colorMode": "cell", "colors": [ "#E0B400", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "decimals": 2, "link": false, "mappingType": 2, "pattern": "Current", "rangeMaps": [ { "from": "0", "text": "Import", "to": "0" }, { "from": "1", "text": "Normal", "to": "Infinity" } ], "thresholds": [ "1", "1" ], "type": "string", "unit": "short" } ], "targets": [ { "expr": "min(tikv_config_rocksdb{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=\"hard_pending_compaction_bytes_limit\"}) by (instance)", "format": "time_series", "instant": false, "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A" } ], "title": "Import/Normal mode", "transform": "timeseries_aggregations", "type": "table" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": false, "title": "Resource usage", "titleSize": "h6" }, { "collapse": false, "height": 223, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_LIGHTNING}", "fill": 1, "id": 13, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(lightning_chunk_parser_read_block_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(lightning_chunk_parser_read_block_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "read block", "refId": "A" }, { "expr": "rate(lightning_apply_worker_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name = \"io\"}[30s]) /rate(lightning_apply_worker_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name = \"io\"}[30s]) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "apply worker", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Chunk parser read block duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_LIGHTNING}", "fill": 1, "id": 15, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(lightning_row_encode_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(lightning_row_encode_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "row encode", "refId": "A" }, { "expr": "rate(lightning_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(lightning_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "block deliver", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "SQL process duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": false, "title": "Dashboard Row", "titleSize": "h6" }, { "collapse": false, "height": 235, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_LIGHTNING}", "fill": 1, "id": 16, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{kind}} deliver rate", "refId": "B" }, { "expr": "sum(rate(lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "total deliver rate", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "SQL process rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_LIGHTNING}", "fill": 1, "id": 17, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "lightning_row_read_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "parser read size", "refId": "A" }, { "expr": "lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{kind}} deliver size", "refId": "B" }, { "expr": "pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"storage_size\"} / ignoring(type) pd_config_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"max_replicas\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "storage_size / replicas", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total bytes", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": false, "title": "Dashboard Row", "titleSize": "h6" }, { "collapse": false, "height": 243, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_LIGHTNING}", "fill": 1, "id": 18, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(tikv_import_range_delivery_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_range_delivery_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "range deliver-importer-{{instance}}", "refId": "A" }, { "expr": "rate(tikv_import_sst_delivery_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_sst_delivery_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "SST file deliver-importer-{{instance}}", "refId": "B" }, { "expr": "rate(lightning_row_kv_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(lightning_row_kv_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "row deliver-lightning-{{instance}}", "refId": "C" }, { "expr": "rate(lightning_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(lightning_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "block deliver-lightning-{{instance}}", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Deliver duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_LIGHTNING}", "fill": 1, "id": 19, "legend": { "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "SST size", "yaxis": 2 } ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(tikv_import_sst_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "legendFormat": "SST size-importer-{{instance}}", "format": "time_series", "intervalFactor": 2, "refId": "A" }, { "expr": "rate(tikv_import_split_sst_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_split_sst_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "legendFormat": "Split SST-importer-{{instance}}", "format": "time_series", "intervalFactor": 2, "refId": "B" }, { "expr": "rate(tikv_import_sst_upload_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_sst_upload_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "legendFormat": "SST upload-importer-{{instance}}", "format": "time_series", "intervalFactor": 2, "refId": "C" }, { "expr": "rate(tikv_import_sst_ingest_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_sst_ingest_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "legendFormat": "SST ingest-importer-{{instance}}", "format": "time_series", "instant": false, "intervalFactor": 2, "refId": "D" }, { "expr": "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", kind=\"split\"}[30s]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", kind=\"split\"}[30s])\n", "legendFormat": "Region split and scatter-lightning-{{instance}}", "format": "time_series", "instant": false, "intervalFactor": 2, "refId": "E" }, { "expr": "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", kind=\"write\"}[30s]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", kind=\"write\"}[30s])\n", "legendFormat": "SST write-lightning-{{instance}}", "format": "time_series", "instant": false, "intervalFactor": 2, "refId": "F" }, { "expr": "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", kind=\"ingest\"}[30s]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", kind=\"ingest\"}[30s])", "legendFormat": "SST ingest-lightning-{{instance}}", "format": "time_series", "instant": false, "intervalFactor": 2, "refId": "G" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "SST process duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": false, "title": "Dashboard Row", "titleSize": "h6" } ], "schemaVersion": 14, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "current": { }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "K8s-cluster", "multi": false, "name": "k8s_cluster", "options": [ ], "query": "label_values(lightning_chunks, k8s_cluster)", "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "tidb_cluster", "multi": false, "name": "tidb_cluster", "options": [ ], "query": "label_values(lightning_chunks{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "Test-Cluster-Lightning", "uid": "t5PjsKUGz", "version": 4 }