{ "title": "Spice.ai Dashboard", "description": "Monitoring dashboard for Spice.ai runtime - covers system health, query performance, data acceleration, caching, AI/ML operations, and Flight protocol metrics.", "widgets": [ { "id": 1, "definition": { "title": "\ud83c\udfe0 System Overview", "background_color": "vivid_blue", "type": "group", "layout_type": "ordered", "widgets": [ { "id": 102, "definition": { "title": "Active Datasets (by instance)", "title_size": "16", "title_align": "left", "type": "toplist", "requests": [ { "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.dataset_active_count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}", "aggregator": "last" } ], "response_format": "scalar", "formulas": [ { "formula": "query1" } ], "sort": { "count": 10, "order_by": [ { "type": "formula", "index": 0, "order": "asc" } ] } } ], "time": { "live_span": "5m" } }, "layout": { "x": 0, "y": 0, "width": 3, "height": 2 } }, { "id": 103, "definition": { "title": "Active Models (by instance)", "title_size": "16", "title_align": "left", "type": "toplist", "requests": [ { "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.model_active_count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}", "aggregator": "last" } ], "response_format": "scalar", "formulas": [ { "formula": "query1" } ], "sort": { "count": 10, "order_by": [ { "type": "formula", "index": 0, "order": "asc" } ] } } ], "time": { "live_span": "5m" } }, "layout": { "x": 3, "y": 0, "width": 3, "height": 2 } }, { "id": 104, "definition": { "title": "Active Embeddings (by instance)", "title_size": "16", "title_align": "left", "type": "toplist", "requests": [ { "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.embeddings_active_count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}", "aggregator": "last" } ], "response_format": "scalar", "formulas": [ { "formula": "query1" } ], "sort": { "count": 10, "order_by": [ { "type": "formula", "index": 0, "order": "asc" } ] } } ], "time": { "live_span": "5m" } }, "layout": { "x": 6, "y": 0, "width": 3, "height": 2 } }, { "id": 105, "definition": { "title": "Active Tools (by instance)", "title_size": "16", "title_align": "left", "type": "toplist", "requests": [ { "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.tool_active_count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}", "aggregator": "last" } ], "response_format": "scalar", "formulas": [ { "formula": "query1" } ], "sort": { "count": 10, "order_by": [ { "type": "formula", "index": 0, "order": "asc" } ] } } ], "time": { "live_span": "5m" } }, "layout": { "x": 9, "y": 0, "width": 3, "height": 2 } } ] }, "layout": { "x": 0, "y": 0, "width": 12, "height": 3 } }, { "id": 2, "definition": { "title": "\ud83d\udcca Query Performance", "background_color": "vivid_green", "type": "group", "layout_type": "ordered", "widgets": [ { "id": 201, "definition": { "title": "Query Duration p99 by Protocol", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millisecond" } }, "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "p99:spiceai.query_duration_ms{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {protocol,service_instance_id}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear", "label": "", "min": "auto", "max": "auto" } }, "layout": { "x": 0, "y": 0, "width": 6, "height": 3 } }, { "id": 202, "definition": { "title": "Query Throughput", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "sum", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.query_executions.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {protocol}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 6, "y": 0, "width": 6, "height": 3 } }, { "id": 203, "definition": { "title": "Concurrent Active Queries", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.query_active_count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {protocol,service_instance_id}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 0, "y": 3, "width": 4, "height": 3 } }, { "id": 204, "definition": { "title": "Query Failures by Error Code", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "sum", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.query_failures.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {err_code,service_instance_id}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "warm", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 4, "y": 3, "width": 4, "height": 3 } }, { "id": 205, "definition": { "title": "Rows Returned per Query", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.query_returned_rows_summary.quantile{$instance,$percentile,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {protocol}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 8, "y": 3, "width": 4, "height": 3 } } ] }, "layout": { "x": 0, "y": 3, "width": 12, "height": 7 } }, { "id": 3, "definition": { "title": "\ud83d\udcc8 Data Throughput", "background_color": "vivid_purple", "type": "group", "layout_type": "ordered", "widgets": [ { "id": 211, "definition": { "title": "Data Throughput Rate", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "Processed B/s", "formula": "per_second(query1)", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "byte" } } }, { "alias": "Returned B/s", "formula": "per_second(query2)", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "byte" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.query_processed_bytes.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name}.as_count()" }, { "data_source": "metrics", "name": "query2", "query": "sum:spiceai.query_returned_bytes.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "area" } ], "yaxis": { "include_zero": true, "scale": "linear" }, "markers": [] }, "layout": { "x": 0, "y": 0, "width": 6, "height": 3 } }, { "id": 212, "definition": { "title": "Query Spill (Memory Pressure)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "sum", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "Spilled B/s", "formula": "per_second(query1)", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "byte" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.query_spilled_bytes.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "orange", "line_type": "solid", "line_width": "normal" }, "display_type": "area" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 6, "y": 0, "width": 6, "height": 3 } } ] }, "layout": { "x": 0, "y": 10, "width": 12, "height": 4 } }, { "id": 4, "definition": { "title": "\u26a1 Accelerated Datasets", "background_color": "vivid_orange", "type": "group", "layout_type": "ordered", "widgets": [ { "id": 302, "definition": { "title": "Time Since Last Refresh", "title_size": "16", "title_align": "left", "type": "toplist", "requests": [ { "queries": [ { "data_source": "metrics", "name": "query1", "query": "min:spiceai.dataset_acceleration_last_refresh_unix_time_ms{$instance,!dataset:runtime.*,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {dataset,service_instance_id}", "aggregator": "last" } ], "response_format": "scalar", "conditional_formats": [ { "comparator": "<", "value": 60000, "palette": "white_on_green" }, { "comparator": "<", "value": 300000, "palette": "white_on_yellow" }, { "comparator": ">=", "value": 300000, "palette": "white_on_red" } ], "formulas": [ { "formula": "(time() * 1000 - query1) * (query1 / query1)", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millisecond" } } } ], "sort": { "count": 10, "order_by": [ { "type": "formula", "index": 0, "order": "desc" } ] } } ], "time": { "live_span": "5m" } }, "layout": { "x": 0, "y": 0, "width": 4, "height": 3 } }, { "id": 304, "definition": { "title": "Refresh Duration", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millisecond" } }, "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.dataset_acceleration_refresh_duration_ms_summary.quantile{$instance,!dataset:runtime.*,$percentile,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {dataset}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 4, "y": 0, "width": 4, "height": 3 } }, { "id": 305, "definition": { "title": "Refresh Errors", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "sum", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.dataset_acceleration_refresh_errors.count{$instance,!dataset:runtime.*,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {dataset,service_instance_id}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "warm", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 8, "y": 0, "width": 4, "height": 3 } }, { "id": 303, "definition": { "title": "Refresh Staleness", "title_size": "16", "title_align": "left", "time": { "live_span": "5m" }, "type": "toplist", "requests": [ { "queries": [ { "data_source": "metrics", "name": "query1", "query": "max:spiceai.dataset_acceleration_ingestion_lag_ms{$instance,!dataset:runtime.*,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {dataset,service_instance_id}", "aggregator": "last" } ], "response_format": "scalar", "conditional_formats": [ { "comparator": "<", "value": 10000, "palette": "white_on_green" }, { "comparator": "<", "value": 60000, "palette": "white_on_yellow" }, { "comparator": ">=", "value": 60000, "palette": "white_on_red" } ], "formulas": [ { "formula": "query1", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millisecond" } } } ], "sort": { "count": 10, "order_by": [ { "type": "formula", "index": 0, "order": "desc" } ] } } ], "style": {} }, "layout": { "x": 0, "y": 3, "width": 6, "height": 3 } }, { "id": 358266188692798, "definition": { "title": "Refresh Timestamp Advancement", "title_size": "16", "title_align": "left", "time": { "live_span": "5m" }, "type": "toplist", "requests": [ { "queries": [ { "name": "query1", "data_source": "metrics", "query": "max:spiceai.dataset_acceleration_refresh_lag_ms{$instance,!dataset:runtime.*,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {dataset,service_instance_id}", "aggregator": "last" } ], "response_format": "scalar", "formulas": [ { "formula": "query1", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millisecond" } } } ], "sort": { "count": 10, "order_by": [ { "type": "formula", "index": 0, "order": "desc" } ] } } ], "style": { "display": { "type": "stacked" } } }, "layout": { "x": 6, "y": 3, "width": 6, "height": 3 } } ] }, "layout": { "x": 0, "y": 14, "width": 12, "height": 7 } }, { "id": 5, "definition": { "title": "\ud83d\udcbe Results Cache", "background_color": "vivid_pink", "type": "group", "layout_type": "ordered", "widgets": [ { "id": 401, "definition": { "title": "Cache Hit Rate", "title_size": "16", "title_align": "left", "type": "timeseries", "requests": [ { "formulas": [ { "alias": "Hit %", "formula": "(query1 / query2) * 100", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "percent" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.results_cache_hits.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name}.as_count()" }, { "data_source": "metrics", "name": "query2", "query": "sum:spiceai.results_cache_requests.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "green", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "show_legend": true, "legend_layout": "vertical", "legend_columns": [ "avg", "min", "value" ], "yaxis": { "include_zero": true, "scale": "linear", "max": "100" } }, "layout": { "x": 0, "y": 0, "width": 4, "height": 3 } }, { "id": 402, "definition": { "title": "Cache Hits and Misses", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "Hits", "formula": "query1" }, { "alias": "Misses", "style": { "palette": "warm", "palette_index": 4 }, "formula": "query2" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.results_cache_hits.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name}.as_count()" }, { "data_source": "metrics", "name": "query2", "query": "sum:spiceai.results_cache_misses.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "green", "line_type": "solid", "line_width": "normal" }, "display_type": "area" } ], "yaxis": { "include_zero": true, "scale": "linear", "max": "auto" } }, "layout": { "x": 4, "y": 0, "width": 4, "height": 3 } }, { "id": 403, "definition": { "title": "Cache Memory Usage", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "Used", "formula": "query1", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "byte" } } }, { "alias": "Max Capacity", "formula": "query2", "style": { "palette": "gray", "palette_index": 4 }, "number_format": { "unit": { "type": "canonical_unit", "unit_name": "byte" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.results_cache_size_bytes{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}" }, { "data_source": "metrics", "name": "query2", "query": "avg:spiceai.results_cache_max_size_bytes{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 8, "y": 0, "width": 4, "height": 3 } }, { "id": 404, "definition": { "title": "Cached Items Count", "title_size": "16", "title_align": "left", "show_legend": false, "type": "timeseries", "requests": [ { "formulas": [ { "alias": "Items", "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.results_cache_items_count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 0, "y": 3, "width": 4, "height": 2 } }, { "id": 405, "definition": { "title": "Cache Evictions by Reason", "title_size": "16", "title_align": "left", "show_legend": true, "type": "timeseries", "requests": [ { "formulas": [ { "alias": "Over capacity (size)", "formula": "query1", "style": { "palette": "warm", "palette_index": 4 } }, { "alias": "Expired (TTL)", "formula": "query2", "style": { "palette": "gray", "palette_index": 2 } }, { "alias": "Invalidated (refresh/DML)", "formula": "query3", "style": { "palette": "cool", "palette_index": 1 } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.results_cache_evictions.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name,reason:size}.as_count()" }, { "data_source": "metrics", "name": "query2", "query": "sum:spiceai.results_cache_evictions.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name,reason:expired}.as_count()" }, { "data_source": "metrics", "name": "query3", "query": "sum:spiceai.results_cache_evictions.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name,reason:invalidated}.as_count()" } ], "response_format": "timeseries", "display_type": "bars" } ], "yaxis": { "include_zero": true, "scale": "linear" }, "legend_layout": "auto", "legend_columns": [ "sum", "value" ] }, "layout": { "x": 4, "y": 3, "width": 4, "height": 2 } }, { "id": 406, "definition": { "title": "Stale-While-Revalidate Activity", "title_size": "16", "title_align": "left", "show_legend": false, "type": "timeseries", "requests": [ { "formulas": [ { "alias": "SWR Background Queries", "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.results_cache_swr_background_query_count.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "cool", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 8, "y": 3, "width": 4, "height": 2 } } ] }, "layout": { "x": 0, "y": 21, "width": 12, "height": 6 } }, { "id": 6025761671556420, "definition": { "title": "\ud83d\udce6 Acceleration Snapshots", "background_color": "vivid_orange", "show_title": true, "type": "group", "layout_type": "ordered", "widgets": [ { "id": 6999096650525327, "definition": { "title": "Creation Duration", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "response_format": "timeseries", "queries": [ { "name": "query1", "data_source": "metrics", "query": "avg:spiceai.dataset_acceleration_snapshot_write_duration_ms_summary.quantile{$instance,$percentile,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {dataset}" } ], "formulas": [ { "formula": "query1", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millisecond" } } } ], "style": { "palette": "dog_classic", "order_by": "values", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 0, "width": 4, "height": 3 } }, { "id": 1548831823273523, "definition": { "title": "File Size", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "response_format": "timeseries", "queries": [ { "name": "query1", "data_source": "metrics", "query": "avg:spiceai.dataset_acceleration_snapshot_write_bytes{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {dataset}" } ], "formulas": [ { "formula": "query1", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "byte" } } } ], "style": { "palette": "dog_classic", "order_by": "values", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 4, "y": 0, "width": 4, "height": 3 } }, { "id": 783195861224473, "definition": { "title": "Bootstrap Duration", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "response_format": "timeseries", "queries": [ { "name": "query1", "data_source": "metrics", "query": "sum:spiceai.dataset_acceleration_snapshot_bootstrap_duration_ms.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {dataset,service_instance_id}.as_count()" } ], "formulas": [ { "formula": "query1", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millisecond" } } } ], "style": { "palette": "dog_classic", "order_by": "values", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" } ] }, "layout": { "x": 8, "y": 0, "width": 4, "height": 3 } } ] }, "layout": { "x": 0, "y": 27, "width": 12, "height": 4 } }, { "id": 6, "definition": { "title": "\ud83c\udf10 HTTP API", "background_color": "vivid_yellow", "type": "group", "layout_type": "ordered", "widgets": [ { "id": 601, "definition": { "title": "HTTP Request Duration", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millisecond" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "max:spiceai.http_requests_duration_ms_summary.quantile{$instance,$percentile,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {method,path,service_instance_id}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 0, "y": 0, "width": 6, "height": 3 } }, { "id": 602, "definition": { "title": "HTTP Request Count by Status", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "sum", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.http_requests.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {method,path,status}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 6, "y": 0, "width": 6, "height": 3 } } ] }, "layout": { "x": 0, "y": 31, "width": 12, "height": 1 } }, { "id": 7, "definition": { "title": "\u2708\ufe0f Arrow Flight Operations", "background_color": "gray", "type": "group", "layout_type": "ordered", "widgets": [ { "id": 701, "definition": { "title": "do_get Duration", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millisecond" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.flight_request_duration_ms_summary.quantile{$instance,method:do_get,$percentile,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {command}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 0, "y": 0, "width": 4, "height": 3 } }, { "id": 702, "definition": { "title": "do_put Duration", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millisecond" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.flight_request_duration_ms_summary.quantile{$instance,method:do_put,$percentile,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {command}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 4, "y": 0, "width": 4, "height": 3 } }, { "id": 703, "definition": { "title": "get_flight_info Duration", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millisecond" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.flight_request_duration_ms_summary.quantile{$instance,method:get_flight_info,$percentile,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {command}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 8, "y": 0, "width": 4, "height": 3 } }, { "id": 704, "definition": { "title": "Flight Requests by Method", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "sum", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.flight_requests.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {method}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 0, "y": 3, "width": 6, "height": 3 } }, { "id": 705, "definition": { "title": "DoExchange Data Updates", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "sum", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "Data Updates Sent", "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.flight_do_exchange_data_updates_sent.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "cool", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 6, "y": 3, "width": 6, "height": 3 } } ] }, "layout": { "x": 0, "y": 32, "width": 12, "height": 1 } }, { "id": 8, "definition": { "title": "\u23e9 Debezium Ingestion", "background_color": "white", "show_title": true, "type": "group", "layout_type": "ordered", "widgets": [ { "id": 908, "definition": { "title": "Debezium Record Lag", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.dataset_kafka_records_lag{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {name}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 0, "y": 0, "width": 6, "height": 3 } }, { "id": 909, "definition": { "title": "Debezium Records/sec", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "per_second(query1)" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.dataset_kafka_records_consumed_total.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {name}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 6, "y": 0, "width": 6, "height": 3 } } ] }, "layout": { "x": 0, "y": 33, "width": 12, "height": 1 } }, { "id": 9, "definition": { "title": "\u23e9 DynamoDB Streams Ingestion", "background_color": "blue", "show_title": true, "type": "group", "layout_type": "ordered", "widgets": [ { "id": 801, "definition": { "title": "DynamoDB Stream Lag", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.dataset_dynamodb_lag_ms{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {name}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "log" } }, "layout": { "x": 0, "y": 0, "width": 6, "height": 3 } }, { "id": 802, "definition": { "title": "DynamoDB Records/sec", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "per_second(query1)" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.dataset_dynamodb_records_consumed_total.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {name}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 6, "y": 0, "width": 6, "height": 3 } }, { "id": 803, "definition": { "title": "DynamoDB Active Shards", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.dataset_dynamodb_shards_active{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {name}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 0, "y": 3, "width": 6, "height": 3 } }, { "id": 804, "definition": { "title": "DynamoDB Transient Errors/min", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "sum", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "per_minute(query1)" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.dataset_dynamodb_errors_transient_total.count{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {name}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "warm", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 6, "y": 3, "width": 6, "height": 3 } } ] }, "layout": { "x": 0, "y": 34, "width": 12, "height": 1 } }, { "id": 11, "definition": { "title": "\u2699\ufe0f Runtime Resources", "background_color": "vivid_yellow", "show_title": true, "type": "group", "layout_type": "ordered", "widgets": [ { "id": 1001, "definition": { "title": "CPU Cores", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.spiced_cpu_budget_cores{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {source,service_instance_id}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 0, "y": 0, "width": 6, "height": 3 } }, { "id": 1002, "definition": { "title": "CPU Entitlement", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1", "alias": "budget", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millicore" } } }, { "formula": "query2", "alias": "limit", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millicore" } } }, { "formula": "query3", "alias": "request", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millicore" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.spiced_cpu_budget_millicores{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {source,service_instance_id}" }, { "data_source": "metrics", "name": "query2", "query": "avg:spiceai.spiced_cpu_limit_millicores{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}" }, { "data_source": "metrics", "name": "query3", "query": "avg:spiceai.spiced_cpu_request_millicores{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 6, "y": 0, "width": 6, "height": 3 } }, { "id": 1003, "definition": { "title": "Worker Threads", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.tokio_runtime_workers{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {runtime,service_instance_id}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 0, "y": 3, "width": 4, "height": 3 } }, { "id": 1004, "definition": { "title": "Active Tasks", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.tokio_runtime_alive_tasks{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {runtime,service_instance_id}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 4, "y": 3, "width": 4, "height": 3 } }, { "id": 1005, "definition": { "title": "Queued Tasks", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.tokio_runtime_global_queue_depth{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {runtime,service_instance_id}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 8, "y": 3, "width": 4, "height": 3 } }, { "id": 1006, "definition": { "title": "Process Memory", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "byte" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.process_resident_memory_bytes{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 0, "y": 6, "width": 4, "height": 3 } }, { "id": 1007, "definition": { "title": "Query Memory", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "byte" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.query_memory_pool_used_bytes{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 4, "y": 6, "width": 4, "height": 3 } }, { "id": 1008, "definition": { "title": "Compaction Memory", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1", "alias": "used", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "byte" } } }, { "formula": "query2", "alias": "budget", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "byte" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:spiceai.cayenne_compaction_memory_pool_used_bytes{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}" }, { "data_source": "metrics", "name": "query2", "query": "avg:spiceai.cayenne_compaction_memory_pool_bytes{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 8, "y": 6, "width": 4, "height": 3 } }, { "id": 1009, "definition": { "title": "Cayenne Segment Cache Memory", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "used", "formula": "query1", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "byte" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:spiceai.cayenne_segment_cache_weighted_bytes{$instance,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {service_instance_id}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 0, "y": 9, "width": 4, "height": 3 } } ] }, "layout": { "x": 0, "y": 35, "width": 12, "height": 10 } }, { "id": 10, "definition": { "title": "\ud83d\udd27 Kubernetes Resource Utilization", "background_color": "vivid_green", "show_title": true, "type": "group", "layout_type": "ordered", "widgets": [ { "id": 910, "definition": { "title": "CPU Usage (by Instance)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1 / 1000000", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "millicore" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:kubernetes.cpu.usage.total{pod_name:$instance.value,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {pod_name}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 0, "y": 0, "width": 6, "height": 3 } }, { "id": 911, "definition": { "title": "Memory Usage (by Instance)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:kubernetes.memory.working_set{pod_name:$instance.value,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {pod_name}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 6, "y": 0, "width": 6, "height": 3 } }, { "id": 912, "definition": { "title": "PVC Storage Usage", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1 - query2", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "byte" } } } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:kubernetes.kubelet.volume.stats.capacity_bytes{pod_name:$instance.value,!persistentvolumeclaim:kube-api-access-*,!persistentvolumeclaim:default-token-*,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {pod_name,persistentvolumeclaim}" }, { "data_source": "metrics", "name": "query2", "query": "avg:kubernetes.kubelet.volume.stats.available_bytes{pod_name:$instance.value,!persistentvolumeclaim:kube-api-access-*,!persistentvolumeclaim:default-token-*,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {pod_name,persistentvolumeclaim}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 0, "y": 3, "width": 6, "height": 3 } }, { "id": 913, "definition": { "title": "PVC Storage Usage %", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "((query1 - query2) / query1) * 100" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:kubernetes.kubelet.volume.stats.capacity_bytes{pod_name:$instance.value,!persistentvolumeclaim:kube-api-access-*,!persistentvolumeclaim:default-token-*,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {pod_name,persistentvolumeclaim}" }, { "data_source": "metrics", "name": "query2", "query": "avg:kubernetes.kubelet.volume.stats.available_bytes{pod_name:$instance.value,!persistentvolumeclaim:kube-api-access-*,!persistentvolumeclaim:default-token-*,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {pod_name,persistentvolumeclaim}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear", "max": "100" } }, "layout": { "x": 6, "y": 3, "width": 6, "height": 3 } }, { "id": 914, "definition": { "title": "CPU Utilization Percentage (by Pod)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "number_format": { "unit": { "type": "canonical_unit", "unit_name": "percent" } }, "formula": "(query1 / 1000000000) / query2 * 100" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:kubernetes.cpu.usage.total{pod_name:$instance.value,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {pod_name}" }, { "data_source": "metrics", "name": "query2", "query": "sum:kubernetes.cpu.requests{pod_name:$instance.value,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {pod_name}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "markers": [ { "label": " > 80% ", "value": "y > 80", "display_type": "error dashed" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 0, "y": 6, "width": 6, "height": 3 } }, { "id": 915, "definition": { "title": "Memory Limit Utilization Percentage (by Pod)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "max", "value" ], "type": "timeseries", "requests": [ { "formulas": [ { "number_format": { "unit": { "type": "canonical_unit", "unit_name": "percent" } }, "formula": "(query1 / query2) * 100" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:kubernetes.memory.working_set{pod_name:$instance.value,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {pod_name}" }, { "data_source": "metrics", "name": "query2", "query": "sum:kubernetes.memory.limits{pod_name:$instance.value,$env,$kube_namespace,$kube_cluster_name,$pod_name} by {pod_name}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "markers": [ { "label": " > 80% ", "value": "y > 80", "display_type": "error dashed" } ], "yaxis": { "include_zero": true, "scale": "linear" } }, "layout": { "x": 6, "y": 6, "width": 6, "height": 3 } } ] }, "layout": { "x": 0, "y": 45, "width": 12, "height": 1 } } ], "template_variables": [ { "name": "instance", "prefix": "service_instance_id", "default": "*" }, { "name": "percentile", "prefix": "quantile", "available_values": [ "0.9", "0.95", "0.99" ], "default": "0.99" }, { "name": "env", "prefix": "env", "available_values": [], "default": "*" }, { "name": "kube_namespace", "prefix": "kube_namespace", "available_values": [], "default": "*" }, { "name": "kube_cluster_name", "prefix": "kube_cluster_name", "available_values": [], "default": "*" }, { "name": "pod_name", "prefix": "pod_name", "available_values": [], "default": "*" } ], "layout_type": "ordered", "notify_list": [], "pause_auto_refresh": false, "reflow_type": "fixed", "tags": [] }