{ "title": "[git] WarpStream Agent", "description": "[[suggested_dashboards]]", "layout_type": "ordered", "template_variables": [ { "name": "env", "prefix": "env", "available_values": [], "default": "*" }, { "name": "service", "prefix": "service", "available_values": [], "default": "*" } ], "widgets": [ { "id": 6270738485521706, "definition": { "title": "Overview", "background_color": "vivid_green", "show_title": true, "type": "group", "layout_type": "ordered", "widgets": [ { "id": 609750671110050, "definition": { "title": "Used Cores By Host", "title_size": "16", "title_align": "left", "show_legend": false, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:container.cpu.usage{$env ,$service} by {image_name,image_tag,host}.fill(0)" } ], "response_format": "timeseries", "style": { "palette": "cool" }, "display_type": "line" } ], "markers": [], "custom_links": [] }, "layout": { "x": 0, "y": 0, "width": 4, "height": 2 } }, { "id": 8174721858613184, "definition": { "title": "Used Cores", "title_size": "16", "title_align": "left", "show_legend": false, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:container.cpu.usage{$env ,$service} by {image_name,image_tag}.fill(0)" } ], "response_format": "timeseries", "style": { "palette": "cool" }, "display_type": "line" } ], "markers": [], "custom_links": [] }, "layout": { "x": 4, "y": 0, "width": 4, "height": 2 } }, { "id": 3739619395586102, "definition": { "title": "Write Throughput", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "autosmooth(query1)" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_segment_batcher_flush_file_size_uncompressed_bytes{$env ,$service}.as_rate()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 8, "y": 0, "width": 4, "height": 2 } }, { "id": 2143763458615196, "definition": { "title": "Error Logs by Service", "title_size": "16", "title_align": "left", "show_legend": false, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "a" } ], "queries": [ { "compute": { "aggregation": "count" }, "data_source": "logs", "group_by": [ { "facet": "service", "limit": 10, "sort": { "aggregation": "count", "order": "desc" } } ], "indexes": [ "*" ], "name": "a", "search": { "query": "$env $service status:error" }, "storage": "hot" } ], "response_format": "timeseries", "style": { "palette": "warm" }, "display_type": "bars" } ] }, "layout": { "x": 0, "y": 2, "width": 4, "height": 2 } }, { "id": 6951111177778210, "definition": { "title": "Memory usage by host", "title_size": "16", "title_align": "left", "show_legend": false, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:container.memory.usage{$env ,$service} by {image_name,image_tag,host}.fill(0)" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "custom_links": [] }, "layout": { "x": 4, "y": 2, "width": 4, "height": 2 } } ] }, "layout": { "x": 0, "y": 0, "width": 12, "height": 5 } }, { "id": 134054655021036, "definition": { "title": "Kafka", "background_color": "vivid_orange", "show_title": true, "type": "group", "layout_type": "ordered", "widgets": [ { "id": 262345380730676, "definition": { "title": "Kafka Inflight Connections", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "horizontal", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_kafka_inflight_connections{$env,$service} by {availability-zone}.weighted()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 0, "width": 4, "height": 2 } }, { "id": 3677101073102824, "definition": { "title": "Kafka Inflight Requests (per Connection)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "horizontal", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "max:warpstream_agent_kafka_inflight_request_per_connection{$env,$service}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 4, "y": 0, "width": 4, "height": 2 } }, { "id": 2209966198844790, "definition": { "title": "Kafka Requests by AZ", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_kafka_request_outcome{$env,$service} by {availability-zone}.as_rate()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 8, "y": 0, "width": 4, "height": 2 } }, { "id": 5276542461782028, "definition": { "title": "Kafka Requests", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_kafka_request_outcome{$env,$service} by {kafka_key,outcome}.as_rate()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 2, "width": 4, "height": 2 } }, { "id": 3324485385515046, "definition": { "title": "Kafka Requests by Host", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_kafka_request_outcome{$env,$service} by {host}.as_rate()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 4, "y": 2, "width": 4, "height": 2 } }, { "id": 8394554529723394, "definition": { "title": "Kafka Latency", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:warpstream_agent_kafka_request_latency{$env,$service,outcome:success} by {kafka_key}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 8, "y": 2, "width": 4, "height": 2 } }, { "id": 8565644663707608, "definition": { "title": "Produce by Host", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_kafka_request_outcome{$env,$service,kafka_key:produce} by {host}.as_rate()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 4, "width": 4, "height": 2 } }, { "id": 2361608786455328, "definition": { "title": "Produce Batch Sizes", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "horizontal", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "max:warpstream_agent_segment_batcher_flush_num_records_distribution{$env,$service} by {availability-zone}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 4, "y": 4, "width": 4, "height": 2 } }, { "id": 4832676350517238, "definition": { "title": "Fetch by Host", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_kafka_request_outcome{$env,$service,kafka_key:fetch} by {host}.as_rate()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 8, "y": 4, "width": 4, "height": 2 } }, { "id": 2284839170400594, "definition": { "title": "Fetch Partial Bytes Due to Errors", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_kafka_fetch_partial_response_error_scenario_num_bytes_distribution{$env} by {source}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 6, "width": 4, "height": 2 } }, { "id": 2823467961259624, "definition": { "title": "Fetch Max Pointers in a Single Request", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "max:warpstream_agent_kafka_fetch_num_pointers_distribution{$env}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 4, "y": 6, "width": 4, "height": 2 } }, { "id": 730212706359752, "definition": { "title": "Speculative retries", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" }, { "formula": "query2" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:warpstream_fast_retrier_execution{name:file_cache_blob_store, $env} by {name}.as_count()" }, { "data_source": "metrics", "name": "query2", "query": "avg:warpstream_fast_retrier_speculative_retry{name:file_cache_blob_store, $env} by {name}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 8, "y": 6, "width": 4, "height": 2 } }, { "id": 3942996347374670, "definition": { "title": "Consumer Groups Lag", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:warpstream_consumer_group_lag{service:warp-agent} by {virtual_cluster_id,consumer_group,topic}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 8, "width": 3, "height": 2 } }, { "id": 2096277892106720, "definition": { "title": "Consumer Groups Lag in Seconds", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:warpstream_consumer_group_estimated_lag_very_coarse_do_not_use_to_measure_e2e_seconds{service:warp-agent} by {virtual_cluster_id,consumer_group,topic}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 3, "y": 8, "width": 3, "height": 2 } }, { "id": 2597620413636628, "definition": { "title": "Consumer Groups Lag by Topic-Partition", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:warpstream_consumer_group_lag{service:warp-agent} by {virtual_cluster_id,consumer_group,topic,partition}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 6, "y": 8, "width": 3, "height": 2 } }, { "id": 5042859662452720, "definition": { "title": "Consumer Groups Lag in Seconds by Topic-Partition", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:warpstream_consumer_group_estimated_lag_very_coarse_do_not_use_to_measure_e2e_seconds{service:warp-agent} by {virtual_cluster_id,consumer_group,topic,partition}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 9, "y": 8, "width": 3, "height": 2 } }, { "id": 5321057773853008, "definition": { "title": "Consumer Groups Max Offset", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:warpstream_consumer_group_max_offset{service:warp-agent} by {virtual_cluster_id,consumer_group,topic}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 10, "width": 6, "height": 2 } }, { "id": 8413144592734232, "definition": { "title": "Prefetcher Concurrency", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:warpstream_agent_kafka_fetch_prefetch_concurrency_distribution{$env, $service}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 6, "y": 10, "width": 6, "height": 2 } }, { "id": 2465561256328970, "definition": { "title": "Aggregate Uncompressed Throughput", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "produce", "formula": "query1" }, { "alias": "consume", "formula": "query2" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_kafka_produce_uncompressed_bytes{$env, $service}.as_rate()" }, { "data_source": "metrics", "name": "query2", "query": "sum:warpstream_agent_kafka_fetch_uncompressed_bytes{$env, $service}.as_rate()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 12, "width": 3, "height": 2 } }, { "id": 337749142947136, "definition": { "title": "Aggregate Compressed Throughput", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "produce", "formula": "query1" }, { "alias": "consume", "formula": "query2" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_kafka_produce_compressed_bytes{$env, $service}.as_rate()" }, { "data_source": "metrics", "name": "query2", "query": "sum:warpstream_agent_kafka_fetch_compressed_bytes{$env, $service}.as_rate()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 3, "y": 12, "width": 3, "height": 2 } }, { "id": 717243196545928, "definition": { "title": "Aggregate Uncompressed Throughput by Topic", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "produce", "formula": "query1" }, { "alias": "consume", "formula": "query2" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_kafka_produce_uncompressed_bytes{$env, $service} by {topic}.as_rate()" }, { "data_source": "metrics", "name": "query2", "query": "sum:warpstream_agent_kafka_fetch_uncompressed_bytes{$env, $service} by {topic}.as_rate()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 6, "y": 12, "width": 3, "height": 2 } }, { "id": 7275133891902460, "definition": { "title": "Aggregate Compressed Throughput by Topic", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "produce", "formula": "query1" }, { "alias": "consume", "formula": "query2" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_kafka_produce_compressed_bytes{$env, $service} by {topic}.as_count()" }, { "data_source": "metrics", "name": "query2", "query": "avg:warpstream_agent_kafka_fetch_compressed_bytes{$env, $service} by {topic}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 9, "y": 12, "width": 3, "height": 2 } } ] }, "layout": { "x": 0, "y": 5, "width": 12, "height": 15 } }, { "id": 2362276188489970, "definition": { "title": "Background Jobs", "background_color": "vivid_blue", "show_title": true, "type": "group", "layout_type": "ordered", "widgets": [ { "id": 7561047216000612, "definition": { "title": "Executed Jobs", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_run_and_ack_job_outcome{$env, service:warp-agent} by {job_type}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "semantic", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" } ] }, "layout": { "x": 0, "y": 0, "width": 4, "height": 2 } }, { "id": 5500570105666992, "definition": { "title": "Compactions by Status", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "compute": { "aggregation": "count" }, "data_source": "logs", "group_by": [ { "facet": "status", "limit": 10, "sort": { "aggregation": "count", "order": "desc" } } ], "indexes": [ "*" ], "name": "query1", "search": { "query": "$env service:warp-agent @stream_job_input.type:COMPACTION_JOB_TYPE" }, "storage": "hot" } ], "response_format": "timeseries", "style": { "palette": "semantic", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" } ] }, "layout": { "x": 4, "y": 0, "width": 4, "height": 2 } }, { "id": 6154741133420570, "definition": { "title": "Compactions by Status and Level", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "compute": { "aggregation": "count" }, "data_source": "logs", "group_by": [ { "facet": "status", "limit": 10, "sort": { "aggregation": "count", "order": "desc" } }, { "facet": "@stream_job_input.compaction.compaction_level", "limit": 10, "sort": { "aggregation": "count", "order": "desc" } } ], "indexes": [ "*" ], "name": "query1", "search": { "query": "$env $service service:warp-agent @stream_job_input.type:COMPACTION_JOB_TYPE status:info" }, "storage": "hot" } ], "response_format": "timeseries", "style": { "palette": "semantic", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" } ] }, "layout": { "x": 8, "y": 0, "width": 4, "height": 2 } }, { "id": 526077819969198, "definition": { "title": "P99 Compaction Duration by Level", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "compute": { "aggregation": "pc99", "metric": "@duration_ms" }, "data_source": "logs", "group_by": [ { "facet": "status", "limit": 10, "sort": { "aggregation": "pc99", "metric": "@duration_ms", "order": "desc" } }, { "facet": "@stream_job_input.compaction.compaction_level", "limit": 10, "sort": { "aggregation": "pc99", "metric": "@duration_ms", "order": "desc" } } ], "indexes": [ "*" ], "name": "query1", "search": { "query": "$env service:warp-agent @stream_job_input.type:COMPACTION_JOB_TYPE status:info" }, "storage": "hot" } ], "response_format": "timeseries", "style": { "palette": "semantic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 2, "width": 4, "height": 2 } }, { "id": 3632330150540384, "definition": { "title": "Compaction File Output Size", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "avg", "formula": "query1" }, { "alias": "p99", "formula": "query2" } ], "queries": [ { "compute": { "aggregation": "avg", "metric": "@stream_job_output.compaction.file_metadatas.index_offset" }, "data_source": "logs", "group_by": [ { "facet": "source", "limit": 10, "sort": { "aggregation": "avg", "metric": "@stream_job_output.compaction.file_metadatas.index_offset", "order": "desc" } }, { "facet": "@stream_job_input.compaction.compaction_level", "limit": 10, "sort": { "aggregation": "avg", "metric": "@stream_job_output.compaction.file_metadatas.index_offset", "order": "desc" } } ], "indexes": [ "*" ], "name": "query1", "search": { "query": "$env $service status:info" }, "storage": "hot" }, { "compute": { "aggregation": "pc99", "metric": "@stream_job_output.compaction.file_metadatas.index_offset" }, "data_source": "logs", "group_by": [ { "facet": "source", "limit": 10, "sort": { "aggregation": "pc99", "metric": "@stream_job_output.compaction.file_metadatas.index_offset", "order": "desc" } }, { "facet": "@stream_job_input.compaction.compaction_level", "limit": 10, "sort": { "aggregation": "pc99", "metric": "@stream_job_output.compaction.file_metadatas.index_offset", "order": "desc" } } ], "indexes": [ "*" ], "name": "query2", "search": { "query": "$env $service status:info" }, "storage": "hot" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 4, "y": 2, "width": 4, "height": 2 } }, { "id": 1127037207500336, "definition": { "title": "Compaction Files per Level (Indicator of Compaction Lag)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query2" } ], "queries": [ { "data_source": "metrics", "name": "query2", "query": "sum:warpstream_files_count{*} by {compaction_level}.weighted()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 8, "y": 2, "width": 4, "height": 2 } }, { "id": 2519482004458078, "definition": { "title": "Dead Files Scanner: Created vs. Deleted Files", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "Files Deleted", "formula": "query2" }, { "alias": "Files Created", "formula": "(query1 + query3)" } ], "queries": [ { "data_source": "metrics", "name": "query2", "query": "sum:warpstream_agent_deadscanner_outcomes{$env,outcome:deleted}.as_count()" }, { "compute": { "aggregation": "count" }, "data_source": "logs", "group_by": [], "indexes": [ "*" ], "name": "query1", "search": { "query": "$env flush_end service:warp-agent" }, "storage": "hot" }, { "compute": { "aggregation": "count" }, "data_source": "logs", "group_by": [], "indexes": [ "*" ], "name": "query3", "search": { "query": "$env service:warp-agent @stream_job_input.type:COMPACTION_JOB_TYPE" }, "storage": "hot" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 4, "width": 4, "height": 2 } }, { "id": 570127757104442, "definition": { "title": "Dead Files Scanner: Checked vs Deleted Files", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "scanned", "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_deadscanner_outcomes{$env} by {outcome,service}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 4, "y": 4, "width": 4, "height": 2 } } ] }, "layout": { "x": 0, "y": 20, "width": 12, "height": 7 } }, { "id": 2603950888359238, "definition": { "title": "Object Storage", "background_color": "vivid_purple", "show_title": true, "type": "group", "layout_type": "ordered", "widgets": [ { "id": 6461710332181784, "definition": { "title": "File Cache Bytes Transferred (Server)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_file_cache_server_get_stream_range_num_bytes_count{$env,$service} by {outcome,host}.as_rate()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 0, "width": 4, "height": 2 } }, { "id": 2970131821893814, "definition": { "title": "File Cache Outcomes (Client)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_file_cache_client_get_stream_range_outcome{$env ,$service} by {outcome}.as_rate()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 4, "y": 0, "width": 4, "height": 2 } }, { "id": 1237049947221546, "definition": { "title": "File Cache Latency (Client)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:warpstream_agent_file_cache_client_get_stream_range_latency{$env,$service ,outcome:success} by {outcome}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 8, "y": 0, "width": 4, "height": 2 } }, { "id": 5724945637510878, "definition": { "title": "S3 Operations", "title_size": "16", "title_align": "left", "show_legend": false, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "GETs", "formula": "a + query1" }, { "alias": "PUTs", "formula": "query2 + query3" } ], "queries": [ { "data_source": "metrics", "name": "a", "query": "count:warpstream_blob_store_operation_latency{$env , operation:get_stream}.as_rate()" }, { "data_source": "metrics", "name": "query1", "query": "count:warpstream_blob_store_operation_latency{$env , operation:get_stream_range}.as_count()" }, { "data_source": "metrics", "name": "query2", "query": "count:warpstream_blob_store_operation_latency{$env , operation:put_stream}.as_count()" }, { "data_source": "metrics", "name": "query3", "query": "count:warpstream_blob_store_operation_latency{$env , operation:put_bytes}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 2, "width": 4, "height": 2 } }, { "id": 7957930293773946, "definition": { "title": "Cache Size (Bytes)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:warpstream_agent_file_cache_server_chunk_cache_curr_size_bytes{$env} by {host}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 4, "y": 2, "width": 4, "height": 2 } }, { "id": 4558404686400542, "definition": { "title": "File Cache Latency (Server)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:warpstream_agent_file_cache_server_get_stream_range_latency{$env,$service,outcome:success} by {outcome,host}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 8, "y": 2, "width": 4, "height": 2 } }, { "id": 766048912614776, "definition": { "title": "Direct or Remote Loads", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_file_cache_client_fetch_local_or_remote_counter{$env} by {source}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 4, "width": 4, "height": 2 } }, { "id": 1498936311293404, "definition": { "title": "File Cache Outcomes (Server)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_file_cache_server_get_stream_range_outcome{$env,$service} by {outcome,host}.as_rate()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 4, "y": 4, "width": 4, "height": 2 } }, { "id": 8384823106866512, "definition": { "title": "", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "1 - (query2 / query1)" } ], "queries": [ { "data_source": "metrics", "name": "query2", "query": "sum:warpstream_agent_file_cache_server_chunk_cache_num_misses{$env} by {host}.as_count()" }, { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_file_cache_server_chunk_cache_num_gets{$env} by {host}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 8, "y": 4, "width": 4, "height": 2 } }, { "id": 8189070773422506, "definition": { "title": "File Cache Per Request Bytes Read Average (Server)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "100KiB", "formula": "query1 * 100000" }, { "alias": "3.5MiB", "formula": "query3 * 3500000" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "count(v: v<=100000):warpstream_agent_file_cache_server_get_stream_range_num_bytes_distribution{$env,$service,outcome:success} by {outcome}.as_count()" }, { "data_source": "metrics", "name": "query2", "query": "count(v: v>500000):warpstream_agent_file_cache_server_get_stream_range_num_bytes_distribution{$env,$service,outcome:success} by {outcome}.as_count()" }, { "data_source": "metrics", "name": "query3", "query": "count(v: v>3500000):warpstream_agent_file_cache_server_get_stream_range_num_bytes_distribution{$env,$service,outcome:success} by {outcome}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 6, "width": 4, "height": 2 } }, { "id": 7233389978686754, "definition": { "title": "Cache Size (Entries)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "max:warpstream_agent_file_cache_server_chunk_cache_num_entries{$env} by {host}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 4, "y": 6, "width": 4, "height": 2 } }, { "id": 7707105228318646, "definition": { "title": "Bytes copied into cache", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_file_cache_server_get_range_copy_chunk_num_bytes_copied{$env} by {host}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 8, "y": 6, "width": 4, "height": 2 } }, { "id": 2709556089046480, "definition": { "title": "Fetch Pointers", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_kafka_fetch_num_pointers_counter{$env}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 8, "width": 4, "height": 2 } }, { "id": 2520572757611152, "definition": { "title": "Num Fetches by Size", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_file_cache_server_fetch_size_counter{$env} by {fetch_size}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 4, "y": 8, "width": 4, "height": 2 } }, { "id": 3682163511252190, "definition": { "title": "File Cache Per Request Bytes Read Average (Server)", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "avg:warpstream_agent_file_cache_server_get_stream_range_num_bytes_distribution{$env,$service,outcome:success} by {outcome,host}" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 8, "y": 8, "width": 4, "height": 2 } }, { "id": 5281090860798178, "definition": { "title": "Num Bytes Fetched by Size", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_file_cache_server_fetch_size_num_bytes_counter{$env} by {fetch_size}.as_count()" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 0, "y": 10, "width": 4, "height": 2 } }, { "id": 5093724151797074, "definition": { "title": "Flush File Size in Bytes - Compressed", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "time": {}, "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_segment_batcher_flush_file_size_compressed_bytes{*}.as_count().rollup(count)" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 4, "y": 10, "width": 4, "height": 2 } }, { "id": 4264862291363380, "definition": { "title": "Flush File Size in Bytes - Uncompressed", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "time": {}, "type": "timeseries", "requests": [ { "formulas": [ { "formula": "query1" } ], "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:warpstream_agent_segment_batcher_flush_file_size_uncompressed_bytes{*}.as_count().rollup(count)" } ], "response_format": "timeseries", "style": { "palette": "dog_classic", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ] }, "layout": { "x": 8, "y": 10, "width": 4, "height": 2 } } ] }, "layout": { "x": 0, "y": 27, "width": 12, "height": 13, "is_column_break": true } } ], "notify_list": [], "reflow_type": "fixed" }