{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "Prometheus datasource for Modelship metrics",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "annotations": {
    "list": []
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "links": [],
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 100,
      "title": "Overview",
      "type": "row"
    },
    {
      "title": "Request Rate",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 1
      },
      "id": 1,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_modelship_request_total{gateway=~\"$gateway\", model=~\"$model\"}[5m])) by (endpoint)",
          "legendFormat": "{{ endpoint }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        }
      }
    },
    {
      "title": "Error Rate",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 1
      },
      "id": 2,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_modelship_request_errors_total{gateway=~\"$gateway\", model=~\"$model\"}[5m])) by (model, error_type)",
          "legendFormat": "{{ model }} - {{ error_type }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        }
      }
    },
    {
      "title": "In-Flight Requests",
      "type": "timeseries",
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 16,
        "y": 1
      },
      "id": 3,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_modelship_request_in_progress{gateway=~\"$gateway\", model=~\"$model\"}",
          "legendFormat": "{{ model }}"
        }
      ]
    },
    {
      "title": "Models Loaded",
      "type": "stat",
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 20,
        "y": 1
      },
      "id": 4,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_modelship_models_loaded{gateway=~\"$gateway\"}"
        }
      ]
    },
    {
      "title": "Auth Failures",
      "type": "timeseries",
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 16,
        "y": 5
      },
      "id": 6,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_modelship_auth_failures_total{gateway=~\"$gateway\"}[5m])) by (reason)",
          "legendFormat": "{{ reason }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        }
      }
    },
    {
      "title": "Client Disconnects",
      "type": "timeseries",
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 20,
        "y": 5
      },
      "id": 5,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_modelship_client_disconnects_total{gateway=~\"$gateway\", model=~\"$model\"}[5m])) by (model)",
          "legendFormat": "{{ model }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        }
      }
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 9
      },
      "id": 101,
      "title": "Latency",
      "type": "row"
    },
    {
      "title": "Request Latency P50 / P95 / P99",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 10
      },
      "id": 10,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.50, sum(rate(ray_modelship_request_duration_seconds_bucket{gateway=~\"$gateway\", model=~\"$model\"}[5m])) by (le, endpoint))",
          "legendFormat": "p50 {{ endpoint }}"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_modelship_request_duration_seconds_bucket{gateway=~\"$gateway\", model=~\"$model\"}[5m])) by (le, endpoint))",
          "legendFormat": "p95 {{ endpoint }}"
        },
        {
          "expr": "histogram_quantile(0.99, sum(rate(ray_modelship_request_duration_seconds_bucket{gateway=~\"$gateway\", model=~\"$model\"}[5m])) by (le, endpoint))",
          "legendFormat": "p99 {{ endpoint }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      }
    },
    {
      "title": "Per-Model Latency P95",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 10
      },
      "id": 11,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_modelship_request_duration_seconds_bucket{gateway=~\"$gateway\", model=~\"$model\"}[5m])) by (le, model))",
          "legendFormat": "p95 {{ model }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      }
    },
    {
      "title": "Inference Latency by Usecase P95",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 10
      },
      "id": 12,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_modelship_generation_duration_seconds_bucket{gateway=~\"$gateway\", model=~\"$model\"}[5m])) by (le, model))",
          "legendFormat": "generate {{ model }}"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_modelship_tts_generation_duration_seconds_bucket{gateway=~\"$gateway\", model=~\"$model\"}[5m])) by (le, model))",
          "legendFormat": "tts {{ model }}"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_modelship_image_generation_duration_seconds_bucket{gateway=~\"$gateway\", model=~\"$model\"}[5m])) by (le, model))",
          "legendFormat": "image {{ model }}"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_modelship_transcription_duration_seconds_bucket{gateway=~\"$gateway\", model=~\"$model\"}[5m])) by (le, model))",
          "legendFormat": "transcription {{ model }}"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_modelship_embedding_duration_seconds_bucket{gateway=~\"$gateway\", model=~\"$model\"}[5m])) by (le, model))",
          "legendFormat": "embedding {{ model }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      }
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 18
      },
      "id": 102,
      "title": "vLLM Engine",
      "type": "row"
    },
    {
      "title": "KV Cache Usage",
      "type": "gauge",
      "gridPos": {
        "h": 8,
        "w": 4,
        "x": 0,
        "y": 19
      },
      "id": 20,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_vllm_kv_cache_usage_perc",
          "legendFormat": "{{ model_name }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "min": 0,
          "max": 1,
          "thresholds": {
            "steps": [
              {
                "value": 0,
                "color": "green"
              },
              {
                "value": 0.8,
                "color": "yellow"
              },
              {
                "value": 0.95,
                "color": "red"
              }
            ]
          }
        }
      }
    },
    {
      "title": "Time to First Token (TTFT)",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 5,
        "x": 4,
        "y": 19
      },
      "id": 21,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.50, sum(rate(ray_vllm_time_to_first_token_seconds_bucket[5m])) by (le))",
          "legendFormat": "p50"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_vllm_time_to_first_token_seconds_bucket[5m])) by (le))",
          "legendFormat": "p95"
        },
        {
          "expr": "histogram_quantile(0.99, sum(rate(ray_vllm_time_to_first_token_seconds_bucket[5m])) by (le))",
          "legendFormat": "p99"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      }
    },
    {
      "title": "Inter-Token Latency (ITL)",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 5,
        "x": 9,
        "y": 19
      },
      "id": 22,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.50, sum(rate(ray_vllm_inter_token_latency_seconds_bucket[5m])) by (le))",
          "legendFormat": "p50"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_vllm_inter_token_latency_seconds_bucket[5m])) by (le))",
          "legendFormat": "p95"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      }
    },
    {
      "title": "Token Throughput",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 5,
        "x": 14,
        "y": 19
      },
      "id": 23,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_vllm_prompt_tokens_total[5m]))",
          "legendFormat": "prefill tok/s"
        },
        {
          "expr": "sum(rate(ray_vllm_generation_tokens_total[5m]))",
          "legendFormat": "decode tok/s"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "tok/s"
        }
      }
    },
    {
      "title": "Queue Depth",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 5,
        "x": 19,
        "y": 19
      },
      "id": 24,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_vllm_num_requests_running",
          "legendFormat": "running"
        },
        {
          "expr": "ray_vllm_num_requests_waiting",
          "legendFormat": "waiting"
        }
      ]
    },
    {
      "title": "Preemptions",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 0,
        "y": 27
      },
      "id": 25,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_vllm_num_preemptions_total[5m]))",
          "legendFormat": "preemptions/s"
        }
      ]
    },
    {
      "title": "Prefix Cache Hit Rate",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 6,
        "y": 27
      },
      "id": 26,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_vllm_prefix_cache_hits_total[5m])) / clamp_min(sum(rate(ray_vllm_prefix_cache_queries_total[5m])), 1)",
          "legendFormat": "hit rate"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit"
        }
      }
    },
    {
      "title": "Queue Wait Time P95",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 12,
        "y": 27
      },
      "id": 27,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_vllm_request_queue_time_seconds_bucket[5m])) by (le))",
          "legendFormat": "p95"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      }
    },
    {
      "title": "E2E Request Latency P95",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 18,
        "y": 27
      },
      "id": 28,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_vllm_e2e_request_latency_seconds_bucket[5m])) by (le))",
          "legendFormat": "p95"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      }
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 35
      },
      "id": 103,
      "title": "GPU & System Resources",
      "type": "row"
    },
    {
      "title": "GPU Utilization",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 0,
        "y": 36
      },
      "id": 30,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_node_gpus_utilization",
          "legendFormat": "GPU {{ GpuIndex }} ({{ GpuDeviceName }})"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "min": 0,
          "max": 100
        }
      }
    },
    {
      "title": "GPU Memory",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 6,
        "y": 36
      },
      "id": 31,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_node_gram_used",
          "legendFormat": "used GPU {{ GpuIndex }}"
        },
        {
          "expr": "ray_node_gram_available",
          "legendFormat": "available GPU {{ GpuIndex }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "decmbytes"
        }
      }
    },
    {
      "title": "CPU Utilization",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 12,
        "y": 36
      },
      "id": 32,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_node_cpu_utilization",
          "legendFormat": "{{ instance }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "min": 0,
          "max": 100
        }
      }
    },
    {
      "title": "System Memory",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 18,
        "y": 36
      },
      "id": 33,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_node_mem_used / ray_node_mem_total * 100",
          "legendFormat": "used %"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "min": 0,
          "max": 100
        }
      }
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 44
      },
      "id": 104,
      "title": "Ray Serve Internals",
      "type": "row"
    },
    {
      "title": "Health Check Latency P95",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 0,
        "y": 45
      },
      "id": 40,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_serve_health_check_latency_ms_bucket[5m])) by (le, deployment))",
          "legendFormat": "p95 {{ deployment }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ms"
        }
      }
    },
    {
      "title": "Request Count by Deployment",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 6,
        "y": 45
      },
      "id": 41,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_serve_handle_request_counter_total[5m])) by (deployment)",
          "legendFormat": "{{ deployment }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        }
      }
    },
    {
      "title": "Deployment Processing Latency P95",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 12,
        "y": 45
      },
      "id": 42,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_serve_deployment_processing_latency_ms_bucket[5m])) by (le, deployment))",
          "legendFormat": "p95 {{ deployment }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ms"
        }
      }
    },
    {
      "title": "HTTP Request Latency P95",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 18,
        "y": 45
      },
      "id": 43,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_serve_http_request_latency_ms_bucket[5m])) by (le))",
          "legendFormat": "p95"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ms"
        }
      }
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 53
      },
      "id": 105,
      "title": "Operational Health",
      "type": "row"
    },
    {
      "title": "Model Load Time",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 0,
        "y": 54
      },
      "id": 50,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_modelship_model_load_duration_seconds_sum{gateway=~\"$gateway\", model=~\"$model\"} / clamp_min(ray_modelship_model_load_duration_seconds_count{gateway=~\"$gateway\", model=~\"$model\"}, 1)",
          "legendFormat": "avg {{ model }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      }
    },
    {
      "title": "Model Load Failures",
      "type": "stat",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 6,
        "y": 54
      },
      "id": 51,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(ray_modelship_model_load_failures_total{gateway=~\"$gateway\", model=~\"$model\"}) by (model)",
          "legendFormat": "{{ model }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "thresholds": {
            "steps": [
              {
                "value": 0,
                "color": "green"
              },
              {
                "value": 1,
                "color": "red"
              }
            ]
          }
        }
      }
    },
    {
      "title": "Resource Cleanup Errors",
      "type": "stat",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 12,
        "y": 54
      },
      "id": 52,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(ray_modelship_resource_cleanup_errors_total{gateway=~\"$gateway\", model=~\"$model\"}) by (model, component)",
          "legendFormat": "{{ model }} ({{ component }})"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "thresholds": {
            "steps": [
              {
                "value": 0,
                "color": "green"
              },
              {
                "value": 1,
                "color": "red"
              }
            ]
          }
        }
      }
    },
    {
      "title": "Streaming Chunks/s",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 18,
        "y": 54
      },
      "id": 53,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_modelship_stream_chunks_total{gateway=~\"$gateway\", model=~\"$model\"}[5m])) by (model)",
          "legendFormat": "{{ model }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "cps"
        }
      }
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 62
      },
      "id": 106,
      "title": "Alerts",
      "type": "row"
    },
    {
      "title": "Error Rate %",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 63
      },
      "id": 60,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_modelship_request_errors_total{gateway=~\"$gateway\", model=~\"$model\"}[5m])) / clamp_min(sum(rate(ray_modelship_request_total{gateway=~\"$gateway\", model=~\"$model\"}[5m])), 1) * 100",
          "legendFormat": "error %"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": 0,
                "color": "green"
              },
              {
                "value": 2,
                "color": "yellow"
              },
              {
                "value": 5,
                "color": "red"
              }
            ]
          },
          "custom": {
            "thresholdsStyle": {
              "mode": "line+area"
            }
          }
        }
      }
    },
    {
      "title": "KV Cache Usage",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 63
      },
      "id": 61,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_vllm_kv_cache_usage_perc * 100",
          "legendFormat": "{{ model_name }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "min": 0,
          "max": 100,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": 0,
                "color": "green"
              },
              {
                "value": 80,
                "color": "yellow"
              },
              {
                "value": 95,
                "color": "red"
              }
            ]
          },
          "custom": {
            "thresholdsStyle": {
              "mode": "line+area"
            }
          }
        }
      }
    },
    {
      "title": "Queue Depth",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 63
      },
      "id": 62,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_vllm_num_requests_waiting",
          "legendFormat": "waiting"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": 0,
                "color": "green"
              },
              {
                "value": 5,
                "color": "yellow"
              },
              {
                "value": 10,
                "color": "red"
              }
            ]
          },
          "custom": {
            "thresholdsStyle": {
              "mode": "line+area"
            }
          }
        }
      }
    },
    {
      "title": "TTFT P99",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 0,
        "y": 71
      },
      "id": 63,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.99, sum(rate(ray_vllm_time_to_first_token_seconds_bucket[5m])) by (le))",
          "legendFormat": "p99"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": 0,
                "color": "green"
              },
              {
                "value": 3,
                "color": "yellow"
              },
              {
                "value": 5,
                "color": "red"
              }
            ]
          },
          "custom": {
            "thresholdsStyle": {
              "mode": "line+area"
            }
          }
        }
      }
    },
    {
      "title": "Client Disconnects",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 6,
        "y": 71
      },
      "id": 64,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_modelship_client_disconnects_total{gateway=~\"$gateway\", model=~\"$model\"}[5m]))",
          "legendFormat": "disconnects/s"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        }
      }
    },
    {
      "title": "Preemptions",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 12,
        "y": 71
      },
      "id": 65,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_vllm_num_preemptions_total[5m]))",
          "legendFormat": "preemptions/s"
        }
      ]
    },
    {
      "title": "GPU Memory Available",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 18,
        "y": 71
      },
      "id": 66,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_node_gram_available",
          "legendFormat": "GPU {{ GpuIndex }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "decmbytes",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": 0,
                "color": "red"
              },
              {
                "value": 1024,
                "color": "yellow"
              },
              {
                "value": 4096,
                "color": "green"
              }
            ]
          },
          "custom": {
            "thresholdsStyle": {
              "mode": "line+area"
            }
          }
        }
      }
    },
    {
      "type": "row",
      "title": "Cluster / HA",
      "collapsed": false,
      "id": 107,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 79
      },
      "panels": []
    },
    {
      "title": "Deploy Lock Held",
      "type": "stat",
      "id": 108,
      "gridPos": {
        "h": 8,
        "w": 4,
        "x": 0,
        "y": 80
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "max(ray_modelship_deploy_lock_held)",
          "legendFormat": ""
        }
      ],
      "fieldConfig": {
        "defaults": {}
      }
    },
    {
      "title": "Deploy Reservations/s by outcome",
      "type": "timeseries",
      "id": 109,
      "gridPos": {
        "h": 8,
        "w": 10,
        "x": 4,
        "y": 80
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_modelship_deploy_reservations_total[5m])) by (result)",
          "legendFormat": "{{ result }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        }
      }
    },
    {
      "title": "Operator Force-Releases (1h)",
      "type": "timeseries",
      "id": 110,
      "gridPos": {
        "h": 8,
        "w": 10,
        "x": 14,
        "y": 80
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(increase(ray_modelship_operator_force_release_total[1h])) by (reason)",
          "legendFormat": "{{ reason }}"
        }
      ],
      "fieldConfig": {
        "defaults": {}
      }
    },
    {
      "title": "Routing Generation: coordinator vs gateway replicas",
      "type": "timeseries",
      "id": 111,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 88
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "max(ray_modelship_coordinator_generation{gateway=~\"$gateway\"}) by (gateway)",
          "legendFormat": "coordinator {{ gateway }}"
        },
        {
          "expr": "min(ray_modelship_gateway_routing_generation{gateway=~\"$gateway\"}) by (gateway)",
          "legendFormat": "gateway min {{ gateway }}"
        }
      ],
      "fieldConfig": {
        "defaults": {}
      }
    },
    {
      "title": "Gateway Watch Errors/s",
      "type": "timeseries",
      "id": 112,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 88
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_modelship_gateway_watch_errors_total{gateway=~\"$gateway\"}[5m])) by (gateway)",
          "legendFormat": "{{ gateway }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        }
      }
    },
    {
      "title": "State Store Op Latency P95",
      "type": "timeseries",
      "id": 113,
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 96
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_modelship_state_store_operation_duration_seconds_bucket[5m])) by (le, backend, op))",
          "legendFormat": "{{ backend }} {{ op }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      }
    },
    {
      "title": "State Store Errors/s",
      "type": "timeseries",
      "id": 114,
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 96
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "sum(rate(ray_modelship_state_store_operations_total{result=\"error\"}[5m])) by (backend, op)",
          "legendFormat": "{{ backend }} {{ op }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        }
      }
    },
    {
      "title": "Deploy Duration P95 / Models Changed",
      "type": "timeseries",
      "id": 115,
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 96
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(ray_modelship_deploy_duration_seconds_bucket{gateway=~\"$gateway\"}[1h])) by (le, gateway))",
          "legendFormat": "p95 {{ gateway }}"
        },
        {
          "expr": "sum(increase(ray_modelship_deploy_models_changed_total{gateway=~\"$gateway\"}[1h])) by (action)",
          "legendFormat": "{{ action }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      }
    },
    {
      "type": "row",
      "title": "Per-Node Resources",
      "collapsed": false,
      "id": 116,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 104
      },
      "panels": []
    },
    {
      "title": "GPU Utilization by Node",
      "type": "timeseries",
      "id": 117,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 105
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_node_gpus_utilization{instance=~\"$node\"}",
          "legendFormat": "{{ instance }} gpu{{ GpuIndex }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent"
        }
      }
    },
    {
      "title": "GPU Memory Used by Node",
      "type": "timeseries",
      "id": 118,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 105
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_node_gram_used{instance=~\"$node\"}",
          "legendFormat": "{{ instance }} gpu{{ GpuIndex }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "decmbytes"
        }
      }
    },
    {
      "title": "CPU Utilization by Node",
      "type": "timeseries",
      "id": 119,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 113
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_node_cpu_utilization{instance=~\"$node\"}",
          "legendFormat": "{{ instance }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent"
        }
      }
    },
    {
      "title": "System Memory % by Node",
      "type": "timeseries",
      "id": 120,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 113
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "targets": [
        {
          "expr": "ray_node_mem_used{instance=~\"$node\"} / ray_node_mem_total{instance=~\"$node\"} * 100",
          "legendFormat": "{{ instance }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent"
        }
      }
    }
  ],
  "schemaVersion": 39,
  "tags": [
    "modelship",
    "inference",
    "gpu",
    "llm"
  ],
  "templating": {
    "list": [
      {
        "name": "gateway",
        "label": "Gateway",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "query": {
          "qryType": 1,
          "query": "label_values(ray_modelship_request_total, gateway)",
          "refId": "gateway"
        },
        "definition": "label_values(ray_modelship_request_total, gateway)",
        "includeAll": true,
        "allValue": ".*",
        "multi": true,
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "refresh": 2,
        "sort": 1
      },
      {
        "name": "model",
        "label": "Model",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "query": {
          "qryType": 1,
          "query": "label_values(ray_modelship_request_total, model)",
          "refId": "model"
        },
        "definition": "label_values(ray_modelship_request_total, model)",
        "includeAll": true,
        "allValue": ".*",
        "multi": true,
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "refresh": 2,
        "sort": 1
      },
      {
        "name": "node",
        "label": "Node",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "query": {
          "qryType": 1,
          "query": "label_values(ray_node_cpu_utilization, instance)",
          "refId": "node"
        },
        "definition": "label_values(ray_node_cpu_utilization, instance)",
        "includeAll": true,
        "allValue": ".*",
        "multi": true,
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "refresh": 2,
        "sort": 1
      }
    ]
  },
  "time": {
    "from": "now-1h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "browser",
  "title": "Modelship Inference Server",
  "uid": "modelship-overview",
  "version": 2
}