# -- Default values for llmstack helm chart # -- Declare variables to be passed into your templates. # -- Serving engine configuration servingEngineSpec: # -- Whether to enable the serving engine enableEngine: true # @schema skipProperties:true # -- Customized labels for the serving engine deployment labels: environment: "test" release: "test" # -- Extra service ports for models extraPorts: [] # -- API key for securing the vLLM models. Can be either a string that will be stored in a generated secret or an object referencing an existing secret. vllmApiKey: # @schema type:[string, object] # -- Name of the existing Kubernetes secret that contains the vLLM API key secretName: "" # -- Key within the secret that contains the vLLM API key secretKey: "" # -- Array of specifications for configuring multiple serving engine deployments running different models. Each entry in the modelSpec array should contain the following fields ## The model below serves as an example. It is not deployed by default. modelSpec: # -- The name of the model. - name: "mistral" # -- Whether to deploy the model. it Default to true if not set enabled: false # -- The annotations to add to the deployment annotations: {} # -- The annotations to add to the pods podAnnotations: {} # -- Additional labels to add to the deployment labels: {} # -- Additional labels to add to the pods podLabels: {} # -- The name of the service account to use for the deployment serviceAccountName: "" # -- The name of the priority class name for the deployment priorityClassName: "" # -- Runtime class for the pod, e.g., "nvidia". If not specified, falls back to servingEngineSpec.runtimeClassName runtimeClassName: "nvidia" # -- The repository of the model repository: "lmcache/vllm-openai" # -- The tag of the model tag: "latest" # -- Name of secret with credentials to private container repository imagePullSecret: "" # -- The URL of the model modelURL: "mistralai/Mistral-7B-Instruct-v0.2" # -- Hugging Face model revision (branch, tag, or commit hash) to pin the model version modelRevision: "" # -- Chat template (Jinga2) specifying tokenizer configuration # chatTemplate: "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ 'Question:\n' + message['content'] + '\n\n' }}{% elif message['role'] == 'system' %}\n{{ 'System:\n' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ 'Answer:\n' + message['content'] + '\n\n' }}{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ 'Answer:\n' }}{% endif %}{% endfor %}" # -- The number of replicas for the model replicaCount: 1 # -- a resources block containing requests and limits. If specified, this takes priority over simplified resource fields resources: {} # -- The number of CPUs requested for the model requestCPU: 6 # -- The amount of memory requested for the model requestMemory: "16Gi" # -- The number of GPUs requested for the model requestGPU: 1 # @schema type:[integer, number, string] # -- The type of GPU requested. If not specified, defaults to "nvidia.com/gpu" requestGPUType: "nvidia.com/mig-4g.71gb" # -- Requires HAMi. The amount of GPU memory requested. Each unit equals to 1M. https://project-hami.io/docs/userguide/NVIDIA-device/specify-device-memory-usage requestGPUMem: "3000" # -- Requires HAMi. The percentage of GPU memory requested, e.g., "80". It cannot be used together with requestGPUMem. requestGPUMemPercentage: "80" # -- Requires HAMi. The percentage of GPU cores requested, e.g., "10". Each unit equals to 1% device cores. https://project-hami.io/docs/userguide/NVIDIA-device/specify-device-core-usage requestGPUCores: "10" # -- The CPU limit for the model. If limitCPU and limitMemory are not specified, only GPU resources will have limits set equal to their requests. limitCPU: 8 # @schema type:[integer, number, string] # -- The memory limit for the model. If limitCPU and limitMemory are not specified, only GPU resources will have limits set equal to their requests. limitMemory: "32Gi" # -- Requires HAMi. The limit of GPU memory. Each unit equals to 1M. https://project-hami.io/docs/userguide/NVIDIA-device/specify-device-memory-usage limitGPUMem: "3000" # -- Requires HAMi. The limit of GPU memory of GPU limitGPUMemPercentage: "80" # -- Requires HAMi. The limit of GPU cores. Each unit equals to 1% device cores. https://project-hami.io/docs/userguide/NVIDIA-device/specify-device-core-usage limitGPUCores: "10" # -- The amount of storage requested for the model pvcStorage: "50Gi" # -- The access mode policy for the mounted volume. pvcAccessMode: - ReadWriteOnce # -- The storage class of the PVC storageClass: "" # -- The labels to match the PVC. pvcMatchLabels: {} # -- The labels to add to the PVC pvcLabels: {} # -- The annotations to add to the PVC pvcAnnotations: {} # Use an externally-managed PersistentVolumeClaim instead of one created by # this chart. When set, the chart skips rendering the PVC and mounts the # named claim directly (useful for GitOps setups where PVCs are managed # outside the chart). Leave unset to have the chart create the PVC. # pvcExistingClaimName: "my-existing-claim" # -- Additional volumes to add to the pod, in Kubernetes volume format extraVolumes: [] # - name: tmp-volume # emptyDir: # medium: "" # sizeLimit: 5Gi # -- Additional volume mounts to add to the container, in Kubernetes volumeMount format extraVolumeMounts: [] # - name: tmp-volume # mountPath: /tmp # -- Additional containers to run alongside the main vLLM container, in Kubernetes container format. Unlike the LoRA sidecar (servingEngineSpec.sidecar), these are added regardless of enableLoRA and support the full container spec (image, args, env, volumeMounts, livenessProbe, resources, etc.). extraContainers: [] # - name: my-sidecar # image: my-registry/my-sidecar:latest # args: ["--flag", "value"] # env: # - name: MY_ENV_VAR # value: "my-value" # volumeMounts: # - name: tmp-volume # mountPath: /tmp # resources: # requests: # cpu: "100m" # memory: "128Mi" # livenessProbe: # httpGet: # path: /healthz # port: 8080 # initialDelaySeconds: 5 # periodSeconds: 10 # -- Optional container command for the main vLLM container. Replaces the default vllm serve invocation. command: [] # @schema item: string # -- Optional container args for the main vLLM container. Used with command to override the default vllm serve invocation. args: [] # @schema item: string # -- The configuration for the init container to be run before the main container. initContainer: # -- The name of the init container name: "init" # -- The Docker image for the init container image: "busybox:latest" # -- The command to run in the init container command: - "sh" - "-c" # -- Additional arguments to pass to the command args: - "ls" # -- List of environment variables to set in the container env: [] # -- The resource requests and limits for the container resources: {} # -- Whether to mount the model's volume mountPvcStorage: true # -- Additional volume mounts to add to the init container, in Kubernetes volumeMount format. https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#volumemount-v1-core extraVolumeMounts: [] # -- The configuration for the VLLM model vllmConfig: # -- Specify to 1 to use vLLM v0, otherwise vLLM v1 is used v0: "1" # -- Enable prefix caching enablePrefixCaching: false # -- Enable chunked prefill. enableChunkedPrefill: false # -- The maximum model length maxModelLen: 16384 # -- The data type dtype: "bfloat16" # -- The degree of tensor parallelism tensorParallelSize: 2 # -- Maximum number of sequences to be processed in a single iteration. maxNumSeqs: 32 # -- The maximum number of LoRA models to be loaded in a single batch maxLoras: 4 # -- The fraction of GPU memory to be used for the model executor, which can range from 0 to 1 gpuMemoryUtilization: 0.95 # @schema type:[integer, number, string] # -- The runner type for the model, can be auto or pooling runner: "pooling" # @schema enum:[auto,pooling] # -- The conversion type for the model, can be "token_embed", "embed", "token_classify", "classify", or "score". convert: "embed" # @schema enum:[token_embed,embed,token_classify,classify,score] # -- Extra command line arguments to pass to vLLM extraArgs: - "--gpu-memory-utilization" - "0.4" # -- The configuration of the LMCache for KV offloading lmcacheConfig: # -- Enable LMCache enabled: true # -- The CPU offloading buffer size cpuOffloadingBufferSize: "30" # @schema type:[integer, number, string] # -- The log level for LMCache, e.g., "DEBUG", "INFO", "WARNING", "ERROR" logLevel: INFO # @schema enum:[DEBUG,INFO,WARNING,ERROR] # -- Hugging Face token configuration. Can be either a string that will be stored in a generated secret or an object referencing an existing secret. hf_token: # @schema type:[string, object] # -- Name of the existing Kubernetes secret that contains the Hugging Face token secretName: "" # -- Key within the secret that contains the Hugging Face token secretKey: "" # -- Reference to an existing Kubernetes Secret from which all key/value pairs will be loaded as environment variables into the container envFromSecret: {} # name: s3-registry # The referenced Secret could look like: # apiVersion: v1 # kind: Secret # metadata: # name: s3-registry # namespace: vllm # data: # AWS_ACCESS_KEY_ID: # AWS_SECRET_ACCESS_KEY: # AWS_ENDPOINT_URL: # -- The environment variables to set in the container, e.g., your HF_TOKEN env: [] # -- Affinity configuration affinity: {} # -- The node selector terms to match the nodes. When both affinity and nodeSelectorTerms are defined, nodeSelectorTerms will be ignored. nodeSelectorTerms: [] # -- Directly assigns a pod to a specific node (e.g., "192.168.56.5"). When both nodeName and nodeSelectorTerms are defined, the preference is given to nodeName. nodeName: "" # -- Per-model tolerations. Interacts with servingEngineSpec.tolerations according to tolerationsPolicy. tolerations: [] # Example: # tolerations: # - key: "dedicated" # operator: "Equal" # value: "gpu-model-a" # effect: "NoSchedule" # -- How per-model tolerations interact with global servingEngineSpec.tolerations. tolerationsPolicy: append # - "append" (default): model tolerations are appended to the global tolerations (union of both lists). # - "override": model tolerations replace the global tolerations entirely for this model. # -- The size of the shared memory shmSize: "20Gi" # -- The model's service annotations serviceAnnotations: {} # -- The model's service labels serviceLabels: {} # -- Whether to enable LoRA enableLoRA: true # -- KEDA autoscaling configuration for this model deployment. Requires KEDA to be installed in the cluster. keda: # -- Whether to enable KEDA autoscaling for this model enabled: true # -- Minimum number of replicas (supports 0 for scale-to-zero) minReplicaCount: 1 # @schema minimum:0 # -- Maximum number of replicas maxReplicaCount: 5 # -- How often KEDA should check the metrics (in seconds) pollingInterval: 15 # -- How long to wait before scaling down after scaling up (in seconds). cooldownPeriod: 360 # -- Number of replicas to scale to when no triggers are active idleReplicaCount: 0 # -- Initial cooldown period in seconds before scaling down after creation initialCooldownPeriod: 60 # -- Fallback configuration when scaler fails fallback: # -- Number of consecutive failures before fallback failureThreshold: 3 # -- Number of replicas to scale to in fallback replicas: 2 # -- List of KEDA trigger configurations triggers: # -- Trigger type type: "prometheus" # -- Trigger metadata metadata: # -- Prometheus server address serverAddress: "http://prometheus-operated.monitoring.svc:9090" # -- Name of the metric to monitor metricName: "vllm:num_requests_waiting" # -- Prometheus query to fetch the metric query: "vllm:num_requests_waiting" # -- Threshold value that triggers scaling threshold: "5" # -- Advanced KEDA configuration advanced: # -- Restore original replica count when ScaledObject is deleted restoreToOriginalReplicaCount: false # -- HPA-specific configuration horizontalPodAutoscalerConfig: # -- Custom name for the HPA resource name: "keda-hpa-{scaled-object-name}" # -- HPA scaling behavior configuration, see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/ behavior: {} # -- Scaling modifiers for composite metrics scalingModifiers: # -- Target value for the composed metric target: "" # -- Activation target for the composed metric activationTarget: "" # -- Metric type (AverageValue or Value) metricType: "AverageValue" # -- Formula to compose metrics together formula: "" # -- RaySec configuration, to deploy the model as a rayCluster instead of a deployment raySpec: # -- Whether to deploy the model as a rayCluster enabled: false # -- Head node configuration headNode: # -- CPU request for the head node requestCPU: 1 # -- Memory request for the head node requestMemory: "1Gi" # -- GPU request for the head node requestGPU: 1 # -- Raw Kubernetes resources block for the head node container. When set, overrides the request*/limit* fields above. # resources: # requests: # cpu: "2" # memory: "8Gi" # nvidia.com/gpu: "1" # limits: # cpu: "4" # memory: "16Gi" # nvidia.com/gpu: "1" # -- Optional init container run before the Ray head container starts. Mirrors modelSpec.initContainer. # initContainer: # name: "init" # image: "busybox:latest" # command: ["sh", "-c"] # args: ["ls"] # env: [] # resources: {} # mountPvcStorage: true # extraVolumeMounts: [] # Worker nodes reuse the top-level modelSpec fields: # - resources come from modelSpec.requestCPU / requestMemory / requestGPU (or modelSpec.resources for a raw override) # - the optional init container comes from modelSpec.initContainer # -- Container port containerPort: 8000 # -- Service port servicePort: 80 # -- Set other environment variables from config map configs: {} # -- deployment strategy strategy: {} # -- Readiness probe configuration startupProbe: # -- Number of seconds after the container has started before startup probe is initiated initialDelaySeconds: 15 # -- How often (in seconds) to perform the startup probe periodSeconds: 10 # -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not ready failureThreshold: 60 # -- Configuration of the Kubelet http request on the server httpGet: # -- Path to access on the HTTP server path: /health # -- Name or number of the port to access on the container, on which the server is listening port: 8000 # @schema type:[integer, string] # -- Liveness probe configuration livenessProbe: # -- Number of seconds after the container has started before liveness probe is initiated initialDelaySeconds: 15 # -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not alive failureThreshold: 3 # -- How often (in seconds) to perform the liveness probe periodSeconds: 10 # -- Configuration of the Kubelet http request on the server httpGet: # -- Path to access on the HTTP server path: /health # -- Name or number of the port to access on the container, on which the server is listening port: 8000 # @schema type:[integer, string] # -- Readiness probe configuration readinessProbe: # -- Number of seconds after the container has started before readiness probe is initiated initialDelaySeconds: 15 # -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not alive failureThreshold: 3 # -- How often (in seconds) to perform the readiness probe periodSeconds: 5 # -- Configuration of the Kubelet http request on the server httpGet: # -- Path to access on the HTTP server path: /health # -- Name or number of the port to access on the container, on which the server is listening port: 8000 # @schema type:[integer, string] # -- Tolerations configuration (when there are taints on nodes) # Example: # tolerations: # - key: "node-role.kubernetes.io/control-plane" # operator: "Exists" # effect: "NoSchedule" tolerations: [] # -- RuntimeClassName configuration, set to "nvidia" if the model requires GPU runtimeClassName: "nvidia" # -- SchedulerName configuration schedulerName: "" # -- image pull policy for the model deployment imagePullPolicy: "Always" # @schema enum:[Always,IfNotPresent,Never] # -- Pod-level security context configuration. https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#podsecuritycontext-v1-core securityContext: {} # -- Run as a non-root user ID # runAsUser: 1000 # -- Run with a non-root group ID # runAsGroup: 1000 # -- Run as non-root # runAsNonRoot: true # -- Set the seccomp profile # seccompProfile: # type: RuntimeDefault # -- Drop all capabilities # capabilities: # drop: # - ALL # -- Set the file system group ID for all containers # fsGroup: 1000 # -- Container-level security context configuration. https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#securitycontext-v1-core containerSecurityContext: # -- Run as non-root runAsNonRoot: false # -- Don't allow privilege escalation # allowPrivilegeEscalation: false # -- Drop all capabilities # capabilities: # drop: # - ALL # -- Read-only root filesystem # readOnlyRootFilesystem: true # -- Sidecar configuration sidecar: # -- Docker image repository and tag for the sidecar image: "lmcache/lmstack-sidecar:latest" # -- image pull policy for the sidecar imagePullPolicy: "Always" # @schema enum:[Always,IfNotPresent,Never] # -- ServiceMonitor configuration to collect serving engine metrics serviceMonitor: # -- Specifies whether to create a ServiceMonitor resource for collecting Prometheus metrics enabled: false # -- Additional labels additionalLabels: {} # -- Interval to scrape metrics interval: 30s # -- Timeout if metrics can't be retrieved in given time interval scrapeTimeout: 25s # -- Let prometheus add an exported_ prefix to conflicting labels honorLabels: false # -- Metric relabel configs to apply to samples before ingestion. [Metric Relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs) metricRelabelings: [] # - action: replace # regex: (.*) # replacement: $1 # sourceLabels: # - exported_namespace # targetLabel: namespace # -- Relabel configs to apply to samples before ingestion. [Relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] # separator: ; # regex: ^(.*)$ # targetLabel: nodename # replacement: $1 # action: replace # -- Router configuration routerSpec: # -- Whether to enable the router service enableRouter: true # -- The docker image of the router. The following values are defaults: repository: "lmcache/lmstack-router" # -- docker tag for the router pod tag: "latest" # -- image pull policy for the router pod imagePullPolicy: "Always" # @schema enum:[Always,IfNotPresent,Never] # -- The name of an existing service account to use for the router deployment. # -- If set, no ServiceAccount will be created and this name will be used directly. serviceAccountName: "" # -- Hugging Face token configuration hf_token: "" # -- Pod-level security context configuration. https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#podsecuritycontext-v1-core securityContext: {} # -- Container-level security context configuration. https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#securitycontext-v1-core containerSecurityContext: {} # -- Additional volumes to add to the router pod, in Kubernetes volume format extraVolumes: [] # - name: tmp-volume # emptyDir: {} # - name: home-volume # emptyDir: {} # -- Additional volume mounts to add to the router container, in Kubernetes volumeMount format extraVolumeMounts: [] # - name: tmp-volume # mountPath: /tmp # - name: home-volume # mountPath: /home # -- Image pull secrets for private container registries # Example: # imagePullSecrets: # - name: my-registry-secret imagePullSecrets: [] # -- Number of replicas replicaCount: 1 # -- List of environment variables to set for the router env: [] # -- autoscaling configuration autoscaling: enabled: false minReplicas: 1 # @schema minimum:0 maxReplicas: 3 # @schema minimum:0 targetCPUUtilizationPercentage: 80 # -- Router Pod Disruption Budget # -- Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ pdb: # -- Deploy a PodDisruptionBudget for the router deployment enabled: false # -- Labels to be added to router pdb labels: {} # -- Annotations to be added to router pdb annotations: {} # -- Number of pods that are available after eviction as number or percentage (eg.: 50%) minAvailable: # @schema type:[string, integer, null];minimum:0 # -- Number of pods that are unavailable after eviction as number or percentage (eg.: 50%). maxUnavailable: # @schema type:[string, integer, null];minimum:0 # -- Priority Class priorityClassName: "" # -- Container port containerPort: 8000 # -- Service type serviceType: ClusterIP # @schema enum:[ClusterIP,NodePort,LoadBalancer,ExternalName] # -- Service annotations if you use a LoadBalancer or NodePort service type serviceAnnotations: {} # -- Fixed NodePort for the router service when serviceType is NodePort. # -- If not set, Kubernetes assigns a random port (range 30000-32767). nodePort: # @schema type:[integer, null];minimum:30000;maximum:32767 # -- Service port servicePort: 80 # -- Service discovery mode, supports "k8s" or "static". Defaults to "k8s" if not set. serviceDiscovery: "k8s" # @schema enum:[k8s,static] # -- Service discovery mode type, supports "pod-ip" or "service-name". Defaults to "pod-ip" if not set. k8sServiceDiscoveryType: "pod-ip" # @schema enum:[pod-ip,service-name] # -- If serviceDiscovery is set to "static", the comma-separated values below are required. There needs to be the same number of backends and models staticBackends: "" staticModels: "" # -- routing logic, supports "roundrobin", "session", "prefixaware", "kvaware", "disaggregated_prefill" or "disaggregated_prefill_orchestrated" routingLogic: "roundrobin" # @schema enum:[roundrobin,session,prefixaware,kvaware,disaggregated_prefill,disaggregated_prefill_orchestrated] # -- Minimum prefix match length required for prefixaware routing to reuse a # matched endpoint. If the longest prefix match is shorter than this value, # the request falls back to least-inflight routing via request_stats (same # load signal as loadaware). Prefix matches are computed in chunks (chunk # size defaults to 128 characters), so this threshold is effectively # quantized to that granularity. Defaults to 0, which disables it. prefixMinMatchLength: 0 # -- session key if using "session" routing logic sessionKey: "" # -- FastAPI root path for hosting under a subpath (e.g. /vllm) rootPath: "" # -- extra router commandline arguments extraArgs: [] # -- extra service ports extraPorts: [] # -- Interval in seconds to scrape the serving engine metrics engineScrapeInterval: 15 # -- Window size in seconds to calculate the request statistics requestStatsWindow: 60 # -- OpenTelemetry tracing configuration # When otelEndpoint is set, tracing is automatically enabled otel: # -- OTLP endpoint for tracing (e.g., "localhost:4317" or "otel-collector:4317") endpoint: "" # -- Service name for traces (default: "vllm-router") serviceName: "vllm-router" # -- Use secure (TLS) connection for OTLP exporter (default: false, i.e., insecure) secure: false # -- deployment strategy strategy: {} # -- API key for securing the vLLM models. Must be an object referencing an existing secret. If not set, it defaults to .Values.servingEngineSpec.vllmApiKey. vllmApiKey: # -- Name of the existing Kubernetes secret that contains the vLLM API key secretName: "" # -- Key within the secret that contains the vLLM API key secretKey: "" # -- lmcache configuration for the router deployment lmcacheConfig: # -- LMCache log level for the router deployment. logLevel: "INFO" # -- lmcache controller port, used when routingLogic is "kvaware" lmcacheControllerPort: "" # @schema type:[integer, string] # -- router resource requests and limits resources: requests: cpu: 400m memory: 1000Mi limits: memory: 1000Mi # @schema skipProperties:true # -- Customized labels for the router deployment labels: environment: "router" release: "router" # -- Customized pod annotations for the router pods podAnnotations: {} # -- Ingress configuration for the router ingress: # -- Enable ingress controller resource enabled: false # -- IngressClass that will be used to implement the Ingress className: "" # -- Additional annotations for the Ingress resource annotations: {} # kubernetes.io/ingress.class: alb # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" # The list of hostnames to be covered with this ingress record. hosts: - host: vllm-router.local paths: - path: / pathType: Prefix # -- The tls configuration for hostnames to be covered with this ingress record. tls: [] # - secretName: vllm-router-tls # hosts: # - vllm-router.local # @schema mergeProperties: true # -- Expose the service via gateway-api HTTPRoute # More routes can be added by adding a dictionary key like the 'main' route. # Requires Gateway API resources and suitable controller installed within the cluster # (see: https://gateway-api.sigs.k8s.io/guides/) route: main: # -- Enables or disables the route enabled: false # -- Set the route apiVersion, e.g. gateway.networking.k8s.io/v1 or gateway.networking.k8s.io/v1alpha2 apiVersion: gateway.networking.k8s.io/v1 # -- Set the route kind kind: HTTPRoute # @schema enum:[HTTPRoute,GRPCRoute,TCPRoute,TLSRoute,UDPRoute] # -- Additional annotations to add to the route annotations: {} # -- Additional labels to add to the route labels: {} # -- List of hostnames for the Route hostnames: [] # - my-filter.example.com # -- Gateway API parentRefs for the Route ## Must reference an existing Gateway parentRefs: [] # - name: acme-gw # -- create http route for redirect (https://gateway-api.sigs.k8s.io/guides/http-redirect-rewrite/#http-to-https-redirects) ## Take care that you only enable this on the http listener of the gateway to avoid an infinite redirect. ## matches, filters and additionalRules will be ignored if this is set to true. httpsRedirect: false # -- Route match configuration matches: - path: type: PathPrefix value: / # -- Filters define the filters that are applied to requests that match this rule. filters: [] # - type: RequestHeaderModifier # requestHeaderModifier: # add: # - name: X-Custom-Header # value: custom-value # -- Additional custom rules that can be added to the route additionalRules: [] # -- Affinity configuration affinity: {} # -- The node selector terms to match the nodes. Will be ignored if affinity is configured. nodeSelectorTerms: [] # - matchExpressions: # - key: nvidia.com/gpu.product # operator: "In" # values: # - "NVIDIA-RTX-A6000" # -- Liveness probe configuration for the router pod. livenessProbe: # -- Number of seconds after the container has started before liveness probe is initiated initialDelaySeconds: 30 # -- How often (in seconds) to perform the liveness probe periodSeconds: 5 # -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not alive failureThreshold: 3 # -- Configuration of the Kubelet http request on the server httpGet: # -- Path to access on the HTTP server path: /health # -- Name or number of the port to access on the container, on which the server is listening port: router-port # @schema type:[integer, string] # Startup probe configuration for the router pod. startupProbe: # -- Number of seconds after the container has started before liveness probe is initiated initialDelaySeconds: 5 # -- How often (in seconds) to perform the startup probe periodSeconds: 5 # -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not ready failureThreshold: 3 # -- Configuration of the Kubelet http request on the server httpGet: # -- Path to access on the HTTP server path: /health # -- Name or number of the port to access on the container, on which the server is listening port: router-port # @schema type:[integer, string] # Readiness probe configuration for the router pod. readinessProbe: # -- Number of seconds after the container has started before liveness probe is initiated initialDelaySeconds: 30 # -- How often (in seconds) to perform the readiness probe periodSeconds: 5 # -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not alive failureThreshold: 3 # -- Configuration of the Kubelet http request on the server httpGet: # -- Path to access on the HTTP server path: /health # -- Name or number of the port to access on the container, on which the server is listening port: router-port # @schema type:[integer, string] # -- ServiceMonitor configuration to collect router metrics serviceMonitor: # -- Specifies whether to create a ServiceMonitor resource for collecting Prometheus metrics enabled: false # -- Additional labels additionalLabels: {} # -- Interval to scrape metrics interval: 30s # -- Timeout if metrics can't be retrieved in given time interval scrapeTimeout: 25s # -- Let prometheus add an exported_ prefix to conflicting labels honorLabels: false # -- Metric relabel configs to apply to samples before ingestion. [Metric Relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs) metricRelabelings: [] # - action: replace # regex: (.*) # replacement: $1 # sourceLabels: # - exported_namespace # targetLabel: namespace # -- Relabel configs to apply to samples before ingestion. [Relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] # separator: ; # regex: ^(.*)$ # targetLabel: nodename # replacement: $1 # action: replace # -- Configuration for the LMCache cache server deployment cacheserverSpec: # -- Whether to enable the cache server enabled: false # -- Image configuration for the cache Server image: # -- Docker image repository for the cache server repository: "lmcache/vllm-openai" # -- Docker image tag for the cache server tag: "latest" # -- Image pull policy for the cache server imagePullPolicy: "IfNotPresent" # @schema enum:[Always,IfNotPresent,Never] # -- Image pull secrets for private container registries imagePullSecrets: [] # -- Number of replicas for the cache server pod replicaCount: 1 # -- Port the cache server container is listening on containerPort: 8000 # -- Kubernetes service type for the cache server serviceType: ClusterIP # @schema enum:[ClusterIP,NodePort,LoadBalancer,ExternalName] # -- Port the cache server service will listen on servicePort: 80 # -- Service annotations for the cache server service serviceAnnotations: {} # -- Resource requests and limits for the cache server container resources: {} # -- Customized annotations for the cache server deployment annotations: {} # -- Customized labels for the cache server deployment and service labels: {} # -- Deployment strategy for the cache server pods strategy: # @schema additionalProperties: false # -- Customize updateStrategy of the cache server deployment. type: RollingUpdate # @schema enum:[RollingUpdate, Recreate] rollingUpdate: maxUnavailable: 0 # @schema type:[integer, string, null] maxSurge: 1 # @schema type:[integer, string, null] # -- Configuration for the liveness probe livenessProbe: # @schema additionalProperties: false # -- The number of seconds to wait before starting the first probe. initialDelaySeconds: 30 # -- The number of seconds to wait between consecutive probes. periodSeconds: 10 # -- The number of consecutive failures allowed before considering the probe as failed. failureThreshold: 3 # -- The number of seconds to wait for a probe response before considering it as failed. timeoutSeconds: 5 # -- Tolerations configuration for the cache server pods tolerations: [] # -- RuntimeClassName configuration for the cache server pods runtimeClassName: "" # -- SchedulerName configuration for the cache server pods schedulerName: "" # -- Pod-level security context configuration securityContext: {} # -- Container-level security context configuration containerSecurityContext: {} # -- Priority class for the cache server pods priorityClassName: "" # -- Node selector for the cache server pods nodeSelector: {} # -- Affinity configuration. If specified, takes precedence over nodeSelectorTerms. affinity: {} # -- Serialization/deserialization format for the cache server serde: "" # -- LoRA Adapters Configuration loraAdapters: # -- The name of the LoRA adapter instance - name: "llama3-nemoguard-adapter" # -- Whether to create le LoRa adapter enabled: false # -- The name of the base model this adapter is for baseModel: llama3-8b-instr # -- API key configuration for vLLM authentication. If not Set, it defaults to servingEngineSpec.vllmApiKey vllmApiKey: # -- Name of the secret secretName: "vllm-api-key" # -- Key in the secret containing the API key secretKey: "VLLM_API_KEY" # -- Configuration for the adapter source adapterSource: # -- Type of adapter source (local, s3, http, huggingface) type: "local" # -- Name of the adapter to apply adapterName: "llama-3.1-nemoguard-8b-topic-control" # -- Path to the LoRA adapter weights adapterPath: "/data/lora-adapters/llama-3.1-nemoguard-8b-topic-control" # -- Repository to get the LoRA adapter from repository: "" # -- Pattern to use for the adapter name pattern: "" # -- Maximum number of adapters to load maxAdapters: 1 # -- Reference to secret with storage credentials ## Can be either a string that will be stored in a generated secret or an object referencing an existing secret. credentials: # @schema type:[string, object] # -- Name of the secret secretName: "secret" # -- Key in the secret containing the credentials secretKey: "key" # -- Configuration for adapter deployment loraAdapterDeploymentConfig: # -- Placement algorithm to use (default, ordered, equalized) algorithm: "default" # -- Number of replicas that should load this adapter replicas: 1 # -- Additional labels for the LoRA adapter labels: {} # -- lora controller Configuration loraController: # -- Whether to enable the Lora Controller enableLoraController: false # -- kubernetes cluster domain kubernetesClusterDomain: "cluster.local" # -- The name of an existing service account to use for the lora controller deployment. # -- If set, no ServiceAccount will be created and this name will be used directly. serviceAccountName: "" # -- Number of lora controller replicas replicaCount: 1 # -- lora controller image configuration image: # -- Docker image repository repository: "lmcache/lmstack-lora-controller" # -- Docker image tag tag: "latest" # -- Image pull policy pullPolicy: "IfNotPresent" # @schema enum:[Always,IfNotPresent,Never] # -- Image pull secrets imagePullSecrets: [] # -- Deployment annotations annotations: {} # -- Deployment labels labels: {} # -- Pod annotations podAnnotations: {} # -- Pod labels podLabels: {} # @schema skipProperties:true # -- Pod security context podSecurityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault # @schema skipProperties:true # -- Container security context containerSecurityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL # -- lora controller resources resources: {} # -- Node selector nodeSelector: {} # -- Affinity affinity: {} # -- Tolerations tolerations: [] # -- Environment variables env: [] # -- Extra arguments for the lora controller extraArgs: [] # -- Lora Controller metrics configuration metrics: # -- Expose lora controller metrics enabled: true # -- loraController Webhook configuration webhook: # -- Deploy a webhook inside lora controller enabled: false # -- loraController Pod Disruption Budget # -- Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ pdb: # -- Deploy a PodDisruptionBudget for the loraController deployment enabled: false # -- Labels to be added to loraController pdb labels: {} # -- Annotations to be added to loraController pdb annotations: {} # -- Number of pods that are available after eviction as number or percentage (eg.: 50%) minAvailable: # @schema type:[string, integer, null];minimum:0 # -- Number of pods that are unavailable after eviction as number or percentage (eg.: 50%). maxUnavailable: # @schema type:[string, integer, null];minimum:0 # -- Configuration for a shared PVC that can be mounted to multiple model deployments. sharedPvcStorage: # -- Whether to enable shared storage for the models enabled: false # -- Size of the shared storage volume size: "100Gi" # -- Access modes for the shared storage volume accessModes: - ReadWriteMany # -- Storage class name for the shared storage volume storageClass: "" # -- Host path for the shared storage volume. Specifying a hostPath or nfs server will create a PersistentVolume. These fields should be omitted to rely on dynamic provisioning of PersistentVolumeClaim. hostPath: "" # -- NFS configuration for the shared storage volume nfs: # -- NFS server address for the shared storage volume server: "" # -- NFS export path for the shared storage volume path: "" # -- Docker image used in the lora Controller as an init container to set permissions in the PVC image: # -- Docker image repository repository: "busybox" # -- Docker image tag tag: "1.37" # -- Array of extra Kubernetes objects to deploy. Each object should be a valid Kubernetes manifest in YAML format. This can be used to deploy additional resources such as ConfigMaps, Secrets, or custom resources that are not directly supported by the chart's built-in configuration. Supports use of custom Helm templates. extraObjects: [] # -- Set to true do deploy dashboards stored in the "dashboards" directory as configmaps. This requires the kube-prometheus-stack to be deployed with Grafana enabled and properly configured to pick up dashboards from configmaps. grafanaDashboards: enabled: false # -- Additional annotations to add to the dashboard configmaps annotations: {} # @schema skipProperties:true # -- Additional labels to add to the dashboard configmaps labels: grafana_dashboard: "1" # @schema skipProperties:true # -- Kube Prometheus Stack dependency chart configuration. More values can be found [here](https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml). kube-prometheus-stack: enabled: false # -- Configure the Prometheus Operator to automatically discover and scrape metrics from the vLLM chart using ServiceMonitors. prometheus: prometheusSpec: serviceMonitorSelector: matchLabels: app.kubernetes.io/part-of: vllm-stack # @schema skipProperties:true # -- Prometheus Adapter dependency chart configuration. More values can be found [here](https://github.com/prometheus-community/helm-charts/blob/main/charts/prometheus-adapter/values.yaml). prometheus-adapter: enabled: false prometheus: url: http://{{ .Release.Name }}-kube-prometheus-stack-prometheus port: 9090 # rules: # default: true # custom: # # Example metric to export for HPA # - seriesQuery: '{__name__=~"^vllm:num_requests_waiting$"}' # resources: # overrides: # namespace: # resource: "namespace" # name: # matches: "" # as: "vllm_num_requests_waiting" # metricsQuery: sum by(namespace) (vllm:num_requests_waiting) # # Export num_incoming_requests_total by model name # - seriesQuery: '{__name__=~"^vllm:num_incoming_requests_total$"}' # resources: # overrides: # namespace: # resource: "namespace" # name: # matches: "" # as: "vllm_num_incoming_requests_total" # metricsQuery: sum by(namespace, model) (vllm:num_incoming_requests_total) # -- Optional runtime AI inventory (k8s-aibom). When enabled, a # lightweight, unprivileged controller generates a CycloneDX 1.6 ML-BOM # for AI workloads in namespaces labeled aibom.k8saibom.dev/enabled=true # — including the vLLM deployments this chart manages. Disabled by # default; nothing is inventoried without both this flag and the # namespace opt-in label. k8s-aibom: enabled: false