package autogen import ( "fmt" "math" "strings" "sync" ) // vllmDefaultGpuUtil is the fallback fraction of the GPU vllm may fill (weights // + KV + activations) when the card's size is unknown, so a utilization can't be // derived from the VRAM budget. vllm self-manages placement inside this budget — // there is no -ngl / --n-cpu-moe layer offload like llama.cpp, so we don't place // layers per-model; we hand vllm a cap and let its allocator fit inside it. Tune // per-model via Override.VllmGpuUtil. const vllmDefaultGpuUtil = 0.90 // vllm utilization bounds. The floor keeps a tiny budget on a big card from // producing a utilization vllm can't even load weights into; the ceiling leaves // the display/compositor room, since vllm measures utilization against TOTAL // card memory and refuses to start if that much isn't actually free. const ( vllmMinGpuUtil = 0.10 vllmMaxGpuUtil = 0.95 ) // vllmOverheadGB is the non-weights, non-KV VRAM vllm needs on top of the model: // activations, CUDA graphs, and the profiling run's peak. A flat figure, unlike // the llama path's modelled compute buffer — vllm's allocator is opaque to us, so // this is a reserve, not an estimate. const vllmOverheadGB = 1.5 // vllmFallbackCtx caps --max-model-len when the KV cost model can't be built // from the gguf header (missing attention dims). Better a conservative window // than the model's full trained context, which is what vllm would otherwise try // to allocate KV for at startup. const vllmFallbackCtx = 32768 // kindClasses maps a backend kind to every model class it can serve, most // representative first, so auto-pick can match a model to a compatible backend. // llama and vllm both serve LLMs. // // Nearly every engine serves exactly one class, and the singular kindClass below // is what the UI labels a backend with. audio.cpp is the exception: one binary // that does speech synthesis AND transcription (and, later, music), so a single // registry row has to be resolvable from more than one class. Use // kindServesClass for "can this backend run that model", never an equality test // against kindClass. func kindClasses(kind string) []string { switch strings.ToLower(strings.TrimSpace(kind)) { case "llama", "llama.cpp", "server", "vllm": return []string{"llm"} case "sd", "sd-server", "image": return []string{"image"} case "tts", "tts-server", "speech", "ttscpp", "tts.cpp", "kokoro": return []string{"tts"} case "asr", "parakeet", "parakeet-server", "transcribe": return []string{"asr"} case "audiocpp", "audio.cpp", "audiocpp-server": return []string{"tts", "asr"} case "sam", "sam3", "segment": return []string{"segment"} case "upscale", "realesrgan", "esrgan": return []string{"upscale"} case "trellis2", "trellis", "3d": return []string{"3d"} } return nil } // kindClass is the class a backend is LABELLED with: the first one it serves. // Matching a model against a backend is kindServesClass's job. func kindClass(kind string) string { if cs := kindClasses(kind); len(cs) > 0 { return cs[0] } return "" } // kindServesClass reports whether a backend of this kind can run models of the // given class. func kindServesClass(kind, class string) bool { for _, c := range kindClasses(kind) { if c == class { return true } } return false } // KindClass exposes the kind→class mapping to callers outside this package (the // managed-backend installer needs it to decide whether a class already has an // entry before marking a freshly installed backend as the class default). func KindClass(kind string) string { return kindClass(kind) } // KindClasses exposes the full class set of a kind outside this package. func KindClasses(kind string) []string { return kindClasses(kind) } // KindServesClass exposes the class-membership test outside this package. It is // what callers deciding "is there already a backend for this class" must use: // an equality test against KindClass silently ignores every class a multi-class // engine serves beyond its first. func KindServesClass(kind, class string) bool { return kindServesClass(kind, class) } // ClassTaken reports whether the registry already holds an entry serving any // class the given kind would serve. It is the "an install never steals ★" test, // asked by all three registration paths (install, activate, first-run setup). // Shared because the multi-class case has to consider EVERY class the new // backend would serve: audio.cpp arriving on a box that already has parakeet // must not take the auto-pick just because nothing claimed the tts class yet. func ClassTaken(list []BackendEntry, kind string) bool { for _, c := range kindClasses(kind) { for _, e := range list { if kindServesClass(e.Kind, c) { return true } } } return false } // resolvedBackend is the backend a model resolves to: its kind (drives which // emitter runs) and exe (the launcher). A zero value => no registry entry // matched; the caller falls back to the legacy derived exe (ServerExe etc.). type resolvedBackend struct { Kind string Exe string ID string } // resolveBackend picks the backend entry for a model of the given class. // Precedence: an explicit Override.Backend (registry entry id) wins; else the // class-default entry (Default=true); else the first entry of that class; else a // zero value so the caller keeps the legacy behaviour. Auto-pick is thus: // "the ★-default backend for this class, or the first one installed." func resolveBackend(s Settings, ov *Override, class string) resolvedBackend { if ov != nil { if id := strings.TrimSpace(ov.Backend); id != "" { for _, e := range s.Backends { if e.ID == id { return resolvedBackend{Kind: e.Kind, Exe: strings.TrimSpace(e.Path), ID: e.ID} } } // A stale id (backend removed from the registry) falls through to // auto-pick rather than emitting a broken command. } } var first *BackendEntry for i := range s.Backends { e := &s.Backends[i] if !kindServesClass(e.Kind, class) { continue } if e.Default { return resolvedBackend{Kind: e.Kind, Exe: strings.TrimSpace(e.Path), ID: e.ID} } if first == nil { first = e } } if first != nil { return resolvedBackend{Kind: first.Kind, Exe: strings.TrimSpace(first.Path), ID: first.ID} } return resolvedBackend{} } // resolveBackendPreferring is resolveBackend with a family hint. Some classes // hold engines that are NOT interchangeable: llama and vllm both serve any LLM // gguf, but qwentts.cpp and TTS.cpp each read their own weights format, so the // ★-default of the "tts" class is the wrong pick for a model of the other // family. preferKind therefore beats Default — an explicit per-model // Override.Backend still wins over both, since that is the user overruling the // auto-pick on purpose. Falls back to plain resolveBackend when the class holds // no entry of the preferred kind. func resolveBackendPreferring(s Settings, ov *Override, class, preferKind string) resolvedBackend { if ov != nil { if id := strings.TrimSpace(ov.Backend); id != "" { for _, e := range s.Backends { if e.ID == id { return resolvedBackend{Kind: e.Kind, Exe: strings.TrimSpace(e.Path), ID: e.ID} } } } } if preferKind != "" { var first *BackendEntry for i := range s.Backends { e := &s.Backends[i] if !kindServesClass(e.Kind, class) || !strings.EqualFold(strings.TrimSpace(e.Kind), preferKind) { continue } if e.Default { return resolvedBackend{Kind: e.Kind, Exe: strings.TrimSpace(e.Path), ID: e.ID} } if first == nil { first = e } } if first != nil { return resolvedBackend{Kind: first.Kind, Exe: strings.TrimSpace(first.Path), ID: first.ID} } } return resolveBackend(s, ov, class) } // vllmCmdLines builds the vllm argv (exe first) for a gguf or an HF folder // served through vllm. Shared by emitVllmModel (YAML emit) and RenderSoloCmd // (editor preview) so the launch-parameters box matches a save. vllm loads the // SAME file QM discovered (--quantization gguf for a gguf; an HF folder names its // own quantization in config.json); no per-model VRAM sizing — vllm's allocator fits inside // --gpu-memory-utilization. Ctx maps to --max-model-len (caps the KV window). func vllmCmdLines(s Settings, row GgufRow, ov *Override, name string, be resolvedBackend, meta Metadata) []string { modelPath := strings.ReplaceAll(row.FullPath, "\\", "/") exe := be.Exe if exe == "" { exe = "vllm" } ctx, _ := vllmMaxModelLen(s, ov, row, meta) util, _ := vllmGpuUtil(s, ov, vllmFootprintGB(s, ov, row, meta, ctx)) lines := []string{ exe, "serve " + imageArg(modelPath), "--host 127.0.0.1", "--port ${PORT}", "--served-model-name " + imageArg(name), } // An HF folder carries its own quantization_config (or none), which vllm reads // itself; forcing gguf there makes it look for a .gguf that is not in it. if !row.IsHF { lines = append(lines, "--quantization gguf") } if !vllmExtraSetsMemory(ov) { lines = append(lines, fmt.Sprintf("--gpu-memory-utilization %g", round2(util))) if kv, ok := vllmKvCacheGB(ov, meta, ctx); ok { lines = append(lines, fmt.Sprintf("--kv-cache-memory-bytes %d", int64(math.Ceil(kv*gib)))) } } if ctx > 0 { lines = append(lines, fmt.Sprintf("--max-model-len %d", ctx)) } // Upstream: "We recommend using the tokenizer from base model instead of GGUF // model. Because the tokenizer conversion from GGUF is time-consuming and // unstable." It is not emitted automatically: GgufRow.Repo is the local folder // name, not a verified Hugging Face id, so deriving one would bake a guessed // remote reference into a launch command. Set it explicitly per model. if ov != nil { if tok := strings.TrimSpace(ov.VllmTokenizer); tok != "" { lines = append(lines, "--tokenizer "+imageArg(tok)) } } if parser, _ := vllmToolParser(ov, meta, name); parser != "" && !vllmExtraSetsToolParser(ov) { lines = append(lines, "--enable-auto-tool-choice", "--tool-call-parser "+parser) } if ov != nil && ov.VllmTensorParallel > 1 { lines = append(lines, fmt.Sprintf("--tensor-parallel-size %d", ov.VllmTensorParallel)) } if ov != nil { if extra := strings.TrimSpace(ov.ExtraArgs); extra != "" { lines = append(lines, extra) } } return lines } // emitVllmModel writes a vllm YAML entry for a gguf LLM. vllm speaks the OpenAI // API natively (/v1/chat/completions, /health) so it needs no capabilities block // (default text chat) and no explicit checkEndpoint (llama parity — the config // default /health probe fits vllm too). Named/ctx-tier variants are NOT emitted // for vllm: the llama profile/KV sizing that produces them doesn't apply here. func emitVllmModel(b *strings.Builder, s Settings, row GgufRow, ov *Override, name string, be resolvedBackend, meta Metadata, emitted *[]string) { if !row.IsHF && isSplitGguf(row) { // Discovery represents a split set by shard 1 alone, which is all // llama.cpp needs — it opens the sibling shards itself. vllm does not: // it would load a fifth of the weights and fail somewhere downstream. // Emit the reason as a comment rather than a command that cannot work, // and leave the model out of `emitted` so nothing routes to it. fmt.Fprintf(b, "\n # %s: skipped (vllm cannot load split gguf shards; merge with llama-gguf-split --merge)\n", name) return } lines := vllmCmdLines(s, row, ov, name, be, meta) ctx, ctxNote := vllmMaxModelLen(s, ov, row, meta) footprint := vllmFootprintGB(s, ov, row, meta, ctx) util, utilNote := vllmGpuUtil(s, ov, footprint) format := "gguf" if row.IsHF { format = "safetensors" } kvNote := "kv=vllm-sized" if kv, ok := vllmKvCacheGB(ov, meta, ctx); ok { kvNote = fmt.Sprintf("kv=%.2fGB pinned", kv) } fmt.Fprintf(b, "\n # arch=%s size=%gGB (vllm, %s, gpu-util=%g [%s], %s, max-model-len=%d [%s])\n", meta.Architecture, row.SizeGB, format, round2(util), utilNote, kvNote, ctx, ctxNote) fmt.Fprintf(b, " %q:\n", name) b.WriteString(" cmd: >\n") for _, line := range lines { fmt.Fprintf(b, " %s\n", line) } fmt.Fprintf(b, " ttl: %d\n", s.TtlSec) // vllm allocates its whole pool (weights + KV) up front from // --gpu-memory-utilization, so its footprint is whatever that flag grants, // not a weights+KV sum: the router must charge what vllm will really hold. writeEstVram(b, vllmReservedGB(util, footprint)) if ov != nil && ov.Unlisted { b.WriteString(" unlisted: true\n") } writeDisplayName(b, s, name) // Advertised in /v1/models as function_calling, so a client can tell a // model that will take `tools` from one vLLM will 400 on. if parser, parserNote := vllmToolParser(ov, meta, name); parser != "" || vllmExtraSetsToolParser(ov) { b.WriteString(" capabilities:\n tools: true\n") } else { fmt.Fprintf(b, " # no tool-call parser (%s): vllm rejects requests carrying tools\n", parserNote) } *emitted = append(*emitted, name) } // vllmToolParsers maps an architecture (HF config.json model_type, or a gguf's // general.architecture) to the vLLM --tool-call-parser for its family's // tool-call format. Deliberately short: a wrong parser does not fail, it // silently leaves tool calls unparsed in the text, which is worse than the 400 // an unmapped model gets. Anything else is picked per model in the editor. var vllmToolParsers = map[string]string{ "qwen2": "hermes", "qwen2_moe": "hermes", "qwen2moe": "hermes", "qwen3": "hermes", "qwen3_moe": "hermes", "qwen3moe": "hermes", "qwen3_next": "hermes", "qwen3next": "hermes", // Qwen3.5 moved to the XML tool format Qwen3-Coder introduced. "qwen3_5": "qwen3_coder", "qwen3_5_moe": "qwen3_coder", "qwen35": "qwen3_coder", "qwen35moe": "qwen3_coder", "llama": "llama3_json", "mistral": "mistral", "mistral3": "mistral", "gemma4": "gemma4", "glm4_moe": "glm45", "glm4moe": "glm45", "deepseek_v3": "deepseek_v3", "granite": "granite", "granitemoe": "granite", "minimax_m2": "minimax_m2", "olmo3": "olmo3", } // vllmToolParser is the --tool-call-parser for a model and why: the override // when set ("none" turns it off), else the architecture's entry. Qwen3 coder // finetunes share model_type with the chat models but emit the XML format, so // the served name breaks that tie. func vllmToolParser(ov *Override, meta Metadata, name string) (parser, note string) { if ov != nil { switch p := strings.TrimSpace(ov.VllmToolParser); { case strings.EqualFold(p, "none"): return "", "turned off" case p != "": return p, "pinned" } } arch := strings.ToLower(strings.TrimSpace(meta.Architecture)) p, ok := vllmToolParsers[arch] if !ok { return "", fmt.Sprintf("no parser known for arch %q; pick one in the model editor", meta.Architecture) } if p == "hermes" && strings.HasPrefix(arch, "qwen3") && strings.Contains(strings.ToLower(name), "coder") { p = "qwen3_coder" } return p, "auto" } // vllmExtraSetsToolParser reports whether the model's hand-written extra args // already name a parser: the workaround before this knob existed. Their pick // stands, and a second copy of the flag in the launch preview reads as a bug. func vllmExtraSetsToolParser(ov *Override) bool { return ov != nil && strings.Contains(ov.ExtraArgs, "--tool-call-parser") } // isSplitGguf reports whether the row's file is one shard of a split set. // Discovery collapses a set to its first shard, so this is the only trace left // that the weights continue in sibling files. func isSplitGguf(row GgufRow) bool { return shardRe.MatchString(strings.ReplaceAll(row.FullPath, "\\", "/")) } // vllmBudgetGB is the VRAM this model may occupy: the per-model cap when set, // else the fleet budget. Same precedence the llama sizer uses. func vllmBudgetGB(s Settings, ov *Override) float64 { if ov != nil && ov.VramTargetGB > 0 { return ov.VramTargetGB } return s.TargetVramGB } // vllmMaxModelLen sizes --max-model-len against the VRAM budget instead of // handing vllm the model's full trained context. vllm allocates its KV pool up // front from --max-model-len, so a 262144-token trained window on a card that // can hold a fraction of it is not a large context — it is a refused or OOMing // startup. An explicit Override.Ctx always wins: a pinned window is the user // stating the requirement, and vllm failing loudly beats silently serving a // smaller one. // // The KV cost model is llama's, evaluated at f16 (vllm's KV is fp16 and paged; // paging changes fragmentation, not the per-token cost), so this is an // approximation of a different engine's allocator — deliberately conservative // rather than exact. note explains the choice for the YAML comment. func vllmMaxModelLen(s Settings, ov *Override, row GgufRow, meta Metadata) (ctx int, note string) { trained := int(meta.ContextLength) if ov != nil && ov.Ctx > 0 { return ov.Ctx, "pinned" } avail := vllmBudgetGB(s, ov) - row.SizeGB - vllmOverheadGB if avail <= 0 { // Weights alone overrun the budget. vllm has no CPU offload to fall back // on, so this model needs a bigger budget or a smaller quant; emit the // floor and say so rather than emitting a window that implies it fits. return RoundedCtx(0), "weights exceed the VRAM budget" } m := GetKvCostModel(meta, "f16", "f16") if !m.OK { ctx = vllmFallbackCtx note = "no KV cost model; conservative default" } else { ctx = RoundedCtx(float64(MaxCtxForBudget(avail, m.SlopeGB, m.ConstGB))) note = fmt.Sprintf("sized to %.1fGB free of budget", avail) } if trained > 0 && ctx > trained { ctx, note = trained, "trained context" } return ctx, note } // vllmFootprintGB is the VRAM this model should be granted: weights + the KV // pool for the chosen --max-model-len + the flat overhead, capped at the budget. // // vllm preallocates its whole --gpu-memory-utilization share and fills whatever // is left after the weights with KV blocks, so handing it the full budget makes // a small model hold the full budget. A 0.5B model with a 32k trained window // needs ~3GB but would have taken 22.8GB of a 24GB card, and the router (which // charges estVramGB) would evict everything else to admit it. When the window // was sized TO the budget this returns the budget anyway, since that is what the // window was computed to fill. No KV cost model => the budget: without one there // is nothing to size a smaller share from. // // With the KV pool pinned (vllmKvCacheGB) the cap is dropped: vllm will hold the // pinned pool however the window was sized, so the honest figure is the sum, and // a share rounded down to the budget would fail vllm's startup free-memory check // less often but charge the router less than the process takes. func vllmFootprintGB(s Settings, ov *Override, row GgufRow, meta Metadata, ctx int) float64 { if kv, ok := vllmKvCacheGB(ov, meta, ctx); ok { return row.SizeGB + kv + vllmOverheadGB } budget := vllmBudgetGB(s, ov) m := GetKvCostModel(meta, "f16", "f16") if !m.OK || ctx <= 0 { return budget } need := row.SizeGB + KvReserveGB(ctx, m.SlopeGB, m.ConstGB) + vllmOverheadGB return math.Min(need, budget) } // vllmKvMarginFrac and vllmKvMarginGB pad the KV pool handed to vllm over what // the llama KV cost model predicts for --max-model-len. vllm refuses to start // when the pool cannot hold one full-length sequence, and its paged layout // (16-token blocks, per-group page sizes on hybrid models) does not match the // llama model byte for byte. A few hundred MB of slack is cheap next to a model // that will not start. const ( vllmKvMarginFrac = 0.10 vllmKvMarginGB = 0.25 ) // vllmKvCacheGB is the KV pool to pin with --kv-cache-memory-bytes: what one // --max-model-len sequence needs, plus the margin above. // // Why pin it at all: without the flag vllm sizes KV as utilization x card minus // everything else it measured (weights, the profiling run's activation peak, // CUDA graphs, non-torch allocations). We can only guess that "everything else" // (vllmOverheadGB), and every GB guessed low comes straight out of KV until the // pool cannot hold one sequence and vllm refuses to start (issue #93: MiniCPM // needed a hand-raised utilization). With the pool pinned, vllm skips that // subtraction and allocates exactly this much KV; the overhead is whatever it // really is. --gpu-memory-utilization is still emitted beside it because vllm // checks free memory >= utilization x card at startup regardless of this flag. // // Not pinned (ok=false) when the user pinned a utilization (they asked vllm to // fill that share with KV, the pre-#93 behaviour and the escape hatch for a vllm // older than 0.11, which lacks the flag) or when there is no KV cost model or // window to size from. func vllmKvCacheGB(ov *Override, meta Metadata, ctx int) (gb float64, ok bool) { if ov != nil && ov.VllmGpuUtil > 0 { return 0, false } m := GetKvCostModel(meta, "f16", "f16") if !m.OK || ctx <= 0 { return 0, false } kv := KvReserveGB(ctx, m.SlopeGB, m.ConstGB) return kv*(1+vllmKvMarginFrac) + vllmKvMarginGB, true } // vllmExtraSetsMemory reports whether the hand-written extra args already size // vllm's memory, the workaround before the KV pin existed. Their numbers stand: // emitting ours too would let --kv-cache-memory-bytes silently override the // utilization the user raised to get the model to start. func vllmExtraSetsMemory(ov *Override) bool { return ov != nil && (strings.Contains(ov.ExtraArgs, "--gpu-memory-utilization") || strings.Contains(ov.ExtraArgs, "--kv-cache-memory")) } // vllmGpuUtil derives --gpu-memory-utilization from the model's footprint (see // vllmFootprintGB) and the card's real size. The old flat 0.90 was a fraction of // TOTAL memory regardless of what the budget said or what the desktop already // holds, so it both ignored a deliberately small budget and could exceed what is // actually free. Falls back to the flat default when the card's size can't be // read (headless, no GPU telemetry) - there is nothing to take a fraction of. // // Rounded UP to the 2 decimals the flag is printed with: rounding a derived // share to nearest can land under the footprint, and a model sized to fit // exactly then refuses to start for lack of KV blocks. func vllmGpuUtil(s Settings, ov *Override, footprintGB float64) (util float64, note string) { if ov != nil && ov.VllmGpuUtil > 0 { return ov.VllmGpuUtil, "pinned" } total, ok := cachedTotalVramGB() if !ok || total <= 0 { return vllmDefaultGpuUtil, "no GPU reading" } util = math.Ceil(footprintGB/total*100-1e-9) / 100 if util < vllmMinGpuUtil { util = vllmMinGpuUtil } if util > vllmMaxGpuUtil { util = vllmMaxGpuUtil } if budget := vllmBudgetGB(s, ov); footprintGB < budget { return util, fmt.Sprintf("%.1fGB needed (weights+KV+overhead) of %.1fGB card, under the %.1fGB budget", footprintGB, total, budget) } return util, fmt.Sprintf("%.1fGB budget of %.1fGB card", footprintGB, total) } // vllmReservedGB is what vllm will actually hold: the granted share of the // card when its size is known (so the floor clamp and a pinned util are // charged too), else the computed footprint. func vllmReservedGB(util, footprintGB float64) float64 { if total, ok := cachedTotalVramGB(); ok && total > 0 { return round2(util * total) } return footprintGB } // cachedTotalVramGB probes the card once per process. Sizing runs per model and // a GPU query is not free, and the card does not change size while we run. The // function variable is the seam tests use to supply a card without a GPU. var cachedTotalVramGB = sync.OnceValues(func() (float64, bool) { // currentProbePolicy, not a Settings: this probe is memoised for the process // and filled lazily during sizing, but ResolveGpuSet has always run by then // (both EnsureConfig and the estimate preview resolve the device set first). return SampleTotalVramGB(autoVramSampleTimeout, currentProbePolicy()) }) // round2 trims float noise out of a derived utilization so the emitted flag // reads 0.72, not 0.7200000000000001. func round2(f float64) float64 { return math.Round(f*100) / 100 }