package autogen import ( "fmt" "slices" "sort" "strings" ) // genMoeShare is Generate-Config.ps1's own expert-weight share table (line 190). // It differs from the planner's table (plan.go) by including qwen35moe; the // generation-side ctx sizing and the force-low-active-MoE recompute use this one. var genMoeShare = map[string]float64{ "gemma4": 0.88, "qwen3": 0.90, "qwen3moe": 0.90, "qwen35moe": 0.90, "llama": 0.80, "lfm2": 0.78, "lfm2moe": 0.78, } // genMoeShareFor mirrors Generate-Config's Get-MoeShare (default 0.85). func genMoeShareFor(arch string) float64 { if s, ok := genMoeShare[strings.ToLower(arch)]; ok { return s } return 0.85 } // profile is one emitted quartermaster entry for a model: the solo variant or an // optional ctx-tier variant. Target/Overhead are the VRAM budget the sizing math // uses; the flags drive ub/spec/reasoning. type profile struct { Name string Target float64 Overhead float64 Unlisted bool Ctx int // 0 = auto-size; >0 forces that ctx via the manual-cap path // CtxExact marks a Ctx that came from a custom launch pin (-c): it is the // window the process will run with, so the sizer must not round it to a // 4096 multiple (RoundedCtx) the way it rounds its own picks. CtxExact bool // IsLong marks a profile whose context is pinned at or above // longCtxThreshold (a ctx tier, a long named variant, or a long Ctx on the // model itself). It tops the VRAM budget's safety slack up to // longCtxHeadroomGB (see longCtxTarget) and drops -ub to 512 on a // small-VRAM card. IsLong bool // Per-variant overrides. Empty/zero => inherit the model-wide override. Set // only by named custom variants (Override.Variants); emitProfile and the // kv-cost sizing prefer these over the model-wide values. KvK, KvV string Spec string ReasoningFmt string Ub int // physical batch size override (0 => default) CpuOffload int // >0 pins layers offloaded to CPU, overriding the sizer // CpuOffloadSet distinguishes a pinned CpuOffload of 0 (every layer on GPU: // `-ngl ` on a dense model) from "no pin". The field's zero value is // otherwise indistinguishable from auto. CpuOffloadSet bool // CtxCheckpoints, when non-nil, emits --ctx-checkpoints N (0 disables the KV // prompt-prefix checkpoint cache). nil => inherit the model-wide value, else // the llama-server default (32). See effectiveCtxCheckpoints. CtxCheckpoints *int // DraftSlopeGB is the per-token VRAM (GB) the baked-in MTP drafter's own KV // cache adds on top of the main model's, at this profile's draft KV quant. // See mtpDraftSlopeGB: the drafter is a second llama_context over the same // model at the SAME n_ctx, so its cost scales with the window and has to be // part of the slope the sizer solves ctx against, not a flat overhead. DraftSlopeGB float64 // CheckpointMinStep, when > 0, is the resolved -cms (checkpoint spacing in // prompt tokens) for this profile. 0 => the arch default from // defaultCheckpointMinStep. Both the emitted flag and the VRAM reserve read // it, so they can't drift apart. CheckpointMinStep int // Variant, when non-nil, is the named-variant source this profile was built // from. Its engine knobs (kvInRam/flash/mmap/mlock/threads/parallel/extraArgs) // layer over the model-wide override at emit so a variant carries the full // launch shape. Solo/ctx-tier profiles leave this nil and use the override. Variant *VariantSpec // Vision marks a profile that loads the CLIP projector: emits --mmproj // and an image-input capabilities block so a client can send // images. EVERY profile of a model with a projector carries this, not just // the "-vision" twin — a served id that silently rejects images is a trap, // since llama-server can only load a projector at spawn time and answers an // image with "image input is not supported" for the life of the process. // The twin remains as the id whose projector is GPU-resident. Vision bool // MmprojGB is the projector footprint (weights + CLIP compute reserve) this // profile charges as VRAM overhead. Zero when CpuMmproj puts it in RAM. MmprojGB float64 // MmprojPin is the resolved projector placement for this profile, "gpu" or // "ram". Defaults differ by profile: the "-vision" twin is GPU-resident (it // exists to encode images fast), every other profile keeps it in RAM (vision // is the exception there, and the text side must not pay for it). The Default // tab's mmproj override sets the former, the "vision" variant's the latter. MmprojPin string // TensorSplit is the resolved per-device layer ratio for a multi-GPU plan, // one entry per eligible device in Index order, or nil for a single-GPU box // (and for every non-llama class, none of which can split). Derived by the // SIZER rather than the renderer because it depends on Overhead, which is // only final after the compute buffer is charged: see gpuset.go. TensorSplit []float64 // MainGpu is the device index --main-gpu pins the non-splittable buffers to. // -1 when there is nothing to pin (single GPU / no telemetry). MainGpu int // CpuMmproj emits --no-mmproj-offload: the CLIP projector runs on the CPU, so // it costs no VRAM and the profile keeps the ctx/layer placement it would // have had without vision, at the price of a slow (host-side) image encode. // That trade is why it is the default everywhere except the "-vision" twin: // the encode is a one-off per image, while VRAM spent on a projector is paid // by every token of every request, image or not. CpuMmproj bool } // Generate discovers models under gf.Settings.ModelsRoot and returns a complete // quartermaster config YAML. Port of Generate-Config.ps1. nowRFC is stamped into // the header comment (passed in so the function stays deterministic/testable). func Generate(gf GenerateFile, nowRFC string) (string, error) { s := gf.Settings rows, err := DiscoverGgufModelsMulti(s.RootList()) if err != nil { return "", err } sort.Slice(rows, func(i, j int) bool { if rows[i].ID != rows[j].ID { return rows[i].ID < rows[j].ID } return rows[i].Publisher < rows[j].Publisher }) // Name-based prompt-enhancer detection, over the same rows the emit loop is // about to walk: it pre-fills an image model that names no enhancer of its // own (see resolveEnhancerIDs). The other half of the same classifier keeps // enhancers OUT of the text-encoder pool, where a same-arch, bigger-file PE // otherwise outranks the real conditioner (see encoderpool.go). s.autoEnhancers = detectEnhancers(rows) // Qwen3-ASR's forced aligner is a companion found anywhere in the scanned // roots, not beside the ASR (see audiocppaligner.go). s.qwen3Aligner = detectQwen3Aligner(rows) var b strings.Builder fmt.Fprintf(&b, "# Quartermaster config - generated %s\n", nowRFC) fmt.Fprintf(&b, "# TargetVramGB=%g MaxRamGB=%g Threads=%d\n", s.TargetVramGB, s.MaxRamGB, s.Threads) b.WriteString("# Regen: Quartermaster startup (hash-gated)\n\n") fmt.Fprintf(&b, "healthCheckTimeout: %d\n\n", s.HealthCheckTimeout) emitVramBudget(&b, s) emitRamBudget(&b, s) emitSlotCache(&b, s.SlotCache) emitAPIKeys(&b, s.APIKeys) emitPromptEnhancers(&b, s.PromptEnhancers) b.WriteString("models:\n") var emitted []string var coexist coexistSets seen := map[string]bool{} for _, row := range rows { ov := ResolveOverride(row, gf.Overrides) if ov != nil && ov.Skip { continue } name := uniqueModelName(row, func(n string) bool { return seen[n] || slices.Contains(emitted, n) }) seen[name] = true // A single unparseable/misdetected gguf must not nuke the whole config // (and with it startup): note it in-band and skip, keeping every other // model servable. if err := emitModel(&b, s, gf, row, ov, name, &emitted, &coexist); err != nil { fmt.Fprintf(&b, "\n # SKIPPED %q: %v\n", name, err) continue } if row.IsSam { coexist.Sam = append(coexist.Sam, name) } } emitExtraImageModels(&b, s, gf.Overrides, seen, &emitted) emitExtraModels(&b, s, seen, &emitted) emitGroupsAndListeners(&b, s, emitted, coexist) return b.String(), nil } // uniqueModelName picks the served id for a discovered row: its own id, else // qualified by publisher, then by repo folder, then numbered. Every candidate is // re-checked, since copies of one quant under one publisher derive the same // publisher tag too, and a reused key makes the loader keep one block and drop // the other. Rows arrive sorted, so the same tree always yields the same names. func uniqueModelName(row GgufRow, taken func(string) bool) string { name := row.ID for _, tag := range []string{row.Publisher, row.Publisher + "-" + row.Repo} { if !taken(name) { return name } if t := slugify(tag); t != "" { name = row.ID + "-" + t } } base := name for n := 2; taken(name); n++ { name = fmt.Sprintf("%s-%d", base, n) } return name } // emitModel reads metadata once and emits every profile (solo, ctx tiers, game) // for one discovered gguf. func emitModel(b *strings.Builder, s Settings, gf GenerateFile, row GgufRow, ov *Override, name string, emitted *[]string, coexist *coexistSets) error { // SAM models are raw *.ggml with no gguf header — route them before the // metadata read (ReadGgufMetadata would fail). Served by sam3_server. if row.IsSam { emitSamModel(b, s, row, ov, name, emitted) return nil } // TRELLIS.2 packages are directories served by trellis2-server, so they route // before the metadata read for the same reason: there is no gguf to open. if row.IsTrellis { return emitTrellisModel(b, s, row, ov, name, emitted) } // HF folders (config.json + safetensors) are a directory with no gguf header, // and vllm is the only backend here that can load one. if row.IsHF { return emitHFModel(b, s, row, ov, name, emitted) } meta, err := ReadGgufMetadataCached(row.FullPath) if err != nil { return fmt.Errorf("%s: %w", name, err) } // Video-generation GGUFs go to sd-server as their own class, and are tested // BEFORE the image path: MiniMax-H3 ships with no metadata whatsoever, and // the only arch string that would route a video model here ("wan") belongs // to ERNIE-Image-Turbo, an image model. Detection is therefore structural // (see video.go), and it has to win over isImageArch. if vid := videoInfoFrom(meta); vid.is() { emitVideoModel(b, s, row, ov, name, effectiveImageArch(meta), vid, meta.CondHidden, emitted) if ov != nil { for _, v := range ov.Variants { if strings.TrimSpace(v.Name) == "" { continue } vov := mergeImageVariant(*ov, v) emitVideoModel(b, s, row, &vov, name+"-"+v.Name, effectiveImageArch(meta), vid, meta.CondHidden, emitted) } } return nil } // Diffusion GGUFs go to sd-server, not llama-server: no KV cache / -ngl // sizing applies. Detect by arch and emit a separate block. Unknown image // archs fall through to the llama path, where the YAML "# arch=..." comment // reveals the real arch name to add to imageArchs. imgArch := effectiveImageArch(meta) if isImageArch(imgArch) { emitImageModel(b, s, row, ov, name, imgArch, meta.HasBakedEncoders, meta.CondHidden, emitted) // Named variants are emitted as separate "-" sd-server // entries. Each inherits the model's component paths/placement and overrides // only its own generation preset. Fleet-wide defaultVariants are llama-shaped // (game/judge) and intentionally skipped for image models. if ov != nil { for _, v := range ov.Variants { if strings.TrimSpace(v.Name) == "" { continue } vov := mergeImageVariant(*ov, v) emitImageModel(b, s, row, &vov, name+"-"+v.Name, imgArch, meta.HasBakedEncoders, meta.CondHidden, emitted) } } return nil } // Decision models go to llama-server's /v1/systemone. Tested ahead of the // embedder check: one built on an encoder can bake a pooling_type, and the // --embeddings launch it would get there serves the wrong API. See decision.go. if IsDecisionModel(meta) { emitDecisionModel(b, s, row, ov, name, meta, emitted) if ov.InCpuPool() { coexist.CPU = append(coexist.CPU, name) } return nil } // Text embedders (BERT-family or any model baking a pooling_type) go to // llama-server with --embeddings: no KV/ctx-tier/spec sizing, and the // embedding capability buckets them out of the chat-able LLM catalog. if IsEmbeddingModel(meta) { emitEmbeddingModel(b, s, row, ov, name, meta, emitted) if ov.InCpuPool() { coexist.CPU = append(coexist.CPU, name) } return nil } // audio.cpp GGUFs go to audiocpp_server, and are tested ahead of both legacy // speech engines: the header states the family outright (general.architecture // = audiocpp), so there is nothing to disambiguate, and a qwentts or TTS.cpp // command built for one would fail at load. The emitter can decline (a music // family, or a conversion carrying no family id), in which case the model is // deliberately dropped from the config with a comment saying why - no other // engine here could serve it either. if IsAudioCppModel(meta) { emitAudioCppModel(b, s, row, ov, name, meta, emitted) return nil } // Speech GGUFs go to a TTS server (OpenAI /v1/audio/speech), not llama-server: // a Qwen3-TTS "talker" emits audio-codec tokens and loads with a paired codec // gguf (qwentts.cpp), a TTS.cpp export is self-contained. Detect ahead of the // LLM path — a talker is a small qwen3 LM that would otherwise route to chat. // emitTTSModel picks the engine from the registry (see ttsBackend). if IsTTSModel(meta, row.FileName) { emitTTSModel(b, s, row, ov, name, meta, emitted) // TTS.cpp runs on the CPU and is charged no VRAM, so it belongs in the // coexistence group rather than the exclusive one: speaking a reply must // not evict the chat model that produced it. qwentts is on the GPU and // stays exclusive. if kind, _ := ttsBackend(s, row, ov, meta); kind == ttsKindTTSCpp { coexist.TTS = append(coexist.TTS, name) } return nil } // Parakeet-family ASR GGUFs go to parakeet.cpp's parakeet-server (OpenAI // /v1/audio/transcriptions), not llama-server: transducer encoder+decoder, no // KV/ctx/spec sizing, and the audio->text capability buckets them out of the // chat-able LLM catalog. if IsASRModel(meta, row.FileName) { emitASRModel(b, s, row, ov, name, meta.Architecture, emitted) // Same reasoning as TTS.cpp above: parakeet runs on the CPU (20-36x // realtime) and is charged no VRAM, so dictating must not evict the chat // model the transcript is headed for. A GPU opt-in via extraArgs keeps // coexisting — the same accepted under-charge emitASRModel already makes // for estVramGB. coexist.ASR = append(coexist.ASR, name) return nil } // Everything left is launched as a chat LLM, which a non-causal encoder or a // classifier-headed file can never be: it attends both ways and ends in a // label head, so it has nothing to generate with. The decision and embedder // checks above already took the ones llama-server knows how to serve, so // this is a classifier converted without decision.type. Drop it with the // reason rather than list a chat model that answers garbage. if meta.NonCausal || meta.ClassifierHead { return fmt.Errorf("non-causal classifier (arch=%s) with no decision.type: llama-server cannot serve it as chat or on /v1/systemone; re-convert it as a decision model", meta.Architecture) } // Resolve which backend serves this LLM. A "vllm" (or other non-llama) kind // emits through its own path — different engine, different args, no KV/-ngl // sizing. A chosen "llama" build just swaps the exe threaded through the llama // path below. No registry match => keep the legacy ServerExe (zero behaviour // change for single-backend setups). be := resolveBackend(s, ov, "llm") if strings.EqualFold(be.Kind, "vllm") { emitVllmModel(b, s, row, ov, name, be, meta, emitted) return nil } if be.Exe != "" { s.ServerExe = be.Exe // this model's chosen llama build wins (local copy) } // Every profile this model emits (solo, ctx tiers, variants, vision twin) // runs on the same device, so a CPU-pool pin files them all in the cpu group. if ov.InCpuPool() { start := len(*emitted) defer func() { coexist.CPU = append(coexist.CPU, (*emitted)[start:]...) }() } // Custom launch arguments are sizing constraints, not only argv overrides: a // pinned -c/-ctk/-ub/... plans the launch (pins.go), so the emitted flags and // the baked estVramGB describe the process that will actually run. The text // is still appended verbatim by ComposeCmd and still wins at spawn; ov here // is a local copy, the caller's override is untouched. if ov != nil { if pins := PinsFromArgs(ov.CustomArgsText()); !pins.Empty() { pinned := pins.ApplyToOverride(*ov) ov = &pinned } } // Charge the draft sidecar only when this spec chain actually attaches it as // -md (matchedDraftSizeGB mirrors the emitter's kind gate). A dir pairing a // DFlash drafter to a model that runs on its baked-in MTP head must fall back // to the flat baked-in charge, not the drafter's weights. modelSpec := effectiveSpec(meta, ov, row.DraftKind) specOh := draftOverheadGB(modelSpec, matchedDraftSizeGB(modelSpec, row.DraftKind, row.DraftSizeGB)) // KV quant: f16 unless it can't buy denseMinCtx in this model's budget, in // which case q8_0 (see defaultKvQuant); settings.kvQuant pins it outright. // Per-model override still wins. Fast flash-attention needs matched K/V and // never iq4_nl. kvTarget := s.TargetVramGB if ov != nil && ov.VramTargetGB > 0 { kvTarget = ov.VramTargetGB } kvDef := defaultKvQuant(s, meta, kvTarget, s.VramOverheadGB+specOh) ovKvK, ovKvV := "", "" if ov != nil { ovKvK, ovKvV = ov.KvK, ov.KvV } kvK, kvV := resolveKvPair(ovKvK, ovKvV, kvDef, kvDef) kvModel := GetKvCostModel(meta, kvK, kvV) perTokGB := 0.0 kvConstGB := 0.0 if kvModel.OK { perTokGB = kvModel.SlopeGB kvConstGB = kvModel.ConstGB } modelMax := nativeCtx(meta) if ov != nil { // Rope scaling is the one knob that lifts the trained-length ceiling. modelMax = ropeCeiling(meta, ov.RopeScaling, ov.Ctx) } kvInRam := ov != nil && ov.KvInRam var ctxVariants []int override := Override{} if ov != nil { override = *ov ctxVariants = ov.CtxVariants } // A prompt enhancer is emitted as ONE profile shaped for its job (see // enhancerCtx): none of the fleet's default variants, no vision twin. peDir, _, isPE := promptEnhancerName(row.ID) // Build the profile set: solo + optional ctx tiers. // Per-model VRAM budget: an override caps it below the fleet default. soloTarget := s.TargetVramGB if override.VramTargetGB > 0 { soloTarget = override.VramTargetGB } profiles := []profile{{ Name: name, Target: soloTarget, Overhead: s.VramOverheadGB + specOh, Unlisted: override.Unlisted, Ctx: override.Ctx, // A model pinned to a long ctx is a long profile like any tier — it earns // the same budget headroom (longCtxTarget) and the same -ub treatment. An // auto ctx (0) is not: the sizer picks the window against the full budget. IsLong: override.Ctx >= longCtxThreshold, CpuOffload: override.CpuOffload, CtxCheckpoints: override.CtxCheckpoints, CheckpointMinStep: override.CheckpointMinStep, }} if isPE && profiles[0].Ctx == 0 { profiles[0].Ctx = enhancerCtx } for _, cv := range ctxVariants { profiles = append(profiles, profile{ Name: fmt.Sprintf("%s-%s", name, formatCtxTag(cv)), // Budget headroom for the long rungs is charged by longCtxTarget off // IsLong, so every long profile (and the editor preview) gets it. Target: s.TargetVramGB, Overhead: s.VramOverheadGB + specOh, Ctx: cv, IsLong: cv >= longCtxThreshold, CtxCheckpoints: override.CtxCheckpoints, CheckpointMinStep: override.CheckpointMinStep, }) } // Named custom variants: per-override ones plus the fleet-wide // settings.DefaultVariants, each emitting "-" with its own // ctx/VRAM/kv/spec; zero fields inherit. Spec affects the VRAM overhead, so // bake the per-variant overhead here. fleetVariants := s.DefaultVariants if isPE { // The fleet's tiers (32k/64k/game) are sized for chat models; on a // rewriter they are three more ids to pick wrong. Per-model variants are // the user asking, and stay. fleetVariants = nil } variantSpecs := append(append([]VariantSpec{}, override.Variants...), fleetVariants...) var visionSpec *VariantSpec for i := range variantSpecs { v := variantSpecs[i] if strings.TrimSpace(v.Name) == "" { continue } // "vision" is reserved: it tunes the auto-generated vision twin below in // place (ctx/vram/unlisted/kv/spec/…) instead of spawning its own profile. if strings.EqualFold(v.Name, "vision") { visionSpec = &variantSpecs[i] continue } vTarget := s.TargetVramGB if v.VramTargetGB > 0 { vTarget = v.VramTargetGB } // Variants inherit the model-wide spec chain unless they set their own, // so charge the draft overhead off the effective spec. vEffSpec := modelSpec if v.Spec != "" { vEffSpec = v.Spec } vSpecOh := draftOverheadGB(vEffSpec, matchedDraftSizeGB(vEffSpec, row.DraftKind, row.DraftSizeGB)) // Standalone: a blank ctx-checkpoints uses the generator default, not the // model-wide override value. vCheckpoints := v.CtxCheckpoints // Spacing DOES inherit the model-wide value (matching the effOv merge at // emit): a blank variant -cms must not diverge from the emitted flag. vMinStep := v.CheckpointMinStep if vMinStep == 0 { vMinStep = override.CheckpointMinStep } profiles = append(profiles, profile{ Name: fmt.Sprintf("%s-%s", name, slugify(v.Name)), Target: vTarget, Overhead: s.VramOverheadGB + vSpecOh, Unlisted: v.Unlisted, Ctx: v.Ctx, IsLong: v.Ctx >= longCtxThreshold, KvK: v.KvK, KvV: v.KvV, Spec: v.Spec, ReasoningFmt: v.ReasoningFmt, Ub: v.Ub, CpuOffload: v.CpuOffload, CtxCheckpoints: vCheckpoints, CheckpointMinStep: vMinStep, Variant: &variantSpecs[i], }) } // Vision twin: a model shipping a sibling mmproj projector gets an extra // "-vision" profile that loads it (--mmproj) and declares image input. Listed // by default (a distinct served id, scopeable by API keys); the config editor's // reserved "vision" variant can still mark it unlisted. Charged the projector's // file size as flat VRAM overhead. // "none" drops the twin outright: a projector the user does not want wired // (a family-inherited one they judge wrong for this finetune, or vision they // simply never use) should cost no served id at all. // The projector is whatever the override names, falling back to discovery. // An explicit path is also what CREATES the twin: a model that paired with // nothing has no twin at all today, and that is the whole point of the field // (one shared mmproj in its own folder, pointed at by several models). // The reserved "vision" variant may name its own, with the usual sentinel. mmprojFile := override.MmprojFile if v := visionSpec; v != nil { mmprojFile = inheritStr(v.MmprojFile, mmprojFile) } mmprojPath, mmprojSizeGB := mmprojFor(row, mmprojFile) modelPin := strings.ToLower(strings.TrimSpace(override.Mmproj)) // A t2i rewriter is never shown an image, so by default it wires no // projector at all; an explicit pin still wins. if isPE && peDir == PEDirText && modelPin == "" { modelPin = "none" } if mmprojPath != "" && modelPin != "none" { mmprojOh := MmprojVramGB(mmprojPath, mmprojSizeGB, s) // Every OTHER profile of this model loads the projector too, in RAM // (--no-mmproj-offload). A projector on the CPU costs no VRAM and so // changes neither the context window nor the layer placement these // profiles would have had without it — the only cost is a one-off encode // per image, on the requests that actually carry one. That makes "wire it // everywhere" strictly better than the alternative, which was a default id // that 500s on an image for the life of the process. The Default tab's // mmproj pin overrides the placement ("gpu" to pay VRAM for a fast encode // here too); "none" above opts the model out of vision entirely. defPin := modelPin if defPin == "" { defPin = "ram" // An i2i rewriter is handed an image on EVERY call, so the RAM-side // projector never banks its saving and pays a host encode each time // (measured: minutes per reference image). Its one profile holds the // projector in VRAM instead of leaving that to a twin. if isPE { defPin = "gpu" } } for i := range profiles { p := &profiles[i] p.Vision = true p.MmprojPin = defPin if defPin == "ram" { p.CpuMmproj = true } else { p.MmprojGB = mmprojOh p.Overhead += mmprojOh } } // The twin is the id that pays VRAM for the projector, so its default is // the opposite: GPU-resident, for an image encode an order of magnitude // faster than the host-side one. The reserved "vision" variant re-pins it. vp := profile{ Name: fmt.Sprintf("%s-vision", name), Target: soloTarget, Overhead: s.VramOverheadGB + specOh + mmprojOh, MmprojGB: mmprojOh, MmprojPin: "gpu", Ctx: override.Ctx, CheckpointMinStep: override.CheckpointMinStep, Vision: true, } // Small text window only for dedicated VL models (Qwen-VL, InternVL, …): // their whole purpose is image chat, so the maxed 32k ctx (~2.5 GB KV) is // wasted. A plain LLM that merely ships an mmproj sidecar keeps full context // on its vision twin (ctx 0 → sizer picks the budget max, same as solo). // A model or vision-variant Ctx override still wins below. if vp.Ctx == 0 && isVLModel(name, meta.Architecture) { vp.Ctx = s.VisionCtx } // A reserved "vision" variant (from the config editor) tunes the twin in // place; blank fields keep the auto defaults, so an untouched seed emits // the same profile as before. Engine knobs layer over the model override // at emit via Variant. if v := visionSpec; v != nil { if v.VramTargetGB > 0 { vp.Target = v.VramTargetGB } if v.Ctx > 0 { vp.Ctx = v.Ctx vp.IsLong = v.Ctx >= longCtxThreshold } // Spec changes the draft overhead; recharge off the variant's spec. if v.Spec != "" { vp.Overhead = s.VramOverheadGB + draftOverheadGB(v.Spec, matchedDraftSizeGB(v.Spec, row.DraftKind, row.DraftSizeGB)) + mmprojOh } vp.Unlisted = v.Unlisted vp.KvK = v.KvK vp.KvV = v.KvV vp.Spec = v.Spec vp.ReasoningFmt = v.ReasoningFmt vp.Ub = v.Ub vp.CpuOffload = v.CpuOffload vp.CtxCheckpoints = v.CtxCheckpoints if v.CheckpointMinStep > 0 { vp.CheckpointMinStep = v.CheckpointMinStep } vp.Variant = v // The vision variant IS the twin, so it is the only thing that repins // the twin's projector; blank keeps the GPU default above. The Default // tab's pin governs the other profiles and never reaches here. if p := strings.ToLower(strings.TrimSpace(v.Mmproj)); p != "" { vp.MmprojPin = p } } // A "ram" pin is applied here rather than at construction so the variant // re-charge above (which rebuilds Overhead from scratch for a vision // variant with its own spec) can't hand the projector back. if vp.MmprojPin == "ram" { vp.Overhead -= mmprojOh vp.MmprojGB = 0 vp.CpuMmproj = true } // "none" from the vision variant lands here rather than at the gate above, // which only sees the model-wide value; either way no twin is emitted. if vp.MmprojPin != "none" && !isPE { profiles = append(profiles, vp) } } // Fold the pins of the text that applies to each profile into its sizing // inputs: a named variant's own text sizes that variant, and a ctx tier's own // window sizes that tier. Runs before the compute-buffer pass below, which // reads the profile's -ub (and before sizing, which reads ctx/placement). for i := range profiles { pinsForProfile(&profiles[i], override, meta) } // Charge the GPU compute buffer (logits + activations + the GPU runtime // constant) per profile; it scales with the physical batch and lives on the // GPU regardless of CPU expert offload, so it's flat VRAM overhead. Replaces // the old flat 0.17 GB ubSoloOh fudge. A baked-in MTP drafter runs a second // llama_context and is charged its own graph on top, at the same ub. gpus := s.GpuSetOrEmpty() for i := range profiles { ub := effectiveUb(meta, profiles[i], ov, profiles[i].Ctx, s.TargetVramGB) pspec := modelSpec if profiles[i].Spec != "" { pspec = profiles[i].Spec } pDraftGB := matchedDraftSizeGB(pspec, row.DraftKind, row.DraftSizeGB) profiles[i].Overhead += computeBufferGB(meta, ub, s.ComputeBufFactor) + mtpDraftComputeGB(meta, pspec, pDraftGB, ub, s.ComputeBufFactor) // Every device past the main one pays its own runtime context, and none // of it splits, so it is overhead against the POOLED budget, charged // before the split ratio is derived from what is left. Order matters: // PlanTensorSplit reads the FINISHED Overhead as the main device's fixed // cost, which is what makes the pooled budget reachable without any one // card going over (gpuset.go), so it has to run after the compute-buffer // and drafter-graph charges above. profiles[i].MainGpu = -1 if gpus.Multi() { // The main device's fixed cost is the overhead as it stands HERE. // ExtraDeviceOverheadGB is the OTHER devices' runtime contexts, which // splitBy already charges to each of them individually; folding it in // as well billed the secondary card's runtime twice, once to the main // card's side of the ratio, and pushed layers off the main GPU onto a // card that had no room for them. mainFixedGB := profiles[i].Overhead profiles[i].Overhead += gpus.ExtraDeviceOverheadGB() profiles[i].TensorSplit = gpus.PlanTensorSplit(mainFixedGB) if profiles[i].TensorSplit != nil { profiles[i].MainGpu = gpus.PlanMainIndex() } } } for _, prof := range profiles { // Per-variant kv quant changes the KV cost model, so resolve effective // kv and re-derive the cost slope/const for this profile (defaults to the // model-wide values when the variant doesn't override kv). // Variants inherit the model-wide kv quant; a variant's own kv wins. // A variant that pins only one side mirrors it; only a genuine mismatch // falls back to the model-wide quant. ekvK, ekvV := resolveKvPair(prof.KvK, prof.KvV, kvK, kvV) ptg, kcg := perTokGB, kvConstGB if ekvK != kvK || ekvV != kvV { if m := GetKvCostModel(meta, ekvK, ekvV); m.OK { ptg, kcg = m.SlopeGB, m.ConstGB } } // A variant may force KV in/out of RAM; otherwise inherit the model-wide // setting. Affects sizing, so resolve before sizeProfile. pkvInRam := kvInRam if prof.Variant != nil && prof.Variant.KvInRam { pkvInRam = true } // A variant can turn rope scaling on (or ask for a bigger ctx) by itself, so // the trained-length ceiling is resolved per profile, not once per model. pModelMax := modelMax if v := prof.Variant; v != nil { rs := v.RopeScaling if rs == "" { rs = override.RopeScaling } if c := ropeCeiling(meta, rs, prof.Ctx); c > pModelMax { pModelMax = c } } // Multi-slot: --kv-unified gives every slot one shared KV pool, so N slots // each holding a full-size conversation cost N x the KV. Charge that here — // sizing then yields a PER-SLOT ctx that N of can actually fit (buildCmdLines // multiplies it back out for -c), and every downstream number (kvReserve, // checkpoint reserve, estVramGB) describes the whole pool. Sizing against the // unscaled cost instead would hand slot 1 the whole card and let slots 2..N // evict it mid-conversation. pSlots := profileParallel(prof, override) if pSlots > 1 { ptg *= float64(pSlots) kcg *= float64(pSlots) } // A baked-in MTP head runs as a second context over the same model at the // same window, so its KV is a per-token cost on top of ptg, not a flat // overhead (draftOverheadGB now only charges its compute pad). Resolve it // per profile: a variant can carry its own spec chain and its own kv quant, // and -ctkd follows the effective main quant. pspec := modelSpec if prof.Spec != "" { pspec = prof.Spec } pdKvK, pdKvV := draftKvPair(override.KvKDraft, override.KvVDraft, ekvK, ekvV) prof.DraftSlopeGB = mtpDraftSlopeFor(meta, pspec, pdKvK, pdKvV, matchedDraftSizeGB(pspec, row.DraftKind, row.DraftSizeGB)) if pSlots > 1 { prof.DraftSlopeGB *= float64(pSlots) } ctx, plan, kvReserve, planCkptGB, err := sizeProfile(meta, s, prof, ptg, kcg, pModelMax, pkvInRam) if err != nil { return err } ngl, ncpuMoe := forceLowActiveMoE(meta, plan, prof, kvReserve) if prof.CpuOffload > 0 || prof.CpuOffloadSet { ngl, ncpuMoe = applyForcedOffload(meta, prof.CpuOffload) } // A CPU-device pin: nothing on the GPU, so the whole footprint the // offload model prices for ngl 0 (its VRAM half is the compute buffer // and overhead, which now live in RAM too) is charged to RAM. cpuDev := override.CpuDevice() if cpuDev { ngl, ncpuMoe = 0, 0 } // Re-price whenever the emitted placement is not the one GetLoadPlan // costed, so the baked estVramGB/estRamGB describe the flags we actually // write (forceLowActiveMoE rewrites them without touching the estimate). if ngl != plan.Ngl || ncpuMoe != plan.NCpuMoe || cpuDev { plan.EstVramGB, plan.EstRamGB = estForOffload(meta, prof, kvReserve, planCkptGB, ngl, ncpuMoe) if cpuDev { plan.EstVramGB, plan.EstRamGB = 0, plan.EstVramGB+plan.EstRamGB } plan.RamExceeded = s.MaxRamGB > 0 && plan.EstRamGB > s.MaxRamGB } // Per-variant spec/reasoning override the model-wide values for emit. // Variants layer their engine knobs OVER the model-wide override (inherit // at generate time: spec/draft chain, kv quant, reasoning budget, etc. flow // down); a variant's own non-blank field still wins, and sidecar edits keep // drifting per-variant freely. effOv := override if prof.Spec != "" { effOv.Spec = prof.Spec } if prof.ReasoningFmt != "" { effOv.ReasoningFmt = prof.ReasoningFmt } // A named variant carries the full launch shape: layer its engine knobs // over the model-wide override (zero/empty => inherit the override value). if v := prof.Variant; v != nil { // Inherit the model's preserve-thinking; an explicit variant value wins. // Both are *bool with the same nil = on meaning, so this is a pointer // assignment, not a deref: a nil variant value must leave the model's // own setting (which may be an explicit false) alone. if v.PreserveThinking != nil { effOv.PreserveThinking = v.PreserveThinking } if v.SlotCache != nil { effOv.SlotCache = v.SlotCache } if v.FlashAttn != "" { effOv.FlashAttn = v.FlashAttn } if v.Mmap != "" { effOv.Mmap = v.Mmap } if v.Mlock { effOv.Mlock = true } if v.Threads > 0 { effOv.Threads = v.Threads } if v.Parallel > 0 { effOv.Parallel = v.Parallel } // Free-form string knobs (chat template, extra args, tensor // placement, draft KV) resolve through the shared sentinel rule: // empty inherits, "none" forces the knob off. See inherit.go. mergeInheritStrings(&effOv, v) // Sampler / speculative sub-knobs: non-zero/non-nil variant value wins. if v.Dry != nil { effOv.Dry = v.Dry } if v.DryMultiplier != 0 { effOv.DryMultiplier = v.DryMultiplier } if v.DryBase != 0 { effOv.DryBase = v.DryBase } if v.DryAllowedLength != 0 { effOv.DryAllowedLength = v.DryAllowedLength } // Sampler defaults: nil => inherit, non-nil wins INCLUDING an explicit // 0 (greedy temp / min-p off are real settings, not "unset"). if v.Temp != nil { effOv.Temp = v.Temp } if v.TopK != nil { effOv.TopK = v.TopK } if v.TopP != nil { effOv.TopP = v.TopP } if v.MinP != nil { effOv.MinP = v.MinP } if v.PresencePenalty != nil { effOv.PresencePenalty = v.PresencePenalty } if v.SpecDraftNMax != 0 { effOv.SpecDraftNMax = v.SpecDraftNMax } if v.SpecDefault { effOv.SpecDefault = true } if v.SpecNgramSizeN != 0 { effOv.SpecNgramSizeN = v.SpecNgramSizeN } if v.SpecNgramSizeM != 0 { effOv.SpecNgramSizeM = v.SpecNgramSizeM } if v.SpecNgramMinHits != 0 { effOv.SpecNgramMinHits = v.SpecNgramMinHits } // Advanced knobs: variant's own non-zero/non-empty value wins. if v.ThreadsBatch != 0 { effOv.ThreadsBatch = v.ThreadsBatch } if v.Prio != 0 { effOv.Prio = v.Prio } if v.DirectIo { effOv.DirectIo = true } if v.NoOpOffload { effOv.NoOpOffload = true } if v.NoRepack { effOv.NoRepack = true } if v.CacheReuse != 0 { effOv.CacheReuse = v.CacheReuse } if v.CacheRamMB != 0 { effOv.CacheRamMB = v.CacheRamMB } if v.CacheIdleSlots != "" { effOv.CacheIdleSlots = v.CacheIdleSlots } if v.SwaFull { effOv.SwaFull = true } if v.CheckpointMinStep != 0 { effOv.CheckpointMinStep = v.CheckpointMinStep } if v.ContextShift != "" { effOv.ContextShift = v.ContextShift } if v.SpecDraftNMin != 0 { effOv.SpecDraftNMin = v.SpecDraftNMin } if v.SlotPromptSimilarity != 0 { effOv.SlotPromptSimilarity = v.SlotPromptSimilarity } if v.RopeScaling != "" { effOv.RopeScaling = v.RopeScaling } if v.RopeScale != 0 { effOv.RopeScale = v.RopeScale } if v.RopeFreqBase != 0 { effOv.RopeFreqBase = v.RopeFreqBase } if v.YarnOrigCtx != 0 { effOv.YarnOrigCtx = v.YarnOrigCtx } if v.SplitMode != "" { effOv.SplitMode = v.SplitMode } if v.MainGpu != 0 { effOv.MainGpu = v.MainGpu } } emitProfile(b, s, meta, row, prof, ctx, ngl, ncpuMoe, plan, ekvK, ekvV, pkvInRam, &effOv) *emitted = append(*emitted, prof.Name) } return nil }