package autogen import ( "fmt" "os" "path/filepath" "reflect" "regexp" "strings" "github.com/quartermaster-labs/quartermaster/internal/config" "gopkg.in/yaml.v3" ) // GenerateFile is the external autogen control surface: tuning knobs (Settings) // plus the per-model Overrides table. It replaces the PowerShell // Generate-Config.ps1 parameters + $Overrides array so models can be tuned // without recompiling. type GenerateFile struct { Settings Settings `yaml:"settings"` Overrides []Override `yaml:"overrides"` } // Settings are the global generation knobs. Defaults mirror the PowerShell // Generate-Config.ps1 parameter defaults; applyDefaults fills any zero value. type Settings struct { ModelsRoot string `yaml:"modelsRoot"` // CategoryRoots are optional per-UI-category extra scan folders (one key per // CategoryOrder entry -> path). Discovery scans the union of // ModelsRoot + these; capability detection still decides each model's // category/engine, so a root is just additional scan scope (organizational), // not a hard tag. Empty/absent => only ModelsRoot is scanned. CategoryRoots map[string]string `yaml:"categoryRoots"` ServerExe string `yaml:"serverExe"` // SdServerExe runs all-in-one diffusion GGUFs (stable-diffusion.cpp's // sd-server). Empty => a sibling "sd-server" of ServerExe, else bare on PATH. SdServerExe string `yaml:"sdServerExe"` // TtsServerExe runs Qwen3-TTS talker GGUFs (qwentts.cpp's tts-server). Empty => // a sibling "tts-server" of ServerExe, else bare on PATH. TtsServerExe string `yaml:"ttsServerExe"` // AsrServerExe runs Parakeet-family speech-to-text GGUFs (parakeet.cpp's // parakeet-server). Empty => a sibling "parakeet-server" of ServerExe, else // bare on PATH. AsrServerExe string `yaml:"asrServerExe"` // Backends is the dashboard's full backend registry (llama / vllm / sd / tts / // asr / custom entries). Populated from the UI sidecar; a model resolves its // backend against this via resolveBackend. The legacy ServerExe/SdServerExe/ // TtsServerExe/AsrServerExe above stay as the fallback when no registry entry // matches a model's class. Backends []BackendEntry `yaml:"backends"` // KvQuant pins the fleet-wide default KV cache type (-ctk/-ctv) for every // llama-backed LLM: f32 | f16 | bf16 | q8_0 | q5_1 | q5_0 | q4_1 | q4_0. // Empty (default) => auto, which prefers f16 and steps down to q8_0 only when // f16 can't reach denseMinCtx inside the VRAM budget (see defaultKvQuant). A // per-model Override.KvK/KvV still wins over either. An unknown value is // ignored (falls back to auto). KvQuant string `yaml:"kvQuant"` TargetVramGB float64 `yaml:"targetVramGB"` // MultiResident emits the `vramBudgetGB` config block and a per-model // `estVramGB`, which switch the router from "every group is exclusive, any // load evicts everything else" to VRAM-budget admission: models stay // co-resident as long as their summed estimates fit TargetVramGB. nil => // default ON. Set false to keep the legacy one-model-at-a-time behaviour // (the per-model estimate is still emitted; only the budget is withheld). MultiResident *bool `yaml:"multiResident"` // MinGpuFraction is the admission floor for the spawn-time sizer: the least // of a model's weight that must land on the GPU before a load is REFUSED // instead of silently degraded to CPU. It exists because multi-resident // loading otherwise always "succeeds" — the sizer just offloads the new // model until it fits the leftovers, and every model ends up crawling. Only // applies when the model's own baked plan was above the floor, so a model // deliberately configured to run mostly on CPU still loads. 0 => default // 0.5; negative disables the floor (pure best-effort degradation). MinGpuFraction float64 `yaml:"minGpuFraction"` // OomGuardReserveGB is extra VRAM held back from the resident model set for // the OTHER GPU clients' growth. The live ceiling the router admits against // is (vramBudgetGB - foreign usage ABOVE the idle baseline - this), so the // reserve is charged only while a foreign client is actually growing: a game // that has just claimed 8 GB is still allocating, and admitting a model into // the exact leftovers means the next allocation of either side lands in // shared memory. An untroubled box is charged nothing. 0 => default 1.0; // negative disables the reserve (admit into the raw leftovers). OomGuardReserveGB float64 `yaml:"oomGuardReserveGB"` // OomGuardEvict enables the post-load watchdog: when foreign VRAM grows into // the resident set's footprint and stays there for OomGuardGraceSec, IDLE // models are unloaded until the set fits again. Without it the only guard is // at spawn, and a model already resident when a game starts is silently // demoted into shared memory by the driver - no error, just collapsed // throughput. nil => default ON. Set false to leave resident models alone and // accept the degradation. OomGuardEvict *bool `yaml:"oomGuardEvict"` // OomGuardGraceSec is how long the resident set must be over the live ceiling // before the watchdog sheds anything. VRAM pressure is spiky - a shader // compile, a video decode, a browser painting - and unloading a model on a // transient spike costs a full reload for nothing. 0 => default 30. OomGuardGraceSec int `yaml:"oomGuardGraceSec"` // AutoVram measures free VRAM at gen time and caps TargetVramGB at it (the raw // reading — VramOverheadGB is charged inside the plan, not deducted here). Only // ever tightens: a static TargetVramGB below the reading stays a user ceiling. // The reading used is the idle high-water mark, not the instantaneous sample, // so re-resolving while a model is resident can't budget into its leftovers. AutoVram bool `yaml:"autoVram"` VramOverheadGB float64 `yaml:"vramOverheadGB"` // MultiGpu pools every eligible adapter into one budget and emits // `-sm layer` / `--main-gpu` / `--tensor-split` so a model splits across the // cards instead of being sized against the largest one and leaving the rest // unused. nil => default ON; it only does anything on a box with more than // one eligible device. Set false to pin every plan to the single card with // the most free VRAM (the pre-multi-GPU behaviour). MultiGpu *bool `yaml:"multiGpu"` // MinGpuVramGB is the eligibility floor: an adapter whose total capacity is // below it is not an inference target. It is applied AFTER an integrated // GPU's shared system memory is counted, so an APU's BIOS carveout alone no // longer decides the question (issue #37). Raise it above a device's total to // keep that device out of the budget entirely. 0 => default 3.0. MinGpuVramGB float64 `yaml:"minGpuVramGB"` // SharedMemory decides whether the system-memory pool a GPU can address // (AMD's GTT, which rocm-smi reports as "GTT Total Memory") counts toward the // VRAM budget: "auto" (default) counts it for a device that looks like an // APU, "on" for every device that reports one, "off" for none. Empty means // auto. The telemetry cannot answer the question by itself — a discrete card // reports a host aperture of the same shape — so "off" is also the escape // hatch when auto guesses wrong. SharedMemory string `yaml:"sharedMemory"` // PoolIntegratedGpu allows an integrated GPU to be a --tensor-split target // beside a dedicated card. nil => default false: the APU still gets the whole // budget when it is the only GPU, but real layers are not split onto one that // borrows the CPU's memory, which is slower than not splitting at all. Set it // true only on a box where that is genuinely wanted. PoolIntegratedGpu *bool `yaml:"poolIntegratedGpu"` // Gpus is the RESOLVED eligible device set for this generate pass, filled by // ResolveGpuSet from live telemetry. Never serialised: it describes the box, // not the user's intent, and a stale copy in a config file would size a plan // for hardware that is no longer there. Gpus GpuSet `yaml:"-"` // ComputeBufFactor scales the modeled compute buffer (logits + activations). // 1.0 = the analytic estimate; tune against the "compute buffer size" llama // prints at load if your build/arch differs. 0 => default 1.0. Measure PEAK // (mid-generation dedicated + SHARED usage), never an idle process — see the // warning above computeBufferGB before lowering this on Vulkan/ROCm. ComputeBufFactor float64 `yaml:"computeBufFactor"` // VisionOverheadGB is the VRAM reserved for a "-vision" twin's CLIP/vision // compute buffer (image-token activations + patch-embed work), on TOP of the // projector's own weights (its gguf size). llama.cpp allocates this when an // image is processed and it is NOT captured by the text compute-buffer model. // Tune against the "CLIP ... compute buffer size" llama prints for a real // image. 0 => default 1.0. ponytail: flat reserve, not resolution-scaled — // one image at a time here; make it image-size-aware if batched vision lands. VisionOverheadGB float64 `yaml:"visionOverheadGB"` // VisionCtx is the default context window (tokens) for the auto-generated // "-vision" twin. Image chats need a small text window — one image is a few // hundred-to-thousand tokens plus a short prompt — so the twin doesn't inherit // the solo model's maxed 32k ctx (that KV is ~2.5 GB on an 8B and buys nothing // for vision). A per-model/variant Ctx override still wins. 0 => default 8192. VisionCtx int `yaml:"visionCtx"` // Groups optionally split the emitted models across named groups bound to // separate listen addresses (use-case agnostic: membership is by model-name // glob, first match wins). Empty => one group, one port (upstream default). Groups []GroupSpec `yaml:"groups"` MaxRamGB float64 `yaml:"maxRamGB"` MoeCtxTarget int `yaml:"moeCtxTarget"` // DefaultVariants are named custom variants emitted for EVERY non-skip model, // in addition to any per-override variants (use-case agnostic). Lets a config // apply a fleet-wide spawn shape (e.g. a low-VRAM coexistence variant) without // repeating it on each row. Same VariantSpec semantics as Override.Variants. DefaultVariants []VariantSpec `yaml:"defaultVariants"` // DisplayNames renames a model's advertised id WITHOUT touching the real // served id (the config key that keys cmd/gguf/slotcache/groups). Keyed by // BASE served id (row.ID); the new name cascades to every derived variant id // via a prefix swap (see resolvePublicName), so "foo"->"bar" also renames // "foo-mtp" to "bar-mtp". Each renamed id emits config name: + a routable // alias; the real id still resolves. UI-owned (sidecar), regen-safe. DisplayNames map[string]string `yaml:"displayNames"` DenseCtxLadder []int `yaml:"denseCtxLadder"` DenseMinCtx int `yaml:"denseMinCtx"` Threads int `yaml:"threads"` TtlSec int `yaml:"ttlSec"` HealthCheckTimeout int `yaml:"healthCheckTimeout"` // DryDefault is the fleet-wide DRY-sampler default. nil => off; set true to // make DRY on by default (a model still flips it via Override.Dry). DryDefault *bool `yaml:"dryDefault"` // APIKeys are the UI-managed API keys emitted into the generated config's // apiKeys / apiKeyModels blocks. Owned by the web UI (sidecar), not the // hand-authored generate file. Empty Models on an entry => full access. APIKeys []APIKeyEntry `yaml:"apiKeys"` // SlotCache enables on-disk slot KV persistence: adds --slot-save-path to each // llama-server cmd and emits the matching slotCache config block the server // reads. Off unless Enable. SlotCache SlotCacheSettings `yaml:"slotCache"` // Encoders declares the shared diffusion component files (VAE / CLIP / T5 / // text-encoder) ONCE, so image models get them auto-attached by architecture // instead of a per-model override each. A bare `--diffusion-model` GGUF carries // none of these; archComponents (image.go) maps each family to the fields it // needs. A per-model Override component path still wins over the pool. Encoders EncoderSet `yaml:"encoders"` // PromptEnhancers are the rewrite models an image model may hand its prompt // to before rendering (Qwen ships one per direction: PE-I2I for edits, PE-T2I // for text-to-image). Declared ONCE here, keyed by the catalog model id, // because the thing that makes an enhancer usable is the fixed system prompt // it was trained against: it is a property of the enhancer, not of the image // model, and pasting it into every image model's override would be the same // string twenty times. An image model names one via Override.PromptEnhancer. // // The model id is NOT validated here. Generation must not fail because an // enhancer was renamed or its gguf removed - the entry simply resolves to // nothing and the image model renders without one, which is the pre-feature // behaviour. PromptEnhancers []PromptEnhancer `yaml:"promptEnhancers"` // LoraDir is the fleet-wide `--lora-model-dir` for image models: the folder // sd-server scans for LoRA .safetensors, which is what `/sdapi/v1/loras` // lists and what a request's `lora: [{path,multiplier}]` refs resolve against. // Empty => each image model defaults to its OWN gguf's directory, so dropping // a LoRA next to the checkpoint it was trained for is zero-config. A per-model // Override.LoraDir still wins over both. LoraDir string `yaml:"loraDir"` // LoraDirs narrows LoraDir per UI category (category id -> path), for the // common case where each kind of LoRA lives in its own tree (a ComfyUI // install keeps models/loras and its video LoRAs apart, and a Wan LoRA is // not loadable by an SDXL checkpoint anyway). Three keys mean something: // // "image", "video" — the sd-server `--lora-model-dir`, whole-directory: // the backend lists everything in it and a REQUEST picks which to apply. // "llm" — the folder a model's Override.Loras entries resolve // bare filenames against. llama-server has no directory flag at all // (`--lora FNAME` names one adapter file, baked in at spawn), so this // key is a browse root and a base path, NOT a flag. On its own it emits // nothing; it only has an effect once a model lists adapters. // // Resolution order is Override.LoraDir -> LoraDirs[category] -> LoraDir -> // the model gguf's own directory, so LoraDir stays the fleet-wide fallback // for anyone who keeps one tree. Keys are CategoryOrder ids. LoraDirs map[string]string `yaml:"loraDirs"` // ExtraImageModels are sd-server image models that autogen's gguf scan can't // discover or header-parse — chiefly single-file .safetensors DiTs whose weights // exceed the gguf 4-dim tensor cap (HiDream-O1's 5-D vision patch-embed can't be // stored in gguf at all, only safetensors). Each entry emits a verbatim sd-server // block from explicit paths; no VRAM planner or arch detection runs. Persisted // here (not a runtime hack) so a regen keeps them. ExtraImageModels []ExtraImageModel `yaml:"extraImageModels"` // ExtraModels are hand-written config.yaml model blocks (name -> block), // emitted as written so a server autogen has no backend kind for (any // OpenAI-compatible engine) survives a regen. Kept as a yaml.Node so the // user's key order and scalar styles round-trip. Generate-file only, never // the sidecar: a UI-editable block is an arbitrary command line. See // extramodels.go. ExtraModels yaml.Node `yaml:"extraModels" json:"-"` // autoEnhancers is the name-based prompt-enhancer pairing table built by // Generate from the rows it just discovered (see promptenhancer.go), read by // the image emitters to pre-fill a model that names no enhancer of its own. // // Unexported on purpose: it is DERIVED from what is on disk this run, so it // must never round-trip through the settings file a user edits. A Settings // built anywhere else (the UI, a test) simply has none, and auto-pairing is // then off rather than stale. autoEnhancers autoEnhancers // qwen3Aligner is the Qwen3 ForcedAligner gguf discovery found this run, // the word-timestamp companion of every qwen3_asr entry. Derived and // unexported for the same reason as autoEnhancers. See audiocppaligner.go. qwen3Aligner *audioCppAligner } // ExtraImageModel is one hand-declared sd-server image model (see // Settings.ExtraImageModels). Only ModelPath is required. ModelFlag defaults to // "-m" (all-in-one checkpoint, the sd.cpp version-detect entry). The component // paths are wired verbatim — nothing is arch-inferred — so a self-contained DiT // that still needs an external VAE (HiDream-O1 bakes its text model but no VAE) // just sets VaePath. Placement tri-states mirror Override ("" => on). type ExtraImageModel struct { Name string `yaml:"name"` ModelPath string `yaml:"modelPath"` ModelFlag string `yaml:"modelFlag"` // "" => "-m"; or "--diffusion-model" VaePath string `yaml:"vaePath"` // --vae LlmPath string `yaml:"llmPath"` // --llm ClipLPath string `yaml:"clipLPath"` // --clip_l ClipGPath string `yaml:"clipGPath"` // --clip_g T5Path string `yaml:"t5Path"` // --t5xxl LoraDir string `yaml:"loraDir"` // --lora-model-dir ("" => settings.loraDir, else the model's own dir) // PromptEnhancer names the settings.promptEnhancers entry this model sends // its prompt through before rendering. "" => none. See Override.PromptEnhancer. PromptEnhancer string `yaml:"promptEnhancer"` // PromptEnhancerEdit is the img2img half of the pair. See // Override.PromptEnhancerEdit. PromptEnhancerEdit string `yaml:"promptEnhancerEdit"` // PromptEnhancerPrompt / PromptEnhancerEditPrompt are this model's own system // prompts for the two directions. See Override.PromptEnhancerPrompt. PromptEnhancerPrompt string `yaml:"promptEnhancerPrompt"` PromptEnhancerEditPrompt string `yaml:"promptEnhancerEditPrompt"` VramTargetGB float64 `yaml:"vramTargetGB"` DefaultCfg float64 `yaml:"defaultCfg"` DefaultSteps int `yaml:"defaultSteps"` DefaultSampler string `yaml:"defaultSampler"` // --sampling-method DefaultWidth int `yaml:"defaultWidth"` DefaultHeight int `yaml:"defaultHeight"` DiffusionFa string `yaml:"diffusionFa"` // "" => on, "off" => off VaeTiling string `yaml:"vaeTiling"` // "" => on, "off" => off // Video VRAM levers. "" means OFF here, NOT auto: an extra model is // hand-declared and never goes through the gguf tensor scan, so nothing on // this path can tell a video DiT from an image one. Set "on" explicitly. // (The discovered path defaults both ON for video models: see Override.) TemporalTiling string `yaml:"temporalTiling"` // "on" => --temporal-tiling StreamLayers string `yaml:"streamLayers"` // "on" => --stream-layers (ignored unless offloaded) TeOnCpu string `yaml:"teOnCpu"` // "" => on (te=cpu), "off" => keep on GPU VaeOnCpu string `yaml:"vaeOnCpu"` // "on" => add vae=cpu to --backend; "" => GPU OffloadToCpu string `yaml:"offloadToCpu"` // "on" => --offload-to-cpu (+ vae=cpu in --backend) Threads int `yaml:"threads"` // CustomArgs / CustomArgsOff / ExtraArgs mirror Override's trio, because an // extra image model is edited through the same launch-argument panes and // composed by the same ComposeCmd. ExtraArgs is the legacy bucket, read only // when CustomArgs is empty (see Override.CustomArgsText). CustomArgs string `yaml:"customArgs"` CustomArgsOff bool `yaml:"customArgsOff"` ExtraArgs string `yaml:"extraArgs"` Unlisted bool `yaml:"unlisted"` // Backend is the registry entry id this extra image model launches with, // carried over from its Override. Empty => auto-pick the class default. Backend string `yaml:"backend"` } // EncoderSet is the pool of shared diffusion component files, each field one // physical file on disk. Models draw from it by architecture (archComponents), // so adding another model of an already-declared family is zero-config. An empty // field means "not on disk"; a model whose arch requires it emits a WARNING in // the generated YAML rather than a silently broken command. type EncoderSet struct { FluxVae string `yaml:"fluxVae"` // --vae for flux.1 / chroma (the flux "ae.safetensors") ClipL string `yaml:"clipL"` // --clip_l (flux, sdxl, sd3) ClipG string `yaml:"clipG"` // --clip_g (sdxl, sd3) T5 string `yaml:"t5"` // --t5xxl (flux, sd3) SdxlVae string `yaml:"sdxlVae"` // --vae for sdxl (optional; full checkpoints bake it) ZimageVae string `yaml:"zimageVae"` // --vae for z-image / lumina QwenLlm string `yaml:"qwenLlm"` // --llm text encoder (z-image, qwen-image, flux.2 klein) Flux2Vae string `yaml:"flux2Vae"` // --vae for flux.2 (32-ch latent, NOT flux.1's fluxVae) VideoVae string `yaml:"videoVae"` // --vae for a 3D video model (MiniMax-H3's transformer VAE) AudioVae string `yaml:"audioVae"` // --audio-vae: decodes the soundtrack latent of an audio-capable video model } // PromptEnhancer is one rewrite model: a catalog model id plus the fixed system // prompt it was trained to run under. // // The system prompt is REQUIRED in practice even though nothing enforces it. A // PE model without its own prompt is not a weaker enhancer, it is a generic // chat model that answers the instruction ("make her smile" -> "Sure, here's how // you could...") instead of rewriting it, and that answer would be pasted // straight into the prompt box. Qwen ships the exact text as system_prompt.txt // in each PE repo. type PromptEnhancer struct { // Model is the catalog model id, exactly as it appears in the generated // config and /v1/models. Case-insensitive on lookup, since the ids are // filename-derived and the UI copies them around. Model string `yaml:"model"` // Name is an optional display label for the picker ("Qwen PE - edits"). // Empty => the UI shows Model. Name string `yaml:"name"` // SystemPrompt is the fixed instruction the model runs under. Multi-line; // written as a YAML block scalar. SystemPrompt string `yaml:"systemPrompt"` // Vision declares that this enhancer reads the INPUT IMAGE as well as the // instruction, so the client attaches the reference image(s) to the rewrite // request. True for PE-I2I, false for PE-T2I, which has no image to look at. // // Declared rather than detected: whether a model has a projector is a fact // about the gguf, but whether it was TRAINED to condition the rewrite on the // picture is a fact about the finetune, and a vision-capable model handed an // image it does not expect rewrites worse, not better. Vision bool `yaml:"vision"` } // SlotCacheSettings mirrors config.SlotCacheConfig; zero values fall back to the // server's defaults (30k tokens / 10 GB / 20 sessions). type SlotCacheSettings struct { Enable bool `yaml:"enable"` Path string `yaml:"path"` MinSaveTokens int `yaml:"minSaveTokens"` // RecurrentSeeds mirrors config.SlotCacheConfig.RecurrentSeeds: also run the // partial-prefix seed paths on hybrid/recurrent archs. Measurement knob. RecurrentSeeds bool `yaml:"recurrentSeeds"` MaxDiskGB float64 `yaml:"maxDiskGB"` MaxSessions int `yaml:"maxSessions"` // MaxIdleDays deletes a snapshot nobody has restored or saved for this many // days. 0 => server default (7). MaxIdleDays int `yaml:"maxIdleDays"` // PreambleCaches mirrors config.SlotCacheConfig.PreambleCaches: the fleet-wide // switch for the preamble (system+tools seed) half of the cache. nil => on. PreambleCaches *bool `yaml:"preambleCaches"` } // APIKeyEntry is one named API key plus the model IDs it may reach. Name is a // UI label only; Key is the secret; empty Models => unrestricted (full access, // and admin rights over the management endpoints). type APIKeyEntry struct { Name string `yaml:"name"` Key string `yaml:"key"` Models []string `yaml:"models"` } // CategoryOrder is the canonical UI-category order; RootList walks CategoryRoots // in this order for deterministic scanning + hashing. // // It MUST stay in sync with MODEL_CATEGORIES in ui-svelte/src/lib/modelUtils.ts: // the Models tab renders its folder picker on every tab and posts that tab's id // to /api/settings/root/pick, so a category missing here is a root the user can // set and see in the toolbar that is then never scanned. The pick handler // rejects anything not listed here so that failure is loud rather than silent. var CategoryOrder = []string{"llm", "image", "video", "3d", "segment", "music", "tts", "transcribe", "embed", "decision"} // IsCategory reports whether name is a known UI category (i.e. a valid // CategoryRoots key). func IsCategory(name string) bool { for _, c := range CategoryOrder { if c == name { return true } } return false } // RootList returns the ordered, de-duplicated set of folders to scan: ModelsRoot // first, then each CategoryRoots value in CategoryOrder. Blank entries are // dropped; duplicates collapse to the first, comparing separator-insensitively // always and case-insensitively only where the filesystem is (config.PathKey). func (s Settings) RootList() []string { seen := map[string]bool{} var out []string add := func(p string) { p = strings.TrimSpace(p) if p == "" { return } key := config.PathKey(p) if seen[key] { return } seen[key] = true out = append(out, p) } add(s.ModelsRoot) for _, c := range CategoryOrder { add(s.CategoryRoots[c]) } return out } // SettingsPatch is a partial Settings override written by the dashboard's // "GPU memory" editor and stored in the UI sidecar. Only non-nil fields apply, // so the hand-authored generate file keeps owning everything not touched in the // UI. Setting a manual TargetVramGB pairs with AutoVram=false so the live-VRAM // sampler doesn't clobber the user's choice. // EVERY field here must be a pointer, and the dashboard PUTs only the fields // its own section owns — see MergeSettingsPatch, which carries the rest forward. // TestSettingsPatch_AllFieldsArePointers enforces the first half of that rule. type SettingsPatch struct { TargetVramGB *float64 `yaml:"targetVramGB,omitempty"` VramOverheadGB *float64 `yaml:"vramOverheadGB,omitempty"` MaxRamGB *float64 `yaml:"maxRamGB,omitempty"` AutoVram *bool `yaml:"autoVram,omitempty"` DryDefault *bool `yaml:"dryDefault,omitempty"` // TtlSec is the idle-eviction timeout (seconds) baked into every model's // `ttl`. 0 => never auto-unload. nil => inherit the generate file / default. TtlSec *int `yaml:"ttlSec,omitempty"` // --- OOM guard (Settings.OomGuard*, consumed by internal/server/vramguard.go) OomGuardEvict *bool `yaml:"oomGuardEvict,omitempty"` OomGuardReserveGB *float64 `yaml:"oomGuardReserveGB,omitempty"` OomGuardGraceSec *int `yaml:"oomGuardGraceSec,omitempty"` // --- GPU usage / admission. The last three are also the device policy // (Settings.DevicePolicy): which adapters count, and whether an integrated // GPU's shared system memory is part of the budget. The Advanced section edits // all three, so clearAdvancedPatch clears them as well. MinGpuFraction *float64 `yaml:"minGpuFraction,omitempty"` MultiResident *bool `yaml:"multiResident,omitempty"` MultiGpu *bool `yaml:"multiGpu,omitempty"` MinGpuVramGB *float64 `yaml:"minGpuVramGB,omitempty"` SharedMemory *string `yaml:"sharedMemory,omitempty"` PoolIntegrated *bool `yaml:"poolIntegratedGpu,omitempty"` // --- Advanced sizer knobs. Wrong values here mis-size every model, which is // why the UI hides them behind a warning and a per-section reset. ComputeBufFactor *float64 `yaml:"computeBufFactor,omitempty"` VisionOverheadGB *float64 `yaml:"visionOverheadGB,omitempty"` VisionCtx *int `yaml:"visionCtx,omitempty"` MoeCtxTarget *int `yaml:"moeCtxTarget,omitempty"` DenseMinCtx *int `yaml:"denseMinCtx,omitempty"` DenseCtxLadder *[]int `yaml:"denseCtxLadder,omitempty"` Threads *int `yaml:"threads,omitempty"` HealthCheckTimeout *int `yaml:"healthCheckTimeout,omitempty"` // --- Fleet-wide model knobs KvQuant *string `yaml:"kvQuant,omitempty"` LoraDir *string `yaml:"loraDir,omitempty"` // LoraDirs is the per-category narrowing of LoraDir. A pointer like every // other field here, so MergeSettingsPatch's nil-means-untouched contract // holds: a bare map would be dropped by the next save from another section. // A pointer to an empty map is the explicit clear. LoraDirs *map[string]string `yaml:"loraDirs,omitempty"` } // MergeSettingsPatch overlays next onto prev field-wise: a nil field in next // keeps prev's value. // // Reflection rather than 20 hand-written nil checks, because the hand-written // version is what this replaces and it had already lost the argument. The // dashboard writes the patch in SECTIONS — the memory form PUTs four fields, the // guard form three, the advanced form eight — and a whole-struct replace means // every save wipes every field the saving section doesn't model. That bug was // live for DryDefault and TtlSec, each fixed by its own bespoke carry-forward // line; adding fifteen more fields to that pattern is fifteen more chances to // forget one, and the symptom (a setting silently reverting a week later) is // close to undebuggable. Here, a new pointer field is carried forward by // construction. func MergeSettingsPatch(prev, next SettingsPatch) SettingsPatch { out := next pv, ov := reflect.ValueOf(prev), reflect.ValueOf(&out).Elem() for i := 0; i < ov.NumField(); i++ { if ov.Field(i).Kind() == reflect.Ptr && ov.Field(i).IsNil() { ov.Field(i).Set(pv.Field(i)) } } return out } // apply overlays the patch's set fields onto s. Mirrors MergeSettingsPatch's // field-wise contract, but writes into the flat Settings (values, not pointers), // so it stays explicit — the target field names differ from the source ones only // by the dereference, and a reflect-based version here would silently do the // wrong thing for the two fields Settings itself stores as pointers. func (p *SettingsPatch) apply(s *Settings) { if p == nil { return } if p.TargetVramGB != nil { s.TargetVramGB = *p.TargetVramGB } if p.VramOverheadGB != nil { s.VramOverheadGB = *p.VramOverheadGB } if p.MaxRamGB != nil { s.MaxRamGB = *p.MaxRamGB } if p.AutoVram != nil { s.AutoVram = *p.AutoVram } if p.DryDefault != nil { s.DryDefault = p.DryDefault } if p.TtlSec != nil { s.TtlSec = *p.TtlSec } if p.OomGuardEvict != nil { s.OomGuardEvict = p.OomGuardEvict } if p.OomGuardReserveGB != nil { s.OomGuardReserveGB = *p.OomGuardReserveGB } if p.OomGuardGraceSec != nil { s.OomGuardGraceSec = *p.OomGuardGraceSec } if p.MinGpuFraction != nil { s.MinGpuFraction = *p.MinGpuFraction } if p.MultiGpu != nil { s.MultiGpu = p.MultiGpu } if p.MinGpuVramGB != nil { s.MinGpuVramGB = *p.MinGpuVramGB } if p.SharedMemory != nil { s.SharedMemory = *p.SharedMemory } if p.PoolIntegrated != nil { s.PoolIntegratedGpu = p.PoolIntegrated } if p.MultiResident != nil { s.MultiResident = p.MultiResident } if p.ComputeBufFactor != nil { s.ComputeBufFactor = *p.ComputeBufFactor } if p.VisionOverheadGB != nil { s.VisionOverheadGB = *p.VisionOverheadGB } if p.VisionCtx != nil { s.VisionCtx = *p.VisionCtx } if p.MoeCtxTarget != nil { s.MoeCtxTarget = *p.MoeCtxTarget } if p.DenseMinCtx != nil { s.DenseMinCtx = *p.DenseMinCtx } if p.DenseCtxLadder != nil { s.DenseCtxLadder = *p.DenseCtxLadder } if p.Threads != nil { s.Threads = *p.Threads } if p.HealthCheckTimeout != nil { s.HealthCheckTimeout = *p.HealthCheckTimeout } if p.KvQuant != nil { s.KvQuant = *p.KvQuant } if p.LoraDir != nil { s.LoraDir = *p.LoraDir } if p.LoraDirs != nil { s.LoraDirs = *p.LoraDirs } } // GroupSpec defines one output group and (optionally) the listen address that // exposes it. Match is a list of model-name globs (PowerShell -like); a model // joins the first group it matches. Listen empty => the group binds no // dedicated port but still groups its members for eviction. // // Coexist turns the group into a coexistence group (emitted swap:false): its own // members stay loaded alongside each other instead of evicting one another, while // the group as a whole still evicts the other (exclusive) groups. That is what an // eval harness wants on a card with room for several small models at once — // several candidates resident, all other catalogs pushed out. Nothing here budgets // VRAM, so the caller decides how many members it loads concurrently; oversubscribe // and the extra llama-server just fails to allocate. type GroupSpec struct { Name string `yaml:"name"` Listen string `yaml:"listen"` Match []string `yaml:"match"` Coexist bool `yaml:"coexist"` } // LoraRef is one llama-server LoRA adapter attached to a model. // // Scale is a POINTER because 0 is a meaningful value here, not an absent one: // llama.cpp loads an adapter at scale 0 as inert-but-present, which is the // documented way to keep several adapters available and pick between them with // a request's `"lora":[{"id":N,"scale":F}]`. Zero-gating would make that // arrangement unexpressible. nil => emit plain `--lora` and take llama.cpp's // own 1.0; set => emit `--lora-scaled `. type LoraRef struct { Path string `yaml:"path"` Scale *float64 `yaml:"scale,omitempty"` } // Override supplies what gguf metadata can't, matched by a path glob against the // gguf's full path (first match wins; optional Quant scopes the match to one // quant of a multi-quant repo). Mirrors the PowerShell $Overrides rows. type Override struct { Match string `yaml:"match"` Quant string `yaml:"quant"` // Backend is the registry entry id (settings.backends) this model launches // with. Empty => auto-pick the class-default backend (see resolveBackend). The // entry's kind selects the emitter: "llama" => the llama-server path below, // "vllm" => vllm.go (a different arg set — the llama knobs are ignored, only // Ctx/Vllm* apply). Config is thus keyed to the backend: switching kind reads // a different subset of these fields; the dormant ones are kept, not wiped. Backend string `yaml:"backend"` // --- vllm knobs (kind "vllm"; ignored by the llama path) --- // VllmGpuUtil => --gpu-memory-utilization (0 => derived from the VRAM budget // and the card's size). VllmTensorParallel => --tensor-parallel-size when >1. // --max-model-len comes from Ctx when pinned, else it is sized against the // same budget. VllmGpuUtil float64 `yaml:"vllmGpuUtil"` VllmTensorParallel int `yaml:"vllmTensorParallel"` // VllmTokenizer => --tokenizer. Upstream recommends pointing vllm at the base // model's tokenizer rather than the one converted out of the gguf ("the // tokenizer conversion from GGUF is time-consuming and unstable"). It is not // derived automatically: the discovered row only knows the model's local // folder name, not a verified Hugging Face repo id, so a guess here would bake // a wrong remote reference into a launch command. A repo id or a local path. VllmTokenizer string `yaml:"vllmTokenizer"` // VllmToolParser => --enable-auto-tool-choice --tool-call-parser X. "" => // picked from the architecture (vllmToolParser), "none" => off. vLLM refuses // every request that carries `tools` without one, and the parser is // per-family, so it cannot be a fleet-wide default. VllmToolParser string `yaml:"vllmToolParser"` Spec string `yaml:"spec"` // "draft-mtp" | "draft-dflash" | "" (=> ngram-mod); chainable with "+" ReasoningFmt string `yaml:"reasoningFmt"` // "auto" | "off" | "" (=> auto) // ReasoningBudget caps thinking tokens (--reasoning-budget N). 0 => omit (no // cap). Inherited by ctx-tier variants; named variants are standalone. ReasoningBudget int `yaml:"reasoningBudget"` CtxVariants []int `yaml:"ctxVariants"` Ctx int `yaml:"ctx"` KvK string `yaml:"kvK"` KvV string `yaml:"kvV"` KvInRam bool `yaml:"kvInRam"` // VramTargetGB caps how much VRAM this model is sized against. 0 => inherit // settings.TargetVramGB (the global budget). Lets one model run leaner than // the fleet default without a separate variant. VramTargetGB float64 `yaml:"vramTargetGB"` // CpuOffload pins how many layers are pushed to CPU, overriding the auto // sizer. 0 => auto. MoE models offload expert layers (--n-cpu-moe N); dense // models drop GPU layers (-ngl = blocks-N). CpuOffload int `yaml:"cpuOffload"` // Device is where the model RUNS: "" / "gpu" = the sized placement above, // "cpu" = -ngl 0 --device none, no GPU context at all (llama-server classes // only). Distinct from CpuOffload at max, which still lets llama.cpp borrow // the GPU for large-batch matmuls (op offload) and so still holds VRAM. Device string `yaml:"device,omitempty"` // SwapGroup is which memory pool the model is admitted against and evicts // within: "" = follow Device, "gpu" = the VRAM-budgeted exclusive group, // "cpu" = the RAM-budgeted cpu group (never evicts or is evicted by a GPU // model for VRAM). A GPU device in the cpu group is refused: it would hold // VRAM the router never charges. SwapGroup string `yaml:"swapGroup,omitempty"` // Mmproj places the CLIP image projector on this model's DEFAULT profile and // its ctx tiers / named variants (the "-vision" twin has its own pin, on the // reserved vision variant). Only meaningful for a model that has a projector // (own or inherited): // // "" RAM (--no-mmproj-offload): images work everywhere at no VRAM cost, // paid as a one-off host-side encode per image // "ram" the same, said out loud // "gpu" pin the projector in VRAM here too: fastest image encode, and every // profile pays for it in ctx/offload on every request, image or not // "none" wire no projector at all - no image input, and no vision twin // // Note "none" is the only value that removes the twin, and it is not the same // as marking the twin unlisted: unlisted still builds and can still be loaded // by id, this removes the profile. Mmproj string `yaml:"mmproj"` // Engine knobs surfaced from llama-server. Zero/empty => the generator's // default (shown in parentheses): // FlashAttn: "" (on) | "on" | "off" | "auto" (-fa; required for quantized KV) // Mmap: "" | "on" | "off" ("" => placement-gated: mmap // only where weights sit on the CPU, --load-mode none otherwise; // see buildCmdLines. "on"/"off" force it either way) // Mlock: false => no --mlock; true => --mlock (lock weights in RAM) // Threads: 0 => settings.Threads (-t) // Parallel: 0 => 1, capped at MaxParallelSlots (--parallel, concurrent // slots). Sizing treats the computed ctx as PER SLOT and charges // N x the KV cost; emit multiplies it back into -c, which // --kv-unified makes the shared pool. So raising this shrinks the // per-conversation window rather than over-committing VRAM. // Ub: 0 => auto (1024, 512 for >=64k ctx) (-ub/-b physical batch) FlashAttn string `yaml:"flashAttn"` Mmap string `yaml:"mmap"` Mlock bool `yaml:"mlock"` Threads int `yaml:"threads"` Parallel int `yaml:"parallel"` Ub int `yaml:"ub"` // CtxCheckpoints, when non-nil, emits --ctx-checkpoints N. 0 disables the KV // prompt-prefix checkpoint cache (llama-server default is 32, each a full KV // snapshot — costly in VRAM and rarely reused for short/divergent prompts). // nil => omit the flag (llama default). Variants inherit this unless they set // their own. CtxCheckpoints *int `yaml:"ctxCheckpoints"` // PreserveThinking emits --reasoning-preserve so the chat template keeps // prior-turn blocks in history instead of stripping them (Qwen3.6+). // No-op when reasoning is off, and when the template has no preserve support // (llama-server warns and carries on). Requires the client to send // reasoning_content back on assistant messages. // // nil => ON. Reasoning amnesia across turns is the wrong default for an // agentic loop: the model re-derives what it already worked out last turn, // and on a template that preserves by default (3.8) omitting the flag was // never "off" anyway - it was "whatever the template does". So the generator // states it, and *false is the only way to get stripping. // // Still a POINTER, not a plain bool: as a bool, false could not be told from // "never set", so every save through the config editor stamped the default // onto the model and no default could ever be changed again. PreserveThinking *bool `yaml:"preserveThinking"` // Dry sampler (repetition penalty). Dry==nil => the fleet default (off); // *Dry==true emits the flags, *Dry==false omits them. DryMultiplier/DryBase/ // DryAllowedLength override the defaults (0.8 / 1.75 / 3); 0 => default. // Mirrored on VariantSpec. Dry *bool `yaml:"dry"` DryMultiplier float64 `yaml:"dryMultiplier"` DryBase float64 `yaml:"dryBase"` DryAllowedLength int `yaml:"dryAllowedLength"` // --- Sampler defaults baked into the launch command --- // llama-server applies each of these only when a request omits that field, so // they are the server-side default for every client. nil => omit the flag and // take llama's own default (temp 0.80 / top-k 40 / top-p 0.95 / min-p 0.05 / // presence 0.0). // // Pointers, not zero-gated floats like the knobs around them: 0 is a // MEANINGFUL, non-default value for every one of these (min-p 0 disables // truncation, temp 0 is greedy, top-k 0 disables top-k), so a plain float64 // could not express the exact settings model cards recommend. // // TopK and MinP are the two that actually bite: neither has an OpenAI-API // field, so almost no client ever sends them and the launch flag is the only // thing that ever sets them. Temp/TopP are overwritten by nearly every // request and are mostly a floor for clients that omit them. Mirrored on // VariantSpec. See defaultSamplerFor for the arch-derived TopK/MinP baseline. Temp *float64 `yaml:"temp"` TopK *int `yaml:"topK"` TopP *float64 `yaml:"topP"` MinP *float64 `yaml:"minP"` PresencePenalty *float64 `yaml:"presencePenalty"` // Speculative-decode sub-knobs, emitted only for the matching Spec backend; // 0/false => omit (llama-server default). SpecDraftNMax defaults to 2 for // draft-mtp, 6 for draft-dflash (a diffusion block tolerates a longer draft // chain than single-token MTP; 6 measured optimal, higher over-drafts); the // SpecNgram* knobs apply to the // ngram-map-k4v backend. SpecDraftNMax int `yaml:"specDraftNMax"` SpecDefault bool `yaml:"specDefault"` SpecNgramSizeN int `yaml:"specNgramSizeN"` SpecNgramSizeM int `yaml:"specNgramSizeM"` SpecNgramMinHits int `yaml:"specNgramMinHits"` // --- Advanced / power-user llama-server knobs --- // All default zero/empty => the flag is omitted, so existing configs emit // unchanged. Variants inherit these from the model-wide override and can // override each (see the effOv merge in generate.go). Grouped under the UI's // "Advanced" collapsible. // ThreadsBatch: 0 => same as -t (-tb, prompt/batch threads) // Prio: 0 => normal (--prio 0..3) // DirectIo: -dio (faster cold load) // NoOpOffload: --no-op-offload // NoRepack: --no-repack // KvKDraft/KvVDraft: "" => match -ctk/-ctv (-ctkd/-ctvd, draft KV quant; also // applies to a baked-in MTP head, whose own context llama // would otherwise run on f16) // CacheReuse: 0 => off (--cache-reuse N, prefix KV-shift reuse) // CacheRamMB: 0 => llama default (8192) (-cram, prompt-cache size MiB) // CacheIdleSlots: "" | "on" | "off" (--cache-idle-slots / --no-) // LogVerbosity: 0 => backend default (-lv N, log verbosity threshold; // raise it to make llama-server print its load-time buffer // report, which some builds hide at their own default) // SwaFull: --swa-full (full SWA cache) // CheckpointMinStep: 0 => llama default (-cms, ctx-checkpoint spacing) // ContextShift: "" | "on" | "off" (--context-shift / --no-) // SpecDraftNMin: 0 => llama default (--spec-draft-n-min) // SlotPromptSimilarity: 0 => omit (-sps, slot reuse threshold) // RopeScaling: "" | none | linear | yarn (--rope-scaling) // RopeScale: 0 => omit (--rope-scale) // RopeFreqBase: 0 => omit (--rope-freq-base) // YarnOrigCtx: 0 => omit (--yarn-orig-ctx) // SplitMode: "" | none|layer|row|tensor (-sm, multi-GPU) // TensorSplit: "" => omit (-ts, e.g. "3,1") // MainGpu: 0 => GPU 0 (-mg) // OverrideTensor: "" => omit (-ot, manual tensor placement) ThreadsBatch int `yaml:"threadsBatch"` Prio int `yaml:"prio"` DirectIo bool `yaml:"directIo"` NoOpOffload bool `yaml:"noOpOffload"` NoRepack bool `yaml:"noRepack"` KvKDraft string `yaml:"kvKDraft"` KvVDraft string `yaml:"kvVDraft"` CacheReuse int `yaml:"cacheReuse"` CacheRamMB int `yaml:"cacheRamMB"` CacheIdleSlots string `yaml:"cacheIdleSlots"` LogVerbosity int `yaml:"logVerbosity"` SwaFull bool `yaml:"swaFull"` CheckpointMinStep int `yaml:"checkpointMinStep"` ContextShift string `yaml:"contextShift"` SpecDraftNMin int `yaml:"specDraftNMin"` SlotPromptSimilarity float64 `yaml:"slotPromptSimilarity"` RopeScaling string `yaml:"ropeScaling"` RopeScale float64 `yaml:"ropeScale"` RopeFreqBase float64 `yaml:"ropeFreqBase"` YarnOrigCtx int `yaml:"yarnOrigCtx"` SplitMode string `yaml:"splitMode"` TensorSplit string `yaml:"tensorSplit"` MainGpu int `yaml:"mainGpu"` OverrideTensor string `yaml:"overrideTensor"` // AudioDevice pins which adapter an audio.cpp model loads onto // (audiocpp_server --device N, an index within the --backend it was built // for). nil => the generator reads the binary's own --list-devices and picks // the first discrete GPU; a NEGATIVE value means "emit nothing", handing the // choice back to audio.cpp's default. A pointer because 0 is the index most // people would pin by hand, so it cannot double as "unset". // // Only audio.cpp reads this: every other backend here either takes a device // LIST (llama's --device, see backenddev.go) or has one device by // construction. AudioDevice *int `yaml:"audioDevice"` // AudioWeightType is how an audio.cpp model's weights are stored once // loaded: "" => auto (AudioCppWeightTypeAuto), "native" => as the file // ships them, "f16" => converted to f16 on load. Only families listed in // AudioCppWeightTypeFamilies read it: audio.cpp's .weight_type takes // a different value set, and defaults differently, per family, so one // fleet-wide knob would hand some of them a value they reject. AudioWeightType string `yaml:"audioWeightType,omitempty"` // CustomArgs is the user's launch-argument text, stored verbatim and applied // as the last layer of the emitted command: for every knob the text sets, the // generated occurrences are dropped, then the text is appended (last wins, // which is llama-server's own rule for a repeated flag). The app never // rewrites it. See internal/autogen/customargs.go and ui-svelte/launch-args.md. CustomArgs string `yaml:"customArgs"` // CustomArgsOff keeps CustomArgs in the sidecar but stops applying it, so the // editor's enable toggle is reversible without retyping. Also suppresses // ExtraArgs while set. CustomArgsOff bool `yaml:"customArgsOff"` // ExtraArgs is the LEGACY free-form bucket written by the old two-way // launch-parameters box ("anything the parser could not map"). It is still // applied when CustomArgs is empty, so sidecars keep working without a // rewrite; a save through the new editor writes CustomArgs instead. ExtraArgs string `yaml:"extraArgs"` // ChatTemplateFile is a path to a .jinja chat template that replaces the // gguf's baked-in one (--chat-template-file). Empty => the baked-in template. // No arch gets a substitute picked for it (see buildCmdLines); the built-in // Qwen 3.5/3.6 fix this comment used to describe is gone. // // On a VARIANT this field is sentinel-aware: empty inherits the model-wide // value and NoneSentinel drops it. See inherit.go. ChatTemplateFile string `yaml:"chatTemplateFile"` // MmprojFile names the vision projector explicitly (--mmproj), for the case // discovery cannot serve: one shared mmproj kept in a folder of its own and // pointed at by several models. Discovery only pairs a projector sitting in // the gguf's own directory, or in a family sibling's (inheritSidecars), and // that rule is deliberately NOT widened - a projector is bound to a specific // vision tower, so an auto-paired stranger yields a twin that loads clean and // hallucinates on every image. Naming the file is the user stating intent. // // Set => used instead of anything discovery found, AND it creates the // "-vision" twin for a model that paired with nothing. Mmproj "none" still // wins (no twin at all). Distinct from Mmproj, which is the projector's // PLACEMENT ("" auto / gpu / ram / none), not its path. // // On a VARIANT this field is sentinel-aware: empty inherits the model-wide // value and NoneSentinel drops it. See inherit.go. MmprojFile string `yaml:"mmprojFile"` // --- Image (diffusion / sd-server) knobs --- // Only consumed for image-arch models (emitImageModel / imageCmdLines); ignored // by the llama-server path. The component paths are the external VAE + text // encoder(s) a diffusion-only GGUF needs (it loads via --diffusion-model, not an // all-in-one checkpoint). Empty => omit the flag. VaePath string `yaml:"vaePath"` // --vae ClipLPath string `yaml:"clipLPath"` // --clip_l ClipGPath string `yaml:"clipGPath"` // --clip_g T5Path string `yaml:"t5Path"` // --t5xxl TextEncoderPath string `yaml:"textEncoderPath"` // --llm (Z-Image / Lumina text encoder) // AudioVaePath is --audio-vae, the second decoder an audio-capable video // model needs to turn its soundtrack latent into PCM. Video models only. AudioVaePath string `yaml:"audioVaePath"` // LlmVision gates --llm_vision, the vision projector (mmproj) that pairs with // the --llm text encoder. "" => auto: edit/reference models (name-detected, see // wantsVisionEncoder) get the projector found beside their encoder, everything // else gets none. "on"/"off" pin it. // RefEdit gates whether this model takes its source image as a REFERENCE // (extra sequence tokens) rather than as an img2img denoise base, which is // what the emitted `in: [text, image]` capability tells the client. "" => // auto (name detection, see IsReferenceEditModel), "on"/"off" pin it. The // distinction is not structural - a base and an edit checkpoint have // identical tensor shapes - so this toggle is the reliable answer for a // model whose name does not follow the "-edit-" convention. RefEdit string `yaml:"refEdit"` LlmVision string `yaml:"llmVision"` // LlmVisionPath pins the --llm_vision projector file, winning over the // directory pairing. Empty => auto (see LlmVision). LlmVisionPath string `yaml:"llmVisionPath"` // PromptEnhancer names the settings.promptEnhancers entry this image model // sends its prompt through before rendering. Empty => no enhancer, the // prompt is rendered as typed. // // A model ID rather than a file path, and that is the whole design: an // enhancer is a llama-server model the ONE router schedules, sized and // evicted alongside everything else. A path here would mean a second // inference process outside the scheduler, with no shared VRAM accounting. // // Consumed by the CLIENT, not by sd-server: it has no such flag, and a // rewrite the user cannot see or edit before it renders is worse than none. // It rides the generated config purely so the playground can read it off the // model listing. PromptEnhancer string `yaml:"promptEnhancer"` // PromptEnhancerEdit is the enhancer used INSTEAD of PromptEnhancer when the // request carries a reference image. Qwen ships the pair (PE-T2I, PE-I2I) and // they are not interchangeable: the edit one rewrites an instruction about an // existing picture, the text one composes a scene from nothing. Empty => the // model uses PromptEnhancer in both directions, which is what a single-model // setup wants and what every model configured before this field existed does. PromptEnhancerEdit string `yaml:"promptEnhancerEdit"` // PromptEnhancerPrompt is the system prompt this model runs its txt2img // enhancer under, winning over the settings.promptEnhancers row (if any). // // Per IMAGE MODEL rather than per enhancer, because that is where it earns its // keep: one rewriter is reused across checkpoints that want very different // output (an SDXL tag soup, a Flux paragraph, a Qwen instruction), and a // rewrite is only ever as good as the target it was told to write for. // Empty => fall back to the shared row, which is the pre-field behaviour. PromptEnhancerPrompt string `yaml:"promptEnhancerPrompt"` // PromptEnhancerEditPrompt is the img2img half, same precedence. Kept apart // from the text one because the two directions are different jobs: one // composes a scene, the other rewrites an instruction about a picture that // already exists. PromptEnhancerEditPrompt string `yaml:"promptEnhancerEditPrompt"` // LoraDir is this model's `--lora-model-dir`. Empty => settings.loraDir, and // if that is empty too, the directory the model gguf itself lives in. // sd-server models only; the llama-server path uses Loras below. LoraDir string `yaml:"loraDir"` // Loras are the LoRA adapters attached to a LLAMA-SERVER model, emitted as // one `--lora`/`--lora-scaled` per entry. Unlike the sd-server side there is // no directory flag: llama.cpp binds adapters at spawn, so which adapters a // model runs with is a per-model decision, not a per-request one. // // A relative Path resolves against settings.loraDirs["llm"], then // settings.loraDir, then the model gguf's own directory (see // resolveLlmLoraDir); an absolute Path is used as written. Empty => no flag, // which is every existing model. Loras []LoraRef `yaml:"loras"` // Placement tri-states: "" => the generator default (shown), "on"/"off" pin it. // OffloadToCpu: "" => auto (sizer offloads when weights+compute don't fit) // TeOnCpu: "" => on (--backend te=cpu); "off" keeps the encoder on GPU // VaeTiling: "" => on (--vae-tiling, caps the VAE decode VRAM spike) // DiffusionFa: "" => on (--diffusion-fa) // VaeOnCpu: "" => off (VAE decodes on GPU); "on" adds vae=cpu to --backend // (bf16 VAE whitens on some GPU backends; CPU is the safe fallback) // TemporalTiling: "" => on for video families whose VAE implements it // StreamLayers: "" => on for VIDEO models that OFFLOAD (--stream-layers) OffloadToCpu string `yaml:"offloadToCpu"` TeOnCpu string `yaml:"teOnCpu"` VaeOnCpu string `yaml:"vaeOnCpu"` VaeTiling string `yaml:"vaeTiling"` DiffusionFa string `yaml:"diffusionFa"` // Video-only VRAM levers, both on by default for a model the scan identified // as a video DiT and emitted for nothing else. // // A video render has TWO peaks. --vae-tiling above caps the decode // SPATIALLY, which is the whole story for a still; a clip's decode also grows // along TIME with --video-frames, and --temporal-tiling is the only flag that // chunks that axis (tune it with --extra-tiling-args // temporal_tile_size=N,temporal_tile_overlap=N via ExtraArgs). // // TemporalTiling "" is on only where the family's VAE actually implements a // tiled decode (Wan today, not MiniMax-H3, which accepts the flag and // processes the full temporal dimension anyway). "on" forces it regardless, // for a backend build that has gained support since. // // --stream-layers attacks the other peak from the side: it streams the // diffusion weights against the --max-vram budget with prefetch instead of // pinning them resident, handing that headroom back to the sampler. // // It has TWO preconditions, and the second one is the one that bites. The // documented one is --max-vram, which every sd-server line emitted here sets. // The undocumented one only shows up at runtime: "--stream-layers has no // effect unless diffusion params backend is cpu; ignoring". Streaming the // weights IN means they have to live somewhere to stream from, and only // --offload-to-cpu puts them there. So "" follows the offload decision rather // than the video one: a clip whose weights fit resident gets nothing from // this flag except a line in the log saying so. "on" forces it regardless. TemporalTiling string `yaml:"temporalTiling"` StreamLayers string `yaml:"streamLayers"` // Generation defaults baked into the sd-server command (applied when a request // omits them). 0/empty => omit (sd-server's own default). DefaultCfg matters: // Z-Image-Turbo blurs unless cfg-scale is pinned to 1.0. DefaultSteps int `yaml:"defaultSteps"` // --steps DefaultCfg float64 `yaml:"defaultCfg"` // --cfg-scale DefaultSampler string `yaml:"defaultSampler"` // --sampling-method DefaultWidth int `yaml:"defaultWidth"` // --width DefaultHeight int `yaml:"defaultHeight"` // --height // Video-only generation defaults. DefaultFrames is --video-frames, which // sd.cpp ALIGNS UP to the family's grid: 17k+5 for MiniMax-H3 (5, 22, 39, // 56...), the largest 4n+1 for everything else. Off-grid numbers are not // rejected, they are silently changed, so prefer an exact one. 0 => the // family default (see videoDefaultsFor). // // DefaultFps is --fps. H3 IGNORES it and logs an override warning: the model // is fixed at 24 fps. It is honored by Wan. DefaultFrames int `yaml:"defaultFrames"` // --video-frames DefaultFps int `yaml:"defaultFps"` // --fps Unlisted bool `yaml:"unlisted"` Skip bool `yaml:"skip"` // SlotCache opts this model into on-disk slot KV persistence: emits // --slot-save-path so the server's slotCache can save/restore its conversation // KV. OPT-IN: nil/absent => off, so enabling the dashboard master switch does // not start littering snapshots and preamble caches for every model in the // fleet; set true on the models whose prefills are actually worth keeping. // No-op unless settings.slotCache.enable is also on (the master switch). SlotCache *bool `yaml:"slotCache"` // SlotCachePreamble opts this model out of the PREAMBLE half only (the shared // system+tools seed minted per agent), keeping conversation snapshots. nil => // on for a model that set SlotCache. Emitted as the model's // slotCachePreamble config key, which the server reads per request. SlotCachePreamble *bool `yaml:"slotCachePreamble"` // Variants are named custom profiles emitted in addition to the solo model: // each becomes "-" with its own ctx/VRAM/kv/spec. Use-case // agnostic — the UI's "create variant" flow writes these. Variants []VariantSpec `yaml:"variants"` } // VariantSpec is one named custom variant of a model (UI-created). Zero/empty // fields inherit: VramTargetGB => settings.TargetVramGB, KvK/KvV/Spec/ // ReasoningFmt => the model-wide override. Name is the suffix appended after // the model id ("-"). // // Ub and Dry are use-case-agnostic serving knobs that let a variant reproduce // any spawn shape (e.g. a low-VRAM "game" coexistence variant, or a short-ctx // "judge" variant with greedy sampling) without the engine knowing those names: // - Ub: physical batch size (-ub/-b). 0 => default (1024, or 512 for >=64k ctx). // - Dry: nil => the fleet default (off); *Dry==true emits the flags, false omits. // // ReasoningFmt: "off" emits "--reasoning off" (plus --reasoning-format none). type VariantSpec struct { Name string `yaml:"name"` Ctx int `yaml:"ctx"` VramTargetGB float64 `yaml:"vramTargetGB"` KvK string `yaml:"kvK"` KvV string `yaml:"kvV"` Spec string `yaml:"spec"` ReasoningFmt string `yaml:"reasoningFmt"` Ub int `yaml:"ub"` Dry *bool `yaml:"dry"` // CtxCheckpoints, when non-nil, emits --ctx-checkpoints N (0 disables). nil => // inherit the model-wide Override.CtxCheckpoints. CtxCheckpoints *int `yaml:"ctxCheckpoints"` Unlisted bool `yaml:"unlisted"` // PreserveThinking is *bool so a standalone variant defaults preserve-on // (nil => emit, matching the Qwen3.6 reasoning default); false disables it. PreserveThinking *bool `yaml:"preserveThinking"` // SlotCache opts this variant into on-disk slot KV persistence. nil => inherit // the model-wide Override.SlotCache; true/false sets it explicitly. Gated by // settings.slotCache.enable like the model-wide flag. SlotCache *bool `yaml:"slotCache"` // Engine knobs mirroring Override, so a variant can carry the full launch // shape (the UI's "full settings page" for a variant). Zero/empty => INHERIT // the model-wide Override; the variant's own non-zero value wins at merge // (see the effOv chain in buildProfiles). This comment used to claim variants // were standalone, which the merge code has never done. // // Because empty means inherit, the free-form string knobs below take // NoneSentinel ("none") to mean "explicitly nothing" - otherwise a variant of // a model that pins a chat template / extra args / tensor placement has no // way to run without it. See inherit.go. KvInRam bool `yaml:"kvInRam"` CpuOffload int `yaml:"cpuOffload"` FlashAttn string `yaml:"flashAttn"` Mmap string `yaml:"mmap"` Mlock bool `yaml:"mlock"` Threads int `yaml:"threads"` Parallel int `yaml:"parallel"` CustomArgs string `yaml:"customArgs"` ExtraArgs string `yaml:"extraArgs"` // ChatTemplateFile mirrors Override; empty => inherit the model-wide value. ChatTemplateFile string `yaml:"chatTemplateFile"` // MmprojFile mirrors Override; empty => inherit the model-wide value, "none" // => no explicit projector (fall back to whatever discovery paired). MmprojFile string `yaml:"mmprojFile"` // Mmproj pins the image projector's placement, but ONLY on the reserved // "vision" variant - it is the one variant that tunes a profile carrying an // mmproj at all. Same vocabulary as Override.Mmproj ("gpu"/"ram"/"none"); // empty inherits the model-wide value, so an untouched variant changes // nothing. Ignored on every other variant, whose profile loads no projector. Mmproj string `yaml:"mmproj"` // Sampler / speculative sub-knobs mirroring Override; 0/empty => inherit the // model-wide value. (Dry on/off is the *bool field above.) DryMultiplier float64 `yaml:"dryMultiplier"` DryBase float64 `yaml:"dryBase"` DryAllowedLength int `yaml:"dryAllowedLength"` // Sampler defaults mirroring Override; nil => inherit the model-wide value // (and, failing that, the arch baseline / llama's default). Non-nil wins at // merge, so a variant can pin greedy sampling (temp 0) on a model whose // default is hot. Temp *float64 `yaml:"temp"` TopK *int `yaml:"topK"` TopP *float64 `yaml:"topP"` MinP *float64 `yaml:"minP"` PresencePenalty *float64 `yaml:"presencePenalty"` SpecDraftNMax int `yaml:"specDraftNMax"` SpecDefault bool `yaml:"specDefault"` SpecNgramSizeN int `yaml:"specNgramSizeN"` SpecNgramSizeM int `yaml:"specNgramSizeM"` SpecNgramMinHits int `yaml:"specNgramMinHits"` // Advanced / power-user knobs mirroring Override; zero/empty => inherit the // model-wide value (a variant's own non-zero/non-empty value wins at merge). ThreadsBatch int `yaml:"threadsBatch"` Prio int `yaml:"prio"` DirectIo bool `yaml:"directIo"` NoOpOffload bool `yaml:"noOpOffload"` NoRepack bool `yaml:"noRepack"` KvKDraft string `yaml:"kvKDraft"` KvVDraft string `yaml:"kvVDraft"` CacheReuse int `yaml:"cacheReuse"` CacheRamMB int `yaml:"cacheRamMB"` CacheIdleSlots string `yaml:"cacheIdleSlots"` SwaFull bool `yaml:"swaFull"` CheckpointMinStep int `yaml:"checkpointMinStep"` ContextShift string `yaml:"contextShift"` SpecDraftNMin int `yaml:"specDraftNMin"` SlotPromptSimilarity float64 `yaml:"slotPromptSimilarity"` RopeScaling string `yaml:"ropeScaling"` RopeScale float64 `yaml:"ropeScale"` RopeFreqBase float64 `yaml:"ropeFreqBase"` YarnOrigCtx int `yaml:"yarnOrigCtx"` SplitMode string `yaml:"splitMode"` TensorSplit string `yaml:"tensorSplit"` MainGpu int `yaml:"mainGpu"` OverrideTensor string `yaml:"overrideTensor"` // Image (sd-server) knobs. Unlike the llama knobs above, an image variant // INHERITS the model-wide Override for anything it leaves empty (component // paths + placement) and overrides only what it sets — the natural use is a // generation preset (e.g. "fast" 8-step cfg 1 vs "quality" 30-step) on the // same model + encoders. See mergeImageVariant. VaePath string `yaml:"vaePath"` ClipLPath string `yaml:"clipLPath"` ClipGPath string `yaml:"clipGPath"` T5Path string `yaml:"t5Path"` TextEncoderPath string `yaml:"textEncoderPath"` AudioVaePath string `yaml:"audioVaePath"` RefEdit string `yaml:"refEdit"` LlmVision string `yaml:"llmVision"` LlmVisionPath string `yaml:"llmVisionPath"` LoraDir string `yaml:"loraDir"` OffloadToCpu string `yaml:"offloadToCpu"` TeOnCpu string `yaml:"teOnCpu"` VaeOnCpu string `yaml:"vaeOnCpu"` VaeTiling string `yaml:"vaeTiling"` TemporalTiling string `yaml:"temporalTiling"` StreamLayers string `yaml:"streamLayers"` DiffusionFa string `yaml:"diffusionFa"` DefaultSteps int `yaml:"defaultSteps"` DefaultCfg float64 `yaml:"defaultCfg"` DefaultSampler string `yaml:"defaultSampler"` DefaultWidth int `yaml:"defaultWidth"` DefaultHeight int `yaml:"defaultHeight"` DefaultFrames int `yaml:"defaultFrames"` DefaultFps int `yaml:"defaultFps"` } // applyDefaults fills zero-valued settings with the PowerShell defaults. // defaultDenseCtxLadder is the dense context ladder used when nothing pins one. // Its TOP rung is the ceiling GetDenseCtx sizes against, so it has to sit at or // above the largest trained window in circulation: a ladder topping out at 128k // pinned every 256k/512k/1M model there and left the VRAM those extra tokens // would have used sitting idle (gemma-4-12B-it-qat is trained to 262144 and was // being loaded at 131072 with 10 GB of a 22.8 GB budget unspent). The rungs // below the top are documentation of the usual tiers: the sizer picks the // largest window that actually fits and rounds it to a 4096 boundary. func defaultDenseCtxLadder() []int { return []int{1048576, 524288, 262144, 131072, 65536, 32768} } // legacyDenseCtxLadder is the ladder shipped before the ceiling was raised. var legacyDenseCtxLadder = []int{131072, 65536, 32768} // dropLegacyCtxLadder clears a ladder that is EXACTLY the old shipped default so // applyDefaults re-seeds the current one. Both the generate file and the UI // settings sidecar persist every knob they were shown, including ones the user // never touched, so an install made before the ceiling moved carries the 128k cap // forever and raising the default alone changes nothing. Any other ladder is a // deliberate cap and is left exactly as written. func (s *Settings) dropLegacyCtxLadder() { if len(s.DenseCtxLadder) != len(legacyDenseCtxLadder) { return } for i, v := range s.DenseCtxLadder { if v != legacyDenseCtxLadder[i] { return } } s.DenseCtxLadder = nil } func (s *Settings) applyDefaults() { if s.ServerExe == "" { s.ServerExe = "llama-server" } if s.MinGpuVramGB == 0 { s.MinGpuVramGB = minInferenceVramGB } if s.SharedMemory == "" { s.SharedMemory = SharedMemoryAuto } if s.SdServerExe == "" { if strings.ContainsAny(s.ServerExe, `/\`) { s.SdServerExe = filepath.Join(filepath.Dir(s.ServerExe), "sd-server") } else { s.SdServerExe = "sd-server" } } if s.TtsServerExe == "" { if strings.ContainsAny(s.ServerExe, `/\`) { s.TtsServerExe = filepath.Join(filepath.Dir(s.ServerExe), "tts-server") } else { s.TtsServerExe = "tts-server" } } if s.AsrServerExe == "" { if strings.ContainsAny(s.ServerExe, `/\`) { s.AsrServerExe = filepath.Join(filepath.Dir(s.ServerExe), "parakeet-server") } else { s.AsrServerExe = "parakeet-server" } } // The budget placeholders. LoadGenerateFile replaces both with a live // hardware reading when the file/sidecar left them unset (seedHardwareBudgets), // so these only survive on a box with no GPU telemetry and no OS memory // reading — deliberately small, because the cost of guessing low is a model // sized conservatively and the cost of guessing high is a load that OOMs. if s.TargetVramGB == 0 { s.TargetVramGB = 7 } // vramOverheadGB is pure allocator slack: targetVramGB is already the free // VRAM, so the desktop's own usage is outside the budget rather than // something this has to reserve for. 1.0 was double-counting that, and on a // 12 GB card it cost a dense model a whole layer for nothing. 0.5 also // matches the example file and longCtxHeadroomGB, which makes the long-context // top-up in longCtxTarget an exact no-op instead of a partial one. if s.VramOverheadGB == 0 { s.VramOverheadGB = 0.5 } if s.MinGpuFraction == 0 { s.MinGpuFraction = 0.5 } else if s.MinGpuFraction < 0 { s.MinGpuFraction = 0 // explicit opt-out: no floor, degrade instead of refuse } if s.OomGuardReserveGB == 0 { s.OomGuardReserveGB = 1.0 } else if s.OomGuardReserveGB < 0 { s.OomGuardReserveGB = 0 // explicit opt-out: admit into the raw leftovers } if s.OomGuardGraceSec == 0 { s.OomGuardGraceSec = 30 } if s.ComputeBufFactor == 0 { s.ComputeBufFactor = 1.0 } if s.VisionOverheadGB == 0 { s.VisionOverheadGB = 1.0 } if s.VisionCtx == 0 { s.VisionCtx = 8192 } if s.MaxRamGB == 0 { s.MaxRamGB = 24 // placeholder; see TargetVramGB above } if s.MoeCtxTarget == 0 { s.MoeCtxTarget = 65536 } s.dropLegacyCtxLadder() if len(s.DenseCtxLadder) == 0 { s.DenseCtxLadder = defaultDenseCtxLadder() } if s.DenseMinCtx == 0 { s.DenseMinCtx = 32768 } if s.Threads == 0 { s.Threads = 7 } if s.TtlSec == 0 { s.TtlSec = 600 } if s.HealthCheckTimeout == 0 { s.HealthCheckTimeout = 300 } // Slot KV-cache persistence is OFF by default (the master switch); the // server-default knobs below are still pre-filled so the dashboard shows real // numbers, not 0. Per-model SlotCache is OPT-IN (nil => off), so the master // switch arms the feature and each model is enrolled from its config editor - // otherwise every model in the fleet starts leaving snapshots and preamble // caches behind on its first big prefill. Enable's zero value (false) is // the default; turn it on via the dashboard (writes enable:true to the // sidecar, which overlays this). if s.SlotCache.MinSaveTokens == 0 { s.SlotCache.MinSaveTokens = 30000 } if s.SlotCache.MaxDiskGB == 0 { s.SlotCache.MaxDiskGB = 10 } if s.SlotCache.MaxSessions == 0 { s.SlotCache.MaxSessions = 20 } if s.SlotCache.MaxIdleDays == 0 { s.SlotCache.MaxIdleDays = 7 } } // MinimalGenerateFile is the control file written when a caller has to create // one from nothing: the setup wizard on a fresh install, and a bare binary // bootstrapping its own config directory. // // applyDefaults fills every zero-valued knob, so a file with nothing but // modelsRoot is fully functional -- what is lost against the shipped example is // its explanatory comments, not behaviour. The pointer to the example is there // so a user who wants the annotated version knows one exists. const MinimalGenerateFile = `# Quartermaster autogen control file. # Every unset knob falls back to its built-in default; see # quartermaster-generate.example.yaml in the repository for the annotated form. settings: modelsRoot: "" ` // LoadGenerateFile reads and validates an autogen control file. Settings // defaults are applied. modelsDirOverride (from --models-dir) wins over the // file's modelsRoot when non-empty. func LoadGenerateFile(path, modelsDirOverride string) (GenerateFile, error) { data, err := os.ReadFile(path) if err != nil { return GenerateFile{}, err } var gf GenerateFile if err := yaml.Unmarshal(data, &gf); err != nil { return GenerateFile{}, fmt.Errorf("parsing %s: %w", path, err) } // Capture "the file said nothing about the memory budgets" BEFORE // applyDefaults writes its placeholders over the zero values, so // seedHardwareBudgets below can tell an unset knob from a deliberate one. vramUnset, ramUnset := gf.Settings.TargetVramGB == 0, gf.Settings.MaxRamGB == 0 // Overlay UI-owned backend-exe paths BEFORE applyDefaults so an empty // sd/tts-server derives as a sibling of the (possibly UI-set) llama exe. be, err := LoadSidecarBackends(path) if err != nil { return GenerateFile{}, err } if be != nil { if be.ServerExe != "" { gf.Settings.ServerExe = be.ServerExe } if be.SdServerExe != "" { gf.Settings.SdServerExe = be.SdServerExe } if be.TtsServerExe != "" { gf.Settings.TtsServerExe = be.TtsServerExe } if be.AsrServerExe != "" { gf.Settings.AsrServerExe = be.AsrServerExe } } // The full backend registry (multi-entry) that per-model resolveBackend reads. // Sidecar wins over any generate-file backends list. if list, err := LoadSidecarBackendList(path); err != nil { return GenerateFile{}, err } else if len(list) > 0 { gf.Settings.Backends = list } gf.Settings.applyDefaults() // Overlay the UI-owned settings patch (dashboard VRAM/headroom edits) ahead of // modelsDir resolution. Absent => no-op. patch, err := LoadSidecarSettings(path) if err != nil { return GenerateFile{}, err } patch.apply(&gf.Settings) // The sidecar overlay lands after applyDefaults, so a legacy ladder persisted // by the dashboard has to be migrated again here or it wins. gf.Settings.dropLegacyCtxLadder() if len(gf.Settings.DenseCtxLadder) == 0 { gf.Settings.DenseCtxLadder = defaultDenseCtxLadder() } if patch != nil { vramUnset = vramUnset && patch.TargetVramGB == nil ramUnset = ramUnset && patch.MaxRamGB == nil } // Measure the box for anything neither the file nor the dashboard pinned. // Done here rather than in applyDefaults so the ONE cached probe covers every // consumer of the merged settings — startup, hot reload, and the editor's // preview all size against the same budget instead of the preview quietly // using the placeholder the emitter replaced. seedHardwareBudgets(&gf.Settings, vramUnset, ramUnset) // UI-owned fleet-wide default variants replace the file's list wholesale. sideDV, err := LoadSidecarDefaultVariants(path) if err != nil { return GenerateFile{}, err } if sideDV != nil { gf.Settings.DefaultVariants = sideDV } // UI-owned display-name renames overlay the file's (per-key, like categoryRoots). sideDN, err := LoadSidecarDisplayNames(path) if err != nil { return GenerateFile{}, err } if len(sideDN) > 0 { if gf.Settings.DisplayNames == nil { gf.Settings.DisplayNames = map[string]string{} } for k, v := range sideDN { gf.Settings.DisplayNames[k] = v } } // UI-owned per-category scan folders overlay the file's (the folder picker // writes these). Stored at the sidecar top level so a VRAM reset can't wipe them. sideRoots, err := LoadSidecarCategoryRoots(path) if err != nil { return GenerateFile{}, err } if len(sideRoots) > 0 { if gf.Settings.CategoryRoots == nil { gf.Settings.CategoryRoots = map[string]string{} } for k, v := range sideRoots { gf.Settings.CategoryRoots[k] = v } } if modelsDirOverride != "" { gf.Settings.ModelsRoot = modelsDirOverride } // Merge the UI-owned sidecar overrides ahead of the hand-authored ones so // per-model edits from the web editor win (override resolution is // first-match). The sidecar is fully owned by the UI and may be absent. sideRows, err := LoadSidecarOverrides(path) if err != nil { return GenerateFile{}, err } if len(sideRows) > 0 { gf.Overrides = append(sidecarExactFirst(sideRows), gf.Overrides...) } // UI-owned API keys replace the file's list wholesale (the UI sends the full // list). nil => inherit the generate file's apiKeys. sideKeys, err := LoadSidecarAPIKeys(path) if err != nil { return GenerateFile{}, err } if sideKeys != nil { gf.Settings.APIKeys = sideKeys } // UI-owned prompt enhancers replace the file's list wholesale, same contract // as apiKeys above. sideEnh, err := LoadSidecarPromptEnhancers(path) if err != nil { return GenerateFile{}, err } if sideEnh != nil { gf.Settings.PromptEnhancers = sideEnh } // UI-added image models EXTEND the file's list (not replace): see the // sidecar field's comment for why. sideExtra, err := LoadSidecarExtraImageModels(path) if err != nil { return GenerateFile{}, err } removedExtra, err := LoadSidecarRemovedExtraImageModels(path) if err != nil { return GenerateFile{}, err } gf.Settings.ExtraImageModels = mergeExtraImageModels(gf.Settings.ExtraImageModels, sideExtra, removedExtra) // UI-owned slot-KV block overlays the generate file's settings.slotCache. sideSlot, err := LoadSidecarSlotCache(path) if err != nil { return GenerateFile{}, err } if sideSlot != nil { gf.Settings.SlotCache = *sideSlot } // A blank modelsRoot is not an error: the server boots with an empty catalog // so a setup UI can point it at a models folder later. Discovery/hashing // short-circuit on empty (no cwd scan). for i, o := range gf.Overrides { if strings.TrimSpace(o.Match) == "" { return GenerateFile{}, fmt.Errorf("overrides[%d]: match is required", i) } // A model-level "none" is redundant (empty already means no flag) but a // user who learned the sentinel on a variant will write it here too; // clear it once, centrally, so it can never reach the emitter as a path. NormalizeNone(&gf.Overrides[i]) } // Record the llama backend's compute runtime for the sizer's per-process VRAM // constant. Here rather than at startup because this is the one function every // sizing path shares - emit, the editor preview, the spawn-time guard - so all // three charge the same constant even when the UI repoints the backend at // runtime and no restart happens. NoteBackendRuntime(gf.Settings.ServerExe) return gf, nil } // LoadBaseSettings returns the generate file's settings with defaults applied // but WITHOUT the UI sidecar patch - i.e. the values a dashboard "reset to // default" reverts to. func LoadBaseSettings(path, modelsDirOverride string) (Settings, error) { data, err := os.ReadFile(path) if err != nil { return Settings{}, err } var gf GenerateFile if err := yaml.Unmarshal(data, &gf); err != nil { return Settings{}, fmt.Errorf("parsing %s: %w", path, err) } gf.Settings.applyDefaults() if modelsDirOverride != "" { gf.Settings.ModelsRoot = modelsDirOverride } return gf.Settings, nil } // sidecarExactFirst stably partitions sidecar overrides so exact-path rows (the // ones the UI writes) precede glob rows. Override resolution is first-match, so // without this a hand-authored glob (e.g. "*35B-MTP-GGUF*") earlier in the file // shadows the UI's exact-path row for the same gguf and silently drops the // variants the editor still shows (it reads the exact-path row). Returns a fresh // slice; the input is not mutated. func sidecarExactFirst(rows []Override) []Override { exact := make([]Override, 0, len(rows)) var globs []Override for _, r := range rows { if strings.ContainsAny(r.Match, "*?") { globs = append(globs, r) } else { exact = append(exact, r) } } return append(exact, globs...) } // ResolveFileOverride returns the hand-authored generate FILE override (the UI // sidecar is EXCLUDED) that the generator resolves for ggufPath, matched by path // glob + detected quant. The config editor uses this as the base for a UI save so // file-only fields the editor doesn't model (ctxVariants, quant, file-defined // variants) are carried into the sidecar row instead of being silently lost when // the sidecar shadows the file row. ok=false (zero Override) when nothing matches. func ResolveFileOverride(generatePath, ggufPath string) (Override, bool, error) { data, err := os.ReadFile(generatePath) if err != nil { return Override{}, false, err } var gf GenerateFile if err := yaml.Unmarshal(data, &gf); err != nil { return Override{}, false, fmt.Errorf("parsing %s: %w", generatePath, err) } row := GgufRow{FullPath: ggufPath, Quant: quantFromName(filepath.Base(ggufPath))} if o := ResolveOverride(row, gf.Overrides); o != nil { return *o, true, nil } return Override{}, false, nil } // ResolveOverride returns the first override whose Match globs the gguf path and // whose optional Quant equals the row's quant. nil when none match. func ResolveOverride(row GgufRow, overrides []Override) *Override { for i := range overrides { o := &overrides[i] if !globLike(o.Match, row.FullPath) { continue } if o.Quant != "" && !strings.EqualFold(o.Quant, row.Quant) { continue } return o } return nil } var globCache = map[string]*regexp.Regexp{} // globLike emulates PowerShell's -like: case-insensitive, '*' matches any run of // characters (including path separators), '?' matches one. Both pattern and // subject are normalized to forward slashes so a pattern written either way // matches a Windows path. func globLike(pattern, s string) bool { re, ok := globCache[pattern] if !ok { var b strings.Builder b.WriteString("(?i)^") for _, r := range strings.ReplaceAll(pattern, "\\", "/") { switch r { case '*': b.WriteString(".*") case '?': b.WriteString(".") default: b.WriteString(regexp.QuoteMeta(string(r))) } } b.WriteString("$") re = regexp.MustCompile(b.String()) globCache[pattern] = re } return re.MatchString(strings.ReplaceAll(s, "\\", "/")) } // MultiGpuEnabled reports whether a plan may span every eligible adapter // (default on). It says nothing about how many the box HAS: a single-GPU // machine resolves to a one-device set and every multi-GPU path no-ops. func (s Settings) MultiGpuEnabled() bool { return s.MultiGpu == nil || *s.MultiGpu } // SharedMemoryMode resolves the sharedMemory setting to one of // SharedMemoryAuto / Off / On; anything unrecognised (including empty) is auto, // so a typo in the generate file cannot silently disable the pool. func (s Settings) SharedMemoryMode() string { switch s.SharedMemory { case SharedMemoryOff, SharedMemoryOn: return s.SharedMemory default: return SharedMemoryAuto } } // PoolIntegratedEnabled reports whether an integrated GPU may be pooled into a // split beside a dedicated card. Default false: see PoolIntegratedGpu. func (s Settings) PoolIntegratedEnabled() bool { return s.PoolIntegratedGpu != nil && *s.PoolIntegratedGpu } // DevicePolicy is the device-selection policy these settings describe, for the // set builders in gpuset.go. Passed explicitly so the sizer, the OOM guard and // the spawn-time retune are all answering with the same one. func (s Settings) DevicePolicy() GpuPolicy { return GpuPolicy{ MinTotalGB: s.MinGpuVramGB, SharedMemory: s.SharedMemoryMode(), PoolIntegrated: s.PoolIntegratedEnabled(), } } // GpuSetOrEmpty is the resolved device set, or nil when telemetry never // answered. Consumers treat nil as "single GPU, budget as before". func (s Settings) GpuSetOrEmpty() GpuSet { if !s.MultiGpuEnabled() { return nil } return s.Gpus }