package config import ( "errors" "fmt" "runtime" ) const ( MODEL_CONFIG_DEFAULT_TTL = -1 ) var validModalities = map[string]struct{}{ "text": {}, "audio": {}, "image": {}, // 3d is the mesh modality: TRELLIS.2 takes an image and returns a GLB. "3d": {}, // video is the moving-picture modality: a video DiT takes a prompt (and // optionally a first/last frame) and returns an encoded container. A model // that also scores its clip declares audio alongside it. "video": {}, } // ModelCapConfig defines what modalities and features a model supports. // Used in /v1/models to inform clients. An empty block (all zero values) is // treated as not configured. type ModelCapConfig struct { In []string `yaml:"in"` Out []string `yaml:"out"` Tools bool `yaml:"tools"` Reranker bool `yaml:"reranker"` Embedding bool `yaml:"embedding"` Segmentation bool `yaml:"segmentation"` // Decision marks a llama.cpp decision model: it answers typed questions on // /v1/systemone and cannot chat. DecisionType is its ".decision.type" // (kev, laya, openjev, ...), which decides per-model limits upstream such as // the number of choice options, so clients get to see it. Decision bool `yaml:"decision"` DecisionType string `yaml:"decisionType"` // VoiceClone marks a speech model that can register NEW voices from a // reference clip (qwentts.cpp). It is not implied by out:[audio] — a // fixed-voice-pack engine like TTS.cpp/Kokoro serves speech but has no // endpoint to clone with, and offering the UI for it produces a 404 at best. VoiceClone bool `yaml:"voiceClone"` // Music marks an audio.cpp music generator (ACE-Step, MiniMax Music): it // answers /v1/music/generations, never /v1/audio/speech, so it must not // borrow the speech capability a text->audio In/Out pair would imply. // MusicEdit adds /v1/music/edits (repaint / complete / cover), which only // some generators implement. Separation marks a stem splitter (HTDemucs) // answering /v1/audio/separations. Music bool `yaml:"music"` MusicEdit bool `yaml:"musicEdit"` Separation bool `yaml:"separation"` // MusicScore names the score notation a music generator can be conditioned // on ("abc" for YuE2); empty = none. Sent as the generation route's typed // `score` field. MusicScore string `yaml:"musicScore"` // MusicTranscribe names the score notation a transcriber writes from a // recording ("abc" for SheetSage2); empty = not a transcriber. Answered on // /v1/music/transcriptions. MusicTranscribe string `yaml:"musicTranscribe"` // ReasoningEffort lists the reasoning-effort levels this model's chat // template accepts (Qwen 3.8: xhigh/medium/low), lowest-cost value LAST as // the template declares them. Non-empty is the signal that an incoming // OpenAI `reasoning_effort` field may be translated into a // chat_template_kwargs entry — a template that doesn't understand the value // raises, which surfaces as a 500, so an unlisted model is never translated // for. Advertised in /v1/models so a client can discover the ladder. ReasoningEffort []string `yaml:"reasoningEffort"` Context int `yaml:"context"` } // Empty returns true when all fields are at their zero values. func (c ModelCapConfig) Empty() bool { return len(c.In) == 0 && len(c.Out) == 0 && !c.Tools && !c.Reranker && !c.Embedding && !c.Segmentation && !c.Decision && c.DecisionType == "" && !c.VoiceClone && !c.Music && !c.MusicEdit && !c.Separation && c.MusicScore == "" && c.MusicTranscribe == "" && len(c.ReasoningEffort) == 0 && c.Context == 0 } // Validate checks that all modality values are recognized and context is // non-negative. Returns an error if any value is invalid. func (c ModelCapConfig) Validate() error { for _, m := range c.In { if _, ok := validModalities[m]; !ok { return fmt.Errorf("capabilities.in: invalid modality %q, must be one of: text, audio, image, 3d, video", m) } } for _, m := range c.Out { if _, ok := validModalities[m]; !ok { return fmt.Errorf("capabilities.out: invalid modality %q, must be one of: text, audio, image, 3d, video", m) } } if c.Context < 0 { return errors.New("capabilities.context: must be >= 0") } return nil } // TimeoutsConfig holds timeout settings for proxy connections // 0 = no timeout type TimeoutsConfig struct { Connect int `yaml:"connect"` KeepAlive int `yaml:"keepalive"` ResponseHeader int `yaml:"responseHeader"` TLSHandshake int `yaml:"tlsHandshake"` ExpectContinue int `yaml:"expectContinue"` IdleConn int `yaml:"idleConn"` } type ModelConfig struct { Cmd string `yaml:"cmd"` CmdStop string `yaml:"cmdStop"` Proxy string `yaml:"proxy"` Aliases []string `yaml:"aliases"` Env []string `yaml:"env"` CheckEndpoint string `yaml:"checkEndpoint"` UnloadAfter int `yaml:"ttl"` Unlisted bool `yaml:"unlisted"` UseModelName string `yaml:"useModelName"` // MissingComponents names the diffusion components (vae, llm, t5xxl, ...) // autogen could not find on disk for this model. Pure data: the process // layer only reports it, because sd-server dies within seconds without // them and its own log is the only other place that says why. MissingComponents []string `yaml:"missingComponents"` // #179 for /v1/models Name string `yaml:"name"` Description string `yaml:"description"` // Limit concurrency of HTTP requests to process ConcurrencyLimit int `yaml:"concurrencyLimit"` // Model filters see issue #174 Filters ModelFilters `yaml:"filters"` // Macros: see #264 // Model level macros take precedence over the global macros Macros MacroList `yaml:"macros"` // Metadata: see #264 // Arbitrary metadata that can be exposed through the API Metadata map[string]any `yaml:"metadata"` // override global setting SendLoadingState *bool `yaml:"sendLoadingState"` // Timeout settings for proxy connections Timeouts TimeoutsConfig `yaml:"timeouts"` // Capabilities defines what modalities and features the model supports. Capabilities ModelCapConfig `yaml:"capabilities"` // EstVramGB is this model's predicted VRAM footprint when loaded (weights + // KV reserve + compute buffer + overhead), as computed by the autogen sizer // at generate time. It is the admission input for VRAM-budget-aware // multi-load (see Config.VramBudgetGB): the router sums it over the resident // set to decide whether a new model fits alongside them or something must be // evicted first. 0 = unknown (hand-written config, or a class the sizer does // not model), which the router treats conservatively — see // internal/router/group.go. EstVramGB float64 `yaml:"estVramGB"` // EstRamGB is the sizer's companion figure: the share of the weights (plus // KV, when it is kept in system memory) that does NOT fit on the GPU and is // served from RAM. 0 for a fully-offloaded model. The router charges it // against Config.RamBudgetGB (every resident, whichever pool) and evicts // ram-pool models to make it fit; the dashboard shows it per model so the // cost of a partial offload is visible before loading. EstRamGB float64 `yaml:"estRamGB"` // SlotCachePreamble opts this model out of PREAMBLE caches (the shared // system+tools KV seed that is minted once per agent and reused by every // load) while leaving its per-conversation snapshots alone. nil/absent => // on, so a model that opted into slot persistence at all gets both halves. // The whole-feature per-model gate stays the cmd's --slot-save-path; this is // the finer knob, because a preamble cache is minted unprompted (a synthetic // prefill) and is exempt from the LRU disk cap. SlotCachePreamble *bool `yaml:"slotCachePreamble"` // AudioCpp carries the one thing audiocpp_server cannot be told on its // command line. It has no --model flag at all: the model is named only in a // JSON config file passed as --config, while every other knob is an argv // flag that overrides the file. Keeping the entry here instead of writing a // JSON file per model at generate time means one source of truth, no // generated files to keep in step with the config, and nothing left behind // when a model is renamed or removed. The spawn hook materializes it (see // internal/server/audiocppconfig.go). AudioCpp AudioCppConfig `yaml:"audiocpp"` // PromptEnhancer names the Config.PromptEnhancers entry this IMAGE model // sends its prompt through before rendering. Empty (every non-image model, // and every image model that has not opted in) => none. // // A dangling id is not an error: an enhancer whose model was deleted resolves // to nothing and the model renders the prompt as typed. Failing config load // over a rewrite convenience would be the wrong trade. PromptEnhancer string `yaml:"promptEnhancer"` // PromptEnhancerEdit is the enhancer used when the request carries a // reference image, replacing PromptEnhancer for that direction only. Empty => // PromptEnhancer covers both. See autogen.Override.PromptEnhancerEdit. PromptEnhancerEdit string `yaml:"promptEnhancerEdit"` // PromptEnhancerPrompt / PromptEnhancerEditPrompt are THIS model's system // prompts for the two directions, overriding the shared // Config.PromptEnhancers entry's. Empty => use the shared one. See // autogen.Override.PromptEnhancerPrompt for why the prompt belongs to the // image model rather than to the rewriter. PromptEnhancerPrompt string `yaml:"promptEnhancerPrompt"` PromptEnhancerEditPrompt string `yaml:"promptEnhancerEditPrompt"` // Copy of HealthCheckTimeout from global config HealthCheckTimeout int `yaml:"healthCheckTimeout"` } // AudioCppConfig is ONE entry of audio.cpp's models[] array: the fields its // server requires (id comes from the model id) plus the task that selects which // route the model answers on. Deliberately not a passthrough for the rest of // that schema - lazy loading, the residency cap, idle unloads and the memory // guard are audio.cpp's own scheduler, which is contained rather than // configured here (see internal/autogen/audiocpp.go). type AudioCppConfig struct { // Family is audio.cpp's model family id (moss_tts_nano, qwen3_asr, ...), // read from the gguf's own audiocpp.model_spec.family header key. Family string `yaml:"family"` // Path is the gguf (or model directory) the server loads. Path string `yaml:"path"` // Task selects which surface this process serves: "tts", "asr", "gen" // (music generation), "sep" (source separation) or "midi" (score // transcription). Blank lets audio.cpp // apply its own default ("tts"). Task string `yaml:"task"` // SessionOptions are audio.cpp's per-model session_options, passed through // verbatim. A directory model whose package holds several quants // (MiniMax Music 3) is told which component files to load here, since its // built-in defaults name a quant that may not be on disk. SessionOptions map[string]string `yaml:"sessionOptions"` } // Empty reports whether this model carries no audio.cpp block at all. Family // and Path are both required by the server, so a half-filled block is as // unusable as an absent one and is treated the same way. func (a AudioCppConfig) Empty() bool { return a.Family == "" || a.Path == "" } func (m *ModelConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { type rawModelConfig ModelConfig defaults := rawModelConfig{ Cmd: "", CmdStop: "", Proxy: "http://localhost:${PORT}", Aliases: []string{}, Env: []string{}, CheckEndpoint: "/health", UnloadAfter: MODEL_CONFIG_DEFAULT_TTL, // use GlobalTTL Unlisted: false, UseModelName: "", ConcurrencyLimit: 0, Name: "", Description: "", // matches http.DefaultTransport Timeouts: TimeoutsConfig{ Connect: 30, KeepAlive: 30, ResponseHeader: 0, TLSHandshake: 10, ExpectContinue: 1, IdleConn: 90, }, } // the default cmdStop to taskkill /f /t /pid ${PID} if runtime.GOOS == "windows" { defaults.CmdStop = "taskkill /f /t /pid ${PID}" } if err := unmarshal(&defaults); err != nil { return err } *m = ModelConfig(defaults) return nil } func (m *ModelConfig) SanitizedCommand() ([]string, error) { return SanitizeCommand(m.Cmd) } // ModelFilters embeds Filters and adds legacy support for strip_params field // See issue #174 type ModelFilters struct { Filters `yaml:",inline"` } func (m *ModelFilters) UnmarshalYAML(unmarshal func(interface{}) error) error { type rawModelFilters ModelFilters defaults := rawModelFilters{} if err := unmarshal(&defaults); err != nil { return err } // Try to unmarshal with the old field name for backwards compatibility if defaults.StripParams == "" { var legacy struct { StripParams string `yaml:"strip_params"` } if legacyErr := unmarshal(&legacy); legacyErr != nil { return errors.New("failed to unmarshal legacy filters.strip_params: " + legacyErr.Error()) } defaults.StripParams = legacy.StripParams } *m = ModelFilters(defaults) return nil } // SanitizedStripParams wraps Filters.SanitizedStripParams for backwards compatibility // Returns ([]string, error) to match existing API func (f ModelFilters) SanitizedStripParams() ([]string, error) { return f.Filters.SanitizedStripParams(), nil }