using System.Diagnostics; using System.Linq; using System.Net.Http.Headers; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using LlamaApp.Common; namespace LlamaApp.Llama; /// /// Manages the local llama (llama.cpp) executable: detects an existing /// installation, and downloads + runs the official /// install.ps1 when none is found. /// /// Mirrors the macOS app's LlamaInstallManager + LlamaBinaries: /// install.ps1 puts llama on the user PATH (its install dir, /// %LOCALAPPDATA%\Microsoft\WindowsApps, is there by default), so the app /// never hardcodes the binary location — it resolves llama.exe with a /// which-style PATH lookup (). A hit under /// the install dir is the app-managed installation (may be installed/emptied); /// a hit anywhere else is the user's own external installation and is left alone. /// The installation is silent (writes under the user profile, no elevation needed). /// /// Call at startup; it adopts a /// running server, launches one, or downloads the binary on demand, and reports /// progress/state via . Once the server is reachable, /// lists locally available models via the /// GET /models REST endpoint. /// /// /// A server the app starts is managed: its PID is written to /// %LOCALAPPDATA%\Llama\.llama.pid right after spawn, so that after /// an app crash the next instance still recognizes the surviving server as its /// own — and kills it on exit (). Servers started any /// other way (manually, whatever the binary) have no PID file and are left /// alone. /// public sealed class LlamaManager { private static LlamaManager? _shared; /// /// Shared singleton, matching the macOS app's .shared. Created by /// with the configured server port — the app calls /// it once at startup (App.OnLaunched) before anything else can touch the /// manager (the MainWindow constructor subscribes to its events). /// public static LlamaManager Shared => _shared ?? throw new InvalidOperationException( "LlamaManager.Initialize(serverPort) must be called once at startup before first use."); /// /// Creates the singleton bound to /// and . /// Must be called once at startup, before the first /// access: both are baked in at construction and /// every health probe / launch argument / REST URL derives from them, so /// changed settings only take effect on the next app launch. A null, /// blank, or unparseable address falls back to loopback. When /// is non-empty it is sent as a Bearer token on /// every request and passed to the server via the LLAMA_API_KEY /// environment variable. /// public static LlamaManager Initialize(int serverPort, string? listenAddress = null, string? apiKey = null) { if (_shared is not null) throw new InvalidOperationException("LlamaManager is already initialized."); if (serverPort is < 1 or > 65535) throw new ArgumentOutOfRangeException(nameof(serverPort), "Port must be in 1..65535."); if (!System.Net.IPAddress.TryParse(listenAddress, out var parsed)) listenAddress = Common.ListenAddresses.Localhost; else if (parsed.AddressFamily != System.Net.Sockets.AddressFamily.InterNetwork) listenAddress = Common.ListenAddresses.Localhost; // IPv4 only — matches the server's bind. _shared = new LlamaManager(serverPort, listenAddress, apiKey); return _shared; } /// URL of the official Windows install script. private static readonly Uri InstallScriptUrl = new(InstallScriptIntegrity.ExpectedScriptUrl); /// /// The install dir install.ps1 targets — on the user PATH by default, /// which is how the script makes llama resolvable. Never probed /// directly (the binary is discovered via ); /// used to classify a PATH hit as managed vs external /// () and to back the Settings "Installation /// Folder" card. /// private static string ManagedDir => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "Microsoft", "WindowsApps"); /// /// The install directory the app manages — surfaced read-only in Settings /// ("Installation Folder" card). External (PATH) installations are never /// managed: their location comes from instead. /// Shared: this is the WindowsApps folder that also holds every /// other packaged app's execution aliases (winget.exe, python.exe, …), /// so only the names in may ever /// be deleted from it. /// public static string ManagedInstallDir => ManagedDir; /// /// %USERPROFILE%\.llama — the one per-user location MSIX /// file-system virtualization does NOT redirect: the packaged app writes /// here and the out-of-package llama.exe reads the same bytes at the same /// path. Everything the server reads from disk — the models-preset INI, /// the API-key file, the managed models dir — must live under this root: /// a %LOCALAPPDATA% write lands in Packages\…\LocalCache /// where the server cannot open it, and it then refuses to start or /// silently sees no models (issue #107 is the API-key instance of this). /// private static string SharedLlamaDir => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".llama"); /// /// The app's local-models directory, passed to the llama server as /// --models-dir: %USERPROFILE%\.llama\models. The router scans /// it on every GET /models, and a .gguf dropped directly in /// it becomes a loadable model named after its file — which is how /// "Open With → Llama" on a .gguf works: the file is linked in here /// (), then loaded by that id. /// /// Under and not /// %LOCALAPPDATA%\Llama: the server scans this directory from OUTSIDE /// the package identity, and an MSIX-virtualized write would land where /// the server cannot see it — the exact trap issue #107 hit with the /// API-key file. Content of the pre-fix location migrates on first use /// (see ). /// public static string ManagedModelsDir => Path.Combine(SharedLlamaDir, "models"); /// The pre-fix models dir, under the MSIX-virtualized /// %LOCALAPPDATA%\Llama; found content is moved to /// on first use. private static string LegacyManagedModelsDir => Path.Combine(Common.AppData.Root, "models"); /// /// Ensures exists, migrating content from /// the pre-fix (virtualized) location first. Idempotent and best-effort; /// a partially migrated dir resumes on the next call. /// internal static void EnsureManagedModelsDir() { MigrateManagedModelsDir(LegacyManagedModelsDir, ManagedModelsDir); Directory.CreateDirectory(ManagedModelsDir); } /// /// Moves everything under into /// — the pre-fix models dir lived under the /// MSIX-virtualized %LOCALAPPDATA%\Llama, invisible to the /// out-of-package llama server. Per-entry best effort and retry-safe: an /// entry whose target name already exists is skipped (a resumed migration /// completes only what is left), and one that cannot move is logged and /// left behind. Returns the number of entries moved. /// internal static int MigrateManagedModelsDir(string legacyDir, string targetDir) { var moved = 0; try { if (!Directory.Exists(legacyDir)) return 0; Directory.CreateDirectory(targetDir); foreach (var entry in Directory.EnumerateFileSystemEntries(legacyDir)) { var dest = Path.Combine(targetDir, Path.GetFileName(entry)); if (File.Exists(dest) || Directory.Exists(dest)) continue; try { if (Directory.Exists(entry)) Directory.Move(entry, dest); else File.Move(entry, dest); moved++; } catch (Exception ex) { Log.Warn(ex, $"model dir migration: '{entry}' could not move; left behind"); } } if (moved > 0) Log.Info($"model dir migration: moved {moved} entr{(moved == 1 ? "y" : "ies")} to '{targetDir}'"); try { Directory.Delete(legacyDir); } catch { /* still has left-behind entries */ } } catch (Exception ex) { Log.Warn(ex, "model dir migration failed; the legacy location is left as-is"); } return moved; } /// /// Brings a model file into for the /// router to load — the "Open With" step for a local .gguf. Returns the /// imported model's id (the file name without extension); see /// for link/copy/no-op semantics. /// public static LocalModelImportResult ImportLocalModel(string sourcePath) { EnsureManagedModelsDir(); return LocalModelImport.Import(sourcePath, ManagedModelsDir); } /// /// File names install.ps1 can leave in . /// The current script moves only llama.exe there (everything else /// lives in the it wipes on exit); older /// generations decompressed and probed directly in the install dir and can /// have left any of the others. Because the folder is shared, "emptying" /// must delete exactly these names and nothing else. /// internal static readonly IReadOnlyList ManagedInstallArtifacts = new[] { "llama.exe", // the installed server binary "llama.exe.old", // re-install backup, if an install died mid-move "unzstd.exe", // .zst decompressor (older scripts dropped it here) "cuda-probe.exe", // hardware probes (older scripts ran them in place) "rocm-probe.exe", "vulkan-probe.exe", "featcode.exe", "tmp.zst", // interrupted .zst extraction }; /// /// install.ps1's scratch directory (%LOCALAPPDATA%\llama-app): /// downloads and probes land here during an install, and the script /// deletes it on the way out — anything left is debris from an /// interrupted install (the partial tmp.zst can be large). /// internal static string InstallScratchDir => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "llama-app"); /// /// Deletes exactly the known llama install artifacts from /// — never other files: the real install dir /// is the shared WindowsApps folder whose other entries belong to other /// apps (deleting "everything in the folder" used to wipe their execution /// aliases). Each artifact is attempted independently, so one locked file /// counts as failed instead of aborting the rest. Returns (deleted, /// failed) counts; directory parameter for tests. /// internal static (int Deleted, int Failed) EmptyManagedInstall(string directory) { var deleted = 0; var failed = 0; foreach (var name in ManagedInstallArtifacts) { try { var path = Path.Combine(directory, name); if (File.Exists(path)) { File.Delete(path); deleted++; } } catch (Exception ex) { failed++; Log.Warn(ex, $"could not delete install artifact '{name}'"); } } return (deleted, failed); } /// /// Removes an interrupted install's scratch directory, if any — the folder /// is created and destroyed by install.ps1 itself, so anything /// present is download debris. Best-effort and logged, like the artifact /// deletion. Directory parameter for tests. /// internal static void DeleteInstallScratch(string directory) { try { if (Directory.Exists(directory)) Directory.Delete(directory, recursive: true); } catch (Exception ex) { Log.Warn(ex, $"could not delete install scratch dir '{directory}'"); } } /// /// The Settings "Empty" action: stops the managed server (llama.exe is /// otherwise locked), removes the known llama artifacts from the shared /// install dir, and clears interrupted-install debris from the scratch /// dir. Other apps' files in the shared folder are never touched. /// public (int Deleted, int Failed) EmptyManagedInstall() { StopServer(); var result = EmptyManagedInstall(ManagedInstallDir); DeleteInstallScratch(InstallScratchDir); return result; } /// /// Where the resolved binary comes from — surfaced in the flyout footer, so /// an external installation isn't mistaken for the app's own (and a stale /// version isn't mistaken for a bug). /// public enum Origin { /// Not yet resolved — no binary found and no install attempted. Unknown, /// App-managed binary found in (what install.ps1 produces). Managed, /// Pre-existing installation found on PATH (not modified by the app). External, } /// Install lifecycle state, surfaced in the UI. public enum InstallState { /// Ready — a usable binary is present (or we haven't needed to act). Idle, /// Downloading/installing the app-managed binary. Installing, /// The installation failed; retry via . Failed, } /// Server lifecycle state surfaced in the UI. public enum ServerState { /// Not started (no binary yet, or stopped). Stopped, /// Launched; waiting for the port to respond. Starting, /// Listening and serving requests. Running, /// The process exited unexpectedly or failed to bind. Failed, } /// /// Default port the local llama server listens on when the user hasn't /// configured one (mirrored by Settings.ServerPort in the app project). /// public const int DefaultServerPort = 9931; /// /// KV cache quantization types accepted as --cache-type-k/ /// --cache-type-v values. Kept in one place so a hand-edited /// settings.json with an unrecognized value never reaches the server as an /// invalid flag. /// internal static readonly string[] SupportedKvCacheTypes = { "f32", "f16", "bf16", "q8_0", "q4_0", "q4_1", "iq4_nl", "q5_0", "q5_1", }; /// /// Port the local llama server listens on (matches the flyout link). Fixed /// at construction via — the supervisor loop, /// health probes, server launch arguments and every REST URL are built /// from it. /// public int ServerPort { get; } /// /// IPv4 address the llama server binds to via --host: the loopback /// 127.0.0.1 (the default), 0.0.0.0 for every interface, or /// a specific interface's address. Fixed at construction via /// , like the port — the app's own REST client /// follows it (loopback when the server binds all interfaces, so the /// client and the bind address never disagree). A changed value takes /// effect on the next app launch. /// public string ListenAddress { get; } /// /// The address the app's own REST client and user-facing URLs connect to: /// the itself, except loopback when the /// server binds all interfaces (0.0.0.0 is not a connectable /// destination). Kept separate from so the /// bind flag and the client never disagree. /// public string ConnectAddress { get; } /// /// API key the app generated for a non-loopback bind, or null for the /// default loopback server (which needs none). Sent as a Bearer token on /// every REST request and handed to the server via the /// LLAMA_API_KEY environment variable (never an argv flag, which any /// local user could read via WMI, and never a file on disk), so a /// network-exposed server isn't an open control API. /// It must never appear in URLs (browser/WebView history, caches, server /// logs): the overlay injects it as a request header, the sample curl uses /// the Authorization header, and Settings shows it for copy if the /// server's WebUI ever asks for it. /// public string? ApiKey { get; } /// /// The address the app's REST client connects to for a given bind address: /// loopback when the server binds all interfaces (0.0.0.0 is not a /// connectable destination), the address itself otherwise. Pure so the /// mapping is unit-testable. /// internal static string ConnectAddressFor(string listenAddress) => listenAddress == Common.ListenAddresses.AllInterfaces ? Common.ListenAddresses.Localhost : listenAddress; /// /// Hugging Face cache directory passed to the server via /// HF_HUB_CACHE so it resolves downloaded models from the same /// location the app scans. Set by the caller (App.OnLaunched reads it from /// Settings.Current.CacheDirectory) — kept here rather than reading /// Settings directly to avoid a circular project dependency. /// public string? CacheDirectory { get; set; } /// /// Hugging Face access token passed to the server via HF_TOKEN so /// it can pull private/gated models on the user's behalf (llama.cpp reads /// the variable and sends it as a Bearer token on Hub requests). Set by /// the caller (App.OnLaunched reads it from /// Settings.Current.HuggingFaceToken) — kept here rather than /// reading Settings directly to avoid a circular project /// dependency. Only affects servers the app launches: an adopted /// already-running server keeps whatever environment it was started with. /// Never logged — presence only. /// public string? HuggingFaceToken { get; set; } /// /// Seconds of idleness after which the server unloads a model from memory, /// passed as --sleep-idle-seconds at launch; -1 (the default) /// disables it. An idled-out model reports sleeping — it stays the /// active model and wakes transparently on the next request. Set by the /// caller (App.OnLaunched reads it from /// Settings.Current.IdleUnloadSeconds) — kept here rather than /// reading Settings directly to avoid a circular project /// dependency. Only affects servers the app launches: an adopted /// already-running server keeps whatever arguments it was started with, /// and a changed value takes effect on the next server start. /// public int IdleUnloadSeconds { get; set; } = -1; /// /// Maximum number of models loaded in parallel in router mode, passed as /// --models-max at launch; 0 (the default) means unlimited. Set by /// the caller (App.OnLaunched reads it from /// Settings.Current.MaxLoadedModels) — kept here rather than reading /// Settings directly to avoid a circular project dependency. Only /// affects servers the app launches: an adopted already-running server /// keeps whatever arguments it was started with, and a changed value takes /// effect on the next server start. /// public int MaxLoadedModels { get; set; } = 0; /// /// KV cache quantization type for the key tensor, passed as /// --cache-type-k at launch; defaults to f16, llama.cpp's /// own default. Supported values are f32, f16, bf16, /// q8_0, q4_0, q4_1, iq4_nl, q5_0, and /// q5_1. Set by the caller (App.OnLaunched reads it from /// Settings.Current.CacheTypeK) — kept here rather than reading /// Settings directly to avoid a circular project dependency. Only /// affects servers the app launches: an adopted already-running server /// keeps whatever arguments it was started with, and a changed value takes /// effect on the next server start. /// public string CacheTypeK { get; set; } = "f16"; /// /// KV cache quantization type for the value tensor, passed as /// --cache-type-v at launch. Same supported values and default as /// ; K and V may be quantized independently. /// public string CacheTypeV { get; set; } = "f16"; /// /// Free-form extra llama serve arguments from the Settings page, /// tokenized at launch () /// and appended AFTER the built-in flags so they can override them. /// Set by the caller (App.OnLaunched reads it from /// Settings.Current.CustomServeArguments) — kept here rather than /// reading Settings directly to avoid a circular project dependency. /// Only affects servers the app launches: an adopted already-running /// server keeps whatever arguments it was started with, and a changed /// value takes effect on the next server start. /// public string? CustomServeArguments { get; set; } /// /// Per-model context-length preferences (keyed by server model id, /// repo:quant), rendered into a llama.cpp --models-preset INI /// at server launch so the router spawns each child with its chosen /// --ctx-size. This INI is the only channel the router honors: its /// /models/load endpoint reads only the model name and silently /// ignores a ctx field in the request body (verified against the llama.cpp /// source and empirically). Set by the caller (App.OnLaunched shares /// Settings.Current.ModelContextLengths by reference, so later edits /// are visible here) — kept here rather than reading Settings /// directly to avoid a circular project dependency. Only affects servers /// the app launches; an adopted already-running server keeps whatever it /// was started with. /// public IReadOnlyDictionary? ModelContextLengths { get; set; } /// /// Per-model speculative-decoding draft heads (keyed by server model id, /// repo:quant; the value is a llama.cpp --spec-type token /// like draft-dflash, or empty for "none shipped"), rendered into /// the same --models-preset INI as /// . With a spec type set, the server's /// child download plan for that model also fetches the matching draft /// sidecar (mtp-/dflash-/dspark-….gguf) from the repo and wires it /// in — speculative decoding, a free up-to-~2x generation speedup with /// identical output (issue #97). Set by the caller (App.OnLaunched shares /// Settings.Current.ModelSpeculativeTypes by reference, so later /// detections are visible here). Only affects servers the app launches. /// public IReadOnlyDictionary? ModelSpeculativeTypes { get; set; } /// /// Folders enrolled for model discovery (Settings → General → Model /// folders), shared by reference from /// Settings.Current.ModelFolders: places the user already keeps /// GGUF models, scanned recursively (see ) /// and rendered as model = path sections in the --models-preset /// INI at every server start and preset reload — nothing is copied or /// moved. The HF cache () and the managed /// models dir are excluded from the scan: llama downloads to the cache, /// discovery is for what the user already has elsewhere. /// public IReadOnlyList? ModelFolders { get; set; } private Process? _serverProcess; // Single-flight guard for EnsureLlamaOrDownloadAsync / StartServerAsync. Called // fire-and-forget from App.OnLaunched and re-entrant via StateChanged // handlers; without it, two concurrent callers can both pass the initial // "no server reachable" probe and both spawn a `llama serve --port 2276`, // leaking processes (one fails to bind and may linger; the second binds and eats // RAM). The gate serializes launches within one process; cross-instance // races are handled by the retrying adoption probe (see WaitForReachableAsync). private readonly SemaphoreSlim _ensureGate = new(1, 1); // Single-flight guard for WriteContextPresetsIniAsync: a server start and // a concurrent reload (settings change racing a launch) both write the // shared --models-preset file, and the loser of File.WriteAllText's // truncate race used to fail the write and return null — the server then // launched WITHOUT --models-preset. Serializing the writes keeps every // call returning the path. private readonly System.Threading.SemaphoreSlim _presetWriteGate = new(1, 1); /// /// The single for every llama-server REST call. /// carries the configured port and /// the selected listen address — loopback (127.0.0.1) whenever the /// server binds all interfaces, so the client always has an address that /// can actually be connected to. The handler bypasses the system proxy: a /// configured proxy/VPN must never intercept local traffic (the classic /// cause of "browser gets 200 OK, HttpClient fails"). The client-level /// timeout is infinite; each call bounds itself with a linked token /// () so SSE streams can run unbounded while /// probes stay snappy. /// private readonly HttpClient _http; /// Current installation state. public InstallState State { get; private set { if (field == value) return; field = value; StateChanged?.Invoke(this, EventArgs.Empty); } } = InstallState.Idle; /// User-facing reason for the state, if any. public string? FailureMessage { get; private set { field = value; StateChanged?.Invoke(this, EventArgs.Empty); } } /// Path to the resolved llama.exe, or null if none. public string? BinaryPath { get; private set { field = value; StateChanged?.Invoke(this, EventArgs.Empty); } } /// Version string reported by the resolved binary, or null. public string? Version { get; private set { field = value; // Keep LlamaRunner.Version in sync so the footer reads live. LlamaRunner.VersionCache = value; StateChanged?.Invoke(this, EventArgs.Empty); } } /// Where the resolved binary comes from. public Origin CurrentOrigin { get; private set { field = value; StateChanged?.Invoke(this, EventArgs.Empty); } } = Origin.Unknown; /// /// Current server state — derived from HTTP API polls by the always-on /// supervisor (see ), never from process /// handles: a spawned server's crash and an adopted server's crash look /// identical to the poll, and a server that appears is adopted the same /// way no matter who started it. /// public ServerState ServerStatus { get; private set { if (field == value) return; field = value; StateChanged?.Invoke(this, EventArgs.Empty); } } = ServerState.Stopped; /// Raised whenever any observable property changes. public event EventHandler? StateChanged; /// /// Raised by the supervisor loop (see ) on a /// background thread with a fresh GET /models snapshot while the server /// is . Publishing is change-gated: the event /// fires when the observable snapshot (ids, load states, paths, …) actually /// changed, once on entering even if the /// snapshot is unchanged, and as a slow heartbeat (see /// ) so subscribers with their own /// retry/backoff timers still wake up while nothing changes. Steady-state /// ticks do not fire — handlers are expected to be idempotent reconciles. /// Handlers should marshal to the UI thread before touching view models. /// public event EventHandler>? ModelsChanged; private LlamaManager(int serverPort, string listenAddress, string? apiKey) { ServerPort = serverPort; ListenAddress = listenAddress; // The app's REST client follows the bind address, except that // 0.0.0.0 means "the server listens on every interface" — for the // client, pick loopback, the one address that is always among them. var connectAddress = ConnectAddressFor(listenAddress); ConnectAddress = connectAddress; ApiKey = string.IsNullOrWhiteSpace(apiKey) ? null : apiKey.Trim(); _http = new HttpClient(new SocketsHttpHandler { UseProxy = false }) { BaseAddress = new Uri($"http://{connectAddress}:{serverPort}"), Timeout = Timeout.InfiniteTimeSpan, }; // Same User-Agent as the internet-facing clients — the loopback // server logs then identify the caller consistently. _http.DefaultRequestHeaders.UserAgent.ParseAdd(HttpUserAgent.Value); // A non-loopback server requires an API key; set it once here so every // probe, poll, SSE stream and chat request carries it. if (ApiKey is not null) _http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", ApiKey); // The supervisor is the ONLY source of truth for server status: it // polls the HTTP API for the app's whole lifetime and derives // ServerStatus from the answers — no process-handle assumptions. // Fire-and-forget: the loop is inert while Stopped and every tick is // guarded, so it can't fault the process. _ = Task.Run(SupervisorLoopAsync); } /// /// Ensures a llama server is reachable at /// : — the app's /// single point of contact for the model REST API. Resolution order: /// /// Probe GET /health. If a server is already running (a /// previous app instance, another tool, or a manual launch), adopt it as the /// client — no binary needed, no process launched. /// Otherwise resolve the llama binary (app-managed or on /// PATH) and launch it. /// If no binary is found, download it via the official /// install.ps1 (see ), then launch the server. /// /// Returns true once the server is reachable. The Available models list /// is then fetched via . Safe to await from the UI /// thread; installs run on a background process. /// public async Task EnsureLlamaOrDownloadAsync(CancellationToken cancel = default) { // Single-flight: a prior or concurrent caller may already be bringing // the server up (or about to). Waiting here means the second caller // finds Running after the first releases the gate — no duplicate spawn. await _ensureGate.WaitAsync(cancel); try { // Re-check after acquiring: a prior caller just brought the server up. if (ServerStatus == ServerState.Running) { Log.Info("llama server already running (gate re-check)"); return true; } // 1. Adopt an already-running server (no binary/process needed). // Probe briefly (a few attempts over ~3s) rather than once: a sibling // app instance / a manual launch / a server that's just binding won't // answer the very first probe, and a single has misused to spawn a // SECOND `llama serve` on the same port here — leaving two processes eating // RAM (the loser fails to bind, but the app would also abandon timed- // out launches alive — see StartServerAsync). A short adoption window // catches the in-flight server and adopts it instead. if (await WaitForReachableAsync(TimeSpan.FromSeconds(3), cancel)) { Log.Info("adopted an already-running llama server"); ServerStatus = ServerState.Running; // Best-effort: resolve the binary so Version is populated for display, // but don't block the client on it. _ = ResolveAndReadVersionAsync(cancel); return true; } // 2/3. Resolve the binary; install if missing; then launch the server. var resolved = Resolve(); Log.Info($"resolved llama binary: kind={resolved.Kind} path={resolved.Path ?? ""}"); switch (resolved.Kind) { case ResolutionKind.Managed: BinaryPath = resolved.Path; CurrentOrigin = Origin.Managed; Version = await ReadVersionAsync(resolved.Path!, cancel); State = InstallState.Idle; return await StartServerAsync(cancel); case ResolutionKind.External: BinaryPath = resolved.Path; CurrentOrigin = Origin.External; Version = await ReadVersionAsync(resolved.Path!, cancel); State = InstallState.Idle; return await StartServerAsync(cancel); default: // Missing — download then launch. if (await InstallAsync(cancel)) return await StartServerAsync(cancel); return false; } } finally { _ensureGate.Release(); } } /// /// Probes GET /health on the server port once. Any HTTP response /// means a server is already listening (connection-refused means not). The /// atomic unit used by the retrying and /// by the last-chance re-probe in . /// /// /// A linked token that cancels after — the /// per-call time budget replacing (the /// shared runs with an infinite timeout so SSE streams /// aren't cut). /// private static CancellationTokenSource WithTimeout(TimeSpan timeout, CancellationToken cancel) { var cts = CancellationTokenSource.CreateLinkedTokenSource(cancel); cts.CancelAfter(timeout); return cts; } private async Task ProbeHealthAsync(CancellationToken cancel) { try { // 5s per-call budget: a refused connection (no server) fails // instantly regardless — the budget only bounds a server that is // listening but slow to answer (busy loading a model). The old // per-call client with a 1s timeout and system-proxy defaults is // what made this return false while a browser got 200 OK. using var budget = WithTimeout(TimeSpan.FromSeconds(5), cancel); using var resp = await _http.GetAsync("/health", budget.Token); return true; } catch { return false; } } /// /// Repeatedly probes /health for up to , /// returning true as soon as a server responds. Used to ADOPT an /// already-running server (a sibling app instance, a manual launch, or one /// that's mid-bind) rather than spawning a duplicate on the same port — the /// fix for several llama serve processes piling up and eating RAM. The window is short, so a genuinely absent server doesn't /// delay startup by much (each refusal is near-instant; the 250ms cadence /// is what bounds the worst case). /// private async Task WaitForReachableAsync(TimeSpan timeout, CancellationToken cancel) { var deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) { cancel.ThrowIfCancellationRequested(); if (await ProbeHealthAsync(cancel)) return true; await Task.Delay(100, cancel); } return false; } /// /// Best-effort binary resolution and version read — populates /// , and /// without launching the server. Used for an /// adopted (external) server's display fields and by the runtime-update /// scheduler, which must know the origin/version before its startup /// install window (it runs before ). /// Never throws. /// public async Task ResolveAndReadVersionAsync(CancellationToken cancel = default) { try { var resolved = Resolve(); if (resolved.Path is { } p && File.Exists(p)) { BinaryPath = p; CurrentOrigin = resolved.Kind == ResolutionKind.Managed ? Origin.Managed : Origin.External; Version = await ReadVersionAsync(p, cancel); } } catch (Exception ex) { Log.Warn(ex, "best-effort version resolve failed"); } } /// /// (Re)installs the app-managed binary by downloading and executing /// . Also, the retry entry point. /// public async Task InstallAsync(CancellationToken cancel = default) { State = InstallState.Installing; try { await DownloadAndRunInstallerAsync(cancel); // Exit code 0 = success (DownloadAndRunInstallerAsync throws // otherwise): llama is now on PATH. Resolve its absolute path // dynamically ("which") instead of assuming a fixed location — // FindOnPath also reads the registry user/machine PATH, which is // what sees a PATH entry the installer just added (a child process // can't update our own environment block). BinaryPath = FindOnPath("llama.exe") ?? throw new IOException("Install script succeeded but 'llama' was not found on PATH."); CurrentOrigin = IsManagedPath(BinaryPath) ? Origin.Managed : Origin.External; Version = await ReadVersionAsync(BinaryPath, cancel); State = InstallState.Idle; return true; } catch (OperationCanceledException) { throw; } catch (Exception ex) { Log.Error(ex, "llama binary install failed"); FailureMessage = ex.Message; State = InstallState.Failed; return false; } } // ---- Server ---- /// /// Builds the argv for a server the app launches. Kept as a pure function /// so the launch flag matrix (which flags are always passed, which are /// conditional on their value, and in what order) is unit-testable without /// spawning a process. The fixed prefix is serve --host <addr> /// --port <port> --jinja; 0/negative /// means unlimited and omits /// --models-max; supported non-empty /// / values add /// --cache-type-k/--cache-type-v; a non-null /// adds --models-preset; a /// non-empty adds /// --models-dir (local models the "Open With" flow imports — /// see and ); /// tokens are /// appended LAST so they can override the built-in flags (llama.cpp honors /// the last occurrence) — except the reserved bind/auth flags, which /// refuses before this is reached. /// internal static List BuildServeArguments( int serverPort, string listenAddress, int idleUnloadSeconds, int maxLoadedModels, string? cacheTypeK, string? cacheTypeV, string? contextPresetPath, IReadOnlyList? customArguments, string? modelsDir = null) { // `serve` is the unified subcommand (replaces the old llama-server). // Router mode hosts the webui and serves requests even with no model // loaded — models load on demand. --host pins the server to the // interface the user picked; --jinja enables chat templates. var args = new List { "serve", "--host", listenAddress, "--port", serverPort.ToString(), "--jinja", }; // The API key is deliberately neither a process argument (argv is // readable by any local user via WMI) nor a file on disk — it is handed // to the server via the LLAMA_API_KEY environment variable (see // StartServerAsync). Nothing to add here; ServeArgumentPolicy reserves // --api-key and --api-key-file so custom arguments cannot inject one. // Local models the "Open With" flow imports. The router scans this // dir on every GET /models, so a .gguf linked here becomes loadable // without restarting the server. App-owned and reserved — a custom // --models-dir would silently break the Open With flow. if (!string.IsNullOrWhiteSpace(modelsDir)) { args.Add("--models-dir"); args.Add(modelsDir); } // Idle model unload. Done server-side because the server is the only // place that sees ALL model traffic — the overlay's WebUI chats // straight with it, invisible to the app, so an app-side idle timer // could unload a model mid-conversation. The router propagates the // flag to each per-model child server. if (idleUnloadSeconds > 0) { args.Add("--sleep-idle-seconds"); args.Add(idleUnloadSeconds.ToString()); } // Max loaded models. Router mode loads each model in its own child // process; 0 (the default) leaves the server's unlimited behavior, // otherwise the router refuses to exceed N simultaneous loads. if (maxLoadedModels > 0) { args.Add("--models-max"); args.Add(maxLoadedModels.ToString()); } // KV cache quantization. K and V are configured independently so a // user can, for example, keep keys in q8_0 while values stay f16 when // quality is more sensitive to one side. Only values the server // understands are forwarded; unrecognized hand-edited settings are // ignored rather than passed as an invalid flag. if (!string.IsNullOrWhiteSpace(cacheTypeK) && SupportedKvCacheTypes.Contains(cacheTypeK, StringComparer.Ordinal)) { args.Add("--cache-type-k"); args.Add(cacheTypeK); } if (!string.IsNullOrWhiteSpace(cacheTypeV) && SupportedKvCacheTypes.Contains(cacheTypeV, StringComparer.Ordinal)) { args.Add("--cache-type-v"); args.Add(cacheTypeV); } // Per-model context lengths (and spec-decoding types) are merged into // each model's child args via --models-preset (see BuildServeArguments). // Always present — even an empty INI — so mid-session preset reloads // reach the running router. if (contextPresetPath is not null) { args.Add("--models-preset"); args.Add(contextPresetPath); } // Custom arguments go last so they win when llama.cpp keeps the // final occurrence of a repeated flag — the user's typed text is the // most specific intent. if (customArguments is { Count: > 0 }) args.AddRange(customArguments); return args; } /// /// Formats the user-facing reason for a failed launch from the captured /// stderr tail. llama.cpp writes its INFO chatter to stderr too, so the /// last line that mentions an error wins; without one, the last non-blank /// line is the best available clue. Kept pure so the phrasing is /// unit-testable; the footer presentation sanitizes it further. /// internal static string FormatStartFailureMessage(string? stderrTail) { if (string.IsNullOrWhiteSpace(stderrTail)) return "The llama server failed to start."; var lines = stderrTail .Split('\n') .Select(line => line.Trim().TrimEnd('\r')) .Where(line => line.Length > 0) .ToArray(); if (lines.Length == 0) return "The llama server failed to start."; var errorLine = lines.LastOrDefault(line => line.Contains("error", StringComparison.OrdinalIgnoreCase)); var chosen = errorLine ?? lines[^1]; return $"The llama server failed to start: {chosen}"; } /// /// Launches llama serve --port 2276 as a background process and polls /// the port until it responds (or times out). Called automatically by /// once a binary is available. No-op (returns /// true) if the server is already running. /// public async Task StartServerAsync(CancellationToken cancel = default) { if (ServerStatus == ServerState.Running) return true; if (BinaryPath is null || !File.Exists(BinaryPath)) { FailureMessage = "The llama server binary is missing."; ServerStatus = ServerState.Failed; return false; } // Last-chance adoption: between EnsureLlamaOrDownloadAsync's probe and // now (esp. after a slow install.ps1 download), a sibling instance or a // manual launch may have brought up a server on our port. Adopting it // here avoids spawning a duplicate that would fail to bind and orphan // — the exact leak that left several servers eating RAM. if (await ProbeHealthAsync(cancel)) { Log.Info("adopted an already-running llama server (pre-start re-probe)"); FailureMessage = null; ServerStatus = ServerState.Running; _ = ResolveAndReadVersionAsync(cancel); return true; } // A live MANAGED server (valid PID file) that isn't responding yet is // still ours — the app may have crashed and restarted while the server // was mid-startup. DON'T kill it: give it a grace window to come up and // adopt it. Only if it never responds (genuinely hung) do we reclaim // it below — it's ours, so killing is safe. if (ReadLiveManagedPid(PidFilePath) is { } managedPid) { Log.Info($"managed llama server (pid {managedPid}) is alive but not reachable yet; waiting for it"); ServerStatus = ServerState.Starting; if (await WaitForReachableAsync(TimeSpan.FromSeconds(15), cancel)) { Log.Info($"adopted the managed llama server (pid {managedPid})"); FailureMessage = null; ServerStatus = ServerState.Running; _ = ResolveAndReadVersionAsync(cancel); // An adopted server already holds the key in memory (it was // started with LLAMA_API_KEY); sweep any plaintext file a // pre-env build left behind while we're here. RemoveLegacyApiKeyFile(SharedLlamaDir); RemoveLegacyApiKeyFile(Common.AppData.Root); return true; } Log.Warn($"managed llama server (pid {managedPid}) never became reachable; killing and relaunching"); } StopServer(); // reclaim any prior instance / port // Tokenize the custom arguments BEFORE flipping to Starting: an open // quote is a settings error, not a launch attempt, and must fail fast // with the parser's message rather than a 15s port timeout. IReadOnlyList customArguments; try { customArguments = Common.ArgumentTokenizer.Tokenize(CustomServeArguments); } catch (FormatException ex) { Log.Error(ex, "custom serve arguments failed to parse"); FailureMessage = ex.Message; ServerStatus = ServerState.Failed; return false; } // Defense in depth: the Settings window also validates reserved flags, // but a hand-edited settings.json must not be able to silently override // --host/--port (exposing the server) or --api-key (disabling auth). if (Common.ServeArgumentPolicy.Validate(customArguments) is { } reservedError) { Log.Error($"custom serve arguments rejected: {reservedError}"); FailureMessage = reservedError; ServerStatus = ServerState.Failed; return false; } // A fresh attempt invalidates whatever the previous launch failed // with; a new reason is set below if this one fails too. FailureMessage = null; ServerStatus = ServerState.Starting; try { var psi = new ProcessStartInfo { FileName = BinaryPath, // Pin the working directory to the binary's own folder so the // server never inherits (or resolves relative custom paths // against) an unpredictable app CWD. WorkingDirectory = Path.GetDirectoryName(BinaryPath), UseShellExecute = false, CreateNoWindow = true, RedirectStandardOutput = true, RedirectStandardError = true, }; // Per-model context lengths are merged into each model's child args // via --models-preset (see BuildServeArguments). Written fresh every // launch from the current preferences. var contextPresetPath = await WriteContextPresetsIniAsync(); // The app-generated API key (non-loopback binds) reaches the server // via the LLAMA_API_KEY environment variable (set on the launch // below) — never argv (WMI-readable) and never a file on disk. The // only thing persisted is the DPAPI-protected form in settings.json // (see Settings.ProtectedServerApiKey), so there is no plaintext to // materialize here. As hygiene, drop any plaintext key file a // pre-env build left behind (earlier dev builds wrote one for // --api-key-file). RemoveLegacyApiKeyFile(SharedLlamaDir); RemoveLegacyApiKeyFile(Common.AppData.Root); // The local-models dir for the "Open With" flow: the router scans // it on every GET /models, and llama.cpp's scan throws (rather than // skipping) on a missing dir, so it must exist before the launch // (and content of the pre-fix virtualized location must be // migrated in first — see EnsureManagedModelsDir). EnsureManagedModelsDir(); foreach (var arg in BuildServeArguments( ServerPort, ListenAddress, IdleUnloadSeconds, MaxLoadedModels, CacheTypeK, CacheTypeV, contextPresetPath, customArguments, ManagedModelsDir)) psi.ArgumentList.Add(arg); // Point the HF cache at the user-configured directory so the server // resolves downloaded models from the same place the app scans. if (!string.IsNullOrEmpty(CacheDirectory) && Directory.Exists(CacheDirectory)) psi.EnvironmentVariables["HF_HUB_CACHE"] = CacheDirectory; // Hand the server the app-generated API key (non-loopback binds) as // an environment variable — the same channel as HF_TOKEN below, and // for the same reason: llama.cpp reads LLAMA_API_KEY into // params.api_keys at startup and enforces it, while env keeps the // secret out of argv (WMI / process listings) and off disk // entirely. Log presence only, never the value. if (ApiKey is not null) { psi.EnvironmentVariables["LLAMA_API_KEY"] = ApiKey; Log.Info("API key configured; passing LLAMA_API_KEY to the llama server"); } // Hand the server the user's HF access token (if any) so it can // download private/gated models. Passed as an environment variable // — llama.cpp has no token CLI flag, and an arg would be visible in // process listings. Log presence only, never the value. var hfToken = HuggingFaceToken?.Trim(); if (!string.IsNullOrEmpty(hfToken)) { psi.EnvironmentVariables["HF_TOKEN"] = hfToken; Log.Info("HF token configured; passing HF_TOKEN to the llama server"); } var customArgsText = customArguments.Count == 0 ? "" : " " + Common.ServeArgumentPolicy.RedactForLog(customArguments); Log.Info($"starting llama server: {BinaryPath} serve --host {ListenAddress} --port {ServerPort} --jinja" + (ApiKey is not null ? " (LLAMA_API_KEY set)" : "") + (IdleUnloadSeconds > 0 ? $" --sleep-idle-seconds {IdleUnloadSeconds}" : "") + (MaxLoadedModels > 0 ? $" --models-max {MaxLoadedModels}" : "") + (!string.IsNullOrWhiteSpace(CacheTypeK) ? $" --cache-type-k {CacheTypeK}" : "") + (!string.IsNullOrWhiteSpace(CacheTypeV) ? $" --cache-type-v {CacheTypeV}" : "") + customArgsText); var proc = new Process { StartInfo = psi, EnableRaisingEvents = true }; proc.Exited += (_, _) => { // Log-only: server STATUS is derived from API polls by the // supervisor (see SupervisorLoopAsync), never from process // handles — an adopted server has no handle to watch, and a // spawned one's death is detected just as fast via refused // connections on the next poll tick. Log.Info($"llama server process exited (code={proc.ExitCode})"); }; if (!proc.Start()) { Log.Error("llama server process failed to start (proc.Start returned false)"); FailureMessage = "The llama server process failed to start."; ServerStatus = ServerState.Failed; return false; } _serverProcess = proc; // Track ownership across app restarts: if the app crashes, the next // instance recognizes this server as managed via the PID file. WritePidFile(proc.Id); // The server's stdout/stderr are redirected; drain them or a chatty // server blocks once the pipe buffer fills. Draining also captures // server errors (e.g. a bad launch flag) into the app log — and, via // the stderr tail, into the user-facing failure reason below. var stderrTail = new ServerOutputTail(); var (stdoutTask, stderrTask) = DrainServerOutput(proc, stderrTail); // Wait for the port to respond — the server takes a moment to bind. // We pass `proc` so the wait fast-fails if the process exits before // becoming ready (e.g. it couldn't bind the port because a sibling // already did) instead of polling for the full 15s timeout. if (await WaitForPortAsync(proc, TimeSpan.FromSeconds(15), cancel)) { Log.Info("llama server is reachable"); FailureMessage = null; ServerStatus = ServerState.Running; return true; } // Timed out (or the process exited early). DON'T leave the spawned // process running: a prior timeout-then-abandon left the server // alive, and a later app start (or this same retry) spawned a // second on the same port → two servers eating RAM. Kill ours so // the port is free for the next attempt. StopServer sets Stopped // before killing (deliberate intent — the supervisor never leaves // Stopped on its own), then we surface the failure. Log.Error("llama server failed to become ready within 15s (port probe timed out)"); // The pipes close when the process exits, so the drain tasks finish // almost immediately in the died-at-startup case; the timeout only // matters for a genuinely hung server, where we surface whatever // stderr arrived before giving up. try { await Task.WhenAll(stdoutTask, stderrTask).WaitAsync(TimeSpan.FromMilliseconds(500)); } catch { /* snapshot whatever arrived */ } FailureMessage = FormatStartFailureMessage(stderrTail.Snapshot()); StopServer(); ServerStatus = ServerState.Failed; return false; } catch (OperationCanceledException) { throw; } catch (Exception ex) { Log.Error(ex, "llama server start threw"); FailureMessage = ex.Message; ServerStatus = ServerState.Failed; return false; } } /// /// Removes any plaintext API-key file that earlier builds materialized for /// --api-key-file. The current design delivers the key to the server /// via the LLAMA_API_KEY environment variable and never writes it to /// disk (DPAPI covers the persisted form in settings.json), so a lingering /// file is pure residue and is swept on every launch. Best-effort and total /// — a missing or locked file is a silent no-op. /// internal static void RemoveLegacyApiKeyFile(string dir) { try { var p = Path.Combine(dir, ".llama.apikey"); if (File.Exists(p)) { File.Delete(p); Log.Debug("removed legacy plaintext API-key file"); } } catch (Exception ex) { Log.Debug($"legacy API-key file could not be removed: {ex.Message}"); } } /// /// Called after the managed install folder was emptied (from Settings): /// drops the stale binary/version/origin so the UI stops advertising an /// installation that no longer exists. The next /// re-resolves from scratch and /// reinstalls on demand. No-op for external installations — they are not /// the app's to remove. /// public void NotifyManagedInstallRemoved() { if (CurrentOrigin != Origin.Managed) return; BinaryPath = null; Version = null; CurrentOrigin = Origin.Unknown; } /// /// Stops the running managed server (if any). Safe to call repeatedly. /// Sets state to before killing: Stopped /// marks the stop as deliberate app intent, and the supervisor never /// probes or transitions out of Stopped on its own (see /// ). /// /// "Managed" = an app instance started the server: either this one /// (we hold the process handle) or a previous one that crashed — proven by /// the .llama.pid file (). A server /// with no valid PID file (started manually by the user, whatever the /// binary) is not ours and is left running. /// public void StopServer() { ServerStatus = ServerState.Stopped; var proc = _serverProcess; _serverProcess = null; if (proc is not null) { // Spawned this session. Clear the PID file only if it still tracks // THIS process — a racing instance may have rewritten it for a // newer server. if (ReadPidFile(PidFilePath) == proc.Id) DeletePidFile(); if (!proc.HasExited) { try { proc.Kill(entireProcessTree: true); } catch (Exception ex) { Log.Warn(ex, "best-effort server kill failed"); } } return; } // Adopted managed server (started by a previous/crashed instance): no // handle of ours, but a valid PID file proves ownership — kill by PID. if (ReadLiveManagedPid(PidFilePath) is { } managedPid) { DeletePidFile(); try { Log.Info($"killing managed llama server by PID file (pid {managedPid})"); Process.GetProcessById(managedPid).Kill(entireProcessTree: true); } catch (Exception ex) { Log.Warn(ex, $"best-effort managed server kill failed (pid {managedPid})"); } } } // ---- Managed-server PID file ---- /// /// Path of the PID file tracking the managed llama server: /// %LOCALAPPDATA%\Llama\.llama.pid. Written by /// right after the server process is spawned; /// read back after an app crash/restart to recognize the surviving server /// as ours (managed) — and therefore safe to stop. Deleted when the managed /// server is stopped or found dead. /// private static string PidFilePath => Path.Combine(AppData.Root, ".llama.pid"); /// /// Path of the llama.cpp --models-preset INI the app generates from /// — one [repo:quant] section with /// ctx-size = N per model that has a custom context length. Rewritten /// on every server start and preference change. /// /// Under deliberately — the server /// reads this file from outside the package identity (see the /// virtualization note there). /// private static string ContextPresetPath => Path.Combine(SharedLlamaDir, "model-context-presets.ini"); /// /// Renders the llama.cpp --models-preset INI: one /// [repo:quant] section per model that has a custom context length /// and/or a speculative-decoding spec type, as /// ctx-size = N / spec-type = draft-… lines. A model with /// both appears once with both lines. Ids that can't head a section /// safely (brackets, control characters) and values that can't sit on a /// line safely are dropped rather than rendered — see /// /. Pure so the /// exact shape — the section spelling and key names the router's INI /// parser accepts (rm_leading_dashes of the CLI long options) — is /// unit-testable. /// internal static string RenderModelsPresetIni( IEnumerable> contextLengths, IEnumerable>? speculativeTypes = null) { var spec = speculativeTypes?.Where(kv => IsSafeIniValue(kv.Value)).ToDictionary() ?? []; var sb = new StringBuilder(); foreach (var (id, ctx) in contextLengths) { if (ctx <= 0 || !IsSafeSectionId(id)) continue; sb.Append('[').Append(id).Append(']').Append('\n'); sb.Append("ctx-size = ").Append(ctx).Append('\n'); if (spec.TryGetValue(id, out var specType)) { sb.Append("spec-type = ").Append(specType).Append('\n'); spec.Remove(id); } sb.Append('\n'); } // Models with only a spec type (no custom context length). foreach (var (id, specType) in spec) { if (!IsSafeSectionId(id)) continue; sb.Append('[').Append(id).Append(']').Append('\n'); sb.Append("spec-type = ").Append(specType).Append('\n').Append('\n'); } return sb.ToString(); } /// /// Whether a model id can head a preset INI section: non-empty, no /// [/] (either would end the section early — a filename may /// legally contain them, and [x]junk] silently misapplies the /// preset to the wrong model) and no control characters (CR/LF would /// inject whole lines). Shared with EnrolledFolderScanner, which /// now vets its folder-derived section names through this sink. Ids /// arrive from server-reported model lists, Open-With /// file stems and hand-edited settings, so the sink validates even though /// Windows filenames can't carry CR/LF. /// internal static bool IsSafeSectionId(string id) => !string.IsNullOrWhiteSpace(id) && id.All(c => c is not ('[' or ']') && !char.IsControl(c)); /// /// Whether a value can follow a preset INI key: non-empty, no control /// characters (CR/LF would inject whole lines — a hand-edited settings /// entry must not be able to append arbitrary keys). /// internal static bool IsSafeIniValue(string value) => !string.IsNullOrWhiteSpace(value) && value.All(c => !char.IsControl(c)); /// /// Writes the --models-preset INI from /// and /// and returns its path — always, /// even when empty: the server is launched with --models-preset /// unconditionally, so a later /// (a settings entry added mid-session — a context length picked, a draft /// head detected, a folder enrolled) reaches the RUNNING router. Without /// the flag at launch, the reload endpoint re-reads nothing and the entry /// would only engage after an app restart. An empty INI parses to zero /// sections server-side. Null only when the write fails (the server then /// launches without the flag — best-effort). /// private async Task WriteContextPresetsIniAsync() { await _presetWriteGate.WaitAsync(); try { var contexts = ModelContextLengths; var specs = ModelSpeculativeTypes; // Snapshot the dictionaries under their locks before rendering: both // are the LIVE settings dictionaries (shared by reference), and the // draft-head detector mutates its dictionary from a poller thread — // an enumeration racing an Add would throw. (A null dictionary has // no entries and no writer.) var contextItems = new List>(); if (contexts is not null) lock (contexts) contextItems.AddRange(contexts); var specItems = new List>(); if (specs is not null) lock (specs) specItems.AddRange(specs); // Enrolled model folders (Settings → General → Model folders): a // recursive scan is disk I/O, so it runs off the caller's thread — // the write happens on server start (possibly the UI thread) and on // every preset reload, and a slow folder (network drive) must never // stall either. The list is snapshotted under its lock first (it is // the LIVE settings list, mutated in place by the settings UI). The // HF cache and the managed models dir are excluded: the server // already lists those, and enrolling a parent of the cache must not // duplicate every cached model. string[]? folders = null; if (ModelFolders is { Count: > 0 } enrolledFolders) lock (enrolledFolders) folders = [.. enrolledFolders]; var enrolled = folders is { Length: > 0 } ? await Task.Run(() => EnrolledFolderScanner.Scan( folders, [CacheDirectory, ManagedModelsDir])) : []; if (enrolled.Count > 0) Log.Info($"model folders: enrolled {enrolled.Count} model(s) from disk"); // Keep the scan's id → path map: the rows' "reveal in Explorer" // resolves enrolled models through it (the server's /models does not // reliably report a path). Reference-swapped for readers. _enrolledModelPaths = enrolled.ToDictionary( m => m.Name, m => m.ModelPath, StringComparer.OrdinalIgnoreCase); // Don't render preset sections for models whose files no longer exist: // a ctx / spec-type entry outlives its model (deleted in Explorer, a // folder un-enrolled while the app was off) and the router re-lists a // preset section as a loadable ghost — a row that can't even be // deleted, because there is nothing on disk left to delete. var listed = LastModelsSnapshot; bool IsBusy(string id) => listed.Any(m => m.Id == id && (m.IsDownloading || m.IsLoading)); bool Exists(string id) => FindModelFilePath(id) is not null; contextItems = FilterExistingModels(contextItems, IsBusy, Exists); specItems = FilterExistingModels(specItems, IsBusy, Exists); try { Directory.CreateDirectory(Path.GetDirectoryName(ContextPresetPath)!); File.WriteAllText(ContextPresetPath, RenderModelsPresetIni(contextItems, specItems) + EnrolledFolderScanner.RenderSections(enrolled)); return ContextPresetPath; } catch (Exception ex) { Log.Warn(ex, "failed to write model context preset file"); return null; } } finally { _presetWriteGate.Release(); } } /// /// Re-renders the models-preset INI (context lengths, spec types, and the /// user's enrolled model folders — the folder scan runs fresh, so newly /// dropped files are picked up too) and asks the running server to /// re-read its presets (GET /models?reload=1 re-runs the router's /// preset load, which re-reads the --models-preset file). Called /// when the user changes a model's context length, enrolls or removes a /// model folder, or when the draft-head detector records a spec type, so /// the next /models/load picks it up. Reloading does not disturb /// already-loaded models. Best-effort: a server the app didn't launch /// (no --models-preset flag) won't read the file — the preference /// then takes effect on the next app-launched start. /// public async Task ReloadModelPresetsAsync(CancellationToken cancel = default) { await WriteContextPresetsIniAsync(); if (ServerStatus != ServerState.Running) return; try { using var budget = WithTimeout(TimeSpan.FromSeconds(10), cancel); using var resp = await _http.GetAsync("/models?reload=1", budget.Token); Log.Info($"model context presets reloaded (HTTP {(int)resp.StatusCode})"); } catch (Exception ex) when (ex is not OperationCanceledException) { Log.Debug("model context preset reload failed: " + ex.Message); } } /// Writes to the PID file. Best-effort. private static void WritePidFile(int pid) { try { Directory.CreateDirectory(Path.GetDirectoryName(PidFilePath)!); File.WriteAllText(PidFilePath, pid.ToString()); } catch (Exception ex) { Log.Warn(ex, "best-effort PID file write failed"); } } /// /// A bounded, thread-safe tail of the server's stderr. The drain task /// appends every line; a failed launch reads the tail back so the footer /// can say WHY the server died (llama.cpp prints its errors to stderr). /// private sealed class ServerOutputTail { private const int Capacity = 20; private readonly object _gate = new(); private readonly Queue _lines = new(); public void Add(string line) { lock (_gate) { _lines.Enqueue(line); while (_lines.Count > Capacity) _lines.Dequeue(); } } /// The captured lines joined with newlines, or null when none. public string? Snapshot() { lock (_gate) { return _lines.Count == 0 ? null : string.Join(Environment.NewLine, _lines); } } } /// /// Drains a spawned server's redirected stdout/stderr on background threads. /// Redirecting without draining lets a chatty server block once the pipe /// buffer fills (a silent stall); draining also keeps server output available /// for diagnosing a failed launch. llama.cpp writes everything (INFO /// included) to stderr, so both streams log at Debug — anything higher would /// spam the app log with per-request noise. The returned tasks let a failed /// launch wait for the pipes to close before snapshotting the stderr tail. /// private static (Task Stdout, Task Stderr) DrainServerOutput( Process proc, ServerOutputTail? stderrTail = null) { Task Drain(StreamReader reader, string tag, ServerOutputTail? tail) { return Task.Run(() => { try { string? line; while ((line = reader.ReadLine()) is not null) { Log.Debug($"[llama{tag}] {line}"); tail?.Add(line); } } catch { /* stream closed when the server exits */ } }); } return (Drain(proc.StandardOutput, "", null), Drain(proc.StandardError, " stderr", stderrTail)); } private static void DeletePidFile() => DeletePidFile(PidFilePath); /// Deletes the PID file if present. Best-effort. private static void DeletePidFile(string pidFilePath) { try { if (File.Exists(pidFilePath)) File.Delete(pidFilePath); } catch (Exception ex) { Log.Warn(ex, "best-effort PID file delete failed"); } } /// /// Raw PID-file parse: the stored PID, or null when the file is /// missing or unreadable. Garbage content is deleted rather than kept. /// private static int? ReadPidFile(string pidFilePath) { string text; try { if (!File.Exists(pidFilePath)) return null; text = File.ReadAllText(pidFilePath).Trim(); } catch (Exception ex) { Log.Warn(ex, "PID file read failed"); return null; } if (int.TryParse(text, out var pid) && pid > 0) return pid; DeletePidFile(pidFilePath); // garbage — don't keep it around return null; } /// /// Crash-safe managed-server check: the PID from /// , but only if that process is still alive /// AND is actually a llama server — guarding against PID reuse (the OS /// recycling our dead server's PID for an unrelated process): the process /// must be named llama and must have started before the PID file was /// written (we write right after ). A stale or /// mismatched file is deleted so the check stays cheap. Internal for tests. /// internal static int? ReadLiveManagedPid(string pidFilePath) { if (ReadPidFile(pidFilePath) is not { } pid) return null; try { using var proc = Process.GetProcessById(pid); var isLlama = string.Equals(proc.ProcessName, "llama", StringComparison.OrdinalIgnoreCase); var startedBeforeWrite = proc.StartTime.ToUniversalTime() <= File.GetLastWriteTimeUtc(pidFilePath) + TimeSpan.FromSeconds(5); if (isLlama && startedBeforeWrite) return pid; } catch (ArgumentException) { /* no such process — stale file */ } catch (Exception ex) { // Couldn't verify (e.g., access denied): don't kill what we can't // identify, but keep the file for a later re-check. Log.Warn(ex, $"managed-server PID check failed (pid {pid})"); return null; } DeletePidFile(pidFilePath); // stale or PID reused — clean up return null; } /// /// Polls GET /health on the configured port until it responds or the timeout /// elapses (or the spawned exits first). The llama /// server exposes a health endpoint once it's bound and ready; this confirms /// the port is actually serving rather than just waiting for a fixed delay. /// Checking proc.HasExited each iteration fast-fails when the process /// died right after launch (e.g., it couldn't bind the port because a /// sibling already did) so we don't sit out the full timeout before tearing /// down — and we don't keep around a dead-but-tracked process reference. /// private async Task WaitForPortAsync(Process proc, TimeSpan timeout, CancellationToken cancel) { var deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) { cancel.ThrowIfCancellationRequested(); if (proc.HasExited) { Log.Warn($"llama server process exited before becoming ready (code={proc.ExitCode})"); return false; } try { // Any HTTP response (even an error code) means the server is // up and listening — a connection-refused means it's not yet. using var budget = WithTimeout(TimeSpan.FromSeconds(2), cancel); using var resp = await _http.GetAsync("/health", budget.Token); return true; } catch { await Task.Delay(250, cancel); } } return false; } // ---- Server-status supervisor ---- /// /// Pure derivation of from a single API probe — /// the ONLY source of truth for server status. No process-handle /// assumptions: a spawned server's crash and an adopted server's crash /// look identical to the poll, and a server that reappears is adopted the /// same way no matter who (re)started it. /// /// The state observed before the probe. /// Whether the HTTP API answered (any HTTP /// response counts; connection-refused/timeout counts as not). public static ServerState DeriveServerStatus(ServerState current, bool apiReachable) => (current, apiReachable) switch { (ServerState.Running, true) => ServerState.Running, // Was running, now unreachable → crashed (or the machine/network did). (ServerState.Running, false) => ServerState.Failed, // A launch is confirmed by the API answering, not by the process // having started. (ServerState.Starting, true) => ServerState.Running, // Still booting — StartServerAsync's own wait bounds the window. (ServerState.Starting, false) => ServerState.Starting, // Auto-recovery: a server (re)appeared — e.g. the user restarted // their own instance after a crash. Adopt it; no relaunch click // needed. (ServerState.Failed, true) => ServerState.Running, (ServerState.Failed, false) => ServerState.Failed, // Stopped is deliberate app intent (startup before the ensure // pipeline runs, the port-reclaim window inside StartServerAsync, // app exit). The supervisor never probes while Stopped and never // leaves it on its own — otherwise it could "adopt" a process // StopServer is in the middle of killing. Leaving Stopped is // always explicit: EnsureLlamaOrDownloadAsync / the relaunch button. (ServerState.Stopped, _) => ServerState.Stopped, }; /// /// Applies a result, with logging on the /// two transitions that matter operationally: declaring a crash /// () and confirming/adopting a server /// (). /// private void ApplyPolledStatus(ServerState derived) { if (derived == ServerStatus) return; if (derived == ServerState.Failed) { Log.Warn("llama server unreachable; declaring it failed (API-polled)"); FailureMessage = "The llama server stopped responding."; } else if (derived == ServerState.Running) { Log.Info($"llama server reachable; {ServerStatus} → Running (API-polled)"); } ServerStatus = derived; } // ---- Supervisor poll pacing / change detection ---- // // The poll is the app's always-on heartbeat, but almost every tick changes // nothing: without gating, a tray app idling with a static model list fires // ModelsChanged (→ UI dispatcher hop + reconcile) and builds a debug string // twice a second, forever. The GET /models fetch stays the liveness probe on // every tick; everything downstream of it is change-gated. /// Tick interval while and a /// transition is fresh (a load/download state just flipped) — fast enough /// that the play → load-ring → open-glyph progression feels immediate. private const int RunningPollFastMs = 500; /// Tick interval while with a /// static snapshot. The fetch still doubles as the liveness probe, so a dead /// server is still noticed within a couple of seconds. private const int RunningPollSteadyMs = 2000; /// How long after the last observable change the fast interval is /// kept before settling to . private static readonly TimeSpan FastPollWindow = TimeSpan.FromSeconds(15); /// Upper bound between publishes even /// when nothing changed, so subscribers with their own retry/backoff timers /// (e.g. DraftHeadDetection's failure backoff) still get a wake-up. private static readonly TimeSpan PublishHeartbeat = TimeSpan.FromSeconds(60); private IReadOnlyList? _lastPublishedModels; private DateTime _lastPublishAt = DateTime.MinValue; private DateTime _lastPollChangeAt = DateTime.MinValue; private ServerState _lastTickStatus = ServerState.Stopped; /// /// Change detection for poll publishing: compares the observable fields of /// two GET /models snapshots. Order-sensitive — a reorder at worst /// causes one redundant publish, never a missed change. Internal for tests. /// internal static bool SnapshotsEqual(IReadOnlyList? a, IReadOnlyList b) { if (a is null || a.Count != b.Count) return false; for (var i = 0; i < b.Count; i++) { var x = a[i]; var y = b[i]; if (!string.Equals(x.Id, y.Id, StringComparison.Ordinal) || !string.Equals(x.Status, y.Status, StringComparison.Ordinal) || !string.Equals(x.Path, y.Path, StringComparison.Ordinal) || !string.Equals(x.Source, y.Source, StringComparison.Ordinal) || x.CanRemove != y.CanRemove || x.SupportsImage != y.SupportsImage) return false; } return true; } /// /// The always-on supervisor loop — started once in the constructor and run /// for the app's whole lifetime. It is the single place that turns API /// answers into transitions: /// /// While : fetch GET /models /// every 500ms while transitions are fresh, settling to every 2s while the /// snapshot is static — the fetch doubles as the liveness probe AND publishes the /// snapshot via (change-gated — see the event). A failed fetch is confirmed /// with a /health probe before declaring death, so a transient /// /models hiccup on a living server doesn't flip the state. /// While or /// : probe /health every second — a /// reachable API confirms a launch or adopts a (re)appeared server. /// While : idle — see /// . /// /// Every tick is guarded: one bad tick doesn't take down the supervisor. /// private async Task SupervisorLoopAsync() { while (true) { var status = ServerStatus; try { switch (status) { case ServerState.Running: { // null = the fetch FAILED (request error, non-success // status, or an unparseable body) — a real signal, now // distinct from a valid-but-empty model list. A failed // fetch is confirmed with a /health probe below before // declaring death, so a transient /models hiccup on a // living server doesn't flip the state. IReadOnlyList? snapshot = null; try { snapshot = await FetchModelsAsync(reload: false, CancellationToken.None); } catch (Exception ex) { Log.Debug($"model poll fetch failed: {ex.Message}"); /* confirmed via /health below */ } if (snapshot is not null) { // Publish only when something observable changed — or on // the first tick after entering Running (rows need a fresh // look regardless of the fingerprint) — plus a slow // heartbeat so backoff-based subscribers still wake up. // Steady-state ticks (the vast majority) now cost one HTTP // fetch and nothing else: no event, no dispatcher hop, no // log-string building. var now = DateTime.UtcNow; var changed = !SnapshotsEqual(_lastPublishedModels, snapshot); var entered = status != _lastTickStatus; if (changed || entered || now - _lastPublishAt >= PublishHeartbeat) { _lastPublishedModels = snapshot; _lastPublishAt = now; try { _lastModelsSnapshot = snapshot; ModelsChanged?.Invoke(this, snapshot); } catch (Exception ex) { Log.Warn(ex, "ModelsChanged handler threw"); /* a handler error doesn't take down the supervisor */ } } if (changed || entered) _lastPollChangeAt = now; // Level-guarded so Release (Info) builds skip the // interpolation entirely; Debug sessions keep the // per-tick heartbeat line. if (snapshot.Count > 0 && Log.Level <= LogLevel.Debug) Log.Debug( $"poll: {snapshot.Count} model(s): {string.Join(", ", snapshot.Select(m => $"{m.Id}={(m.Status ?? "?")}"))}"); } else { ApplyPolledStatus(DeriveServerStatus(status, await ProbeHealthAsync(CancellationToken.None))); } break; } case ServerState.Starting: case ServerState.Failed: ApplyPolledStatus(DeriveServerStatus(status, await ProbeHealthAsync(CancellationToken.None))); break; // Stopped: no probe, no transition — see DeriveServerStatus. } } catch (Exception ex) { Log.Warn(ex, "supervisor tick threw"); /* one bad tick doesn't take down the supervisor */ } _lastTickStatus = status; await Task.Delay(status == ServerState.Running ? (DateTime.UtcNow - _lastPollChangeAt < FastPollWindow ? RunningPollFastMs : RunningPollSteadyMs) : 1000); } } // ---- Device probing / model fit ---- /// Cached device probe result + timestamp (see ). private (DateTime At, DeviceProbe? Probe) _devicesCache; /// /// Probes the compute devices available to llama.cpp by running /// llama cli --list-devices (see ). An /// empty list means no accelerator devices — callers fall back to system /// CPU/RAM () for fit decisions. The result is /// cached for a short window: the probe spawns a process, preflight /// checks can come in bursts, and free-VRAM numbers don't need to be /// fresher than a minute. A failed probe returns an empty list — it /// must never throw or block (fail-open convention). /// public async Task> ListDevicesAsync(CancellationToken cancel = default) => (await ProbeDevicesAsync(cancel)).Devices; /// /// The full probe result, including whether the probe succeeded — see /// . Callers that only budget memory can use /// (fail-open list); the footer's device /// indicator needs the success flag to tell "CPU-only machine" apart /// from "probe failed / binary not resolved yet". /// public async Task ProbeDevicesAsync(CancellationToken cancel = default) { const double CacheTtlSeconds = 60; var cache = _devicesCache; if (cache.Probe is not null && (DateTime.UtcNow - cache.At).TotalSeconds < CacheTtlSeconds) return cache.Probe; var binary = BinaryPath; if (binary is null || !File.Exists(binary)) binary = FindOnPath("llama.exe"); if (binary is null) { Log.Debug("list-devices skipped: no llama binary resolved yet"); return new DeviceProbe(false, []); } var probe = await DeviceQuery.ListDevicesAsync(binary, cancel); _devicesCache = (DateTime.UtcNow, probe); Log.Info(probe.Devices.Count == 0 ? probe.Succeeded ? "list-devices: no accelerator devices (CPU/RAM fallback)" : "list-devices: probe failed (unreadable output)" : $"list-devices: {string.Join(", ", probe.Devices.Select(d => $"{d.Id} '{d.Name}' {d.FreeBytes / (1 << 20)} MiB free"))}"); return probe; } /// /// Decides whether a model can run on this machine BEFORE it is /// downloaded or loaded — from the catalog metadata (parameter count, /// quant, GGUF file size) against a live device probe /// () and system RAM /// (). See for the /// decision rules. Never throws; unknown inputs fail open. /// /// Catalog params label, e.g. "20B", /// "26B-A4B", "E4B" — empty/null when unknown. /// Quant label, e.g. "Q4_K_M", "mxfp4". /// GGUF size in bytes (0 when unknown) — the /// best weight-size signal when present. public async Task CheckModelFitAsync( string? parameterCount, string? quant, ulong fileSizeBytes, CancellationToken cancel = default) { var devices = await ListDevicesAsync(cancel); ulong? availableRam = SystemMemory.TryGet(out _, out var avail) ? avail : null; var estimate = ModelMemoryEstimator.Estimate(parameterCount, quant, fileSizeBytes); var result = MemoryFit.Check(estimate, devices, availableRam); Log.Info($"fit check: params={parameterCount ?? ""} quant={quant ?? ""} " + $"file={fileSizeBytes}B → fits={result.Fits} target={result.Target} ({result.Details})"); return result; } /// /// The three-level counterpart of : /// classifies a model build as fit / might / no way (the green/yellow/red /// verdicts the variant picker shows) from the same catalog metadata, /// against the same live device + RAM probes. See /// for the decision rules. Never /// throws; unknown inputs land on "might", never block. /// /// Catalog params label, e.g. "20B", /// "26B-A4B", "E4B" — empty/null when unknown. /// Quant label, e.g. "Q4_K_M", "mxfp4". /// GGUF size in bytes (0 when unknown) — the /// best weight-size signal when present. public async Task CheckHardwareFitAsync( string? parameterCount, string? quant, ulong fileSizeBytes, CancellationToken cancel = default) { var devices = await ListDevicesAsync(cancel); ulong? availableRam = SystemMemory.TryGet(out _, out var avail) ? avail : null; var estimate = ModelMemoryEstimator.Estimate(parameterCount, quant, fileSizeBytes); var result = HardwareFitHeuristic.Classify(estimate, devices, availableRam); Log.Info($"hardware fit: params={parameterCount ?? ""} quant={quant ?? ""} " + $"file={fileSizeBytes}B → level={result.Level} " + $"(needs {result.RequiredBytes}B of {result.BudgetBytes}B: {result.Note})"); return result; } /// /// Cached fit-params probe results + timestamps, keyed by (model path, /// context tokens) — see . /// private readonly Dictionary<(string Path, int Ctx), (DateTime At, FitParamsEstimate? Estimate)> _fitParamsCache = new(); /// /// Asks the CLI's fit-params tool for llama.cpp's own memory /// estimate of an on-disk model at a context length (see /// ) — the accurate, GPU-aware counterpart /// of the catalog-metadata heuristic () /// used once the GGUF is actually present. null means "no /// verdict" (no binary, CPU-era build without the tool, unreadable /// model) — callers keep their heuristic then, never block. Results are /// cached per (path, context) for a short window: the context-length /// picker probes a burst of options when it opens, and the estimate /// (unlike free VRAM) doesn't change between them. /// public async Task QueryFitParamsAsync( string modelPath, int contextTokens, CancellationToken cancel = default) { const double CacheTtlSeconds = 60; var key = (modelPath, contextTokens); lock (_fitParamsCache) { if (_fitParamsCache.TryGetValue(key, out var cached) && (DateTime.UtcNow - cached.At).TotalSeconds < CacheTtlSeconds) return cached.Estimate; } var binary = BinaryPath; if (binary is null || !File.Exists(binary)) binary = FindOnPath("llama.exe"); if (binary is null) { Log.Debug("fit-params skipped: no llama binary resolved yet"); return null; } var estimate = await FitParamsQuery.QueryAsync(binary, modelPath, contextTokens, cancel); lock (_fitParamsCache) { _fitParamsCache[key] = (DateTime.UtcNow, estimate); } Log.Debug(estimate is null ? $"fit-params: no verdict for {Path.GetFileName(modelPath)} at ctx {contextTokens}" : $"fit-params: {Path.GetFileName(modelPath)} at ctx {contextTokens} needs " + $"{MemoryFit.FormatBytes(estimate.TotalBytes)} ({estimate.Devices.Count} device(s) + host)"); return estimate; } /// /// The memory budget a context-length option is compared against: /// the sum of free VRAM across all accelerator devices plus the usable /// share of free system RAM (). /// The budgets are added, not tried one after the other as in /// , because a loaded model can split its layers /// across VRAM and RAM (partial offload) — so a context option fits when /// its total requirement fits the combined pool. Fails open exactly like /// the fit checks: no devices AND an unknown RAM probe returns /// (gray nothing out on a measurement /// error); with devices but no RAM number, the VRAM sum alone stands. /// public async Task ContextMemoryBudgetAsync(CancellationToken cancel = default) { var devices = await ListDevicesAsync(cancel); var vram = MemoryFit.SumFreeVram(devices); if (!SystemMemory.TryGet(out _, out var availableRam)) return devices.Count > 0 ? vram : ulong.MaxValue; return vram + (ulong)(availableRam * MemoryFit.CpuRamBudgetFraction); } // ---- Model download ---- /// /// Downloads a model by asking the running llama server to fetch it. The /// server (router mode) handles the actual Hugging Face transfer; this /// method just POSTs {"model": "<name>"} to /// /models and tracks progress via the /models/sse stream. /// Flow: /// /// Open an SSE connection to /models/sse and start parsing /// events. /// POST the model name to /models — the server kicks off the /// download and emits download_progress SSE events. /// Sum the per-URL done/total bytes from each progress /// event and report them via . /// Complete (return) when a download_finished or /// download_failed event arrives for the model. /// /// /// The model to download; /// is the Hugging Face repo id with its :<quant> suffix (e.g. /// ggml-org/gpt-oss-20b-GGUF:Q4_0). The quant suffix is what tells /// the server which GGUF variant to fetch — a bare repo id leaves the /// choice to the server's own default resolution, which can download a /// different (typically larger) quant than the one the user picked. /// Receives updates /// as the server streams them. May be null. /// Cancels the download (closes the SSE stream and /// asks the server to stop via POST /models/unload — see /// CancelServerDownloadAsync for why not DELETE). /// true if the download finished successfully; /// false on failure or cancellation. public async Task DownloadModelAsync(IModel model, IProgress? progress = null, CancellationToken cancel = default) { Log.Info($"downloading model {model.Name}"); if (ServerStatus != ServerState.Running) { progress?.Report(new ModelDownloadProgress( model.Name, 0, 0, Done: false, Failed: true, Message: "Server is not running")); return false; } // The canonical repo:quant id — the quant suffix pins the exact GGUF // variant the user picked; POSTing a bare repo id lets the server // resolve its own default quant (observed: Q4_0 requested, Q8_0 landed // in the cache). var modelName = model.ServerModelId; // Open the SSE stream first so we don't miss the earliest progress events. // HttpCompletionOption.ResponseHeadersRead lets us read the body as it // arrives rather than buffering the whole (infinite) stream. // `using` so the response (and its underlying connection / Content stream) // is released on EVERY exit path — the early returns from a POST failure // and the throw on cancellation used to skip the only Dispose() call, // leaking an HTTP connection per failed/canceled download. using var sseResponse = await _http.GetAsync( "/models/sse", HttpCompletionOption.ResponseHeadersRead, cancel); sseResponse.EnsureSuccessStatusCode(); // Read the SSE stream on a background task; it feeds events into a // channel we consume below. This decouples line-by-line parsing from // the POST + completion logic. var stream = await sseResponse.Content.ReadAsStreamAsync(cancel); var reader = new StreamReader(stream); using var sseCts = CancellationTokenSource.CreateLinkedTokenSource(cancel); // POST the model name to /models — the server starts the download. var payload = BuildModelRequestBody(modelName); using var content = new StringContent(payload, Encoding.UTF8, "application/json"); try { using var budget = WithTimeout(TimeSpan.FromSeconds(30), cancel); using var postResp = await _http.PostAsync("/models", content, budget.Token); if (!postResp.IsSuccessStatusCode) { var body = await postResp.Content.ReadAsStringAsync(cancel); await sseCts.CancelAsync(); progress?.Report(new ModelDownloadProgress( modelName, 0, 0, Done: false, Failed: true, Message: $"Server rejected the request ({(int)postResp.StatusCode}): {body}", HttpStatus: (int)postResp.StatusCode)); return false; } } catch (Exception ex) when (ex is not OperationCanceledException) { Log.Error(ex, "download POST threw"); await sseCts.CancelAsync(); progress?.Report(new ModelDownloadProgress( modelName, 0, 0, Done: false, Failed: true, Message: ex.Message, ExceptionType: ex.GetType().Name)); return false; } // Consume SSE events until the download finishes or fails for our model. // ParseSseStreamAsync is an async iterator that yields events as they // arrive from the stream — no Task.Run needed since IAsyncEnumerable is // inherently lazy/streaming. // // On completion, we `break` out of the loop rather than cancelling the // SSE stream in-place: cancelling sseCts mid-iteration would make the // next ReadLineAsync throw OperationCanceledException, and since that // exception comes from sseCts (not the user's `cancel` token) it would // escape the `when (cancel.IsCancellationRequested)` guard below and // propagate out of the method — masking a successful download as a // cancellation and skipping the post-download load. Breaking lets the final // block cancel + dispose the stream cleanly with no thrown exception. var success = false; var completed = false; try { await foreach (var (evt, modelId, data) in ParseSseStreamAsync(reader, sseCts.Token)) { cancel.ThrowIfCancellationRequested(); // The SSE "model" field may carry the bare repo or the // repo:quant form depending on the server build and how the // download was started — match on the repo part so neither // form is missed for this download. if (!SameDownloadModel(modelId, modelName)) continue; // another model's event switch (evt) { case "download_progress": var (downloaded, total) = SumProgress(data); progress?.Report(new ModelDownloadProgress( modelName, downloaded, total, Done: false, Failed: false)); break; case "download_finished": success = true; progress?.Report(new ModelDownloadProgress( modelName, 0, 0, Done: true, Failed: false, Message: "Download complete")); completed = true; break; case "download_failed": // Capture the server's real error payload (a string // error/message field) instead of discarding it — the // raw payload is logged verbatim and, when present, // carried in Message for classification/diagnosis. // llama.cpp master currently sends data:{} with no // error field, so fall back to the existing literal. var detail = ExtractFailureDetail(data); var rawPayload = data.ValueKind == JsonValueKind.Undefined ? "" : data.GetRawText(); Log.Warn($"server reported download_failed for {modelName}; data: {rawPayload}"); progress?.Report(new ModelDownloadProgress( modelName, 0, 0, Done: false, Failed: true, Message: detail ?? "Download failed")); completed = true; break; } if (completed) break; // exit to await foreach; finally cleans up } } catch (OperationCanceledException) when (cancel.IsCancellationRequested) { // User canceled — ask the server to stop the download. await CancelServerDownloadAsync(modelName); progress?.Report(new ModelDownloadProgress( modelName, 0, 0, Done: false, Failed: false, Message: "Cancelled")); throw; } finally { // sseResponse is disposed by its `using` at scope exit; only cancel // the linked token here so an in-flight ReadLineAsync unwinds. await sseCts.CancelAsync(); } // The stream ended without a terminal event (server dropped the // connection mid-download): nothing was reported, so the caller's // failure path has no detail to show. Log it so the "check the app // log" guidance has substance. No new progress report — the return // contract and row-state ownership are unchanged. if (!completed && !success) Log.Warn($"SSE stream ended before a terminal download event for {modelName}"); return success; } /// /// Watches a download the app did not start (e.g. triggered from the /// WebUI or the CLI) and reports its byte progress until it finishes. Unlike /// nothing is POSTed — the download is /// already in flight — and cancellation only stops the watch; it never /// cancels the server-side download. /// There is deliberately no idle timeout: the caller (the /models /// poller) owns the watch's lifetime and cancels it as soon as the model /// leaves the downloading state, so a quiet stream (a stalled but /// living download) is waited out rather than second-guessed. /// /// The server model id of the download to watch — /// the Hugging Face repo id as the server keys it, bare /// (ggml-org/gemma-3-4b-it-GGUF) or with its quant suffix /// (…GGUF:Q4_0). Either form works: SSE events are matched on the /// repo part (see ). /// Receives updates /// as the server streams them. May be null. /// Stops the watch (does not affect the download). /// true if the download finished while watching; /// false if it failed, the stream ended, or the watch was canceled. public async Task WatchDownloadAsync( string repoName, IProgress? progress = null, CancellationToken cancel = default) { if (ServerStatus != ServerState.Running) return false; // Same pattern as DownloadModelAsync: the shared long-lived client, the // body read as it arrives, and `using` on the response so the connection // is released on every exit path. A dead stream degrades to "no // progress" — the poller keeps the row's state truthful regardless. try { using var sseResponse = await TryOpenSseAsync(cancel); if (sseResponse is null) return false; using var reader = new StreamReader(await sseResponse.Content.ReadAsStreamAsync(cancel)); await foreach (var (evt, modelId, data) in ParseSseStreamAsync(reader, cancel)) { // Match on the repo part: the SSE "model" field may or may not // carry the quant suffix depending on how (and by whom) the // download was started — an exact compare of the bare repo id // misses repo:quant events and leaves the row on an // indeterminate ring for the whole download. if (!SameDownloadModel(modelId, repoName)) continue; // another model's event ("*" broadcasts carry no progress) switch (evt) { case "download_progress": var (downloaded, total) = SumProgress(data); progress?.Report(new ModelDownloadProgress( repoName, downloaded, total, Done: false, Failed: false)); break; case "download_finished": progress?.Report(new ModelDownloadProgress( repoName, 0, 0, Done: true, Failed: false, Message: "Download complete")); return true; case "download_failed": // Same capture as DownloadModelAsync: keep the server's // real error payload (logged verbatim) rather than // discarding it. Row-state transitions stay with the // poller — this only enriches the report. var watchDetail = ExtractFailureDetail(data); var watchRawPayload = data.ValueKind == JsonValueKind.Undefined ? "" : data.GetRawText(); Log.Warn($"server reported download_failed for {repoName}; data: {watchRawPayload}"); progress?.Report(new ModelDownloadProgress( repoName, 0, 0, Done: false, Failed: true, Message: watchDetail ?? "Download failed")); return false; } } } catch (OperationCanceledException) when (cancel.IsCancellationRequested) { // The poller canceled the watch (download completed, failed, or // vanished) — not an error, and the download itself is deliberately // left alone. } catch (Exception ex) when (ex is IOException or HttpRequestException or ObjectDisposedException) { // ObjectDisposedException covers the race where the poller cancels // and disposes the token source before the first GetAsync registers // the token. Log.Warn(ex, $"download watch for {repoName} ended early"); } return false; } /// /// Asks the running llama server to load (launch) a model into memory via /// POST /models/load. In router mode, the server spawns a child /// process for the model. /// /// The model to load; /// is the canonical id the server knows (the HF repo id with its /// :<quant> suffix, e.g. ggml-org/gemma-3-4b-it-GGUF:Q4_K_M) — /// /models/load requires the quant suffix, so the bare repo id won't do. /// Optional sink for the load fraction (0..1). When /// provided, the /models/sse stream is opened BEFORE the POST (a small /// model can finish loading in under a second — opening it after would miss /// the whole load) and status_change events are watched until the /// model reaches a terminal state, reporting each event's fraction. When /// null, the method is fire-and-forget: it returns as soon as the /// load request is accepted. /// Optional context size (ctx_size) /// for the spawned model process — the per-model preference chosen in the /// model details view. null lets the server use its own default. Unknown /// JSON fields are ignored by the server's body parser, so an older llama.cpp /// that predates per-model load options simply loads with its default. /// Cancellation token. /// false only when the load definitely didn't happen — the /// POST was rejected, or the server rolled the model back to unloaded /// (a failed load). true means accepted; the /// poller confirms the final loaded transition. public async Task LoadModelAsync(IModel model, IProgress? progress = null, int? contextLengthTokens = null, CancellationToken cancel = default) { if (ServerStatus != ServerState.Running) return false; var modelId = model.ServerModelId; // Open the SSE stream before the POST when progress is wanted. If it // can't be opened (older server without /models/sse), degrade to // fire-and-forget — the /models poller still reconciles the state. using var sseResponse = progress is null ? null : await TryOpenSseAsync(cancel); using var reader = sseResponse is null ? null : new StreamReader(await sseResponse.Content.ReadAsStreamAsync(cancel)); try { Log.Info($"loading model {modelId}" + (contextLengthTokens is { } ctx ? $" (ctx_size={ctx})" : "")); var payload = contextLengthTokens is { } ctxSize ? BuildModelRequestBody(modelId, ctxSize) : BuildModelRequestBody(modelId); using var content = new StringContent(payload, Encoding.UTF8, "application/json"); using var budget = WithTimeout(TimeSpan.FromSeconds(30), cancel); using var resp = await _http.PostAsync("/models/load", content, budget.Token); if (!resp.IsSuccessStatusCode) { Log.Warn($"server rejected model load ({(int)resp.StatusCode})"); return false; } } catch (Exception ex) when (ex is not OperationCanceledException) { Log.Error(ex, "model load request threw"); return false; } if (reader is null || progress is null) return true; // accepted; the poller takes it from here // Watch status_change events until the model reaches a terminal state. // The timeout only guards against a server that never sends one (a hung // child process): the request WAS accepted, so a timeout still returns // true and leaves the state to the poller. using var watchCts = CancellationTokenSource.CreateLinkedTokenSource(cancel); watchCts.CancelAfter(LoadWatchTimeout); try { await foreach (var (evt, evtModel, data) in ParseSseStreamAsync(reader, watchCts.Token)) { if (evt != "status_change" || !string.Equals(evtModel, modelId, StringComparison.OrdinalIgnoreCase)) continue; // another model's event var (status, fraction) = ParseStatusChange(data); switch (status) { case "loading": progress.Report(fraction); break; case "loaded": progress.Report(1.0); return true; case "unloaded": // Rolled back — the load failed server-side (e.g. the // child process died while mapping the weights). Log.Warn($"load of {modelId} rolled back to unloaded"); return false; } } } catch (OperationCanceledException) when (!cancel.IsCancellationRequested) { Log.Warn($"timed out waiting for load events for {modelId}"); } catch (Exception ex) when (ex is IOException or HttpRequestException or ObjectDisposedException) { // The server died mid-load: its SSE stream broke, so the load // definitely didn't complete — report failure so the caller drops // the row back to the play glyph instead of spinning forever. Log.Warn(ex, $"load watch for {modelId} broke (server died?)"); return false; } return true; } // How long the load-progress SSE watch waits for a terminal status_change // before deferring to the /models poller. Generous because mapping a very // large model from a slow disk can take minutes. private static readonly TimeSpan LoadWatchTimeout = TimeSpan.FromMinutes(5); /// /// Opens the /models/sse event stream for progress watching. /// Returns null (and logs) when the stream can't be opened — the /// caller then degrades to poller-only state tracking. /// private async Task TryOpenSseAsync(CancellationToken cancel) { try { var resp = await _http.GetAsync( "/models/sse", HttpCompletionOption.ResponseHeadersRead, cancel); if (resp.IsSuccessStatusCode) return resp; Log.Warn($"SSE stream rejected ({(int)resp.StatusCode}); progress falls back to the poller"); resp.Dispose(); return null; } catch (Exception ex) when (ex is not OperationCanceledException) { Log.Warn(ex, "SSE stream unavailable; progress falls back to the poller"); return null; } } /// /// Asks the running llama server to unload a model from memory via /// POST /models/unload. In router mode, the server stops the model's /// child process; this returns once the unload request is accepted. Track /// the transition via the poller, which reports the /// server's field flipping from loaded /// to unloaded. /// /// The model to unload; /// is the canonical id the server knows. /// Cancellation token. /// true if the server accepted the unload request. public async Task UnloadModelAsync(IModel model, CancellationToken cancel = default) { if (ServerStatus != ServerState.Running) return false; try { Log.Info($"unloading model {model.ServerModelId}"); var payload = BuildModelRequestBody(model.ServerModelId); using var content = new StringContent(payload, Encoding.UTF8, "application/json"); using var budget = WithTimeout(TimeSpan.FromSeconds(30), cancel); using var resp = await _http.PostAsync("/models/unload", content, budget.Token); if (!resp.IsSuccessStatusCode) Log.Warn($"server rejected model unload ({(int)resp.StatusCode})"); return resp.IsSuccessStatusCode; } catch (Exception ex) when (ex is not OperationCanceledException) { Log.Error(ex, "model unload request threw"); return false; } } // ---- Model listing (GET /models) ---- /// /// Asks the running llama server to remove a model from its cache via /// DELETE /models?model={name} (the model name is passed as a query /// param, not in the path — only cached, non-preset models can be deleted). /// The server deletes the on-disk GGUF and drops it from the model list; /// the poller will surface the removal on its /// next tick (the server also emits a model_remove SSE event). /// Returns false (without throwing) when the server isn't running or /// rejects the request. /// /// The model to delete; /// is the canonical id the server knows. /// Cancellation token. /// true if the server accepted the delete request. public async Task DeleteModelAsync(IModel model, CancellationToken cancel = default) { if (ServerStatus != ServerState.Running) return false; try { Log.Info($"deleting model {model.ServerModelId}"); var url = $"/models?model={Uri.EscapeDataString(model.ServerModelId)}"; using var budget = WithTimeout(TimeSpan.FromSeconds(30), cancel); using var resp = await _http.DeleteAsync(url, budget.Token); if (!resp.IsSuccessStatusCode) { Log.Warn($"server rejected model delete ({(int)resp.StatusCode})"); } return resp.IsSuccessStatusCode; } catch (Exception ex) when (ex is not OperationCanceledException) { Log.Error(ex, "model delete request threw"); return false; } } /// /// A model the running llama server knows about (router /models list): /// its canonical (repo:quant), on-disk , /// load (loaded/unloaded), and the /// flag derived from /// architecture.input_modalities. /// public sealed record ServerModel { /// Server model id, e.g. ggml-org/gemma-3-4b-it-GGUF:Q4_K_M. public string Id { get; init; } = ""; /// Absolute path to the GGUF on disk, when known. public string? Path { get; init; } /// Load state reported by the server: unloaded, downloading, loading, loaded, or sleeping. public string Status { get; init; } = ""; /// True when is loaded (model resident in a child process) /// or sleeping (freed after the idle timeout but still the active model — it wakes /// transparently on the next request, so for every caller it counts as loaded). public bool IsLoaded => string.Equals(Status, "loaded", StringComparison.OrdinalIgnoreCase) || string.Equals(Status, "sleeping", StringComparison.OrdinalIgnoreCase); /// True when is sleeping: the server freed the model's /// memory after --sleep-idle-seconds of idleness; the next request wakes it. public bool IsSleeping => string.Equals(Status, "sleeping", StringComparison.OrdinalIgnoreCase); /// True when is loading (load in progress: child process spawning / weights mmapping). public bool IsLoading => string.Equals(Status, "loading", StringComparison.OrdinalIgnoreCase); /// True when is downloading (the server is fetching the /// model's files; the model is id'd by its bare repo until the download completes and the /// quant is resolved). public bool IsDownloading => string.Equals(Status, "downloading", StringComparison.OrdinalIgnoreCase); /// True when the server reports the last load attempt as failed — /// the model process crashed or refused the model. Distinct from /// (which is unloaded again): without this a /// failed load is indistinguishable from a never-started one. public bool IsFailed { get; init; } /// The failed child process's exit code, when the server reports /// one (see ). public long? LoadExitCode { get; init; } /// True when architecture.input_modalities contains image. public bool SupportsImage { get; init; } /// All declared input modalities (e.g. text, image). public IReadOnlyList InputModalities { get; init; } = []; /// Where the server found the model, e.g. cache. public string? Source { get; init; } /// Whether the server allows removing this model. public bool CanRemove { get; init; } } /// /// Fetches the server's model list (GET /models) — the authoritative /// set of locally available (cached) models, with each model's load state and /// architecture (vision capability). Returns an empty list when the server /// isn't running or the request fails — see /// for the failure-vs-empty distinction the supervisor's liveness logic /// needs but callers don't. /// public async Task> GetModelsAsync(CancellationToken cancel = default) => await FetchModelsAsync(reload: false, cancel) ?? []; /// /// , but first tells the router to rescan its /// model sources — the HF cache and the --models-dir local-models /// folder. Older servers only rescan when asked (GET /models?reload=1); /// newer ones rescan on every fetch, where the parameter is simply ignored. /// This is what makes a freshly imported file () /// loadable without restarting the server. /// public async Task> RefreshModelsAsync(CancellationToken cancel = default) => await FetchModelsAsync(reload: true, cancel) ?? []; /// /// One GET /models round-trip with the failure-vs-empty distinction /// the supervisor's death detection needs: null means the fetch /// FAILED (request error, non-success status, or an unparseable body — e.g. /// a proxy error page or a zombie listener), while a list — possibly empty — /// is a valid answer from a living server. Not running is not a failure: /// callers get a valid empty list. Public callers keep the non-throwing /// empty-list-on-anything contract via . /// private async Task?> FetchModelsAsync(bool reload, CancellationToken cancel) { if (ServerStatus != ServerState.Running) return []; try { using var budget = WithTimeout(TimeSpan.FromSeconds(10), cancel); using var resp = await _http.GetAsync(reload ? "/models?reload=1" : "/models", budget.Token); resp.EnsureSuccessStatusCode(); await using var stream = await resp.Content.ReadAsStreamAsync(cancel); return await ParseModelsAsync(stream, cancel); } catch { return null; // failed fetch — deliberately distinct from an empty model list } } /// /// Parses a GET /models body into a snapshot. null means the /// body was not a valid models response (malformed JSON, or no /// data array) — an error, deliberately distinct from a valid /// response with an empty data array (genuinely no models). Internal /// for tests. /// internal static async Task?> ParseModelsAsync(Stream body, CancellationToken cancel = default) { try { var dto = await JsonSerializer.DeserializeAsync(body, LlamaJsonContext.Default.ModelsResponseDto, cancellationToken: cancel); return dto?.Data is { } data ? data.Select(Map).ToList() : null; } catch (JsonException) { return null; } } internal static ServerModel Map(ServerModelDto d) => new() { Id = d.Id ?? "", Path = d.Path, Status = d.Status?.Value ?? "", IsFailed = d.Status?.Failed ?? false, LoadExitCode = d.Status?.ExitCode, SupportsImage = d.Architecture?.InputModalities != null && d.Architecture.InputModalities.Contains("image", StringComparer.OrdinalIgnoreCase), InputModalities = (IReadOnlyList?)d.Architecture?.InputModalities ?? [], Source = d.Source, CanRemove = d.CanRemove, }; // ---- Chat completion (POST /v1/chat/completions, SSE) ---- /// /// The model the spotlight overlay should prompt: the first server-reported /// loaded model, or null when none is resident (the overlay /// shows its disabled hint in that case). Cached from the latest poller /// snapshot, so a hotkey press doesn't block on GET /models. /// private IReadOnlyList _lastModelsSnapshot = []; /// Latest known loaded model id, or null when none is loaded. public string? LoadedModelId => (from m in _lastModelsSnapshot where m.IsLoaded select m.Id).FirstOrDefault(); /// /// True when the loaded model accepts image input (per the poller's /// latest snapshot) — the gate for attaching a shared image in the chat. /// False while a text-only model is loaded; the share waits for the user /// to load an image-capable model. /// public bool LoadedModelSupportsImages => (from m in _lastModelsSnapshot where m.IsLoaded select m.SupportsImage).FirstOrDefault(); /// /// The most recent GET /models snapshot (refreshed by the 1s poller /// while the server runs). Lets detail views read per-model metadata (e.g. /// the on-disk ) without an extra HTTP round /// trip; may be empty or one poll cycle stale. /// public IReadOnlyList LastModelsSnapshot => _lastModelsSnapshot; /// Enrolled-folder scan's id → file map (the last /// WriteContextPresetsIniAsync run). See /// . private volatile IReadOnlyDictionary _enrolledModelPaths = new Dictionary(StringComparer.OrdinalIgnoreCase); /// /// Best-effort "where is this model's file on disk" for the rows' reveal /// action. The server's /models path is not reliably /// populated, so this resolves locally, in order: the enrolled-folder scan /// map (that scan already walked the trees — and covers stem ids and /// shards the cache layouts don't), the cache/managed flat layouts /// probes, and the managed models dir's /// direct {id}.gguf (Open-With imports). Null when nothing on disk /// matches — the row then simply hides its reveal button. /// public string? FindModelFilePath(string serverModelId) => FindModelFilePath( serverModelId, _enrolledModelPaths, ModelFileLocator.DefaultCacheRoots(), ManagedModelsDir); /// /// with its sources injectable — /// internal so tests can drive it against temp directories. Ids come from /// remote payloads, so every disk probe is containment-checked. /// internal static string? FindModelFilePath( string serverModelId, IReadOnlyDictionary enrolledPaths, IReadOnlyList cacheRoots, string managedDir) { if (string.IsNullOrWhiteSpace(serverModelId)) return null; // 1. Enrolled folders: the scanner's own id (stem) → path map. if (enrolledPaths.TryGetValue(serverModelId, out var enrolledPath) && File.Exists(enrolledPath)) return enrolledPath; // 2. HF-cache / managed flat layouts (owner/repo:quant ids). if (ModelFileLocator.TryFind(serverModelId, cacheRoots) is { PrimaryFilePath: { } located }) return located; // 3. Open-With imports land as {id}.gguf in the managed models dir — // or as a directory ({id}/model.gguf) for multi-file imports. The // directory-style pick must carry real GGUF content (an import // fixture's 2-byte "model.gguf" must not make a zombie // resolvable) and the main file wins over sidecars. if (ModelFileLocator.IsSafeModelId(serverModelId)) { var direct = Path.Combine(managedDir, serverModelId + ".gguf"); if (ModelFileLocator.IsUnder(managedDir, direct) && File.Exists(direct)) return direct; var folder = Path.Combine(managedDir, serverModelId); if (ModelFileLocator.IsUnder(managedDir, folder) && Directory.Exists(folder)) { foreach (var file in Directory .EnumerateFiles(folder, "*.gguf", SearchOption.AllDirectories) .OrderBy(f => f, StringComparer.OrdinalIgnoreCase)) { if (Common.GgufFile.HasGgufMagic(file) && !LocalModelImport.IsMmprojPath(file)) return file; } } } return null; } /// /// Removes a leftover artifact from the app-owned managed models dir: /// the zombie behind rows the server lists but that can never load — a /// directory-import fixture whose {id}/model.gguf files are not /// real GGUF, or a .gguf-named stub. The server's DELETE chokes on /// such entries and the row returns on every rescan, so the source /// artifact itself has to go. Refuses everything else: unsafe ids, paths /// outside the managed dir, and anything that still holds real GGUF /// content (a model, not an artifact — those resolve via /// and never reach this). /// public bool RemoveManagedModelArtifact(string serverModelId) => RemoveManagedModelArtifact(serverModelId, ManagedModelsDir); /// /// with the managed dir /// injectable — internal so tests can drive it against temp directories. /// internal static bool RemoveManagedModelArtifact(string serverModelId, string managedDir) { // The id names an entry INSIDE the managed dir — never trust it into // a path: no separators, no traversal, no wildcards/quotes/control. if (string.IsNullOrWhiteSpace(serverModelId) || serverModelId is "." or ".." || serverModelId.IndexOfAny(['/', '\\', ':', '*', '?', '"', '<', '>', '|']) >= 0 || serverModelId.Any(char.IsControl)) return false; var folder = Path.Combine(managedDir, serverModelId); var direct = Path.Combine(managedDir, serverModelId + ".gguf"); var target = Directory.Exists(folder) ? folder : File.Exists(direct) ? direct : null; if (target is null || !ModelFileLocator.IsUnder(managedDir, target)) return false; // Real GGUF content means it is a model, not an artifact — never // delete it here. IEnumerable ggufs = Directory.Exists(target) ? Directory.EnumerateFiles(target, "*.gguf", SearchOption.AllDirectories) : [target]; if (ggufs.Any(Common.GgufFile.HasGgufMagic)) return false; try { if (Directory.Exists(target)) Directory.Delete(target, recursive: true); else File.Delete(target); Log.Info($"removed leftover model artifact: {target}"); return true; } catch (Exception ex) { Log.Warn(ex, "could not remove leftover model artifact: " + target); return false; } } /// /// Drops preset entries for models whose files no longer exist (the ghost /// guard behind WriteContextPresetsIniAsync), keeping entries for /// models that are merely busy (mid-download / mid-load — no file yet) or /// whose file exists. Deliberately non-destructive: the settings entries /// themselves are untouched, so a temporarily unavailable folder (an /// asleep network share) gets its preferences back when the file does. /// Internal for tests. /// internal static List> FilterExistingModels( IEnumerable> entries, Func isBusy, Func fileExists) => entries.Where(kv => isBusy(kv.Key) || fileExists(kv.Key)).ToList(); /// /// The speculative-type counterpart of the context-length filter — same /// ghost guard. /// internal static List> FilterExistingModels( IEnumerable> entries, Func isBusy, Func fileExists) => entries.Where(kv => isBusy(kv.Key) || fileExists(kv.Key)).ToList(); /// /// Streams an OpenAI-compatible chat completion for /// against the currently loaded model, yielding delta.content chunks as /// they arrive from POST /v1/chat/completions (SSE). Throws if the /// server isn't running or no model is loaded. The caller cancels to abort. /// public async IAsyncEnumerable StreamChatAsync( string userMessage, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancel) { if (ServerStatus != ServerState.Running) throw new InvalidOperationException("llama server is not running."); var model = LoadedModelId ?? throw new InvalidOperationException("No model is loaded. Load one from the flyout first."); var body = $$"""{"model":{{JsonString(model)}},"stream":true, "return_progress": true, "messages":[{"role":"user","content":{{JsonString(userMessage)}}}]}"""; // SendAsync with ResponseHeadersRead returns as soon as the response // headers arrive, so we can read the SSE body incrementally below. // PostAsync (the default ResponseContentRead) would buffer the entire // response before completing — defeating streaming and making the // overlay hang until the whole generation finished. using var req = new HttpRequestMessage(HttpMethod.Post, new Uri("/v1/chat/completions", UriKind.Relative)) { Content = new StringContent(body, Encoding.UTF8, "application/json"), }; Log.Info($"chat completion → POST /v1/chat/completions (model={model}, prompt={userMessage.Length} chars)"); using var resp = await _http.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, cancel); Log.Info($"chat completion ← HTTP {(int)resp.StatusCode} {resp.StatusCode}"); resp.EnsureSuccessStatusCode(); // Reuse the same SSE line framing as /models/sse: data: {json} lines, // terminated by a blank line / "data: [DONE]". We parse incrementally so // tokens surface as soon as the server flushes them. // The default buffer size matches the working /models/sse path. await using var stream = await resp.Content.ReadAsStreamAsync(cancel); using var reader = new StreamReader(stream); var yielded = 0; var loggedLines = 0; Log.Info("chat completion: reading SSE stream"); while (await reader.ReadLineAsync(cancel) is { } line) { cancel.ThrowIfCancellationRequested(); // Log the first few raw lines verbatim (truncated) so we can see // the exact framing the server uses — prefix, line breaks, JSON // shape. Capped to avoid spamming the log on long generations. if (loggedLines < 20) { var preview = line.Length > 200 ? string.Concat(line.AsSpan(0, 200), "…") : line; Log.Debug($"chat sse raw[{loggedLines}]: '{preview}'"); loggedLines++; } if (line.Length == 0) continue; if (!line.StartsWith("data:", StringComparison.Ordinal)) continue; var value = line[5..].TrimStart(); if (value == "[DONE]") { Log.Info($"chat completion done: {yielded} chunk(s) yielded"); yield break; } if (value.Length == 0) continue; using var doc = JsonDocument.Parse(value); var root = doc.RootElement; if (!root.TryGetProperty("choices", out var choices) || choices.GetArrayLength() == 0) continue; var delta = choices[0].TryGetProperty("delta", out var d) ? d : default; if (delta.ValueKind != JsonValueKind.Object || !delta.TryGetProperty("content", out var c) || c.ValueKind != JsonValueKind.String) continue; var text = c.GetString(); if (string.IsNullOrEmpty(text)) continue; yielded++; yield return text; } // ReadLineAsync returned null: the server closed the stream without // sending [DONE]. Log so we can tell a hang (no log) from a clean // close with zero parsed chunks (this line). Log.Info($"chat completion stream ended without [DONE]: {yielded} chunk(s) yielded"); } /// /// Builds a {"model":…} request body (optionally with /// ctx_size), JSON-escaping the id. Model ids originate in remote /// payloads (Hub search, the catalog, an adopted server's /models), /// so they must never be piped raw into a hand-written JSON body — a quote /// or backslash would corrupt the request and could inject fields. Pure so /// the escaping is unit-testable. /// internal static string BuildModelRequestBody(string modelId, int? ctxSize = null) => ctxSize is { } ctx ? $$"""{"model":{{JsonString(modelId)}},"ctx_size":{{ctx}}}""" : $$"""{"model":{{JsonString(modelId)}}}"""; /// Minimal JSON string escaper for embedding user text in a raw body. private static string JsonString(string s) { var sb = new StringBuilder(s.Length + 2); sb.Append('"'); foreach (var ch in s) { switch (ch) { case '"': sb.Append("\\\""); break; case '\\': sb.Append(@"\\"); break; case '\n': sb.Append("\\n"); break; case '\r': sb.Append("\\r"); break; case '\t': sb.Append("\\t"); break; case '\b': sb.Append("\\b"); break; case '\f': sb.Append("\\f"); break; default: if (ch < 0x20) sb.Append($"\\u{(int)ch:X4}"); else sb.Append(ch); break; } } sb.Append('"'); return sb.ToString(); } // ---- /models JSON DTOs ---- internal sealed class ModelsResponseDto { [JsonPropertyName("data")] public List? Data { get; init; } } internal sealed class ServerModelDto { [JsonPropertyName("id")] public string Id { get; set; } = ""; [JsonPropertyName("path")] public string? Path { get; set; } [JsonPropertyName("status")] public ModelStatusDto? Status { get; set; } [JsonPropertyName("architecture")] public ArchitectureDto? Architecture { get; set; } [JsonPropertyName("source")] public string? Source { get; set; } [JsonPropertyName("can_remove")] public bool CanRemove { get; set; } } internal sealed class ModelStatusDto { [JsonPropertyName("value")] public string Value { get; set; } = ""; /// The load failed (the child process crashed or refused the /// model) — set by the server after a load attempt; the row must /// surface it instead of spinning forever. [JsonPropertyName("failed")] public bool Failed { get; set; } /// The child process's exit code when it died — e.g. /// 0xC0000409 (fail-fast, often a GPU OOM abort in the CUDA backend). /// [JsonPropertyName("exit_code")] public long? ExitCode { get; set; } } internal sealed class ArchitectureDto { [JsonPropertyName("input_modalities")] public List? InputModalities { get; set; } [JsonPropertyName("output_modalities")] public List? OutputModalities { get; set; } } /// /// Parses an SSE stream line-by-line, yielding (event, model, /// data JSON) tuples. Standard SSE framing: data: lines carry /// the payload, a blank line dispatches the event. /// internal static async IAsyncEnumerable<(string Event, string Model, JsonElement Data)> ParseSseStreamAsync( StreamReader reader, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancel) { var pendingData = new StringBuilder(); while (!cancel.IsCancellationRequested) { var line = await reader.ReadLineAsync(cancel); if (line is null) { // Stream closed — flush any partially accumulated event. foreach (var tuple in FlushAsync()) yield return tuple; yield break; } if (line.Length == 0) { // Blank line = dispatch the accumulated event. foreach (var tuple in FlushAsync()) yield return tuple; continue; } // Accumulate data: lines (may span multiple for a single event). if (!line.StartsWith("data:", StringComparison.Ordinal)) continue; var value = line[5..].TrimStart(); if (pendingData.Length > 0) pendingData.Append('\n'); pendingData.Append(value); // Ignore event:/id:/retry: lines — the server bundles the event // type inside the JSON data payload ("event" field). } yield break; // Dispatch whatever has been accumulated so far as a single event. // A well-formed SSE stream terminates every event with a blank line. However, // we also flush on EOF so a trailing event without a final blank // line (e.g., a server that dropped the connection mid-event, or a test // fixture) is not silently dropped. IEnumerable<(string Event, string Model, JsonElement Data)> FlushAsync() { if (pendingData.Length == 0) yield break; var json = pendingData.ToString(); pendingData.Clear(); using var doc = System.Text.Json.JsonDocument.Parse(json); var root = doc.RootElement; var evt = root.TryGetProperty("event", out var e) ? e.GetString() ?? "" : ""; var mdl = root.TryGetProperty("model", out var m) ? m.GetString() ?? "" : ""; // Clone detaches the element from the JsonDocument so callers // can safely consume it after the enumerator is disposed. var data = root.TryGetProperty("data", out var d) ? d.Clone() : default; if (evt.Length > 0) yield return (evt, mdl, data); } } /// /// Sums done/total bytes across all URLs in a /// download_progress data payload (a repo can have multiple files). /// internal static (long downloaded, long total) SumProgress(JsonElement data) { long downloaded = 0, total = 0; if (data.ValueKind != JsonValueKind.Object) return (0, 0); if (!data.TryGetProperty("progress", out var progress) || progress.ValueKind != JsonValueKind.Object) return (downloaded, total); foreach (var url in progress.EnumerateObject().Where(url => url.Value.ValueKind == JsonValueKind.Object)) { // TryGetInt64 throws InvalidOperationException on a non-Number // element (e.g. a string) — it only returns false for numbers // that don't fit — so the ValueKind must be checked first. if (url.Value.TryGetProperty("done", out var done) && done.ValueKind == JsonValueKind.Number && done.TryGetInt64(out var d)) downloaded += d; if (url.Value.TryGetProperty("total", out var tot) && tot.ValueKind == JsonValueKind.Number && tot.TryGetInt64(out var t)) total += t; } return (downloaded, total); } /// /// Extracts the human-readable error text from a download_failed SSE /// event's data payload, tolerating every shape: a string /// error field, an error object with a string message, /// or a string message field. Returns null when no such field /// is present (llama.cpp master currently sends data:{}) — no text /// is fabricated. Total: malformed/absent payloads (including a default /// , whose ValueKind is /// Undefined) yield null rather than throwing. /// internal static string? ExtractFailureDetail(JsonElement data) { try { // TryGetProperty throws on a non-Object element, and a default // JsonElement (no `data` field) is Undefined — guard on ValueKind // first, like SumProgress does. if (data.ValueKind != JsonValueKind.Object) return null; if (data.TryGetProperty("error", out var error)) { if (error.ValueKind == JsonValueKind.String) { var text = error.GetString(); if (!string.IsNullOrWhiteSpace(text)) return text; } else if (error.ValueKind == JsonValueKind.Object && error.TryGetProperty("message", out var nested) && nested.ValueKind == JsonValueKind.String) { var text = nested.GetString(); if (!string.IsNullOrWhiteSpace(text)) return text; } } if (data.TryGetProperty("message", out var message) && message.ValueKind == JsonValueKind.String) { var text = message.GetString(); if (!string.IsNullOrWhiteSpace(text)) return text; } return null; } catch { return null; } } /// /// Parses a status_change data payload into the model's new /// status (loading/loaded/unloaded) and, for /// loading, an overall 0..1 fraction. The server's progress object /// carries the load stages, the current stage, and that /// stage's 0..1 value; the overall fraction weights the value by /// the current stage's position — (stageIndex + value) / stageCount, /// which reduces to value for the common single-stage load. /// internal static (string status, double fraction) ParseStatusChange(JsonElement data) { if (data.ValueKind != JsonValueKind.Object || !data.TryGetProperty("status", out var s) || s.ValueKind != JsonValueKind.String) return ("", 0); var status = s.GetString() ?? ""; double fraction = 0; // TryGetDouble throws InvalidOperationException on a non-Number element // (same as TryGetInt64), so the ValueKind guard must come first. if (data.TryGetProperty("progress", out var progress) && progress.ValueKind == JsonValueKind.Object && progress.TryGetProperty("value", out var v) && v.ValueKind == JsonValueKind.Number && v.TryGetDouble(out var value)) { fraction = value; // Multi-stage load (e.g. text_model + mmproj): weight the current // stage's value by how many stages are already behind it. if (progress.TryGetProperty("stages", out var stages) && stages.ValueKind == JsonValueKind.Array && stages.GetArrayLength() > 1 && progress.TryGetProperty("current", out var cur) && cur.ValueKind == JsonValueKind.String) { var current = cur.GetString(); var index = -1; var i = 0; foreach (var stage in stages.EnumerateArray()) { if (stage.ValueKind == JsonValueKind.String && string.Equals(stage.GetString(), current, StringComparison.Ordinal)) { index = i; break; } i++; } if (index >= 0) fraction = (index + value) / stages.GetArrayLength(); } } return (status, Math.Clamp(fraction, 0, 1)); } /// /// Whether an SSE event's model field belongs to the download /// identified by : exact match, a * /// broadcast, or the same repo part — the server ids a model by its bare /// repo while the download is in flight (the quant resolves only on /// completion), while is the repo:quant form /// the download was started with. /// internal static bool SameDownloadModel(string eventId, string wantedId) { if (eventId == "*") return true; if (string.Equals(eventId, wantedId, StringComparison.OrdinalIgnoreCase)) return true; var eventRepo = eventId.Split(':')[0]; var wantedRepo = wantedId.Split(':')[0]; return string.Equals(eventRepo, wantedRepo, StringComparison.OrdinalIgnoreCase); } /// /// Asks the server to cancel an in-flight download via /// POST /models/unload. Best-effort — the server may have already /// finished or the request may fail; either way the SSE stream is closed /// by the caller's cancellation. /// /// Why unload, not DELETE: the router routes DELETE at /// /models?model=… (a path-form DELETE /models/{name} 404s /// and the download runs on), and its DELETE handler also removes the /// model entry and blocks until the download child exits — on current /// builds that wait can wedge the whole router (all /models /// requests stall). /models/unload aborts the download promptly /// without the remove-from-cache step, and answers as soon as the state /// flips — the transient entry then disappears from /models on the /// server's next reload sweep. Note the server drops the partial bytes on /// abort, so the next attempt re-downloads from scratch. /// public async Task CancelServerDownloadAsync(string modelName) { try { // Downloads are keyed by the id they were started with // (repo:quant), but the server may also know the entry by its // bare repo while the download is in flight — try the full id // first, then the bare repo when that misses. if (await TryUnloadAsync(modelName)) return; var repo = modelName.Split(':')[0]; if (!string.Equals(repo, modelName, StringComparison.Ordinal)) await TryUnloadAsync(repo); } catch { /* Best-effort — don't surface cancel cleanup failures. */ } async Task TryUnloadAsync(string id) { try { var payload = BuildModelRequestBody(id); using var content = new StringContent(payload, Encoding.UTF8, "application/json"); using var budget = WithTimeout(TimeSpan.FromSeconds(10), CancellationToken.None); using var resp = await _http.PostAsync("/models/unload", content, budget.Token); return resp.IsSuccessStatusCode; } catch { return false; } } } // ---- Resolution ---- private enum ResolutionKind { Missing, Managed, External } private record Resolution(ResolutionKind Kind, string? Path); /// /// Resolves the active llama binary with a single which-style /// PATH lookup (). A hit under the install /// dir is the app-managed installation; a hit anywhere else is the user's /// own external installation; no hit is . /// private static Resolution Resolve() { if (FindOnPath("llama.exe") is { } found) return new Resolution( IsManagedPath(found) ? ResolutionKind.Managed : ResolutionKind.External, found); return new Resolution(ResolutionKind.Missing, null); } /// /// which llama.exe: resolves the absolute path of /// over the effective PATH — the process /// PATH plus the user and machine PATH read from the registry. The registry /// reads are what make a just-installed binary visible: install.ps1 /// runs as a child process and cannot update our own environment block, so /// a PATH entry it adds only shows up in the registry (and in the process /// PATH of the next login shell). /// private static string? FindOnPath(string exeName) => FindOnPath( exeName, Environment.GetEnvironmentVariable("PATH"), Environment.GetEnvironmentVariable("PATH", EnvironmentVariableTarget.User), Environment.GetEnvironmentVariable("PATH", EnvironmentVariableTarget.Machine)); /// /// Pure core of : searches the three PATH /// lists in order (process, user, machine), first hit wins, directories /// deduped case-insensitively, quoted/whitespace-padded entries normalized, /// malformed entries skipped. Internal for tests. /// internal static string? FindOnPath(string exeName, string? processPath, string? userPath, string? machinePath) { var seen = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (var pathEnv in new[] { processPath, userPath, machinePath }) { if (string.IsNullOrEmpty(pathEnv)) continue; foreach (var rawDir in pathEnv.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries)) { var dir = rawDir.Trim().Trim('"'); if (dir.Length == 0 || !seen.Add(dir)) continue; try { var candidate = Path.Combine(dir, exeName); if (File.Exists(candidate)) return candidate; } catch { /* Malformed PATH entry — skip. */ } } } return null; } /// /// True when sits directly in the app-managed /// install dir () — i.e., it's the /// installation install.ps1 produced, not the user's own. /// internal static bool IsManagedPath(string binaryPath) => string.Equals( Path.GetFullPath(Path.GetDirectoryName(binaryPath)!), Path.GetFullPath(ManagedDir), StringComparison.OrdinalIgnoreCase); // ---- Install ---- /// /// Absolute path of Windows PowerShell 5.1 — the interpreter the install /// script runs under. Internal so tests can pin the invariant. /// internal static string PowerShellExePath => Path.Combine(Environment.SystemDirectory, "WindowsPowerShell", "v1.0", "powershell.exe"); /// /// Downloads and runs it with PowerShell /// — straight from memory: the body is executed via /// -EncodedCommand and is never written to disk, and nothing is /// cached between runs, so every install is a fresh download over HTTPS /// (system certificate store validation, no custom cert callback). The /// download is pinned to the exact HTTPS URL and bounded in size (see /// ); redirects are refused rather /// than followed, and the body's SHA-256 is logged so an install is /// auditable after the fact; it is deliberately not pinned to a /// build-time hash — install.ps1 is revised independently of the app, and /// a stale pin blocked runtime upgrades. Throws on any integrity failure, /// a non-zero exit code, or a download failure. /// private static async Task DownloadAndRunInstallerAsync(CancellationToken cancel) { byte[] scriptBytes; // Deliberately NOT the shared _http: this is an internet download // (llama.app), not a loopback call — the system proxy is welcome // here, and the BaseAddress wouldn't apply. Auto-redirect is off // so a 30x is surfaced (and refused) instead of silently followed // to another host. using (var handler = new SocketsHttpHandler { AllowAutoRedirect = false }) using (var client = new HttpClient(handler)) { client.Timeout = TimeSpan.FromSeconds(30); client.DefaultRequestHeaders.UserAgent.ParseAdd(HttpUserAgent.Value); // One 30s budget across headers AND body: with // ResponseHeadersRead, the HttpClient timeout only covers the // headers, and a hostile trickling endpoint could otherwise // stretch the body read indefinitely (bytes are capped, time // wasn't). using var budget = WithTimeout(TimeSpan.FromSeconds(30), cancel); using var resp = await client.GetAsync( InstallScriptUrl, HttpCompletionOption.ResponseHeadersRead, budget.Token); var responseVerdict = InstallScriptIntegrity.ValidateResponse( (int)resp.StatusCode, resp.RequestMessage?.RequestUri, resp.Content.Headers.ContentLength); if (!InstallScriptIntegrity.IsOk(responseVerdict)) throw new IOException( $"Refusing to run install.ps1: {InstallScriptIntegrity.Describe(responseVerdict)}."); await using var stream = await resp.Content.ReadAsStreamAsync(budget.Token); scriptBytes = await InstallScriptIntegrity.ReadBoundedAsync(stream, cancel: budget.Token); } var sha256 = InstallScriptIntegrity.Sha256Hex(scriptBytes); // Logged so a failed install can be correlated with exactly what ran. // The script itself never touches the disk — see EncodeCommand. Log.Info($"install.ps1 downloaded from {InstallScriptUrl}: {scriptBytes.Length} bytes, sha256={sha256}"); // Fileless execution: base64(UTF-16LE) of the validated script for // PowerShell's -EncodedCommand. Nothing is written to disk — the // body travels from the (hash-logged) response through memory into // the interpreter. The payload is // public script content, so its brief visibility on the child's // command line (WMI/process listings) leaks nothing; the size // guard keeps it inside the CreateProcess command-line limit. var encodedScript = InstallScriptIntegrity.EncodeCommand(scriptBytes); var psi = new ProcessStartInfo { // Absolute path: with UseShellExecute=false a bare // "powershell.exe" resolves through the CreateProcess search // order (the app's directory and the CWD before %SystemRoot%), // so a planted executable in either would run the install // script regardless of every hash/URL/size check above. FileName = PowerShellExePath, WorkingDirectory = Path.GetTempPath(), UseShellExecute = false, CreateNoWindow = true, RedirectStandardOutput = true, RedirectStandardError = true, }; // -EncodedCommand runs the in-memory payload without a script // file. -ExecutionPolicy Bypass is kept for parity with the old // -File launch (and in case the script ever invokes nested // scripts); it never changes the machine/user policy. -NoProfile // keeps the user's PowerShell profile (which can run arbitrary // code) out of the install. psi.ArgumentList.Add("-NoProfile"); psi.ArgumentList.Add("-ExecutionPolicy"); psi.ArgumentList.Add("Bypass"); psi.ArgumentList.Add("-EncodedCommand"); psi.ArgumentList.Add(encodedScript); Log.Info($"running install.ps1 from {InstallScriptUrl}"); using var proc = new Process(); proc.StartInfo = psi; if (!proc.Start()) throw new InvalidOperationException("Could not start the install script."); // Stream output to debug traces for diagnostics; not surfaced to the UI. var stdoutTask = proc.StandardOutput.ReadToEndAsync(cancel); var stderrTask = proc.StandardError.ReadToEndAsync(cancel); await proc.WaitForExitAsync(cancel); Log.Debug($"install.ps1 exit code {proc.ExitCode}"); var stdout = await stdoutTask; var stderr = await stderrTask; if (stdout.Length > 0) Log.Debug($"install.ps1 stdout: {stdout.Trim()}"); if (stderr.Length > 0) Log.Debug($"install.ps1 stderr: {stderr.Trim()}"); if (proc.ExitCode != 0) throw new IOException($"install.ps1 exited with code {proc.ExitCode}.\n{stderr}"); } // ---- Version ---- /// /// Reads the binary's version string by running llama --version and /// capturing the first non-empty line. Returns null if it can't be /// read (the server still runs with an unreadable version — fail open). /// private static async Task ReadVersionAsync(string binaryPath, CancellationToken cancel) { try { var psi = new ProcessStartInfo { FileName = binaryPath, Arguments = "--version", UseShellExecute = false, CreateNoWindow = true, RedirectStandardOutput = true, RedirectStandardError = true, }; using var proc = new Process(); proc.StartInfo = psi; if (!proc.Start()) return null; var stdoutTask = proc.StandardOutput.ReadToEndAsync(cancel); await proc.WaitForExitAsync(cancel); var stdout = await stdoutTask; // llama.cpp prints e.g. "llama-server (llama) b9553 (...) // version header (build: 9553)". The first non-empty line is the tag line. return stdout.Split('\n', StringSplitOptions.RemoveEmptyEntries).Select( line => line.Trim()).FirstOrDefault(trimmed => trimmed.Length > 0 ); } catch { return null; } } }