"use strict"; const { app, BrowserWindow, Menu, Tray, Notification, shell, protocol, ipcMain, dialog, utilityProcess, nativeImage, globalShortcut, } = require("electron"); const fs = require("fs"); const path = require("path"); // Explicit require: global `crypto` is absent on older Electron/node. // eslint-disable-next-line no-redeclare const crypto = require("crypto"); const { Readable } = require("stream"); const { MODELS, DEFAULT_MODEL_ID, getModel, resolveModelId, } = require("./indexer/models.js"); const { SAMPLE_QUERIES, buildFilenamePhrase, } = require("./indexer/memory-embedding-utils.js"); const { sortPathsByMtime } = require("./indexer/import-order-utils.js"); const { SEGMENT_ROW_PER, mergeChunkSegments, partitionChunkReply, shouldParkEnrichment, suspectedTruncated, ENRICH_CHUNK_TIMEOUT_MS, ENRICH_STALL_TIMEOUT_MS, ENRICH_MAX_RETRIES, } = require("./indexer/segment-store-utils.js"); const { TRANSCRIPT_ROW_PER, partitionTranscriptReply, attachQueryVec, applyTranscriptReply, exactDialogueSearch, transcribeBackfillList, TRANSCRIBE_CHUNK_TIMEOUT_MS, TRANSCRIBE_MAX_RETRIES, TRANSCRIBE_LOAD_MAX_RETRIES, transcribeExitLabel, transcribeFailureLabel, isLoadPhaseCrash, parseWhisperModel, transcribeTimeoutsFor, } = require("./indexer/transcript-store-utils.js"); const { probeScreenshotMetadata } = require("./screenshot-probe.js"); const { initSettings, APP_ICON_IDS, DEFAULT_APP_ICON, parseAppIcon, appIconFileFor, readAppIcon, readSettings, writeSettings, readVideoQuality, videoBudgetForCurrentQuality, readWhisperModel, cleanupRemovedWhisperWeights, readOcrLangs, resolveOcrLangString, readOcrLangsMigrated, shouldMigrateOcrLangs, readMenuBarOnly, readNotifyOnDone, readStartHidden, readOnboardingSeen, } = require("./main-lib/settings.js"); const { initFailedImports, failedImportsFile, isSystemIndexerError, rememberImportFailure, isKnownImportFailure, retireImportFailure, clearFailedImports, } = require("./main-lib/failed-imports.js"); const { initCategoryOverrides, CATEGORY_OVERRIDE_VALUES, loadCategoryOverrides, saveCategoryOverrides, clearCategoryOverrides, } = require("./main-lib/category-overrides.js"); const { initLlmConfig, readLlmConfig, writeLlmConfig, llmModelInfos, } = require("./main-lib/llm/config.js"); const { initLlmServer, llmServerSnapshot, binaryDownloaded: llmBinaryDownloaded, modelDownloaded: llmModelDownloaded, downloadBinary: llmDownloadServerBinary, downloadModel: llmDownloadModelWeights, llmChat, stopServer: stopLlmServer, chatThreadCount: llmChatThreadCount, } = require("./main-lib/llm/server.js"); const { validateScopedFilenames, partitionScoped, buildAskEvidence, } = require("./main-lib/ask/retrieve.js"); const { buildAskMessages } = require("./main-lib/ask/prompts.js"); const { initLibraryStore, resetLibraryCaches, libraryGenerationNumber, embedFileFor, phraseFileFor, segmentsMetaFileFor, segmentsBinFileFor, transcriptsMetaFileFor, thumbFor, loadSampleQueries, loadLibrary, readBin, libraryIndex, libraryNorms, enqueuePersist, cachedLibraryBins, cachedSegmentsBin, cachedTranscriptsBin, writeBins, saveLibrary, readEmbedMeta, setEmbedIncomplete, isZeroVector, planDeltaReuse, backfillSourceMtimes, loadSegments, saveSegments, removeSegmentsForAllModels, loadTranscripts, saveTranscripts, removeTranscriptsForAllModels, repairUntranscribedWithAudio, loadModelState, resetModelStateCache, saveModelState, thresholdsFor, binInfo, cosine, } = require("./main-lib/library-store.js"); const { initEmbeddingVersions, listVersions, createVersion, renameVersion, deleteVersion, restoreVersion, } = require("./main-lib/embedding-versions.js"); const { initLibraryReset, resetLibraryFiles, } = require("./main-lib/library-reset.js"); const { cosineSim, RANK_MIN_SCORE, RANK_RELATIVE_KEEP, scoreLibrary, } = require("./main-lib/rank-search.js"); const youtube = require("./main-lib/youtube.js"); const SCHEME = "app"; // Test isolation: point the whole app (library, models, single-instance // lock) at a scratch dir. Must happen before the lock/ready paths resolve // any app paths. if (process.env.MEMORIES_DATA_DIR) { app.setPath("userData", process.env.MEMORIES_DATA_DIR); } // --------------------------------------------------------------------------- // Persistent file logging: the packaged app's stdout goes nowhere on macOS, // so a native crash (SIGTRAP/SIGSEGV in Electron Framework) leaves zero // context about what the app was doing. Every console line is therefore also // appended to userData/logs/main-YYYYMMDD.log (rotated by size, pruned by // age). All best-effort: logging must never throw, block, or grow unbounded. // --------------------------------------------------------------------------- const LOG_DIR = path.join(app.getPath("userData"), "logs"); const LOG_MAX_BYTES = 2 * 1024 * 1024; const LOG_KEEP_DAYS = 14; function initFileLogging() { try { fs.mkdirSync(LOG_DIR, { recursive: true }); const day = new Date().toISOString().slice(0, 10); const file = path.join(LOG_DIR, `main-${day}.log`); try { const stat = fs.statSync(file); if (stat.size >= LOG_MAX_BYTES) { try { fs.renameSync(file, `${file}.1`); } catch { /* overwrite below */ } } } catch { /* no log yet today */ } // Prune logs older than the window (filename-dated, lexicographic). try { const cutoff = new Date(Date.now() - LOG_KEEP_DAYS * 86400_000) .toISOString() .slice(0, 10); for (const entry of fs.readdirSync(LOG_DIR)) { const m = /^main-(\d{4}-\d{2}-\d{2})\.log(\.1)?$/.exec(entry); if (m && m[1] < cutoff) { try { fs.unlinkSync(path.join(LOG_DIR, entry)); } catch { /* best-effort */ } } } } catch { /* best-effort */ } const stream = fs.createWriteStream(file, { flags: "a" }); stream.on("error", () => {}); const stamp = () => new Date().toISOString(); for (const level of ["log", "warn", "error"]) { const orig = console[level].bind(console); console[level] = (...args) => { orig(...args); try { const line = args .map((a) => (typeof a === "string" ? a : JSON.stringify(a))) .join(" "); stream.write(`${stamp()} [${level}] ${line}\n`); } catch { /* logging must never throw */ } }; } stream.write(`${stamp()} [log] --- SCM start (pid ${process.pid}) ---\n`); } catch { /* file logging unavailable; console still works */ } } initFileLogging(); const RENDERER_DIR = path.join(__dirname, "dist"); const DATA_DIR = path.join(app.getPath("userData"), "library"); // settings.json is the source of truth for Appearance etc. Must be defined // before any helper that reads it (app-icon loader) — otherwise TDZ makes // the first APP_ICON load fall back to default even when a choice is persisted. const SETTINGS_FILE = path.join(app.getPath("userData"), "settings.json"); initSettings({ userDataDir: app.getPath("userData") }); initFailedImports({ dataDir: DATA_DIR }); initCategoryOverrides({ dataDir: DATA_DIR }); initLibraryStore({ userDataDir: app.getPath("userData") }); initEmbeddingVersions({ userDataDir: app.getPath("userData") }); initLibraryReset({ dataDir: DATA_DIR }); initLlmConfig({ userDataDir: app.getPath("userData") }); youtube.initYoutube({ userDataDir: app.getPath("userData") }); // LLMs-mode sidecar (main-lib/llm/server.js): status events ride the same // memories:status channel as every other tray. Idle download/spawn work is // broadcast-only — the tray logic deliberately never reads "llm" events. initLlmServer({ userDataDir: app.getPath("userData"), broadcast: (payload) => { for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", payload); } }, }); // App icon (Dock + BrowserWindow). Users pick one of the 1024×1024 PNGs in // public/images/app-icons/ via Settings → Appearance. Persisted as // settings.json `appIcon` (validated; defaults to scm-vhs). Loaded through fs // rather than by path: nativeImage cannot read from inside the packaged // app.asar, while fs.readFileSync is asar-aware. The packaged bundle already // carries SCM.icns, so this is mostly a dev-mode/runtime Dock nicety — and it // must never be able to block the app from opening. // C-01 Wave 2 (S1): APP_ICON_IDS, DEFAULT_APP_ICON live in main-lib/settings.js (required at top). // In-memory staging for hidden-Dock case. Must be defined before // APP_ICON init which runs at module load (TDZ guard). let menuBarOnlyEnabled = false; let pendingDockIcon = null; let pendingDockIconId = null; let currentAppIconId = DEFAULT_APP_ICON; // Dev-time drift guard: main.js and src/lib/appIcons.ts must list the same // ids. The renderer parses the same ids; a drift would make the main reject // an id the UI offers (or vice versa) and the "didn't change" bug would look // like a hidden-Dock bug. In dev we read the TS source and warn. try { const tsPath = path.join(__dirname, "src/lib/appIcons.ts"); if (fs.existsSync(tsPath)) { const ts = fs.readFileSync(tsPath, "utf-8"); const m = ts.match(/APP_ICON_IDS\s*=\s*\[([\s\S]*?)\]/); if (m) { const ids = [...m[1].matchAll(/"([^"]+)"/g)].map((x) => x[1]); const mainSet = new Set(APP_ICON_IDS); const tsSet = new Set(ids); const missingInMain = ids.filter((id) => !mainSet.has(id)); const missingInTs = APP_ICON_IDS.filter((id) => !tsSet.has(id)); if (missingInMain.length || missingInTs.length) { console.warn( `[app-icon] APP_ICON_IDS drift: missingInMain=${missingInMain.join(",") || "—"} missingInSrc=${missingInTs.join(",") || "—"}`, ); } } } } catch { /* drift guard is best-effort dev only */ } // C-01 Wave 2 (S1): parseAppIcon live in main-lib/settings.js (required at top). // C-01 Wave 2 (S1): appIconFileFor live in main-lib/settings.js (required at top). function loadAppIconImage(id) { const safeId = parseAppIcon(id); const filename = appIconFileFor(safeId); const candidates = [ path.join(__dirname, "dist/images/app-icons", filename), path.join(__dirname, "public/images/app-icons", filename), // Legacy fallback (single icon before the chooser existed) path.join(__dirname, "public/images/icns/SCM.png"), path.join(__dirname, "dist/images/icns/SCM.png"), ]; for (const file of candidates) { try { const buf = fs.readFileSync(file); const img = nativeImage.createFromBuffer(buf); if (!img.isEmpty()) return img; } catch { /* try next candidate */ } } return null; } // C-01 Wave 2 (S1): readAppIcon live in main-lib/settings.js (required at top). function currentAppIconImage() { return loadAppIconImage(readAppIcon()); } // NOTE: an earlier applyAppIcon() helper was fully superseded by the // memories:set-app-icon IPC handler below (which adds validation, settings // persistence, and tray feedback) — it was deleted in Phase 0 (N-04) as // dead code. Icon application lives in the handler now. let APP_ICON = currentAppIconImage(); try { currentAppIconId = readAppIcon(); } catch { currentAppIconId = DEFAULT_APP_ICON; } const PHOTOS_DIR = path.join(DATA_DIR, "photos"); const POSTERS_DIR = path.join(DATA_DIR, "posters"); const THUMBS_DIR = path.join(DATA_DIR, "thumbs"); // In-app preview proxies for searchable-only videos (mkv/hevc/...): short // mp4/h264 clips transcoded on demand around the matched scene timestamp so // Chromium can play them. Cached on disk, pruned by age/size (see below). const PREVIEWS_DIR = path.join(DATA_DIR, "previews"); const PREVIEW_SECONDS = 30; const PREVIEW_LEAD_SECONDS = 5; const PREVIEW_TIMEOUT_MS = 120000; const PREVIEW_MAX_BYTES = 500 * 1024 * 1024; const PREVIEW_MAX_AGE_MS = 7 * 24 * 3600 * 1000; const previewPending = new Map(); // preview filename → Promise const INDEX_FILE = path.join(DATA_DIR, "memories-index.json"); // Pre-v4 libraries wrote one bin pair for the default model; v4+ writes a // pair PER MODEL (memory-embeddings-.bin) so switching models never // destroys another model's embeddings. The legacy names are still read as a // fallback when a model's own bins are absent (one-time upgrade). // (Paths live in main-lib/library-store.js; this file keeps only INDEX_FILE.) // Per-model persisted state: calibrated ranking thresholds etc. // C-01 Wave 2 (S4): loadSampleQueries live in main-lib/library-store.js (required at top). // C-01 Wave 2 (S4): sidecar path helpers live in main-lib/library-store.js (required at top). // Same set as the site's build (scripts/build-memory-embeddings-core.js) const IMAGE_EXTENSIONS = new Set([ ".jpg", ".jpeg", ".png", ".gif", ".webp", ".heic", ".heif", ".JPG", ".JPEG", ".PNG", ".GIF", ".WEBP", ".HEIC", ".HEIF", ]); // Videos import through ffmpeg frame extraction (see indexer/video-utils.js, // the same list must stay in sync with src/lib/media.ts). Everything ffmpeg // can decode becomes searchable; renderer playability is a narrower set. const VIDEO_EXTENSIONS = new Set([ ".mp4", ".mov", ".m4v", ".webm", ".mkv", ".avi", ".ts", ".m2ts", ".mts", ".mpg", ".mpeg", ".wmv", ".flv", ".3gp", ".MP4", ".MOV", ".M4V", ".WEBM", ".MKV", ".AVI", ".TS", ".M2TS", ".MTS", ".MPG", ".MPEG", ".WMV", ".FLV", ".3GP", ]); const IMPORT_EXTENSIONS = new Set([...IMAGE_EXTENSIONS, ...VIDEO_EXTENSIONS]); // Poster sidecar naming: every imported video gets a generated JPEG in // POSTERS_DIR with the same basename, so the renderer can derive the URL // purely from the filename (no index change needed). function posterFor(filename) { return path.join( POSTERS_DIR, path.basename(filename, path.extname(filename)) + ".jpg", ); } // Photo thumbnail sidecar: a ~480px JPEG in THUMBS_DIR, derived from the // filename exactly like posters (basename minus extension + ".jpg"). The // grid renders this instead of the original — see ensureThumb. // C-01 Wave 2 (S4): thumbFor live in main-lib/library-store.js (required at top). function isVideo(filename) { return VIDEO_EXTENSIONS.has(path.extname(filename).toLowerCase()); } const MIME_TYPES = { ".html": "text/html; charset=utf-8", ".js": "text/javascript; charset=utf-8", ".mjs": "text/javascript; charset=utf-8", ".css": "text/css; charset=utf-8", ".json": "application/json; charset=utf-8", ".woff2": "font/woff2", ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".svg": "image/svg+xml", ".webp": "image/webp", ".ico": "image/x-icon", ".wasm": "application/wasm", ".bin": "application/octet-stream", ".mp4": "video/mp4", ".m4v": "video/x-m4v", ".mov": "video/quicktime", ".webm": "video/webm", ".mkv": "video/x-matroska", ".avi": "video/x-msvideo", ".ts": "video/mp2t", ".m2ts": "video/mp2t", ".mts": "video/mp2t", ".mpg": "video/mpeg", ".mpeg": "video/mpeg", ".wmv": "video/x-ms-wmv", ".flv": "video/x-flv", ".3gp": "video/3gpp", }; // Magic-byte sniffing for served/stored images. Files arrive with arbitrary // extensions (screenshots saved as .png that are really WebP, downloads // renamed by hand), and the app:// protocol historically answered with the // MIME for the EXTENSION. Blink picks its image decoder from that MIME, so // WebP bytes served as image/png route into the PNG decoder — a wrong-decoder // hazard that has crashed Electron/Chromium on macOS Tahoe (EXC_BREAKPOINT / // SIGTRAP on a ThreadPoolForegroundWorker inside the framework's Rust image // code; the stripped frames all alias to ares_dns_rr_get_ttl; Sep-2026 // production incident on 0.2.1). The bytes, not the name, must decide the // Content-Type — and unrecognized bytes must never inherit an image/* claim // (see imageContentType). See indexer/mime-sniff.js. const { sniffImageMime, canonicalImageExt, imageContentType, } = require("./indexer/mime-sniff.js"); protocol.registerSchemesAsPrivileged([ { scheme: SCHEME, privileges: { standard: true, secure: true, supportFetchAPI: true, stream: true, }, }, ]); // C-01 Wave 2 (S4): library store live in main-lib/library-store.js (required at top). // Queries with no semantic content: the top cosine any one of them can reach // against the library is the noise ceiling for the ACTIVE model's score band. const NOISE_QUERIES = [ "zzz nothing to see here", "qwerty gibberish", "random noise photo", "this is not a real query", "unrelated nonsense text", "aaa bbb ccc ddd", ]; // Measure a model's score band on the CURRENT in-memory library and persist // per-model thresholds: minSemanticScore sits between the noise ceiling // (worst top-1 cosine of a nonsense query) and the weakest genuine match // (SAMPLE_QUERIES' worst top-1), with a generous floor between them. Each // model's anisotropy differs, so the CLIP-tuned 0.04 must not be assumed. // Returns the thresholds, or null when the library is too small to measure. async function calibrateThresholds(modelId) { const l = loadLibrary(); // A retired-model library temporarily names its replacement so the worker // can warm it, but its stored rows are still from the old vector space. // Never calibrate or persist any replacement state until the migration // commits its complete new bin pair. if (l.pendingModelMigration) return null; if (l.filenames.length === 0 || l.embeddings.length !== l.filenames.length) { return null; } const noise = []; const genuine = []; for (const query of [...NOISE_QUERIES, ...SAMPLE_QUERIES]) { const reply = await askIndexer({ type: "embed-query", text: query }); if (!reply.vec || reply.vec.length !== l.dim) continue; let top = 0; for (const row of l.embeddings) { let dot = 0; for (let k = 0; k < l.dim; k++) dot += reply.vec[k] * row[k]; if (dot > top) top = dot; } (NOISE_QUERIES.includes(query) ? noise : genuine).push(top); } if (noise.length === 0) return null; const noiseCeiling = Math.max(...noise); let minSemanticScore = Math.min(0.1, Math.max(0.01, noiseCeiling * 2)); if (genuine.length > 0) { const weakestGenuine = Math.min(...genuine); if (minSemanticScore > weakestGenuine * 0.5) { minSemanticScore = Math.max(0.01, weakestGenuine * 0.5); } } const thresholds = { minSemanticScore: Math.round(minSemanticScore * 10000) / 10000, relativeKeep: 0.6, }; loadModelState().models[modelId] = { ...loadModelState().models[modelId], thresholds, calibratedAt: new Date().toISOString(), }; saveModelState(); console.log( `[memories] calibrated ${modelId}: noiseCeiling=${noiseCeiling.toFixed(4)} ` + `minSemanticScore=${thresholds.minSemanticScore} ` + `(weakestGenuine=${genuine.length ? Math.min(...genuine).toFixed(4) : "n/a"})`, ); return thresholds; } // A text-model outage at import time persists all-zero phrase rows (the // fallback in importPaths), silently disabling the phrase boost; a bin left // --------------------------------------------------------------------------- // Stale-entry cleanup: library rows whose photo files no longer exist on // disk (deleted manually or when the source folder was removed). These // ghost entries cause endless "thumbnail failed" warnings and waste OCR // backfill cycles. Runs once on startup, before the backfill passes. // --------------------------------------------------------------------------- async function pruneStaleLibraryEntries() { const l = loadLibrary(); if (!l || !Array.isArray(l.filenames) || l.filenames.length === 0) return; const total = l.filenames.length; // First pass: count missing files. A mass-missing signal (the photos // dir was emptied by a "cleaner", a volume is unmounted, a backup // restore is half-applied) means the right move is to STOP, not to wipe // the index to match — pruning only makes sense when a FEW rows are // genuinely stale. PHOTOS_DIR always exists (loadLibrary mkdirs it), so // an empty dir surfaces here as "every row's file is missing". let missing = 0; for (let i = 0; i < total; i++) { const filename = l.filenames[i]; if (!filename) continue; if (!fs.existsSync(path.join(PHOTOS_DIR, filename))) missing++; } if (missing === 0) return; const threshold = Math.max(1, Math.ceil(total / 2)); if (missing >= threshold) { // Refuse to wipe: persist nothing, surface the state, let the user // resolve the underlying cause (remount, restore, re-point) and // retry on the next launch. This is the guard against the silent // library wipe documented in C-05. console.warn( `[memories] stale-entry cleanup ABORTED: ${missing}/${total} photos missing (≥50%) — refusing to prune to avoid a silent library wipe`, ); for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "prune-aborted", missing, total, }); } return; } let removed = 0; // Walk backwards so splicing doesn't shift unvisited indices. for (let i = l.filenames.length - 1; i >= 0; i--) { const filename = l.filenames[i]; if (!filename) continue; if (fs.existsSync(path.join(PHOTOS_DIR, filename))) continue; // File is gone — remove the entire row (copy, index arrays, // sidecars). removeLibraryRow tolerates a missing copy. if (removeLibraryRow(filename)) removed++; } if (removed > 0) { await saveLibrary(); console.log( `[memories] stale-entry cleanup: removed ${removed} row(s) with missing files`, ); } } // misaligned by a partial write is unusable too. Detect and re-embed those // rows through the indexer once the text model is ready. Returns the number // healed. // // Detection: a real filename-phrase embedding is a CLIP TEXT vector, which // never equals its photo's IMAGE vector (dot ≈ 0..0.3), so dot > 0.9999 is // an exact image copy; norm ≈ 0 is the all-zero fallback. A phrase bin that // failed to load at all (count mismatch) is rebuilt in full. async function healPhraseBin() { if (migrationState) { // The library is mid-flip; healing the ACTIVE model's bin now could // interleave with the migration's writes. Skip — the next startup // (or a switch back) heals it. return { healed: 0, attempted: 0 }; } const l = loadLibrary(); // See calibrateThresholds: a pending retirement migration must preserve // the old index on disk until its replacement vectors are complete. if (l.pendingModelMigration) return { healed: 0, attempted: 0 }; if (l.filenames.length === 0) return { healed: 0, attempted: 0 }; let corrupt; if (l.phrases.length !== l.filenames.length) { // Unusable bin (wrong count/dim): rebuild every row. corrupt = l.filenames.slice(); } else { corrupt = []; for (let i = 0; i < l.filenames.length; i++) { const ph = l.phrases[i]; const im = l.embeddings[i]; if (!ph || !im || ph.length === 0) continue; let dot = 0; let norm = 0; for (let k = 0; k < ph.length; k++) { dot += ph[k] * (im[k] || 0); norm += ph[k] * ph[k]; } if (norm < 1e-6 || dot > 0.9999) corrupt.push(l.filenames[i]); } } if (corrupt.length === 0) return { healed: 0, attempted: 0 }; console.log(`[memories] healing ${corrupt.length} corrupted phrase rows…`); let healed = 0; for (const filename of corrupt) { try { const reply = await askIndexer({ type: "embed-phrase", filename }); if (reply && reply.vec && reply.vec.length === l.dim) { // Rows can shift while healing runs (a delete or import can // land mid-heal); resolve the row by filename so the vector // lands on the right entry — a vanished file is skipped. const j = l.filenames.indexOf(filename); if (j !== -1) { l.phrases[j] = new Float32Array(reply.vec); healed++; } } } catch (err) { console.warn( `[memories] phrase heal failed for ${filename}: ${err.message}`, ); } } if (healed > 0) { await saveLibrary(); console.log(`[memories] healed ${healed}/${corrupt.length} phrase rows`); } return { healed, attempted: corrupt.length }; } // --------------------------------------------------------------------------- // Indexer worker pool (utilityProcess — plain Node, exactly the environment // the site's build script runs in). // // A small pool of workers, each holding its own CLIP model, lets import // batches embed several files at once and keeps search queries from queueing // behind a long video embed. The first worker ("primary") is spawned eagerly // and drives the AI status LED + first-run download; extra workers are forked // only after the primary's model is ready (a first launch must never race the // download). Dispatch is round-robin over live workers; import batches // collect replies in file order (see importPaths). // // Note: on this machine ONNX Runtime already saturates all cores for a // single run and ignores thread-count env vars, so measured E2E import time // is fastest with a single worker (7.0s vs 7.5s at 2-3 workers on the smoke // batch). Pool size > 1 buys query responsiveness during imports (a search // no longer queues behind a multi-second video embed) and headroom for GPU // workers, at the cost of aggregate embed throughput. The default follows // the measurement; INDEXER_POOL_SIZE=2/3 opts into the pool. // --------------------------------------------------------------------------- const INDEXER_POOL_SIZE = Math.max( 1, Number(process.env.INDEXER_POOL_SIZE) || 1, ); let indexerWorkers = []; // { process, primary, ready, dead, pending } let indexerReady = null; // primary worker's init-done promise // Set when the primary's init handshake rejects (model load failed) so // import batches can fail fast without copying files that will never embed. // Cleared on a fresh spawn (a respawn retries init from scratch). let indexerInitFailed = null; let requestId = 0; let pickRoundRobin = 0; // Latest AI-model status (loading/ready/error + download %). Cached so a // window that mounts after events fired still learns the current state. let modelStatus = null; // True while the ACTIVE model's weights are absent from the model cache — // i.e. the user's first launch with this model. Computed once per worker // spawn and stamped onto every model status so the UI can show a one-time // download hint without flapping mid-download (partial files appear in the // cache early). let modelFirstRun = false; // The model the pool was forked for. Normally follows the library's active // model (null); a migration pins it to the TARGET model while the library // still names the old one, so the respawned pool embeds in the new model // before the index flips. Cleared when the flip completes. let workerModelId = null; // Which model the next worker fork should load: the migration target when // one is pinned, else the library's active model. function poolModelId() { return workerModelId || resolveModelId(loadLibrary().modelId); } // --------------------------------------------------------------------------- // Orphan-proof ffmpeg PID tracking (2026-09-28 ffmpeg-CPU fix). // // Workers report every ffmpeg spawn/exit ({type:ffmpeg-spawn/exit,pid}) so // main can SIGKILL grandchildren by PID even AFTER the worker died (the old // code only killed the Node worker; ffmpeg survived to PID 1 with no timers // left — the 500%+ CPU after "done" and after quit). Maps survive worker // restarts; entries are best-effort (ESRCH on kill = already gone). // --------------------------------------------------------------------------- const indexerFfmpegPids = new Map(); // pid -> { firstSeen, label } const transcribeFfmpegPids = new Map(); function trackFfmpegSpawn(map, pid, label) { if (!Number.isInteger(pid) || pid <= 0) return; if (!map.has(pid)) { map.set(pid, { firstSeen: Date.now(), label: String(label || "") }); } } function untrackFfmpegExit(map, pid) { if (Number.isInteger(pid)) map.delete(pid); } // SIGKILL every PID in the map. Never throws. Returns killed count. function killTrackedFfmpegPids(map, reason) { let killed = 0; for (const pid of [...map.keys()]) { try { process.kill(pid, "SIGKILL"); killed++; } catch { /* ESRCH = already gone, EPERM = not ours — both fine */ } finally { map.delete(pid); } } if (killed > 0) { console.warn( `[memories] killed ${killed} tracked ffmpeg child(ren) (${reason || "unknown"})`, ); } return killed; } // Ask a live worker to kill its own ffmpeg children first (fast path when // the worker is still alive), then belt-and-braces kill tracked PIDs by PID // (covers the worker-already-dead case). Never throws, never awaits. function requestWorkerKillFfmpeg(worker, map, reason) { try { if (worker && worker.process && !worker.dead) { try { worker.process.postMessage({ type: "kill-ffmpeg", reason }); } catch { /* worker gone — PID sweep below still applies */ } } } finally { if (map) killTrackedFfmpegPids(map, reason); } } // Startup orphan reaper (2026-09-28): previous runs (pre-fix, or crashes) // may have left ffmpeg children reparented to PID 1 still decoding. They // share our bundled binary path and usually the library path. Best-effort: // list processes via `ps`, SIGKILL matches, never throw, never kill anything // that does not look like ours. function reapStaleFfmpegOrphans() { try { const { execFileSync } = require("child_process"); let out = ""; try { out = execFileSync("ps", ["-ax", "-o", "pid=,ppid=,command="], { encoding: "utf8", timeout: 5000, windowsHide: true, }); } catch { return 0; } let killed = 0; for (const line of String(out).split("\n")) { const m = line.trim().match(/^(\d+)\s+(\d+)\s+(.*)$/); if (!m) continue; const pid = Number(m[1]); const ppid = Number(m[2]); const cmd = m[3] || ""; if (!Number.isInteger(pid) || pid <= 0 || pid === process.pid) continue; // Our bundled binary only: packaged Resources/ffmpeg or // ffmpeg-static under node_modules. Never touch system ffmpeg. const isOurs = cmd.includes("Contents/Resources/ffmpeg") || cmd.includes("node_modules/ffmpeg-static"); if (!isOurs) continue; // Extra guard: only orphans (PPID 1) or library-path decodes. // A live worker's ffmpeg has a live parent and is tracked; the // dangerous leftovers are exactly the PPID-1 ones. const touchesLibrary = cmd.includes("library/photos") || cmd.includes("memories-transcribe-"); if (ppid !== 1 && !touchesLibrary) continue; try { process.kill(pid, "SIGKILL"); killed++; } catch { /* already gone */ } } if (killed > 0) { console.warn( `[memories] reaped ${killed} stale ffmpeg orphan(s) from previous runs`, ); } return killed; } catch { return 0; } } // transformers.js writes the ONNX weights last, so a cache holding only // partial files is an interrupted FIRST download — that must still count as // first run. Both quantized weights (vision + text) must be on disk for the // model to count as downloaded. Per-model: the cache root may already hold // ANOTHER model's weights, which says nothing about the active one's. function countOnnx(dir) { try { let n = 0; for (const entry of fs.readdirSync(dir)) { const full = path.join(dir, entry); if (fs.statSync(full).isDirectory()) { n += countOnnx(full); } else if (entry.endsWith(".onnx")) { n++; } } return n; } catch { return 0; } } // transformers.js (v2 and v3 alike) caches each repo under // ///onnx/ (the FileCache keys on the request path, e.g. // Xenova/clip-vit-base-patch32/onnx/…), so weights downloaded by either // version are found. Legacy v1 caches used models----; both are // checked so a model downloaded by any version counts as downloaded. function modelCacheDir(modelId) { const repo = (getModel(resolveModelId(modelId)) || {}).repo || ""; return path.join(app.getPath("userData"), "models", ...repo.split("/")); } function modelDownloaded(modelId) { const config = getModel(resolveModelId(modelId)) || {}; if (config.minimumWeightBytes) { const root = modelCacheDir(modelId); return Object.entries(config.minimumWeightBytes).every( ([relativePath, minimumBytes]) => { try { return ( fs.statSync(path.join(root, relativePath)).size >= minimumBytes ); } catch { return false; } }, ); } const current = countOnnx(modelCacheDir(modelId)); if (current >= 2) return true; // Legacy transformers v1 layout (models----). const repo = (getModel(resolveModelId(modelId)) || {}).repo || ""; const legacy = path.join( app.getPath("userData"), "models", `models--${repo}`, ); return countOnnx(legacy) >= 2; } function indexerEnv() { const ffmpegPath = process.env.FFMPEG_PATH || (process.resourcesPath && fs.existsSync(path.join(process.resourcesPath, "ffmpeg")) ? path.join(process.resourcesPath, "ffmpeg") : undefined); return { ...process.env, INDEXER_MODEL: poolModelId(), TRANSFORMERS_CACHE: path.join(app.getPath("userData"), "models"), // Phase 1.4 detect cache: per-file shot plans live next to other // userData state so re-import / re-analysis skips the ffmpeg scene // pass. MEMORIES_DATA_DIR smoke runs already isolate userData; this // makes the packaged app (which does not set MEMORIES_DATA_DIR) // cache too. SCM_DETECT_CACHE_DIR: path.join(app.getPath("userData"), "detect-cache"), // EP A/B knob (plan 0.3): pass through if set in the main process; // default remains cpu inside the worker when unset. ...(process.env.SCM_EXECUTION_PROVIDERS ? { SCM_EXECUTION_PROVIDERS: process.env.SCM_EXECUTION_PROVIDERS } : {}), ...(ffmpegPath ? { FFMPEG_PATH: ffmpegPath } : {}), }; } // Kill the whole pool (and any in-flight requests). Used when the model // must change: the workers hold the old model in memory, so a model switch // is a pool respawn, never a reload inside a worker. // // Orphan fix: each worker is asked to SIGKILL its ffmpeg children FIRST, // then tracked PIDs are killed by PID (covers workers already dead), and // only then is the Node worker itself killed. The old order (kill worker // only) orphaned ffmpeg to PID 1. function stopIndexerPool() { for (const worker of indexerWorkers) { requestWorkerKillFfmpeg(worker, indexerFfmpegPids, "stopIndexerPool"); if (worker.process) { try { worker.process.kill(); } catch { /* already gone */ } } } indexerWorkers = []; indexerReady = null; indexerInitFailed = null; modelStatus = null; modelFirstRun = false; } // Fork one worker and wire its message/exit handlers. The worker serializes // its own message handling (see indexer.js) — the pool just feeds it requests. function createIndexerWorker({ primary = false } = {}) { const worker = { process: null, primary, ready: null, dead: false, pending: new Map(), }; worker.ready = new Promise((resolve, reject) => { // Kept so the exit handler can settle init when the process dies // before answering (kill-during-fork): without this, an // `await indexerReady` that captured this promise hangs forever — // every other indexer call carries a timeout, this one never did. worker.readyReject = reject; worker.process = utilityProcess.fork( path.join(__dirname, "indexer", "indexer.js"), [], { env: indexerEnv(), serviceName: primary ? "clip-indexer" : `clip-indexer-${indexerWorkers.length}`, }, ); // From the moment a worker forks the model is warming; the primary // drives the renderer's status (a first-run download is its job; the // extras load from the already-populated cache and stay quiet). if (primary) { modelFirstRun = !modelDownloaded(poolModelId()); modelStatus = { type: "model", phase: "loading", progress: null, firstRun: modelFirstRun, modelId: poolModelId(), }; updateTrayState(); for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", modelStatus); } } const onInit = (msg) => { if (msg && msg.type === "init-done") { worker.process.off("message", onInit); if (msg.ok) { // A worker for the CURRENTLY ACTIVE model owns the library's // textMean (centering is per-model). During a migration the // pool runs the TARGET model while the library still names // the old one, so its textMean is NOT adopted here — the // migration fetches it explicitly at the flip. if ( primary && poolModelId() === loadLibrary().modelId && !loadLibrary().pendingModelMigration ) { if (Array.isArray(msg.textMean)) { loadLibrary().textMean = new Float32Array(msg.textMean); void saveLibrary(); } // The ONNX output dim is the truth: align the library's // dim with it (and warn when the persisted dim disagreed // — a stale/wrong-model bin must not rank silently). if (Number(msg.dim) > 0 && loadLibrary().dim !== msg.dim) { if (loadLibrary().filenames.length === 0) { loadLibrary().dim = msg.dim; } else { console.warn( `[memories] indexed dim ${loadLibrary().dim} does not match ` + `the ${loadLibrary().modelId} model output (${msg.dim}); ` + "re-embed the library (switch models or re-import) to fix ranking.", ); } } } resolve(msg); // The active model's weights just finished downloading (first // run): let the picker refresh its downloaded flags via the // same channel the preload path uses. if (primary && modelFirstRun) { for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "model-preload", modelId: poolModelId(), phase: "ready", progress: 100, }); } } } else { const err = new Error(msg.error || "Indexer init failed"); if (primary) indexerInitFailed = err; // A worker that cannot load its model can never serve a // request. Drop it from dispatch and free the half-loaded // model so the pool rebuilds cleanly: the primary's exit // resets indexerReady/status (the next spawn retries the // load — an interrupted first download resumes from its // cache, which is the recovery path), and a failed extra // just disappears from round-robin instead of poisoning // every Nth request with a 180s timeout. worker.dead = true; // Demote before killing so a late exit event can't promote a // stale worker's ready promise onto a newer pool. worker.primary = false; try { worker.process.kill(); } catch { /* already gone */ } reject(err); } } }; worker.process.on("message", (event) => { const msg = event && event.data !== undefined ? event.data : event; // Orphan-fix PID tracking: workers report every ffmpeg spawn/exit // so main can SIGKILL grandchildren by PID after the worker dies. if (msg && msg.type === "ffmpeg-spawn") { trackFfmpegSpawn(indexerFfmpegPids, msg.pid, msg.label); return; } if (msg && msg.type === "ffmpeg-exit") { untrackFfmpegExit(indexerFfmpegPids, msg.pid); return; } if (msg && msg.type === "ffmpeg-killed") { // Worker confirmed its own sweep; PID map entries for exited // children arrive separately as ffmpeg-exit. Best-effort sync: // drop any PIDs the worker says are gone (none named = noop). return; } if (msg && (msg.type === "status" || msg.type === "model")) { if (msg.type === "model" && primary) { modelStatus = { ...msg, firstRun: modelFirstRun, modelId: poolModelId(), }; updateTrayState(); } for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", msg); } } if (msg && msg.type === "enrich-progress") { // Background scene analysis → renderer tray (Phase 4). The // worker omits the request id so this can never collide with // the pending-map resolution below; the queue length rides // along so the tray can show "N more waiting". enrichEventsObserved++; lastEnrichProgress = msg; // Heartbeat clock: every embed tick (including heartbeat-only // ticks that repeat the same done count) proves the worker is // alive inside a long ffmpeg decode. pumpEnrichment uses the // silence duration to tell slow-but-alive from wedged. lastEnrichHeartbeatAt = Date.now(); if (msg.phase === "embed" && Number.isFinite(msg.gi)) { lastEnrichDetail = { gi: msg.gi, t: msg.t, step: msg.step || null, }; } else if (msg.phase !== "embed") { lastEnrichDetail = null; } if (msg.phase === "detect") { enrichPhaseCounts.detect++; if (!(msg.pct >= 0 && msg.pct <= 1)) enrichProgressViolations++; } else if (msg.phase === "embed") { enrichPhaseCounts.embed++; // Heartbeat ticks repeat the previous done count by design // (same segment still decoding) — only validate real // forward progress, never the liveness pings. if (!msg.heartbeat) { if (!(msg.total > 0 && msg.done >= 1 && msg.done <= msg.total)) { enrichProgressViolations++; } } } for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { ...msg, type: "enrich", pending: Math.max(0, enrichQueue.length - 1), }); } } onInit(msg); if (msg && msg.id && worker.pending.has(msg.id)) { const { resolve: r, reject: j, timer } = worker.pending.get(msg.id); worker.pending.delete(msg.id); clearTimeout(timer); if (msg.ok) r(msg); else j(new Error(msg.error || "Indexer request failed")); } }); worker.process.on("exit", (...args) => { // Forensic log: a dead worker with no reason in the log is // undebuggable from CI artifacts. Arg shape varies by Electron // version, so dump everything (code-first vs event-first). // Pair with the app-level child-process-gone listener below, // whose `reason` names oom/crashed/killed explicitly. try { console.log(`[indexer] worker exited args=${JSON.stringify(args)}`); } catch { console.log(`[indexer] worker exited (unserializable args)`); } worker.dead = true; indexerWorkers = indexerWorkers.filter((w) => w !== worker); for (const [, req] of worker.pending) { clearTimeout(req.timer); req.reject(new Error("Indexer exited")); } worker.pending = new Map(); // Init never answered and never will — settle it so a captured // `await indexerReady` fails fast (every awaiter try/catches) // instead of hanging on a dead fork. A later spawn assigns a // fresh promise, so this rejection can't poison the new pool. if (worker.readyReject) { const reject = worker.readyReject; worker.readyReject = null; reject(new Error("Indexer exited during init")); } if (indexerWorkers.length === 0) { // Whole pool gone: let a fresh spawn rebuild everything. indexerReady = null; indexerInitFailed = null; modelStatus = null; modelFirstRun = false; } else if (worker.primary) { // The primary died but extras survive: promote the oldest // remaining worker so indexerReady always points at a live // worker's (already settled) promise — a dead primary's ready // could otherwise hang embed-query/healPhraseBin forever. indexerWorkers[0].primary = true; indexerReady = indexerWorkers[0].ready; } }); worker.process.postMessage({ type: "init" }); }); // Extras' ready promises are never awaited (only the primary's is, via // indexerReady); without a handler, a failed extra's rejection would be // an unhandled rejection, which crashes the main process on modern Node. worker.ready.catch(() => {}); return worker; } function spawnIndexer() { if (indexerWorkers.some((w) => !w.dead)) { // Pool already alive — top it up (e.g. after a worker exited). void growPool(); return; } indexerWorkers = []; indexerInitFailed = null; const primary = createIndexerWorker({ primary: true }); indexerWorkers.push(primary); indexerReady = primary.ready; void growPool(); } // Fork the extra workers once the primary's model is ready (on a first run // that means after the ~150MB download, so the extras never race it). Extra // workers that fail init are simply dropped from dispatch — the pool keeps // serving from whatever is alive. async function growPool() { try { await indexerReady; } catch { return; // primary init failed; extras can't help } while (indexerWorkers.filter((w) => !w.dead).length < INDEXER_POOL_SIZE) { indexerWorkers.push(createIndexerWorker({ primary: false })); } } // Round-robin dispatch across live workers; queries and batch embeds share // the pool (a query landing on a busy worker is an edge case, not the norm). function askIndexer(message) { const live = indexerWorkers.filter((w) => !w.dead && w.process); if (live.length === 0) { return Promise.reject(new Error("Indexer not running")); } const worker = live[pickRoundRobin++ % live.length]; return new Promise((resolve, reject) => { const id = `r${++requestId}`; const timer = setTimeout(() => { worker.pending.delete(id); reject(new Error("Indexer request timed out")); }, 180000); worker.pending.set(id, { resolve, reject, timer }); worker.process.postMessage({ ...message, id }); }); } // Enrichment is pinned to the PRIMARY worker so chunked scene data reuses // the worker's per-path plan cache — a round-robin would bounce chunks // across workers and re-run the ffmpeg scene pass on every chunk. // `timeoutMs` overrides the 180 s default: enrich-video chunks of a 2–3 h // Ultra film (16 ffmpeg seeks + 16 CLIP embeds) are legitimately slower // than a single photo embed, so pumpEnrichment passes ENRICH_CHUNK_TIMEOUT_MS. function askPrimaryIndexer(message, timeoutMs = 180000) { const primary = indexerWorkers.find((w) => !w.dead && w.process && w.primary); if (!primary) return askIndexer(message); return new Promise((resolve, reject) => { const id = `r${++requestId}`; const timer = setTimeout(() => { primary.pending.delete(id); const err = new Error("Indexer request timed out"); err.code = "INDEXER_TIMEOUT"; err.timeoutMs = timeoutMs; reject(err); }, timeoutMs); primary.pending.set(id, { resolve, reject, timer }); primary.process.postMessage({ ...message, id }); }); } // Restart a wedged primary worker: a chunk that timed out on the main side // may still be grinding inside the worker (its queue stays busy), so every // later chunk would queue behind the zombie and time out too — the exact // 16/1024-forever signature. // // Orphan fix (2026-09-28): killing the Node worker does NOT drop ffmpeg — // POSIX reparents it to PID 1 with no timers left. So the worker is asked to // SIGKILL its ffmpeg children first AND main kills tracked PIDs by PID // (covers the worker-already-dead case) before the worker itself is killed. // The exit handler promotes an extra (or spawnIndexer rebuilds a fresh // primary when the pool was size 1), and the plan cache rebuilds on the next // chunk's scene pass. The job's `off` is preserved by the caller, so the // retry resumes where it stalled. function restartPrimaryIndexer(reason) { try { const primary = indexerWorkers.find( (w) => w.process && w.primary && !w.dead, ); if (primary) { console.warn(`[memories] restarting wedged indexer worker (${reason})`); requestWorkerKillFfmpeg(primary, indexerFfmpegPids, `restart:${reason}`); primary.dead = true; primary.primary = false; try { primary.process.kill(); } catch { /* already gone */ } } else { // No live primary (already dead): tracked ffmpeg PIDs may still // be grinding — sweep them so a dead worker can't leave orphans. killTrackedFfmpegPids(indexerFfmpegPids, `restart-no-primary:${reason}`); } } finally { try { spawnIndexer(); } catch { /* spawn retries on next pump */ } } } // --------------------------------------------------------------------------- // Model switching: re-embed the whole library in another model. // // The worker pool is respawned with INDEXER_MODEL pinned to the target; each // row is re-embedded from its ORIGINAL source file (falling back to the // app-managed library copy) through the same embed-photo path imports use, // so GIF middle-frame averaging and phrase centering match exactly. The new // pair is written to the target model's own bin files as the pump advances // (checkpointed, but the index still names the OLD model — a crash mid-run // leaves a consistent old-model library; the partial new bins are simply // overwritten on retry). At the end the library flips: modelId, dim, and // textMean come from the worker's own outputs, saveLibrary persists the new // index + bins, and the model is calibrated against its real score band. // The old model's bins stay on disk (memory-embeddings-.bin), so // switching back is a re-embed, never a loss. // --------------------------------------------------------------------------- // --------------------------------------------------------------------------- // Model preload ("Download all models"): download a model's weights WITHOUT // switching the library to it. The user can fetch every model ahead of time // so switching later is instant (no download wait). Each preload forks a // THROWAWAY indexer worker pinned to the target model (INDEXER_MODEL) whose // normal init downloads + warms that model; its download progress relays to // the renderer as { type: "model-preload", modelId, phase, progress }, and // the worker is killed once init lands. The library is untouched — the // weights land in the shared cache, nothing else changes. // --------------------------------------------------------------------------- let modelPreloads = new Map(); // modelId → { process, progress } // Fork one worker, feed it INDEXER_MODEL=modelId so its init downloads the // target's weights, relay progress, then kill it on init-done. Resolves // { modelId, downloaded } when the model is on disk (or was already). function preloadModelWeights(modelId) { const config = getModel(modelId); if (!config) return Promise.reject(new Error(`Unknown model: ${modelId}`)); if (modelDownloaded(modelId)) { // Already cached — nothing to do (the UI calls this for every model // in "download all"; skipped ones resolve immediately). return Promise.resolve({ modelId, downloaded: true, skipped: true }); } // The pool owns the ACTIVE model's download on first run, and a migration // pool owns its TARGET's download. Two processes writing the same cache // files would race (FileCache.put is a plain write in transformers.js v2 // and v3 alike — no atomic temp+rename), so never preload those. This // guard lives HERE (not just in preloadAllModels) so the single-model IPC // can't bypass it. if (modelId === poolModelId()) { return Promise.resolve({ modelId, skipped: true, note: "active model — the engine is downloading it", }); } if (migrationState && migrationState.targetId === modelId) { return Promise.resolve({ modelId, skipped: true, note: "already downloading — the model switch owns it", }); } if (modelPreloads.has(modelId)) return modelPreloads.get(modelId).promise; const entry = { process: null, progress: null, promise: null, }; entry.promise = new Promise((resolve, reject) => { entry.process = utilityProcess.fork( path.join(__dirname, "indexer", "indexer.js"), [], { env: { ...indexerEnv(), INDEXER_MODEL: modelId }, serviceName: `clip-preload-${modelId}`, }, ); entry.process.on("message", (event) => { const msg = event && event.data !== undefined ? event.data : event; if (!msg) return; if (msg.type === "model") { entry.progress = msg.progress ?? null; for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "model-preload", modelId, phase: msg.phase, progress: entry.progress, }); } // The worker posts phase "error" BEFORE the ok:false reply that // carries the real message; don't reject on the bare event or // the detailed error is lost. return; } if (msg.ok === false) { // Download/load failed — drop the worker and report the real error. entry.process.kill(); modelPreloads.delete(modelId); reject(new Error(msg.error || `Preload of ${modelId} failed`)); return; } if (msg.type === "init-done" && msg.ok) { entry.process.kill(); modelPreloads.delete(modelId); resolve({ modelId, downloaded: modelDownloaded(modelId), skipped: false, }); } }); entry.process.on("exit", () => { // Only fatal if the promise is still pending (the normal kill // after init-done already resolved it). if (modelPreloads.has(modelId)) { modelPreloads.delete(modelId); reject(new Error(`Preload worker for ${modelId} exited unexpectedly`)); } }); entry.process.postMessage({ type: "init" }); }); modelPreloads.set(modelId, entry); return entry.promise; } // Fetch every model the app knows about that isn't already on disk. Runs // sequentially so a slow link doesn't saturate bandwidth across models, and // each already-downloaded model resolves instantly. The ACTIVE model is // skipped when its weights are missing — the primary pool owns that download // on first run, and a parallel writer to the same cache files would race it. // Resolves to an array of per-model results ({ modelId, downloaded, skipped? }). async function preloadAllModels() { // preloadModelWeights itself skips the active/migration-owned models, so // this loop is a plain sequential fan-out over the registry. const results = []; for (const modelId of Object.keys(MODELS)) { try { results.push(await preloadModelWeights(modelId)); } catch (err) { results.push({ modelId, error: err.message }); } } return results; } // A migration is in flight; further switches are refused until it settles. // A background delta tail (kind "fill") is the exception: a new switch // cancels it (generation bump — its checkpoints stay on disk and the next // delta resumes them) and proceeds. let migrationState = null; let migrationFillGeneration = 0; // How often (rows) a running migration writes the target's bins to disk. const MIGRATE_CHECKPOINT = 25; // The file a row's embedding was computed from at import. The recorded // source may have moved away; the app-managed copy is the fallback. function rowSourcePath(i) { const l = loadLibrary(); const source = l.sources && l.sources[i]; if (source && fs.existsSync(source)) return source; return path.join(PHOTOS_DIR, l.filenames[i]); } // Re-embed every row of the library into `targetId`'s model. Resolves to // { modelId, reembedded, failures } or throws on a blocking error. Progress // is broadcast as { type: "migrate", phase, done, total, modelId }. // Bounded-concurrency pump, sized like the import pipeline (pool×2). const MIGRATE_CONCURRENCY = Math.max(2, INDEXER_POOL_SIZE * 2); async function reembedToModel(targetId) { const target = getModel(targetId); if (!target) throw new Error(`Unknown model: ${targetId}`); // An in-flight import embeds rows through the pool, which a migration // pins to the TARGET model — those rows would land as target-model // vectors in a library that still names the old one. Refuse to start // until the batch settles. if (importInFlight) { throw new Error( "An import is in progress — switch models after it finishes.", ); } const l = loadLibrary(); const pendingRetirement = l.pendingModelMigration; if (pendingRetirement && pendingRetirement.targetId !== targetId) { throw new Error( `The retired ${pendingRetirement.fromId} index must finish migrating to ${pendingRetirement.targetId} first.`, ); } if (l.modelId === targetId && !pendingRetirement) return { modelId: targetId, reembedded: 0, failures: [] }; if (migrationState) { if (migrationState.kind === "fill") { migrationFillGeneration++; migrationState = null; } else { throw new Error("A model migration is already running"); } } if (l.filenames.length === 0) { // No rows to re-embed — the switch is just a metadata flip. l.modelId = targetId; l.dim = getModel(targetId).dim || l.dim; l.textMean = null; l.pendingModelMigration = null; await saveLibrary(); console.log(`[memories] switched empty library to ${targetId}`); for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "library-updated" }); } return { modelId: targetId, reembedded: 0, failures: [] }; } const total = l.filenames.length; migrationState = { targetId, done: 0, total }; const broadcast = (phase, extra = {}) => { for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "migrate", phase, modelId: targetId, done: migrationState.done, total, ...extra, }); } }; broadcast("embedding", { done: 0 }); // Respawn the pool for the target model and wait for its model (a first // use downloads the weights here — the status LED shows that progress). stopIndexerPool(); workerModelId = targetId; spawnIndexer(); let initMsg; try { initMsg = await indexerReady; } catch (err) { // The respawned pool runs (or failed to run) the TARGET model. Put it // down so the next import/query re-forks for the library's actual // model — a live target-model pool here would embed later rows in the // wrong model. stopIndexerPool(); workerModelId = null; migrationState = null; throw new Error(`Failed to load model ${targetId}: ${err.message}`, { cause: err, }); } const dim = Number(initMsg.dim) > 0 ? initMsg.dim : l.dim; // Fast path — the target model's bins already exist and match this // library's row count and dim (a previous switch, or a pre-v4 legacy // pair). Flip to them instead of re-embedding every row, so switching is // truly instant (the picker advertises exactly this). The pool is already // running the target model (respawned above), so textMean and dim come // from its live output; the target's own phrase bin is used when present // (otherwise the renderer degrades to image-only ranking, same as the // heal path). The old model's pair is untouched — switching back and // forth never re-does work or loses data. const existing = binInfo(targetId); if ( existing && existing.rows === l.filenames.length && existing.dim === dim ) { const embeddings = readBin(embedFileFor(targetId), l.filenames.length, dim); const phrases = readBin(phraseFileFor(targetId), l.filenames.length, dim); // The phrase bin must ALSO match: the slow migration always writes both // pairs, so a target with matching embed rows but a missing/mismatched // phrase bin is a partial/corrupt state — flipping to it would silently // degrade ranking to image-only. Fall through to the re-embed (which // rebuilds and heals) instead. if ( embeddings.length === l.filenames.length && phrases.length === l.filenames.length ) { let textMean = null; try { const tm = await askIndexer({ type: "get-text-mean" }); if (tm.ok && Array.isArray(tm.textMean)) { textMean = new Float32Array(tm.textMean); } } catch (err) { console.warn( `[memories] textMean fetch failed (${err.message}); uncentered queries`, ); } l.embeddings = embeddings; l.phrases = phrases; l.dim = dim; l.textMean = textMean; l.modelId = targetId; l.pendingModelMigration = null; workerModelId = null; await saveLibrary(); console.log( `[memories] switched ${l.filenames.length} rows to ${targetId} from existing bins ` + "(instant, no re-embed)", ); migrationState = null; for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "library-updated" }); } try { await calibrateThresholds(targetId); } catch (err) { console.warn(`[memories] threshold calibration failed: ${err.message}`); } try { backfillEnrichment(); backfillTranscription(); } catch (err) { console.warn( `[memories] post-migration scene backfill skipped: ${err.message}`, ); } return { modelId: targetId, reembedded: 0, failures: [], instant: true }; } } // Delta path — the target's bins exist with a row-identity snapshot from // a previous switch (same pipeline version): flip to the reused rows NOW // and embed only the missing tail in the background, so a switch after N // new imports costs N embeds instead of the whole library. Falls through // to the foreground slow path when there is nothing reusable (first // switch, pipeline change, or unreadable bins). const delta = planDeltaReuse(targetId, dim, [...l.filenames]); if (delta.usable && delta.reused > 0) { return flipWithBackgroundFill(targetId, dim, delta, broadcast, total); } // Snapshot the row set: if an import lands mid-migration the flip below // aborts rather than misaligning (the old model's library stays intact). // The snapshot ALSO pairs every checkpoint's bins with the row identity // they align to (delta resumes from them), and the flip verifies identity // — not just length — so a same-length delete+import can't silently // misalign vectors. const rowCount = l.filenames.length; const rowFilenames = [...l.filenames]; // Dense from the start: checkpointed writeBins mid-migration must never // see sparse holes (.map skips holes, so Buffer.concat would throw on // the unwritten tail). Unwritten rows read as zero vectors until their // reply lands — same contract as failed rows below. const newEmbeddings = new Array(rowCount).fill(null); const newPhrases = new Array(rowCount).fill(null); const failures = []; let next = 0; let inFlight = 0; // Checkpoints below carry partial target-model vectors: stamp them // incomplete so a crash resumes the tail (delta) instead of serving // zeros as final. Cleared before the final save at the flip. setEmbedIncomplete(targetId, true); await new Promise((resolveBatch) => { const pump = () => { while (inFlight < MIGRATE_CONCURRENCY && next < rowCount) { const i = next++; inFlight++; (async () => { try { const filename = l.filenames[i]; // Videos go through the video path (ffmpeg frame // extraction); sharp cannot decode a video file, so // embedding one as a photo always fails and zeroes // the row. Photos use the same embed path imports do // (GIF middle-frame averaging included). const reply = await askIndexer({ type: isVideo(filename) ? "embed-video" : "embed-photo", path: rowSourcePath(i), filename, }); // Failed rows persist as explicit zero vectors (not // undefined) so every row keeps a real, aligned array // and encodeBin has nothing to guess about. if (reply.vec && reply.vec.length === dim) { newEmbeddings[i] = new Float32Array(reply.vec); } else { failures.push(`${filename}: empty or wrong-dim vector`); newEmbeddings[i] = new Float32Array(dim); } if (reply.phrase && reply.phrase.length === dim) { newPhrases[i] = new Float32Array(reply.phrase); } else { if (reply.phrase) { failures.push(`${filename}: wrong-dim phrase`); } newPhrases[i] = new Float32Array(dim); } } catch (err) { failures.push(`${l.filenames[i]}: ${err.message}`); } finally { inFlight--; migrationState.done++; if (migrationState.done % MIGRATE_CHECKPOINT === 0) { void writeBins(targetId, newEmbeddings, newPhrases, dim, dim, { filenames: rowFilenames, incomplete: true, }); } broadcast("embedding"); pump(); if (migrationState.done >= rowCount && inFlight === 0) { resolveBatch(); } } })(); } }; pump(); }); if (failures.length === rowCount) { // Same as the model-load failure: never leave a pool forked for the // target model behind a library that still names the old one. stopIndexerPool(); workerModelId = null; migrationState = null; throw new Error( `Migration to ${targetId} failed for every row: ${failures[0]}`, ); } // Flip. If the library changed while we worked, refuse the flip — the // new rows have no vectors in the target model yet (the old model stays). // Identity, not just length: a delete+import of equal length would pass a // length check with misaligned vectors. if ( l.filenames.length !== rowCount || rowFilenames.some((f, i) => f !== l.filenames[i]) ) { // Same as the model-load failure: never leave a pool forked for the // target model behind a library that still names the old one. // (The checkpoints already on disk stay resumable via delta.) stopIndexerPool(); workerModelId = null; migrationState = null; throw new Error( `Library changed during migration (${rowCount} → ${l.filenames.length}); ` + "run the model switch again", ); } // The worker computed the target model's shared text direction at load. let textMean = null; try { const tm = await askIndexer({ type: "get-text-mean" }); if (tm.ok && Array.isArray(tm.textMean)) { textMean = new Float32Array(tm.textMean); } } catch (err) { console.warn( `[memories] textMean fetch failed (${err.message}); uncentered queries`, ); } l.embeddings = newEmbeddings; l.phrases = newPhrases; l.dim = dim; l.textMean = textMean; l.modelId = targetId; l.pendingModelMigration = null; workerModelId = null; // All rows landed: the final save stamps the meta complete. setEmbedIncomplete(targetId, false); await saveLibrary(); console.log( `[memories] migrated ${rowCount - failures.length}/${rowCount} rows to ${targetId} ` + `(dim ${dim}, failures ${failures.length})`, ); // The final "indexing" event reads migrationState.done (== total), so it // must fire BEFORE the state is cleared. broadcast("indexing"); migrationState = null; for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "library-updated" }); } // Calibrate the target's thresholds against its real score band (no-op // when the library is too small to measure). try { await calibrateThresholds(targetId); } catch (err) { console.warn(`[memories] threshold calibration failed: ${err.message}`); } // Phase 5: segment sidecars are per-model, so the target's is empty // after a migration — re-run the launch backfill so videos regain scene // coverage without waiting for the next app launch. Enrichment is gated // on !migrationState, so it queues safely behind nothing here. try { backfillEnrichment(); backfillTranscription(); } catch (err) { console.warn( `[memories] post-migration scene backfill skipped: ${err.message}`, ); } return { modelId: targetId, reembedded: rowCount - failures.length, failures, }; } // Delta switch (Option A): flip the library to the target model using cached // rows NOW, then embed only the missing tail in the background. The switch // promise resolves at the flip — search works immediately across reused rows // (missing rows read as zero vectors until their tail lands, the same // contract as failed rows) — while migrate-phase "embedding" events keep the // tray informed until the tail completes with "indexing" + library-updated. // Resolves to { modelId, reembedded: 0, reused, pending, failures: [], // delta: true }; callers that need the settled state await // migrationSettled(). async function flipWithBackgroundFill(targetId, dim, delta, broadcast, total) { const l = loadLibrary(); const missing = [...delta.missing]; // Dense flip: reused rows keep their cached vectors; missing rows are // explicit zero vectors until the background tail lands. l.embeddings = delta.reusedEmbeddings.map((v) => v || new Float32Array(dim)); l.phrases = delta.reusedPhrases.map((v) => v || new Float32Array(dim)); let textMean = null; try { const tm = await askIndexer({ type: "get-text-mean" }); if (tm.ok && Array.isArray(tm.textMean)) { textMean = new Float32Array(tm.textMean); } } catch (err) { console.warn( `[memories] textMean fetch failed (${err.message}); uncentered queries`, ); } l.dim = dim; l.textMean = textMean; l.modelId = targetId; l.pendingModelMigration = null; workerModelId = null; // Stamp incomplete BEFORE the save so the on-disk meta says resumable // even if the app quits mid-tail. setEmbedIncomplete(targetId, true); await saveLibrary(); console.log( `[memories] switched ${l.filenames.length} rows to ${targetId} ` + `(delta: ${delta.reused} reused, ${missing.length} filling in background)`, ); const gen = ++migrationFillGeneration; migrationState = { targetId, done: delta.reused, total, kind: "fill", gen, }; broadcast("embedding"); for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "library-updated" }); } // Detached: the switch is done at the flip; the tail reports through the // same migrate events and settles migrationState when it lands. void runDeltaFill(targetId, dim, missing, gen, broadcast); return { modelId: targetId, reembedded: 0, reused: delta.reused, pending: missing.length, failures: [], delta: true, }; } // Background tail of a delta switch: embed the missing filenames through the // target-model pool (already running — the switch respawned it), resolving // each row's index LIVE by filename so imports/deletes/reorders mid-fill // can't misalign vectors. Rows that gained real vectors meanwhile (an import // embedding through the active pool) are skipped, never clobbered. async function runDeltaFill(targetId, dim, missing, gen, broadcast) { const failures = []; let next = 0; let inFlight = 0; let done = 0; const alive = () => gen === migrationFillGeneration; await new Promise((resolveFill) => { const pump = () => { if (!alive()) return resolveFill(); if (missing.length === 0) return resolveFill(); while (inFlight < MIGRATE_CONCURRENCY && next < missing.length) { const filename = missing[next++]; inFlight++; (async () => { try { if (!alive()) return; const cur = loadLibrary(); const idx = cur.filenames.indexOf(filename); if (idx === -1) return; // deleted mid-fill const existing = cur.embeddings[idx]; // An import embedded this row through the active // (target-model) pool while we queued — keep it. if (existing && !isZeroVector(existing)) return; const reply = await askIndexer({ type: isVideo(filename) ? "embed-video" : "embed-photo", path: rowSourcePathAt(cur, idx), filename, }); if (!alive()) return; const after = loadLibrary(); const at = after.filenames.indexOf(filename); if (at === -1) return; // deleted while embedding if (reply.vec && reply.vec.length === dim) { after.embeddings[at] = new Float32Array(reply.vec); } else { failures.push(`${filename}: empty or wrong-dim vector`); after.embeddings[at] = new Float32Array(dim); } if (reply.phrase && reply.phrase.length === dim) { after.phrases[at] = new Float32Array(reply.phrase); } else { if (reply.phrase) { failures.push(`${filename}: wrong-dim phrase`); } after.phrases[at] = new Float32Array(dim); } } catch (err) { if (alive()) failures.push(`${filename}: ${err.message}`); } finally { inFlight--; } if (!alive()) { if (inFlight === 0) resolveFill(); } else { done++; if (migrationState) migrationState.done++; if (done % MIGRATE_CHECKPOINT === 0) { const snap = loadLibrary(); void writeBins( targetId, snap.embeddings, snap.phrases, dim, dim, { filenames: [...snap.filenames], incomplete: true, }, ); } broadcast("embedding"); pump(); if (done >= missing.length && inFlight === 0) resolveFill(); } })(); } }; pump(); }); // Superseded by a newer switch — it owns the state now; our checkpoints // stay on disk for the next delta to resume. if (!alive()) return; if (loadLibrary().modelId !== targetId) return; const filled = missing.length - failures.length; setEmbedIncomplete(targetId, false); await saveLibrary(); console.log( `[memories] delta fill for ${targetId} complete: ${filled}/${missing.length} rows ` + `(failures ${failures.length})`, ); // The final "indexing" event reads migrationState.done, so it must fire // BEFORE the state is cleared (same contract as the slow path). broadcast("indexing"); migrationState = null; for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "library-updated" }); } // Calibrate against the now-complete vectors (skipped at the flip: zeros // would have skewed the score band). try { await calibrateThresholds(targetId); } catch (err) { console.warn(`[memories] threshold calibration failed: ${err.message}`); } // Scene/transcript sidecars are per-model: re-queue coverage for the // target now that its vectors are final (same as the slow path). try { backfillEnrichment(); backfillTranscription(); } catch (err) { console.warn( `[memories] post-migration scene backfill skipped: ${err.message}`, ); } } // Scripts/tests: resolve once no migration or delta fill is in flight (the // delta switch promise resolves at the flip; this waits for the tail). function migrationSettled() { if (!migrationState) return Promise.resolve(); return new Promise((resolve) => { const t = setInterval(() => { if (!migrationState) { clearInterval(t); resolve(); } }, 250); }); } // rowSourcePath for a live library snapshot + resolved index (the delta fill // re-resolves indices by filename, so it can't use the index-based helper). function rowSourcePathAt(lib, idx) { const source = lib.sources && lib.sources[idx]; if (source && fs.existsSync(source)) return source; return path.join(PHOTOS_DIR, lib.filenames[idx]); } // Launch resume: a previous run quit (or crashed) mid-fill, leaving // incomplete:true in the active model's meta. Resume the tail instead of // serving zeros as final. Called after the pool is warm, beside the scene // backfills. async function maybeResumeDeltaFill() { if (migrationState) return; let l; try { l = loadLibrary(); } catch { return; } if (l.pendingModelMigration) return; if (l.filenames.length === 0 || !(l.dim > 0)) return; let meta; try { meta = readEmbedMeta(l.modelId); } catch { return; } if (!meta || meta.incomplete !== true) return; const delta = planDeltaReuse(l.modelId, l.dim, [...l.filenames]); if (!delta.usable || delta.missing.length === 0) { // Nothing to resume. When the vectors are complete (missing==0) the // stale flag is wrong — clear it and persist complete. When unusable // (no meta match), leave the flag: the next switch falls back to the // foreground path, which repairs everything. if (delta.usable) { setEmbedIncomplete(l.modelId, false); try { await saveLibrary(); } catch { /* best-effort flag repair */ } } return; } if (delta.reused === 0) return; // degenerate: leave it to a real switch const gen = ++migrationFillGeneration; setEmbedIncomplete(l.modelId, true); migrationState = { targetId: l.modelId, done: delta.reused, total: l.filenames.length, kind: "fill", gen, }; console.log( `[memories] resuming interrupted fill for ${l.modelId}: ` + `${delta.reused} rows ok, ${delta.missing.length} to embed`, ); const resumeModelId = l.modelId; const resumeDim = l.dim; const broadcast = (phase, extra = {}) => { for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "migrate", phase, modelId: resumeModelId, done: migrationState ? migrationState.done : l.filenames.length, total: l.filenames.length, ...extra, }); } }; void runDeltaFill( resumeModelId, resumeDim, [...delta.missing], gen, broadcast, ); } // Retired/unknown model ids are detected by loadLibrary without changing the // persisted index. Once the replacement worker is ready, run the normal // checkpointed migration. A failed download or re-embed leaves that original // disk state untouched, so the next launch can retry safely. async function migratePendingModel() { const pending = loadLibrary().pendingModelMigration; if (!pending) return null; console.log( `[memories] migrating retired ${pending.fromId} library to ${pending.targetId}`, ); return reembedToModel(pending.targetId); } // The manifest the renderer fetches from /memories-models.json: registry // info + per-model state, enough to render the model picker and rank with // the active model's thresholds. function modelsManifest() { const l = loadLibrary(); return { version: 1, activeModelId: l.modelId, models: Object.entries(MODELS).map(([id, config]) => { const bin = binInfo(id); return { id, label: config.label, description: config.description, dim: config.dim, inputSize: config.inputSize, license: config.license, speed: config.speed, quality: config.quality, thresholds: thresholdsFor(id), binCount: bin ? bin.rows : 0, calibrated: Boolean(thresholdsFor(id)), // True when the model's weights are on disk (drives the picker's // per-model Download chip + the "download all" affordance). downloaded: modelDownloaded(id), }; }), }; } // --------------------------------------------------------------------------- // Import // --------------------------------------------------------------------------- // Recursively collect every supported media file under `dir`. Symlinked // directories are skipped to avoid cycles; unreadable entries are ignored. function scanDirForMedia(dir, out = []) { let entries; try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return out; } for (const entry of entries) { const full = path.join(dir, entry.name); if (entry.isDirectory()) { // Never recurse into symlinked directories (cycle risk); real // subdirectories are scanned recursively. if (!entry.isSymbolicLink()) scanDirForMedia(full, out); } else if (entry.isSymbolicLink()) { // Follow symlinks that resolve to media files (common in iCloud / // cloud-sync folders); statSync resolves the target, unlike the // lstat-based Dirent flags. try { if ( fs.statSync(full).isFile() && IMPORT_EXTENSIONS.has(path.extname(entry.name).toLowerCase()) ) { out.push(full); } } catch { /* dangling symlink — ignore */ } } else if ( entry.isFile() && IMPORT_EXTENSIONS.has(path.extname(entry.name).toLowerCase()) ) { out.push(full); } } return out; } // Collapse a list of picked/dropped paths into importable files: directories // expand to their media contents (recursive); plain files pass through so the // per-file importer can still report unsupported ones. function expandMediaPaths(paths) { const out = []; for (const p of paths) { let stat; try { stat = fs.statSync(p); } catch { out.push(p); // let importPaths report "File not found" continue; } if (stat.isDirectory()) scanDirForMedia(p, out); else out.push(p); } return out; } function uniqueName(base, taken) { const ext = path.extname(base); const stem = path.basename(base, ext); let candidate = base; let n = 2; while (taken.has(candidate)) { candidate = `${stem} (${n})${ext}`; n++; } taken.add(candidate); return candidate; } // SHA-256 of a file's bytes (hex). Content-based duplicate detection: a file // renamed inside a watched folder is a NEW source path (path-dedupe can't see // it) but its bytes are already in the library — the hash proves it, so the // sync skips the copy+embed instead of growing the library with duplicates. // Stall-bounded like copyFileWithWatchdog: a source on a hung network volume // must not freeze the pipeline during the read either. function hashFile(file, { stallMs = 30_000 } = {}) { return new Promise((resolve, reject) => { const h = crypto.createHash("sha256"); let lastData = Date.now(); const s = fs.createReadStream(file); const timer = setInterval(() => { if (Date.now() - lastData >= stallMs) { clearInterval(timer); s.destroy( new Error(`Read stalled (no progress for ${stallMs / 1000}s)`), ); } }, 2_000); s.on("error", (err) => { clearInterval(timer); reject(err); }); s.on("data", (c) => { lastData = Date.now(); h.update(c); }); s.on("end", () => { clearInterval(timer); resolve(h.digest("hex")); }); }); } // copyFile has no abort and no progress signal, so it is the ONE unbounded // await in the import critical path: a stalled network volume (SMB/iCloud) or // a locked file could otherwise freeze the whole pipeline forever (the embed // phase is bounded by askIndexer's timeout, the copy was not). Watch the // destination grow instead of using a wall-clock timeout — a legitimately // large copy on a slow disk must never be killed just for being slow, only // for making NO progress. On a stall the copy keeps running in the background // (its late rejection is swallowed); the error outcome unlinks the partial // destination in applyOutcome. function copyFileWithWatchdog( src, dst, { stallMs = 30_000, tickMs = 2_000 } = {}, ) { return new Promise((resolve, reject) => { let settled = false; let lastSize = -1; let lastGrow = Date.now(); const timer = setInterval(() => { if (settled) return; let size; try { size = fs.statSync(dst).size; } catch { size = -1; // copy hasn't created the destination yet } if (size !== lastSize) { // Any change counts as progress — including a shrink, which is // copyFile truncating a stale destination before rewriting it. lastSize = size; lastGrow = Date.now(); } else if (Date.now() - lastGrow >= stallMs) { clearInterval(timer); settled = true; reject(new Error(`Copy stalled (no progress for ${stallMs / 1000}s)`)); } }, tickMs); fs.promises .copyFile(src, dst) .then((r) => { if (settled) return; clearInterval(timer); settled = true; resolve(r); }) .catch((err) => { if (settled) return; // watchdog already reported the stall clearInterval(timer); settled = true; reject(err); }); }); } // C-01 Wave 2 (S2/S3): import-failure cache live in main-lib/failed-imports.js and main-lib/category-overrides.js (required at top). // C-01 Wave 2 (S2/S3): category-overrides store live in main-lib/failed-imports.js and main-lib/category-overrides.js (required at top). ipcMain.handle( "memories:set-category-override", (_event, filename, category) => { if (typeof filename !== "string" || !filename) { return { ok: false, error: "Invalid filename" }; } if (category !== null && !CATEGORY_OVERRIDE_VALUES.has(category)) { return { ok: false, error: "Invalid category" }; } const l = loadLibrary(); const idx = l.filenames.indexOf(filename); if (idx === -1) return { ok: false, error: "Unknown file" }; const hash = l.hashes && l.hashes[idx]; if (!hash) { return { ok: false, error: "File has no content hash yet — try again shortly", }; } const overrides = loadCategoryOverrides(); if (category === null) delete overrides[hash]; else overrides[hash] = category; saveCategoryOverrides(); // The grid re-derives categories from the index on this broadcast. for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "library-updated" }); } return { ok: true }; }, ); // C-01 Wave 2 (S2/S3): isSystemIndexerError live in main-lib/failed-imports.js and main-lib/category-overrides.js (required at top). // C-01 Wave 2 (S2/S3): rememberImportFailure live in main-lib/failed-imports.js and main-lib/category-overrides.js (required at top). // C-01 Wave 2 (S2/S3): isKnownImportFailure live in main-lib/failed-imports.js and main-lib/category-overrides.js (required at top). // True while an import batch is running. The migration refuses to start // while a batch is in flight (its rows would embed in the wrong model), and // a second batch is refused too (uniqueName dedupe and the ordered flush // assume one writer at a time). let importInFlight = false; // `fromWatch` marks a watched-folder sync (launch / poll / live fs event). // The import-failure cache is only consulted for these: a manual re-import is // the user explicitly asking for a fresh attempt, so it always runs. async function importPaths(rawPaths, { fromWatch = false } = {}) { // A model migration re-embeds the whole library and pins the worker // pool to the target model. Rows imported mid-migration would be // embedded in the WRONG model — they'd land as target-model vectors in // a library that still names the old one (or misalign it when the flip // aborts). Refuse until it settles. if (migrationState) { const first = Array.isArray(rawPaths) && rawPaths[0] ? String(rawPaths[0]) : "import"; return { added: [], skipped: [], errors: [ { file: path.basename(first), error: "A model switch is in progress — import again once it finishes.", }, ], watched: [], }; } if (importInFlight) { const first = Array.isArray(rawPaths) && rawPaths[0] ? String(rawPaths[0]) : "import"; return { added: [], skipped: [], errors: [ { file: path.basename(first), error: "An import is already running — try again when it finishes.", }, ], watched: [], }; } importInFlight = true; updateTrayState(); let result; try { result = await importPathsInner(rawPaths, { fromWatch }); } finally { importInFlight = false; updateTrayState(); } // Hidden-window completion note (watched-folder syncs with nothing new // stay silent — only batches that actually added photos ping). if (result && Array.isArray(result.added) && result.added.length > 0) { const n = result.added.length; const errs = Array.isArray(result.errors) ? result.errors.length : 0; maybeNotify( "Import complete", `${n} photo${n === 1 ? "" : "s"} added to your library` + (errs > 0 ? ` (${errs} file${errs === 1 ? "" : "s"} failed)` : "") + ".", ); } return result; } async function importPathsInner(rawPaths, { fromWatch = false } = {}) { spawnIndexer(); // Any path may be a directory (folder picker / folder drop): expand it to // the media files inside so the batch counter below counts real files. const paths = expandMediaPaths(rawPaths); // Auto-watch folders: any DIRECTORY in the batch joins the persisted // watch list (the expand above already flattened it to files — rawPaths // still name the folders). Individual files are imported once, never // watched. The newly-watched names ride the result so the renderer can // say "now watching …" without a separate round trip. const watchedNow = []; for (const raw of rawPaths) { try { if (fs.statSync(raw).isDirectory() && addWatchedFolder(raw)) { watchedNow.push(path.basename(raw)); } } catch { /* unreadable/missing — the import itself reports it */ } } // Model warm-up now overlaps the copy phase: embed requests sent before // the model is ready queue in the workers (their message listeners attach // at module load, and every request awaits ensureModel()). Only a FAILED // model load short-circuits the batch — spawnIndexer sets the module-level // indexerInitFailed flag, which each file checks synchronously. const l = loadLibrary(); const taken = new Set(l.filenames); // Source paths already recorded in the library — re-importing the same // folder (or re-picking/dragging the same files) must skip these instead // of importing renamed copies. Skipped paths are keyed by path (then // flattened to basenames) so one file dragged twice can't inflate the // skipped count. const importedSources = new Set(l.sources); // Content hashes of rows ALREADY in the library (from past sessions) — a // renamed file (new source path) whose bytes match one of these is a // duplicate and is skipped instead of copied+embedded again. The set is // fixed for the batch on purpose: two same-content files imported // together are two entries in the user's folder and both import, exactly // as before — dedupe targets renames BETWEEN syncs, not within a batch. const libraryHashes = new Set(l.hashes.filter(Boolean)); const skippedPaths = new Set(); const result = { added: [], skipped: [], errors: [], watched: watchedNow }; // A path listed twice in one drop must import once. The sequential loop // used to catch this at iteration time; the concurrent pipeline checks // the source set at dispatch, so identical paths are collapsed here — // the first occurrence wins and the rest are reported skipped. const seenPaths = new Set(); let uniquePaths = []; for (const p of paths) { if (seenPaths.has(p)) { skippedPaths.add(p); continue; } seenPaths.add(p); uniquePaths.push(p); } // A watched-folder scan has no chronological ordering guarantee. The // ordered flush below persists this sequence, and the All/File UI reads // the tail first, so establish source mtime order before dispatching work. uniquePaths = sortPathsByMtime(uniquePaths, fs.statSync); // Video posters are generated off the import critical path (see below); // all jobs must settle before the library-updated broadcast so the // renderer never sees a tile whose thumbnail is still missing. const posterJobs = []; // Disk persistence is coalesced: a per-file save rewrites the whole // index trio (O(n²) total I/O over a batch) while nothing reads the disk // mid-batch — the renderer only reloads on library-updated (batch end). // Poster extractions each spawn ffmpeg; cap how many run at once so a // video-heavy batch cannot saturate the machine while it is off the // critical path. Failures are swallowed (the video stays searchable; // its tile falls back to the "missing media" placeholder). const POSTER_CONCURRENCY = 2; const posterSlots = Array.from({ length: POSTER_CONCURRENCY }, () => Promise.resolve(), ); let posterSlotIndex = 0; const pushPoster = (filename, make) => { const slot = posterSlotIndex % POSTER_CONCURRENCY; posterSlotIndex++; const run = posterSlots[slot].then(make).catch((err) => { console.warn(`[memories] poster failed for ${filename}: ${err.message}`); }); posterSlots[slot] = run; posterJobs.push(run); }; // Per-file AI-processing status: main.js owns the batch counters so the // renderer can draw a real "N/total + phase" progress bar (copy → CLIP // embed → video frames/poster). `done` = files completed IN ORDER before // the current one, so the bar shows done+1/total while it's in flight. const total = uniquePaths.length; // Scale the checkpoint with batch size: a large import (3000 rows) // checkpoints every 25 rows = 120 saves, each encoding the full // growing bin pair (~18 MB at row 3000). Target ~20 checkpoints // instead to cut encoding+I/O by ~6× while still protecting // against mid-batch crashes. const CHECKPOINT_INTERVAL = Math.max(25, Math.ceil(total / 20)); let sinceCheckpoint = 0; let done = 0; const sendImport = (phase, filename) => { for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "import", phase, filename, done, total, }); } }; // The batch embeds run CONCURRENTLY across the worker pool; outcomes are // applied in file order so library rows stay 1:1 with filenames. const pendingResults = new Map(); let next = 0; // Apply one file's outcome in order: add rows, checkpoint-save, or record // the skip/error. `done` doubles as the flush cursor — every path yields // exactly one outcome, so when done reaches total the batch is complete. const applyOutcome = (outcome) => { if (outcome.kind === "skipped") { skippedPaths.add(outcome.filePath); return; } if (outcome.kind === "error") { // The failure may predate the copy (validation, embed timeout); // unlink only what actually landed. try { fs.unlinkSync(outcome.dest); } catch { /* never copied — nothing to clean up */ } result.errors.push({ file: outcome.filename, error: outcome.err.message, }); return; } l.filenames.push(outcome.filename); l.sources.push(outcome.filePath); if (!l.sourceMtimes) l.sourceMtimes = []; l.sourceMtimes.push(outcome.sourceMtime ?? null); // OCR text is extracted in the background after the batch (posters, // screenshots); the slot starts null so the backfill pass knows. if (!l.ocr) l.ocr = []; l.ocr.push(null); if (!l.ocrWords) l.ocrWords = []; l.ocrWords.push(null); // Content hash (parallel to filenames) — the rename-duplicate dedupe // of future syncs compares new files against this set. if (!l.hashes) l.hashes = []; l.hashes.push(outcome.hash || null); // Screenshot metadata hint (parallel to filenames) — the Screenshots // tab's rename-proof signal. false = probed, no metadata found. if (!l.screenshotHints) l.screenshotHints = []; l.screenshotHints.push(outcome.screenshotHint ?? null); importedSources.add(outcome.filePath); // A successful import retires any failure-cache entry for the file // (it was transient after all — the cache is only for poison files). retireImportFailure(outcome.filePath); l.dim = l.dim || outcome.reply.vec.length; l.embeddings.push(new Float32Array(outcome.reply.vec)); // Phrase vectors must never be null: saveLibrary writes the bin // 1:1 with filenames, and a null would crash encodeBin after the // index file was already persisted (leaving an inconsistent trio). const phraseVec = outcome.reply.phrase ? new Float32Array(outcome.reply.phrase) : new Float32Array(outcome.reply.vec.length); l.phrases.push(phraseVec); // Persist periodically instead of after every file (see the // checkpoint note above); the final save happens at batch end. if (++sinceCheckpoint >= CHECKPOINT_INTERVAL) { sinceCheckpoint = 0; void saveLibrary(); } result.added.push(outcome.filename); }; const flushResults = () => { while (pendingResults.has(done)) { const outcome = pendingResults.get(done); pendingResults.delete(done); applyOutcome(outcome); done++; } }; // Process one file: validate, copy, then embed through the worker pool. // Always returns an outcome object (never throws) so the ordered flush // sees a row for every path. const processOne = async (seq) => { const filePath = uniquePaths[seq]; let dest = null; try { // A failed model load (broken weights / download failure) must // not copy+unlink every remaining file: report the rest and bail. if (indexerInitFailed) { return { kind: "error", filename: path.basename(filePath), dest, err: new Error( `AI indexing unavailable: ${indexerInitFailed.message}`, ), }; } // Already imported from this exact source path: skip instead of // creating a "name (2).ext" copy. The source PATH (not the basename) // decides, so a same-named file from a different folder still imports. if (importedSources.has(filePath)) { return { kind: "skipped", filePath }; } // A file that failed import MAX_IMPORT_FAILURES times UNCHANGED is // poison — don't re-copy + re-attempt it on every poll/event. Only // watched-folder syncs consult the cache: a manual re-import is the // user explicitly asking for a fresh attempt. if (fromWatch && isKnownImportFailure(filePath)) { return { kind: "skipped", filePath }; } if (!fs.existsSync(filePath)) { return { kind: "error", filename: filePath, dest, err: new Error("File not found"), }; } const sourceStat = fs.statSync(filePath); if (!sourceStat.isFile()) { return { kind: "error", filename: filePath, dest, err: new Error("File not found"), }; } const sourceMtime = Number.isFinite(sourceStat.mtimeMs) ? sourceStat.mtimeMs : null; const ext = path.extname(filePath).toLowerCase(); if (!IMPORT_EXTENSIONS.has(ext)) { return { kind: "error", filename: path.basename(filePath), dest, err: new Error("Unsupported file type"), }; } // Content dedupe: a file whose bytes are already in the library is // a duplicate even though its source path is new (the rename case). // Hashed before the copy so the expensive part is skipped entirely. const srcHash = await hashFile(filePath); if (libraryHashes.has(srcHash)) { return { kind: "skipped", filePath }; } const isVideoFile = VIDEO_EXTENSIONS.has(ext); // Rename-proof Screenshots-tab signal: the file's own PNG/JPEG // text metadata ("Screenshot" in Software/UserComment) is read // straight from the source before the copy — a bounded read (see // screenshot-probe.js), so it never stalls the batch. const screenshotHint = isVideoFile ? false : await probeScreenshotMetadata(filePath); let filename = uniqueName(path.basename(filePath), taken); dest = path.join(PHOTOS_DIR, filename); sendImport("copying", filename); // Async copy: a multi-GB video copied synchronously would freeze // the main process (and every IPC/status event with it) for the // duration of the copy. The watchdog bounds it: a stalled network // copy must not freeze the pipeline forever (see the helper). await copyFileWithWatchdog(filePath, dest); // Extension normalization: a file whose BYTES disagree with its // extension (WebP saved as .png, JPEG saved as .webp) is renamed // to match its content before anything else references it. This // keeps the served Content-Type, the decoder Chromium picks, and // the filename truthful with each other (see sniffImageMime — // mislabeled images are a wrong-decoder crash hazard). Videos are // untouched; same-type spellings (.jpeg vs .jpg) are untouched. if (!isVideoFile) { const sniffed = sniffImageMime(dest); const canon = sniffed ? canonicalImageExt(sniffed) : null; if (canon) { const cur = path.extname(filename).toLowerCase(); const sameType = canon === cur || (canon === ".jpg" && cur === ".jpeg"); if (!sameType) { const corrected = uniqueName( path.basename(filename, path.extname(filename)) + canon, taken, ); try { fs.renameSync(dest, path.join(PHOTOS_DIR, corrected)); console.log( `[memories] normalized ${filename} → ${corrected} (content is ${sniffed})`, ); filename = corrected; dest = path.join(PHOTOS_DIR, filename); } catch (err) { console.warn( `[memories] extension normalization failed for ${filename}: ${err.message}`, ); } } } } sendImport("embedding", filename); const reply = await askIndexer({ type: isVideoFile ? "embed-video" : "embed-photo", path: dest, filename, }); if (!reply.vec) throw new Error("Embedding produced no vector"); if (isVideoFile) { sendImport("poster", filename); // Poster generation never touches the index, so it runs // concurrently with the rest of the batch instead of // stalling the import loop; all jobs settle just before // the library-updated broadcast below. pushPoster(filename, () => generatePoster(dest)); } return { kind: "added", filename, filePath, reply, dest, hash: srcHash, sourceMtime, screenshotHint, }; } catch (err) { // The failure predates the copy (validation, hash, embed timeout) // or the copy stalled — remember it so the watched-folder sync // stops retrying a poison file every poll. indexerInitFailed and // "File not found" return before the try's risky section, so only // real file-level failures land here; worker-lifecycle errors are // transient and system-wide, and must not poison individual files. if (!isSystemIndexerError(err)) rememberImportFailure(filePath); return { kind: "error", filename: path.basename(filePath), dest, err }; } }; // Bounded-concurrency pump: keep a couple of files in flight per worker // so a slow video on one worker doesn't idle the others; each worker // serializes its own requests (see indexer.js). The batch completes when // every path has been dispatched AND every in-flight task has settled // (its outcome flushed) — a plain Promise.all over the first wave would // miss tasks launched later by pump(). const BATCH_CONCURRENCY = Math.max(2, INDEXER_POOL_SIZE * 2); const inflight = new Set(); let resolveBatch = null; const batchDone = new Promise((r) => { resolveBatch = r; }); const pump = () => { while (inflight.size < BATCH_CONCURRENCY && next < total) { const seq = next++; const task = processOne(seq).then( (outcome) => { pendingResults.set(seq, outcome); flushResults(); }, (err) => { // Defensive guard for the alignment invariant: processOne // always returns an outcome today, but an unexpected // rejection must still produce a row so the ordered flush // never stalls below total (which would misalign the bins). pendingResults.set(seq, { kind: "error", filename: path.basename(uniquePaths[seq]), dest: null, err, }); flushResults(); }, ); inflight.add(task); void task.finally(() => { inflight.delete(task); pump(); // Every task's .then (flush) runs before its .finally, so when // this fires with nothing left in flight the batch is complete. if (next >= total && inflight.size === 0) resolveBatch(); }); } }; pump(); // Empty batch: nothing was dispatched, so nothing will ever resolve it. if (total === 0) resolveBatch(); await batchDone; for (const p of skippedPaths) { result.skipped.push(path.basename(p)); } // Let every video's poster land before the renderer learns about the // batch (poster jobs swallow their own errors — see pushPoster). await Promise.all(posterJobs); // Terminal event: all UI-visible work (copy → embed → poster) is done. // The renderer clears its progress card on this, so a main-initiated // sync (watched-folder launch/poll/live event) can't leave the card // stuck on screen — only the user-initiated paths used to clear it. sendImport("done", ""); if (result.added.length > 0) { // Guarantee the disk matches memory (the loop checkpointed periodically). await saveLibrary(); for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "library-updated" }); } // Scene enrichment: new videos are analyzed in the background AFTER // the batch broadcast, so import wall-clock is untouched. enqueueEnrichment( result.added .filter((name) => isVideo(name)) .map((name) => ({ filename: name, path: path.join(PHOTOS_DIR, name), })), ); // Speech transcription: same videos, queued behind visual // enrichment (transcribeAllowed serializes on enrich drain). enqueueTranscription( result.added .filter((name) => isVideo(name)) .map((name) => ({ filename: name, path: path.join(PHOTOS_DIR, name), })), ); // OCR text extraction: new photos are recognized in the background // too (posters, screenshots — the visible text CLIP can't read). enqueueOcr(result.added.filter((name) => !isVideo(name))); } return result; } // --------------------------------------------------------------------------- // Watched folders ("Import Photos" auto-watch). // // Importing a FOLDER (native picker or drag & drop) adds it to a persisted // watch list — individual files are imported once and never watched. The // library then stays in sync with each watched folder on its own: // • every launch re-scans each folder and imports anything new, // • while running, fs events (debounced) and a periodic safety-net poll // do the same for files that land mid-session. // Re-scanning is cheap by construction: importPaths dedupes by recorded // source path, so already-imported files short-circuit before any copy or // AI work and only genuinely new files are embedded. // --------------------------------------------------------------------------- const WATCHED_FOLDERS_FILE = path.join(DATA_DIR, "watched-folders.json"); // Watched directory paths, in import order (persisted across launches). let watchedFolders = []; // Live fs.watch handles keyed by folder, for teardown and re-arm. const folderWatchers = new Map(); // Debounce per folder: a burst of fs events (a copy tool writing several // files at once) collapses into a single re-scan. const watchDebounceTimers = new Map(); // Folders whose fs.watch could not be armed at all (permissions, exotic // volumes). Sync re-arm attempts skip these so the poll doesn't retry a // known failure every cycle (and spam the log); a fresh user import of the // folder tries again from scratch. const watchFailedFolders = new Set(); let watchPollTimer = null; function loadWatchedFolders() { try { const parsed = JSON.parse(fs.readFileSync(WATCHED_FOLDERS_FILE, "utf8")); if (Array.isArray(parsed.folders)) { watchedFolders = parsed.folders.filter((p) => typeof p === "string"); } } catch { /* no watch list yet */ } } function saveWatchedFolders() { fs.mkdirSync(DATA_DIR, { recursive: true }); fs.writeFileSync( WATCHED_FOLDERS_FILE, JSON.stringify({ version: 1, folders: watchedFolders }, null, "\t"), ); } // Add a folder to the watch list (idempotent) and arm its live watcher. // Returns true when the folder is newly watched. Paths are canonicalized // (path.resolve strips trailing slashes and resolves relative paths), so // "/Photos" and "/Photos/" are the same watched folder. function addWatchedFolder(dir) { if (typeof dir !== "string" || !dir) return false; dir = path.resolve(dir); if (watchedFolders.includes(dir)) return false; watchedFolders.push(dir); saveWatchedFolders(); watchFolder(dir); return true; } // Window after a folder change before re-syncing it (see the debounce note // above). const WATCH_DEBOUNCE_MS = 3000; // fs events can be missed entirely on network/synced volumes (iCloud // placeholders materialize quietly); the poll re-scans everything on a // schedule so nothing is stranded until the next launch. const WATCH_POLL_INTERVAL_MS = 60_000; // Re-scan ONE folder and import anything new. Skips folders that vanished // (they stay on the watch list — they may come back). While an import or // migration owns the pipeline, importPaths would refuse, so retry shortly // instead of dropping the change. Resolves when the (possibly empty) sync // import settles. function syncWatchedFolder(folder) { // A scheduled retry can outlive the watch itself (the user removed the // folder while an import was busy). Never sync — or worse, re-arm and // importPaths (which auto-watches folders in the batch) would resurrect // the folder the user just unwatched. if (!watchedFolders.includes(folder)) return Promise.resolve(); if (importInFlight || migrationState) { setTimeout(() => syncWatchedFolder(folder), 5000); return Promise.resolve(); } let stat; try { stat = fs.statSync(folder); } catch { return Promise.resolve(); } if (!stat.isDirectory()) return Promise.resolve(); // Re-arm a watcher that errored out earlier (poll-only coverage) — but // not one that could never be armed (the set below gates the retry). if (!folderWatchers.has(folder) && !watchFailedFolders.has(folder)) { watchFolder(folder); } return importPaths([folder], { fromWatch: true }).catch((err) => { console.warn( `[memories] watched-folder sync failed for ${folder}: ${err.message}`, ); }); } // Re-scan every watched folder, in order. Runs on launch and on the poll; // awaiting each import keeps the pipeline single-writer. async function syncWatchedFolders() { for (const folder of watchedFolders.slice()) { await syncWatchedFolder(folder); } } // Arm one folder's fs.watch. Recursive on macOS/Windows (catches files // landing in subfolders); elsewhere it falls back to a top-level watch, and // the poll covers whatever the platform watch can't see. function watchFolder(folder) { if (folderWatchers.has(folder)) return; const onChange = () => { clearTimeout(watchDebounceTimers.get(folder)); watchDebounceTimers.set( folder, setTimeout(() => { watchDebounceTimers.delete(folder); syncWatchedFolder(folder); }, WATCH_DEBOUNCE_MS), ); }; let watcher; try { watcher = fs.watch(folder, { recursive: true }, onChange); } catch { try { watcher = fs.watch(folder, onChange); } catch (err) { console.warn(`[memories] cannot watch ${folder}: ${err.message}`); watchFailedFolders.add(folder); return; } } watchFailedFolders.delete(folder); watcher.on("error", () => { // The watch died (folder unmounted / permissions). Drop it — the // poll still syncs, and the next sync re-arms it. folderWatchers.delete(folder); }); folderWatchers.set(folder, watcher); } // Load the persisted list, arm every watcher, and start the safety-net poll. function startWatchedFolderWatchers() { loadWatchedFolders(); for (const folder of watchedFolders) watchFolder(folder); watchPollTimer = setInterval( () => void syncWatchedFolders(), WATCH_POLL_INTERVAL_MS, ); } function stopWatchedFolderWatchers() { if (watchPollTimer) { clearInterval(watchPollTimer); watchPollTimer = null; } for (const timer of watchDebounceTimers.values()) clearTimeout(timer); watchDebounceTimers.clear(); for (const watcher of folderWatchers.values()) { try { watcher.close(); } catch { /* already closed */ } } folderWatchers.clear(); } // The watched folders the renderer's panel lists. `exists` tells the UI a // folder has vanished (unmounted/renamed) so it can say so instead of // implying it is still being scanned. function watchedFolderList() { return watchedFolders.map((p) => ({ path: p, exists: fs.existsSync(p) })); } // The on-disk path "Reveal in Finder" should reveal for a watched folder: // only folders on the watch list that still exist. The renderer supplies // the path; this guard keeps an arbitrary string from opening a location of // the renderer's choosing (same contract as resolveRevealTarget for tiles). function watchedFolderRevealTarget(folder) { if (typeof folder !== "string" || !folder) return null; if (!watchedFolders.includes(folder) || !fs.existsSync(folder)) return null; return folder; } // Stop watching a folder: drop it from the list, close its watcher, cancel // any pending debounce, and persist. Photos already imported stay in the // library — only future auto-imports stop. Returns false when the folder // wasn't watched. function removeWatchedFolder(dir) { const i = watchedFolders.indexOf(dir); if (i === -1) return false; watchedFolders.splice(i, 1); saveWatchedFolders(); const watcher = folderWatchers.get(dir); if (watcher) { try { watcher.close(); } catch { /* already closed */ } folderWatchers.delete(dir); } const timer = watchDebounceTimers.get(dir); if (timer) { clearTimeout(timer); watchDebounceTimers.delete(dir); } watchFailedFolders.delete(dir); return true; } // Extract a representative frame from an imported video into POSTERS_DIR, // named after the video's basename (posterFor()). The tile renders this // instead of the raw video, so nothing else needs to change. // --------------------------------------------------------------------------- // Scene-segment enrichment queue (Phase 1 of scene search). // // New videos are enriched in the BACKGROUND after their import batch ends — // never on the import critical path. Dispatch is paced and gated: nothing // runs while an import or migration is in flight or a query IPC is pending, // so search and import latency are unaffected by construction (see // DESIGN-SCENE-SEARCH.md §3). Chunks are ≤ SEGMENTS_PER_CHUNK segments, and // the worker's priority lane lets queries preempt between chunks. // --------------------------------------------------------------------------- // Chunk size lives in indexer/segment-store-utils.js (SEGMENTS_PER_CHUNK); // main.js never chunks directly — the worker's priority lane lets queries // preempt between chunks. const ENRICH_GAP_MS = 250; // Retry delay between chunk attempts for the same video (backoff base). const ENRICH_RETRY_DELAY_MS = 2000; let enrichQueue = []; // [{ filename, path, off, retries, lastError }] let enrichInFlight = false; let enrichPendingQuery = false; let enrichTimer = null; let lastEnrichStart = 0; // User-controlled pause for ALL background pumps (scene analysis, // transcription, text extraction), toggled from the tray pills via // memories:set-background-paused. Session-only: a relaunch resumes work, // so nobody can strand their library paused forever. While set, the three // *Allowed() gates refuse new jobs; the in-flight chunk (if any) finishes // on its own, then everything idles until resumed. let backgroundPaused = false; // Purge generation: bumped by memories:purge-background. Jobs capture it // at pump start and refuse to commit when it moved — a chunk in flight // across a purge discards its reply instead of landing partial sidecars // that the launch backfills would then skip forever as "covered". let purgeEpoch = 0; // Per-file tombstones: filenames whose queued/in-flight work was cancelled // by delete/purge/reset (dropQueuedJobsFor, purgeBackgroundWork). Every // pump's commit path checks this alongside libraryHasRow/purgeEpoch and, // on hit, discards the reply, drops the job, and deletes the entry — so // the set stays bounded: an entry lives only until its orphan settles. // Purge/reset clear it wholesale after draining (stale entries would // otherwise discard legitimate re-queued work). const deadJobs = new Set(); // True when this file's work was cancelled mid-flight (delete/purge). // Consumes the entry on hit so tombstones never outlive their orphan — // call it only on the discard path, never speculatively. function isJobDead(filename) { if (!deadJobs.has(filename)) return false; deadJobs.delete(filename); return true; } // Phase 4: live progress for the renderer's enrichment tray. The worker's // `enrich-progress` messages (deliberately id-less, so they can't trip the // pending-request map) land in lastEnrichProgress; enrichEventsObserved backs // the E2E's assertion that progress actually flows. let lastEnrichProgress = null; let lastEnrichHeartbeatAt = 0; let lastEnrichDetail = null; // { gi, t, step } of the current segment let enrichEventsObserved = 0; // Videos queued since the last drain, for the hidden-window completion // notification (incremented in enqueueEnrichment, consumed at the drain). let enrichNotifyCount = 0; // Per-phase progress counters + violation flag: backing the Phase-4 deep // test (detect pct must stay in [0,1]; embed done must be 1..total). let enrichPhaseCounts = { detect: 0, embed: 0 }; let enrichProgressViolations = 0; function broadcastEnrichEvent(extra) { for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "enrich", ...extra }); } // Menu-bar tray icon follows background work (no-op when the mode is off). updateTrayState(); } // Current background-enrichment snapshot (Phase 4 tray). Served by the // enrich-state IPC and replayed on did-finish-load so the tray is correct // even when broadcasts raced the renderer's subscription. // // Stale-clear (long-film fix): progress is keyed by filename — a leftover // tick from a finished/dropped video must never render as the NEXT job's // done/total (the old code showed a stale 16/1024 after a drop). When the // in-flight job doesn't match the last tick's filename, the snapshot reports // the job with zeroed progress instead of someone else's numbers. function enrichSnapshot() { const job = enrichInFlight ? enrichQueue[0] : null; const raw = lastEnrichProgress; const prog = job && raw && raw.filename === job.filename ? raw : null; const heartbeatAgeMs = job && lastEnrichHeartbeatAt > 0 ? Date.now() - lastEnrichHeartbeatAt : null; return { type: "enrich", paused: backgroundPaused, phase: job ? (prog?.phase === "embed" ? "embed" : "detect") : "queued", filename: job ? job.filename : null, pct: job && prog ? (prog.pct ?? null) : null, done: job && prog ? (prog.done ?? 0) : 0, total: job && prog ? (prog.total ?? 0) : 0, pending: enrichQueue.length - (job ? 1 : 0), active: Boolean(job), // Liveness detail for the tray/debug: which segment is decoding, in // which step (ffmpeg vs clip), and how long since the last heartbeat. gi: prog && Number.isFinite(prog.gi) ? prog.gi : (lastEnrichDetail?.gi ?? null), t: prog && Number.isFinite(prog.t) ? prog.t : (lastEnrichDetail?.t ?? null), step: prog?.step ?? lastEnrichDetail?.step ?? null, heartbeatAgeMs, stalled: heartbeatAgeMs !== null && heartbeatAgeMs > ENRICH_STALL_TIMEOUT_MS, }; } // Drop a stale progress tick that belongs to a finished/dropped video so it // can never render as another job's numbers. function clearEnrichProgressFor(filename) { if (lastEnrichProgress && lastEnrichProgress.filename === filename) { lastEnrichProgress = null; lastEnrichDetail = null; lastEnrichHeartbeatAt = 0; } } // Phase 4 backfill: videos that predate scene enrichment (or whose sidecar // never landed) are enqueued on launch — one at a time, under the same idle // gates as import-time enrichment — so a big library catches up without ever // touching search or import latency. Videos whose plan was empty are // recorded with an empty segment list, so they are not retried on every // launch (enqueueEnrichment's dedupe guards this too). function backfillEnrichment() { const l = loadLibrary(); if (!l || !Array.isArray(l.filenames)) return; const covered = loadSegments(l.modelId).videos; const missing = []; for (const name of l.filenames) { if (!isVideo(name)) continue; if (covered && covered.has(name)) continue; missing.push({ filename: name, path: path.join(PHOTOS_DIR, name) }); } if (missing.length > 0) { console.log( `[memories] backfill: ${missing.length} video(s) missing scene data`, ); enqueueEnrichment(missing); } } function enqueueEnrichment(files) { if (!Array.isArray(files)) return; let added = 0; for (const f of files) { if ( f && f.filename && f.path && !enrichQueue.some((q) => q.filename === f.filename) ) { enrichQueue.push({ filename: f.filename, path: f.path, off: 0 }); added++; } } if (added > 0) { // Counted for the drain notification (reset when it fires below). enrichNotifyCount += added; // Tray: a fresh batch queued (import or launch backfill). broadcastEnrichEvent({ phase: "queued", pending: enrichQueue.length }); } scheduleEnrichment(); } function scheduleEnrichment() { if (enrichTimer) return; enrichTimer = setTimeout(() => { enrichTimer = null; void pumpEnrichment(); }, ENRICH_GAP_MS); } function enrichAllowed() { return ( !backgroundPaused && !enrichInFlight && !importInFlight && !migrationState && !enrichPendingQuery && !transcribePendingQuery && enrichQueue.length > 0 && Date.now() - lastEnrichStart >= ENRICH_GAP_MS ); } async function pumpEnrichment() { if (!enrichAllowed()) { // While user-paused nothing will change until resume (which kicks // explicitly) — don't spin the gap timer for it. if (enrichQueue.length > 0 && !backgroundPaused) scheduleEnrichment(); return; } const job = enrichQueue[0]; // Row deleted while queued — drop without work, retry, or tray noise. if (job && !libraryHasRow(job.filename)) { console.log( `[memories] enriched ${job.filename}: row deleted while queued — dropping job`, ); enrichQueue.shift(); clearEnrichProgressFor(job.filename); broadcastEnrichEvent({ phase: enrichQueue.length === 0 ? "idle" : "done", filename: job.filename, pending: enrichQueue.length, }); scheduleEnrichment(); return; } if (typeof job.retries !== "number") job.retries = 0; job.purgeEpoch = purgeEpoch; enrichInFlight = true; lastEnrichStart = Date.now(); // Heartbeat baseline for THIS chunk: the worker's per-segment + interval // ticks refresh lastEnrichHeartbeatAt; silence past ENRICH_STALL_TIMEOUT_MS // means the chunk is wedged, not merely slow. lastEnrichHeartbeatAt = Date.now(); try { const modelId = loadLibrary().modelId; const reply = await askPrimaryIndexer( { type: "enrich-video", path: job.path, fromIndex: job.off || 0, filename: job.filename, // The worker persists each segment's midpoint frame here so the // renderer can swap the card thumbnail to the best-scene poster. postersDir: POSTERS_DIR, // Scene density for the user's Settings → Video search preset // (resolved per job so a preset switch takes effect without a // worker restart; the worker keys its cached plan on it). budget: videoBudgetForCurrentQuality(), }, ENRICH_CHUNK_TIMEOUT_MS, ); const c = loadSegments(modelId); // Row deleted — or purged — while this chunk ran: committing would // resurrect sidecar entries (for a row that no longer exists) or // land partial fragments the backfill skips as "covered". Discard // the reply and drop the job (same finish shape as the tail below; // finally still clears the in-flight flag). // isJobDead FIRST: it consumes the tombstone, and short-circuiting // on a row/epoch hit first would leak the entry (a later re-import // of the same filename would then discard one live chunk). if ( isJobDead(job.filename) || !libraryHasRow(job.filename) || job.purgeEpoch !== purgeEpoch ) { console.log( `[memories] enriched ${job.filename}: row deleted or purged mid-flight — discarding chunk`, ); enrichQueue.shift(); clearEnrichProgressFor(job.filename); broadcastEnrichEvent({ phase: enrichQueue.length === 0 ? "idle" : "done", filename: job.filename, pending: enrichQueue.length, }); scheduleEnrichment(); return; } // One corrupt GOP must not kill a 1024-seg film: the worker returns // parallel arrays (segments[i] ↔ vecs[i], null = skipped) and still // advances fromIndex past failures. Partition here and persist only // the successful rows; skipped segments simply leave a searchable gap. const { okSegments, okVecs, skipped } = partitionChunkReply( reply.segments, reply.vecs, ); // Segment rows share the index's model dim; persist it so the // renderer's header validation (header[1] === meta.dim) passes. The // cache starts dim-less (fresh, no prior file) — the index is the // authoritative source, the first SUCCESSFUL vector length the backup // (a fully-skipped chunk carries no usable dim). if (c.dim === 0) { const firstVec = okVecs.length > 0 ? okVecs[0] : null; c.dim = loadLibrary().dim || (firstVec ? firstVec.length : 0); } if (reply.ok && okVecs.length > 0) { // Append this chunk's rows and stamp the file's segment offsets // (poster = the segment's scene-poster index). mergeChunkSegments // ACCUMULATES into the file's list because the worker returns // only the slice it embedded THIS call — replacing the entry // here used to leave only the LAST chunk's segments in the // sidecar, making a long film searchable only in its final // minutes (pinned by test/pump-enrichment.test.js and the // phase-4 multi-chunk check). const segments = mergeChunkSegments( c.videos.get(job.filename) || [], okSegments, c.rows.length, SEGMENT_ROW_PER, ); c.rows.push(...okVecs.map((v) => new Float32Array(v))); c.videos.set(job.filename, segments); c.loaded = true; saveSegments(); job.retries = 0; job.okTotal = (job.okTotal || 0) + okVecs.length; job.skipTotal = (job.skipTotal || 0) + skipped; console.log( `[memories] enriched ${job.filename}: ${reply.fromIndex}/${reply.total} segments` + (skipped > 0 ? ` (${skipped} skipped)` : ""), ); } else if (reply.ok && reply.total === 0) { // Nothing detectable (failed probe / too short) — record an empty // entry so the file is not retried on every launch. c.videos.set(job.filename, []); c.loaded = true; saveSegments(); console.log( `[memories] enriched ${job.filename}: 0 segments (plan empty)`, ); } else if (reply.ok && skipped > 0 && okVecs.length === 0) { // Whole chunk skipped (e.g. a run of corrupt GOPs): still advance // past it — dropping the film here would discard the 16 good // segments before it and freeze the tray at the chunk boundary. job.retries = 0; job.skipTotal = (job.skipTotal || 0) + skipped; console.warn( `[memories] enriched ${job.filename}: ${reply.fromIndex}/${reply.total} segments (all ${skipped} skipped in this chunk, continuing)`, ); } else { console.warn( `[memories] enrichment returned no segments for ${job.filename}: ${reply.error || "unknown"}`, ); } if (reply.ok && !reply.done) { job.off = reply.fromIndex || 0; scheduleEnrichment(); return; } if ( shouldParkEnrichment({ done: reply.done, ok: reply.ok, okTotal: job.okTotal, skippedTotal: job.skipTotal, alreadyCovered: c.videos.has(job.filename), }) ) { // Poison pill: every segment skipped across every chunk (undecodable // frames, 2026-09-26: parseRawFrameSize missing the SAR-less // `WxH, q=` form). Without an entry, backfillEnrichment re-queues // this file on every launch and every model switch forever and the // tray never goes idle. Park it like the empty-plan path — an empty // entry means "no scene data, don't retry". A re-imported copy // arrives under removeLibraryRow/dropQueuedJobsFor cleanup, so a // genuinely fixed file gets another chance on re-import. c.videos.set(job.filename, []); c.loaded = true; saveSegments(); console.warn( `[memories] enriched ${job.filename}: 0/${reply.total} segments (all ${job.skipTotal} skipped across every chunk — parking, will not retry until re-import)`, ); } } catch (err) { // Row deleted — or purged — mid-chunk: the missing file (or the // purge) explains the failure — retrying would burn the whole // budget on it. Drop instead. // isJobDead FIRST: it consumes the tombstone, and short-circuiting // on a row/epoch hit first would leak the entry (a later re-import // of the same filename would then discard one live chunk). if ( isJobDead(job.filename) || !libraryHasRow(job.filename) || job.purgeEpoch !== purgeEpoch ) { console.log( `[memories] enrichment chunk for ${job.filename} failed after the row was deleted or purged — dropping job`, ); enrichQueue.shift(); clearEnrichProgressFor(job.filename); broadcastEnrichEvent({ phase: enrichQueue.length === 0 ? "idle" : "done", filename: job.filename, pending: enrichQueue.length, }); scheduleEnrichment(); return; } // Never-stall retry (long-film fix): a chunk timeout/error keeps the // job at its current offset and retries with backoff instead of // dropping a 2–3 h film for one slow segment. The timed-out worker // may still be grinding the orphaned chunk (its queue stays busy — // the 16/1024-forever signature), so restart the primary to drop the // zombie ffmpeg work with it. After ENRICH_MAX_RETRIES the video is // dropped for this session (backfill retries it); enrichment failure // never touches imports or search. job.retries = (job.retries || 0) + 1; job.lastError = err.message; if (job.retries <= ENRICH_MAX_RETRIES) { console.warn( `[memories] enrichment chunk ${job.off || 0} for ${job.filename} failed ` + `(attempt ${job.retries}/${ENRICH_MAX_RETRIES}): ${err.message} — retrying`, ); if (err.code === "INDEXER_TIMEOUT") { restartPrimaryIndexer(`chunk timeout after ${err.timeoutMs}ms`); } enrichInFlight = false; if (enrichTimer) clearTimeout(enrichTimer); enrichTimer = setTimeout(() => { enrichTimer = null; void pumpEnrichment(); }, ENRICH_RETRY_DELAY_MS * job.retries); return; } console.warn( `[memories] enrichment failed for ${job.filename} after ${job.retries} attempts: ${err.message} (dropping for this session)`, ); if (err.code === "INDEXER_TIMEOUT") { restartPrimaryIndexer(`final chunk timeout for ${job.filename}`); } // Timeout-poison parking (2026-09-28 orphan-loop fix): a film whose // chunk 0 never completes (e.g. 2 h HEVC whose full-rate detect alone // exceeds the chunk timeout) was dropped "for this session" but // backfillEnrichment re-queued it on EVERY launch — leaking a fresh // 500%-CPU ffmpeg each time while the tray claimed idle. When the job // made zero progress across all retries, park it like the empty-plan // path (empty segment list = covered, never retried until re-import). try { const noProgress = !job.off && !job.okTotal && !job.skipTotal; if (noProgress && job.filename) { const parkModelId = loadLibrary().modelId; const sc = loadSegments(parkModelId); if (sc && !sc.videos.has(job.filename)) { sc.videos.set(job.filename, []); sc.loaded = true; saveSegments(); console.warn( `[memories] parked ${job.filename} after ${job.retries} timeouts with zero progress (will not retry until re-import)`, ); } } } catch { /* parking is best-effort — the drop below still applies */ } clearEnrichProgressFor(job.filename); } finally { enrichInFlight = false; } const finished = enrichQueue.shift(); // The finished video's last tick must not leak into the next job's // snapshot (stale 16/1024 after a drop) — enrichSnapshot also guards by // filename, this clears the stored tick itself. if (finished) clearEnrichProgressFor(finished.filename); // Tray: this video finished (or failed) — the next job's progress takes // over, or the queue drained into "idle". broadcastEnrichEvent({ phase: enrichQueue.length === 0 ? "idle" : "done", filename: job.filename, pending: enrichQueue.length, }); if (enrichQueue.length === 0 && enrichNotifyCount > 0) { const n = enrichNotifyCount; enrichNotifyCount = 0; maybeNotify( "Scene analysis complete", `${n} video${n === 1 ? "" : "s"} analyzed — scene search is up to date.`, ); } scheduleEnrichment(); } async function generatePoster(videoPath) { const videoUtils = require("./indexer/video-utils.js"); const ffmpeg = videoUtils.resolveFfmpeg(); fs.mkdirSync(POSTERS_DIR, { recursive: true }); const poster = posterFor(path.basename(videoPath)); await videoUtils.extractPoster(ffmpeg, videoPath, poster, 480); return poster; } // --------------------------------------------------------------------------- // In-app preview proxies for searchable-only videos (mkv/hevc/...). // // Chromium cannot demux MKV (and Electron 43 cannot reliably decode HEVC // 10-bit), so the lightbox shows a poster + "Playback not supported" notice // for those files. A preview is a short mp4/h264 clip transcoded on demand // around the matched scene timestamp — playable in Chromium, seekable, // cheap (30 s at 720p, cached on disk under a deterministic name so repeat // views never re-transcode). // // The transcode runs in the MAIN process through the tracked video-utils // spawner (same orphan-proof registry as the workers — nunca raw spawn, so // the ffmpeg-CPU fix covers previews too) with its own timeout. Failures // resolve to an error (the lightbox falls back to the poster); they never // throw into the protocol handler. // --------------------------------------------------------------------------- function previewFileFor(filename, t) { const stem = path.basename(filename, path.extname(filename)); const at = Math.max(0, Math.round(Number(t) || 0)); return `${stem}-t${at}-30s.mp4`; } // Best-effort LRU prune: drop previews older than a week, then oldest-first // past the size cap. Never throws; a full disk must not break playback of // the originals. function prunePreviews() { try { if (!fs.existsSync(PREVIEWS_DIR)) return; const now = Date.now(); const entries = []; for (const name of fs.readdirSync(PREVIEWS_DIR)) { if (!name.endsWith(".mp4")) continue; const full = path.join(PREVIEWS_DIR, name); try { const st = fs.statSync(full); if (now - st.mtimeMs > PREVIEW_MAX_AGE_MS) { try { fs.unlinkSync(full); } catch { /* best-effort */ } continue; } entries.push({ full, size: st.size, mtimeMs: st.mtimeMs }); } catch { /* vanished mid-prune */ } } let total = entries.reduce((a, e) => a + e.size, 0); if (total <= PREVIEW_MAX_BYTES) return; entries.sort((a, b) => a.mtimeMs - b.mtimeMs); for (const e of entries) { if (total <= PREVIEW_MAX_BYTES) break; try { fs.unlinkSync(e.full); total -= e.size; } catch { /* best-effort */ } } } catch { /* pruning must never break playback */ } } function ensurePreview(filename, t) { const previewName = previewFileFor(filename, t); const cached = previewPending.get(previewName); if (cached) return cached; const job = (async () => { const videoUtils = require("./indexer/video-utils.js"); if (!libraryHasRow(filename)) { return { ok: false, error: "unknown file" }; } const src = path.join(PHOTOS_DIR, path.basename(filename)); if (!fs.existsSync(src)) { return { ok: false, error: "file missing" }; } fs.mkdirSync(PREVIEWS_DIR, { recursive: true }); const dest = path.join(PREVIEWS_DIR, previewName); try { const st = fs.statSync(dest); if (st.size > 0) { prunePreviews(); return { ok: true, path: dest }; } } catch { /* not cached — transcode below */ } let ffmpeg; try { ffmpeg = videoUtils.resolveFfmpeg(); } catch (err) { return { ok: false, error: err.message }; } const start = Math.max(0, (Number(t) || 0) - PREVIEW_LEAD_SECONDS); const args = [ "-y", "-ss", String(start), "-i", src, "-t", String(PREVIEW_SECONDS), "-vf", "scale=-2:720", "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "128k", "-movflags", "+faststart", dest, ]; try { await new Promise((resolve, reject) => { const child = videoUtils.spawnFfmpeg(ffmpeg, args, { stdio: ["ignore", "pipe", "pipe"], }); let stderr = ""; child.stderr.on("data", (d) => { stderr += d.toString(); }); const timer = setTimeout(() => { try { child.kill("SIGKILL"); } catch { /* best-effort */ } reject( new Error(`preview transcode timed out: ${stderr.slice(-300)}`), ); }, PREVIEW_TIMEOUT_MS); child.on("error", (err) => { clearTimeout(timer); reject(err); }); child.on("close", (code) => { clearTimeout(timer); if (code === 0) resolve(); else reject( new Error( `preview transcode exited ${code}: ${stderr.slice(-300)}`, ), ); }); }); } catch (err) { try { fs.unlinkSync(dest); } catch { /* partial output must not poison the cache */ } console.warn( `[memories] preview failed for ${filename} at ${start}s: ${err.message}`, ); return { ok: false, error: err.message }; } try { const st = fs.statSync(dest); if (!st.size) { try { fs.unlinkSync(dest); } catch { /* best-effort */ } return { ok: false, error: "empty preview" }; } } catch { return { ok: false, error: "preview missing" }; } prunePreviews(); console.log( `[memories] preview ready for ${filename} at ${start}s (${previewName})`, ); return { ok: true, path: dest }; })(); previewPending.set(previewName, job); job.finally(() => { previewPending.delete(previewName); }); return job; } // --------------------------------------------------------------------------- // Speech transcription queue (transcript sidecar for scene search). // // New videos are transcribed in the BACKGROUND after visual enrichment — // never on the import critical path, never in parallel with it on 8 GB M1 // (enrich first for posters/grid, transcribe second). Dispatch is paced and // gated like enrichment + OCR: nothing runs while an import/migration is in // flight or a query IPC is pending. Each job uses a DEDICATED // transcribe-worker utilityProcess (whisper.cpp, CPU-heavy, unrelated to // onnxruntime), then embeds chunk texts through the PRIMARY indexer worker // (same text tower + centering as queries). Results land in the per-model // transcript sidecar; empty chunk lists are valid (silent film) and never // retried. Transcription failure never touches imports or search. // --------------------------------------------------------------------------- const TRANSCRIBE_GAP_MS = 250; const TRANSCRIBE_RETRY_DELAY_MS = 2000; let transcribeQueue = []; // [{ filename, path, off, retries, lastError, hadProgress }] let transcribeInFlight = false; let transcribePendingQuery = false; let transcribeTimer = null; let lastTranscribeStart = 0; let lastTranscribeProgress = null; let lastTranscribeHeartbeatAt = 0; let lastTranscribeModel = null; let transcribeWorker = null; let transcribeWorkerDead = false; let transcribePending = new Map(); let transcribeRequestId = 0; function broadcastTranscribeEvent(extra) { for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "transcribe", ...extra }); } updateTrayState(); } function transcribeSnapshot() { const job = transcribeInFlight ? transcribeQueue[0] : null; const raw = lastTranscribeProgress; const prog = job && raw && raw.filename === job.filename ? raw : null; return { type: "transcribe", paused: backgroundPaused, phase: job ? "transcribe" : transcribeQueue.length > 0 ? "queued" : "idle", filename: job ? job.filename : null, done: job && prog ? (prog.done ?? 0) : 0, total: job && prog ? (prog.total ?? 0) : 0, pending: transcribeQueue.length - (job ? 1 : 0), active: Boolean(job), }; } function clearTranscribeProgressFor(filename) { if (lastTranscribeProgress && lastTranscribeProgress.filename === filename) { lastTranscribeProgress = null; lastTranscribeHeartbeatAt = 0; } } function backfillTranscription() { const l = loadLibrary(); if (!l || !Array.isArray(l.filenames)) return; let videos; let progress = {}; try { const c = loadTranscripts(l.modelId); videos = c.videos; progress = c.progress || {}; } catch { videos = null; } // Prune progress for rows no longer in the library (deleted videos). let pruned = false; for (const name of Object.keys(progress)) { if (!l.filenames.includes(name)) { delete progress[name]; pruned = true; } } // Missing entries start at 0; entries with partial progress (crash or // drop mid-film) resume where they stalled. Complete entries skip. const videosOnly = l.filenames.filter((name) => isVideo(name)); const jobs = transcribeBackfillList(videosOnly, videos, progress).map( (j) => ({ filename: j.filename, path: path.join(PHOTOS_DIR, j.filename), off: j.off, }), ); if (pruned) saveTranscripts(); if (jobs.length > 0) { const resumed = jobs.filter((j) => j.off > 0).length; console.log( `[memories] backfill: ${jobs.length} video(s) missing transcript data` + (resumed > 0 ? ` (${resumed} resuming mid-film)` : ""), ); enqueueTranscription(jobs); } } function enqueueTranscription(files) { if (!Array.isArray(files)) return; let added = 0; for (const f of files) { if ( f && f.filename && f.path && !transcribeQueue.some((q) => q.filename === f.filename) ) { // off carries resume offsets (crash mid-film); fresh jobs start 0. const off = Number.isInteger(f.off) && f.off > 0 ? f.off : 0; transcribeQueue.push({ filename: f.filename, path: f.path, off }); added++; } } if (added > 0) { broadcastTranscribeEvent({ phase: "queued", pending: transcribeQueue.length, }); } scheduleTranscription(); } function scheduleTranscription() { if (transcribeTimer) return; transcribeTimer = setTimeout(() => { transcribeTimer = null; void pumpTranscription(); }, TRANSCRIBE_GAP_MS); } function transcribeAllowed() { return ( !backgroundPaused && !transcribeInFlight && !importInFlight && !migrationState && !transcribePendingQuery && !enrichPendingQuery && !ocrPendingQuery && // Serialize with visual enrichment on 8 GB M1: posters/grid first. !enrichInFlight && enrichQueue.length === 0 && transcribeQueue.length > 0 && Date.now() - lastTranscribeStart >= TRANSCRIBE_GAP_MS ); } function ensureTranscribeWorker() { if (transcribeWorker && !transcribeWorkerDead) return transcribeWorker; transcribeWorkerDead = false; const worker = { process: null, dead: false }; worker.process = utilityProcess.fork( path.join(__dirname, "indexer", "transcribe-worker.js"), [], { // Shared model cache + ffmpeg resolution, exactly like the // indexer pool (whisper weights land beside the CLIP weights in // ~/Library/Application Support/scm/models, never in the asar). env: indexerEnv(), serviceName: "transcribe-worker", }, ); worker.process.on("message", (event) => { const msg = event && event.data !== undefined ? event.data : event; if (!msg) return; if (msg.type === "ffmpeg-spawn") { trackFfmpegSpawn(transcribeFfmpegPids, msg.pid, msg.label); return; } if (msg.type === "ffmpeg-exit") { untrackFfmpegExit(transcribeFfmpegPids, msg.pid); return; } if (msg.type === "ffmpeg-killed") return; if (msg.type === "transcribe-progress") { lastTranscribeProgress = msg; lastTranscribeHeartbeatAt = Date.now(); if (typeof msg.model === "string" && msg.model) lastTranscribeModel = msg.model; // Mark load-vs-decode attribution for the active job: a // transcribe-phase tick proves decode started; model-phase ticks // (or silence) mean a later exit was a load-phase crash. if ( msg.phase === "transcribe" && transcribeQueue.length > 0 && msg.filename === transcribeQueue[0].filename ) { transcribeQueue[0].hadProgress = true; } return; } if (!msg.id) return; const req = transcribePending.get(msg.id); if (!req) return; transcribePending.delete(msg.id); clearTimeout(req.timer); if (msg.ok) req.resolve(msg); else req.reject(new Error(msg.error || "Transcribe failed")); }); worker.process.on("exit", (code, signal) => { worker.dead = true; transcribeWorkerDead = true; transcribeWorker = null; // Diagnose, don't mask: the old handler discarded code/signal, so // every OOM kill and every clean crash read identically as // "Transcribe worker exited". The exit label (with a SIGKILL→OOM // hint) rides both the log line and the rejection, so the pump's // retry/drop lines can attribute load-vs-decode. const label = transcribeExitLabel(code, signal); const active = transcribeQueue.length > 0 ? transcribeQueue[0] : null; const hbAge = lastTranscribeHeartbeatAt > 0 ? Date.now() - lastTranscribeHeartbeatAt : null; console.warn( `[memories] transcribe worker ${label}` + (active ? ` (job ${active.filename} off ${active.off || 0})` : " (idle)") + (lastTranscribeModel ? ` model ${lastTranscribeModel}` : "") + (hbAge !== null ? `, last heartbeat ${Math.round(hbAge / 1000)}s ago` : ", no heartbeat yet"), ); for (const [, req] of transcribePending) { clearTimeout(req.timer); const err = new Error(`Transcribe worker ${label}`); err.code = "TRANSCRIBE_EXIT"; err.exitCode = code; err.signal = signal; req.reject(err); } transcribePending = new Map(); }); transcribeWorker = worker; return worker; } function askTranscribe(message, timeoutMs = TRANSCRIBE_CHUNK_TIMEOUT_MS) { const worker = ensureTranscribeWorker(); return new Promise((resolve, reject) => { const id = `t${++transcribeRequestId}`; const timer = setTimeout(() => { transcribePending.delete(id); const err = new Error("Transcribe request timed out"); err.code = "TRANSCRIBE_TIMEOUT"; err.timeoutMs = timeoutMs; reject(err); }, timeoutMs); transcribePending.set(id, { resolve, reject, timer }); worker.process.postMessage({ ...message, id }); }); } async function pumpTranscription() { if (!transcribeAllowed()) { if (transcribeQueue.length > 0 && !backgroundPaused) scheduleTranscription(); return; } const job = transcribeQueue[0]; // Row deleted while queued — drop without work, retry, or tray noise. if (job && !libraryHasRow(job.filename)) { console.log( `[memories] transcribed ${job.filename}: row deleted while queued — dropping job`, ); transcribeQueue.shift(); clearTranscribeProgressFor(job.filename); broadcastTranscribeEvent({ phase: transcribeQueue.length === 0 ? "idle" : "done", filename: job.filename, pending: transcribeQueue.length, }); scheduleTranscription(); return; } if (typeof job.retries !== "number") job.retries = 0; job.purgeEpoch = purgeEpoch; transcribeInFlight = true; lastTranscribeStart = Date.now(); lastTranscribeHeartbeatAt = Date.now(); // Announce the active job immediately so the tray pill appears even // before the worker's first per-slice progress tick (model load can take // a while on first run). broadcastTranscribeEvent({ phase: "transcribe", filename: job.filename, done: 0, total: 0, pending: transcribeQueue.length - 1, active: true, }); try { const modelId = loadLibrary().modelId; const whisperModel = readWhisperModel(); const timeouts = transcribeTimeoutsFor(whisperModel); const reply = await askTranscribe( { type: "transcribe-video", path: job.path, filename: job.filename, fromIndex: job.off || 0, modelId: whisperModel, }, timeouts.chunkTimeoutMs, ); const replyModel = parseWhisperModel(reply.model || whisperModel); // Cancelled (deleted/purged) outranks stale-model: re-running a // dead job under the new model would burn a chunk on a file that // is gone. isJobDead runs FIRST because it consumes the tombstone // — short-circuiting on a row/epoch hit first would leak the entry // (a later re-import of the same filename would then discard one // live chunk). if ( isJobDead(job.filename) || !libraryHasRow(job.filename) || job.purgeEpoch !== purgeEpoch ) { console.log( `[memories] transcribed ${job.filename}: row deleted or purged mid-flight (stale-model reply) — discarding chunk`, ); transcribeQueue.shift(); clearTranscribeProgressFor(job.filename); broadcastTranscribeEvent({ phase: transcribeQueue.length === 0 ? "idle" : "done", filename: job.filename, pending: transcribeQueue.length, }); scheduleTranscription(); return; } if (replyModel !== whisperModel) { // Settings switched mid-flight: drop this stale-model reply // (no commit, no progress write) and re-run the same offset // under the new model. The early return keeps job[0] queued. console.log( `[memories] speech model changed mid-flight (${whisperModel} → ${replyModel}) — re-running ${job.filename} from offset ${job.off || 0}`, ); scheduleTranscription(); return; } const c = loadTranscripts(modelId); // Row deleted — or purged — while this chunk ran: committing would // resurrect sidecar entries or land partial fragments the backfill // skips as "covered". Discard the reply and drop the job (finally // still clears the in-flight flag). // isJobDead FIRST: it consumes the tombstone, and short-circuiting // on a row/epoch hit first would leak the entry (a later re-import // of the same filename would then discard one live chunk). if ( isJobDead(job.filename) || !libraryHasRow(job.filename) || job.purgeEpoch !== purgeEpoch ) { console.log( `[memories] transcribed ${job.filename}: row deleted or purged mid-flight — discarding chunk`, ); transcribeQueue.shift(); clearTranscribeProgressFor(job.filename); broadcastTranscribeEvent({ phase: transcribeQueue.length === 0 ? "idle" : "done", filename: job.filename, pending: transcribeQueue.length, }); scheduleTranscription(); return; } if (c.dim === 0) { c.dim = loadLibrary().dim || 0; } // Stamp the engine that produced these rows (a Settings switch // invalidates; pre-stamp sidecars read back as tiny.en). c.whisperModel = replyModel; if (reply.ok && reply.chunks && reply.chunks.length > 0) { // Embed chunk texts through the primary indexer (same text tower // + centering as queries) before committing rows. let vecs = []; try { spawnIndexer(); await indexerReady; const emb = await askPrimaryIndexer( { type: "embed-texts", texts: reply.chunks.map((ch) => ch.text) }, 180000, ); vecs = emb.vecs || []; } catch (embErr) { console.warn( `[memories] transcript embed failed for ${job.filename}: ${embErr.message} — retrying`, ); throw embErr; } // Accumulate utterance-level full-text rows (exact-search index) // alongside the embedded chunks — same accumulation contract, // validated lines only, persisted by saveTranscripts. if (Array.isArray(reply.utterances) && reply.utterances.length > 0) { if (!(c.utterances instanceof Map)) c.utterances = new Map(); const prev = c.utterances.get(job.filename) || []; const seen = new Set( prev.map((u) => `${u.t0.toFixed(2)}:${u.t1.toFixed(2)}:${u.text}`), ); for (const u of reply.utterances) { if ( !u || !Number.isFinite(u.t0) || !Number.isFinite(u.t1) || u.t1 <= u.t0 ) continue; if (typeof u.text !== "string" || u.text.trim().length === 0) continue; const line = { t0: u.t0, t1: u.t1, text: String(u.text) }; const key = `${line.t0.toFixed(2)}:${line.t1.toFixed(2)}:${line.text}`; if (seen.has(key)) continue; seen.add(key); prev.push(line); } prev.sort((a, b) => a.t0 - b.t0 || a.t1 - b.t1); c.utterances.set(job.filename, prev); } const { okChunks, okVecs, skipped } = partitionTranscriptReply( reply.chunks, vecs, ); if (c.dim === 0 && okVecs.length > 0) { c.dim = okVecs[0].length || loadLibrary().dim || 0; } if (okVecs.length > 0) { // Production commit path, extracted pure as // applyTranscriptReply (unit-tested in // test/transcript-fusion.test.js): accumulate + stamp // offsets at rows.length BEFORE pushing. const applied = applyTranscriptReply( c.videos, c.rows, job.filename, okChunks, okVecs, TRANSCRIPT_ROW_PER, ); c.videos = applied.videos; c.rows = applied.rows; c.loaded = true; saveTranscripts(); job.retries = 0; console.log( `[memories] transcribed ${job.filename}: ${reply.fromIndex}/${reply.total} chunks` + (skipped > 0 ? ` (${skipped} skipped)` : ""), ); } else if (skipped > 0) { job.retries = 0; console.warn( `[memories] transcribed ${job.filename}: ${reply.fromIndex}/${reply.total} chunks (all ${skipped} skipped in this chunk, continuing)`, ); } } else if ( reply.ok && (reply.total === 0 || (reply.chunks || []).length === 0) ) { // Silent / no-audio / no-engine: valid empty entry, never retried. c.videos.set(job.filename, []); if (c.utterances instanceof Map) c.utterances.delete(job.filename); if (c.progress && typeof c.progress === "object") delete c.progress[job.filename]; c.loaded = true; saveTranscripts(); console.log( `[memories] transcribed ${job.filename}: 0 chunks (silent or no engine)`, ); } else { console.warn( `[memories] transcription returned no chunks for ${job.filename}: ${reply.error || "unknown"}`, ); } if (reply.ok && !reply.done) { job.off = reply.fromIndex || 0; // Persist the resume point: a crash/exit mid-film re-queues // from here instead of abandoning the partial sidecar forever. if (!c.progress || typeof c.progress !== "object") c.progress = {}; c.progress[job.filename] = { done: reply.fromIndex || 0, total: reply.total || 0, }; saveTranscripts(); scheduleTranscription(); return; } if (reply.ok && reply.done) { // Film complete: clear any resume point (silent films cleared it // above; transcribed films clear it here). if (c.progress && typeof c.progress === "object") delete c.progress[job.filename]; saveTranscripts(); } } catch (err) { // Row deleted — or purged — mid-chunk: the missing file (or the // purge) explains the failure — retrying would burn the whole // budget on it. Drop instead. // isJobDead FIRST: it consumes the tombstone, and short-circuiting // on a row/epoch hit first would leak the entry (a later re-import // of the same filename would then discard one live chunk). if ( isJobDead(job.filename) || !libraryHasRow(job.filename) || job.purgeEpoch !== purgeEpoch ) { console.log( `[memories] transcription chunk for ${job.filename} failed after the row was deleted or purged — dropping job`, ); transcribeQueue.shift(); clearTranscribeProgressFor(job.filename); broadcastTranscribeEvent({ phase: transcribeQueue.length === 0 ? "idle" : "done", filename: job.filename, pending: transcribeQueue.length, }); scheduleTranscription(); return; } job.retries = (job.retries || 0) + 1; job.lastError = err.message; // Attribute the failure: model (last tick's stamp, else current // setting), load-vs-decode phase (any transcribe-phase tick for this // job proves decode started), exit cause, and heartbeat silence. // The old line logged only err.message, so four identical // "Transcribe worker exited" retries were indistinguishable from // four different transient slices. const matchingProg = lastTranscribeProgress && lastTranscribeProgress.filename === job.filename ? lastTranscribeProgress : null; const modelForLabel = (matchingProg && matchingProg.model) || lastTranscribeModel || readWhisperModel(); const phaseForLabel = job.hadProgress ? "transcribe" : matchingProg ? matchingProg.phase || null : null; const loadPhase = isLoadPhaseCrash({ hadProgress: job.hadProgress, phase: phaseForLabel, }); const hbAge = lastTranscribeHeartbeatAt > 0 ? Date.now() - lastTranscribeHeartbeatAt : null; let sizeLabel = null; try { const st = fs.statSync(job.path); if (st && Number.isFinite(st.size)) sizeLabel = `${(st.size / 1048576).toFixed(1)}MB`; } catch { sizeLabel = "size unknown"; } const detail = transcribeFailureLabel({ model: modelForLabel, phase: loadPhase ? "load" : "decode", code: err.exitCode, signal: err.signal, heartbeatAgeMs: hbAge, }); // Load-phase crashes are systematic (same weights, same death): one // retry, not three full model reloads. Decode-phase keeps the budget. const maxForJob = loadPhase ? TRANSCRIBE_LOAD_MAX_RETRIES : TRANSCRIBE_MAX_RETRIES; if (job.retries <= maxForJob) { console.warn( `[memories] transcription chunk ${job.off || 0} for ${job.filename} failed ` + `(attempt ${job.retries}/${maxForJob}${loadPhase ? ", load-phase" : ""}) [${detail}]` + (sizeLabel ? ` [${sizeLabel}]` : "") + `: ${err.message} — retrying`, ); transcribeInFlight = false; if (transcribeTimer) clearTimeout(transcribeTimer); transcribeTimer = setTimeout(() => { transcribeTimer = null; void pumpTranscription(); }, TRANSCRIBE_RETRY_DELAY_MS * job.retries); return; } console.warn( `[memories] transcription failed for ${job.filename} after ${job.retries} attempts [${detail}]` + (sizeLabel ? ` [${sizeLabel}]` : "") + `: ${err.message} (dropping for this session; backfill retries next launch)`, ); clearTranscribeProgressFor(job.filename); } finally { transcribeInFlight = false; } const finished = transcribeQueue.shift(); if (finished) clearTranscribeProgressFor(finished.filename); broadcastTranscribeEvent({ phase: transcribeQueue.length === 0 ? "idle" : "done", filename: job.filename, pending: transcribeQueue.length, }); scheduleTranscription(); } // --------------------------------------------------------------------------- // OCR text extraction queue. // // New photos (and launch-backfill photos that predate OCR) are recognized in // the BACKGROUND after their import batch ends — never on the import critical // path. Dispatch is paced and gated exactly like scene enrichment: nothing // runs while an import or migration is in flight or a query IPC is pending, // so search and import latency are unaffected. Each job forks (or reuses) a // DEDICATED ocr-worker utilityProcess — tesseract is CPU-heavy and unrelated // to the CLIP model, so it must never share the indexer worker's serialized // inference queue. Results land in library.ocr[i] (parallel to filenames; // "" = OCR'd but no text found, null = not yet OCR'd) and persist with the // index on the next save. // --------------------------------------------------------------------------- const OCR_GAP_MS = 250; // Persist OCR results every N photos so a crash doesn't lose hours of // progress. saveLibrary writes ~18MB of bin files in <20ms on SSD, // so every 50 photos adds negligible overhead (~60 saves for 3000). const OCR_CHECKPOINT_INTERVAL = 50; let ocrQueue = []; // filenames (photos only — videos are scene-searchable) let ocrInFlight = false; // Purge generation captured for the in-flight photo — mirrors // job.purgeEpoch on the enrich/transcribe pumps (OCR jobs are bare // strings, so the epoch rides alongside the flag instead of the job). let ocrInFlightEpoch = 0; let ocrPendingQuery = false; let ocrTimer = null; let lastOcrStart = 0; let ocrSinceCheckpoint = 0; // One OCR worker at a time (tesseract holds a ~100MB traineddata + wasm // core; a pool would multiply memory for zero throughput gain — a single // worker saturates the CPU it runs on). Forked lazily on the first job of a // drain, killed when the queue empties. let ocrWorker = null; let ocrWorkerDead = false; let ocrRequestId = 0; let ocrPending = new Map(); // id → { resolve, reject, timer } // Bumped by every deliberate worker kill (purge, language-change requeue, // drain). The in-flight photo captures it at arm time; a reject that lands // after a generation change is a kill, not a recognition failure, so the // catch skips the drain-summary failure count (the re-queued attempt, if // any, produces the real verdict). let ocrWorkerGeneration = 0; // OCR progress bookkeeping for the renderer tray (mirrors enrichSnapshot). let ocrDoneCount = 0; let ocrTotalCount = 0; // Failed recognitions in the current drain (worker crash, timeout, model // download failure). Failed rows keep their prior text and are reported in // the drain summary — and they block the upgrade marker so the next launch // retries them instead of stamping them as migrated. let ocrFailedCount = 0; let ocrFailedFiles = []; let lastOcrFile = null; function ocrDataDir() { return path.join(app.getPath("userData"), "ocr-data"); } function broadcastOcrEvent(extra) { for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "ocr", ...extra }); } // Menu-bar tray icon follows background work (no-op when the mode is off). updateTrayState(); } function ocrSnapshot() { const job = ocrInFlight ? ocrQueue[0] : null; return { type: "ocr", paused: backgroundPaused, phase: job ? "ocr" : "queued", filename: job ? job : lastOcrFile, done: ocrDoneCount, total: ocrTotalCount, pending: ocrQueue.length - (job ? 1 : 0), active: Boolean(job), }; } // Content hashes (rename-duplicate dedupe) are stored per row at import; // libraries that predate the feature have none, so a rename wouldn't be // recognized as a duplicate. Hash every library file once in the background // (rows keep their hash across launches, so a big library resumes where it // stopped). Pure file I/O — no model, no indexer needed. async function backfillHashes() { const l = loadLibrary(); if (!l || !Array.isArray(l.filenames)) return; if (!l.hashes) l.hashes = []; let changed = false; for (let i = 0; i < l.filenames.length; i++) { if (l.hashes[i]) continue; const p = path.join(PHOTOS_DIR, l.filenames[i]); if (!fs.existsSync(p)) { l.hashes[i] = null; continue; } try { l.hashes[i] = await hashFile(p); changed = true; } catch { l.hashes[i] = null; } } if (changed) { await saveLibrary(); console.log(`[memories] hash backfill: ${l.filenames.length} rows hashed`); } } // Screenshot-hint backfill: rows that predate the metadata probe (or whose // probe slot is still null) get one bounded read of their app-managed copy — // byte-identical to the original at import time, so no source lookup is // needed and the hint works even after the original was moved or deleted. // Rows keep their hint across launches, so a big library resumes where it // stopped. Pure file I/O — no model, no indexer needed. async function backfillScreenshotHints() { const l = loadLibrary(); if (!l || !Array.isArray(l.filenames)) return; if (!l.screenshotHints) l.screenshotHints = []; let changed = false; let probed = 0; for (let i = 0; i < l.filenames.length; i++) { if (l.screenshotHints[i] === true || l.screenshotHints[i] === false) continue; if (isVideo(l.filenames[i])) { l.screenshotHints[i] = false; changed = true; continue; } try { l.screenshotHints[i] = await probeScreenshotMetadata( path.join(PHOTOS_DIR, l.filenames[i]), ); probed++; changed = true; } catch { // Leave null — retried on the next launch (bounded: pruneStale // removes rows whose copy is gone, so nulls drain to zero). } } if (l.screenshotHints.length > l.filenames.length) { l.screenshotHints.length = l.filenames.length; changed = true; } if (changed) { await saveLibrary(); console.log(`[memories] screenshot-hint backfill: probed ${probed} row(s)`); } } // Photos that never got an OCR pass (pre-OCR imports, or rows whose slot is // still null after a failed pass) are enqueued on launch, one at a time, // under the same idle gates as import-time OCR. Videos are skipped — their // content is scene-searchable; OCR targets the text-heavy stills. A language // stamp mismatch (library OCR'd under different langs than settings) also // re-queues EVERY photo: old rows are stale, not just missing. function backfillOcr() { const l = loadLibrary(); if (!l || !Array.isArray(l.filenames)) return; let currentLangs = null; try { currentLangs = resolveOcrLangString(); } catch { /* backfill covers missing rows even without a stamp */ } // One-time upgrade migration: rows that predate the CJK rollout were all // recognized eng-only (no shipped version ever produced CJK rows), so the // first launch that sees photo rows without the migration marker // force-re-queues the whole library once under the current language set. // The stamp alone can't prove freshness (pre-migration builds stamped // without re-queuing), so only the marker gates this — and only a // zero-failure drain sets it (see pumpOcr). let ocrMigrated = false; try { ocrMigrated = readOcrLangsMigrated(); } catch { /* missing settings → not migrated */ } let ocrPhotoCount = 0; for (const name of l.filenames) { if (!isVideo(name)) ocrPhotoCount++; } if ( currentLangs && shouldMigrateOcrLangs({ photoCount: ocrPhotoCount, migrated: ocrMigrated }) ) { console.log( `[memories] OCR upgrade migration: re-queuing ${ocrPhotoCount} photo(s) under ${currentLangs} (pre-multilang rows were eng-only)`, ); l.ocrLangs = currentLangs; void saveLibrary(); requeueAllPhotosForOcr("upgrade"); return; } // Language change across launches: stamp the new set and re-OCR all. if (currentLangs && l.ocrLangs && l.ocrLangs !== currentLangs) { console.log( `[memories] OCR language change ${l.ocrLangs} → ${currentLangs}: re-queuing library`, ); l.ocrLangs = currentLangs; void saveLibrary(); requeueAllPhotosForOcr("language-change"); return; } // First stamp: adopt the current setting without re-queuing (fresh or // pre-stamp libraries keep their eng-only rows until the user changes // languages — the set-langs IPC path handles the full re-OCR then). if (currentLangs && !l.ocrLangs && l.filenames.length > 0) { l.ocrLangs = currentLangs; void saveLibrary(); } const missing = []; for (let i = 0; i < l.filenames.length; i++) { if (isVideo(l.filenames[i])) continue; const words = l.ocrWords ? l.ocrWords[i] : null; if (words === null) { // Never OCR'd with geometry (pre-geometry row, failed pass). missing.push(l.filenames[i]); continue; } // INCONSISTENT row: searchable text exists but no word geometry // stands behind it. This is the raw-fallback signature — an older // pass hit tesseract's no-blocks path, whose cleanText branch keeps // UNFILTERED text while normalizeWords returns []. Such rows rank // in the OCR tab but can never draw a highlight box. Re-OCR them; // genuinely-empty rows (no text AND no words) stay skipped, so a // text-free photo is never retried forever. if ( words.length === 0 && typeof l.ocr[i] === "string" && l.ocr[i].length > 0 ) { missing.push(l.filenames[i]); } } if (missing.length > 0) { console.log( `[memories] OCR backfill: ${missing.length} photo(s) missing word geometry`, ); enqueueOcr(missing); } } function enqueueOcr(files, options) { if (!Array.isArray(files)) return; const force = Boolean(options && options.force); const l = loadLibrary(); let added = 0; for (const name of files) { const idx = l.filenames.indexOf(name); // Unknown row, already queued, or already OCR'd with geometry AND // consistent text (a repeat import must not revisit a healthy row; // text-but-no-geometry rows are re-enqueued deliberately by the // backfill's inconsistency pass). force:true (language change) // bypasses the healthy-row skip — old rows are stale by definition. if (idx === -1) continue; if (ocrQueue.includes(name)) continue; if (!force) { const words = l.ocrWords ? l.ocrWords[idx] : null; if (words !== null && words.length > 0) continue; if ( words !== null && words.length === 0 && !(typeof l.ocr[idx] === "string" && l.ocr[idx].length > 0) ) { // Empty geometry on a genuinely text-free row — nothing to fix. continue; } } ocrQueue.push(name); added++; } if (added > 0) { ocrTotalCount = ocrQueue.length; broadcastOcrEvent({ phase: "queued", pending: ocrQueue.length }); } scheduleOcr(); return added; } // Re-queue every photo for OCR (language change): old text stays visible // until each row's re-OCR lands — rows are overwritten in place by pumpOcr, // never cleared. Returns the queued count for the settings toast. function requeueAllPhotosForOcr(reason) { const l = loadLibrary(); if (!l || !Array.isArray(l.filenames)) return 0; const photos = l.filenames.filter((name) => !isVideo(name)); // A language change restarts the worker so the next job loads the new // traineddata set. The kill rejects the in-flight photo's pending // request; its catch sees the generation change and skips the failure // count, and the force re-queue below retries it under the new worker. killOcrWorker(); const added = enqueueOcr(photos, { force: true }) || 0; if (added > 0) { console.log( `[memories] OCR re-queue (${reason || "manual"}): ${added} photo(s)`, ); } return added; } function scheduleOcr() { if (ocrTimer) return; ocrTimer = setTimeout(() => { ocrTimer = null; void pumpOcr(); }, OCR_GAP_MS); } function ocrAllowed() { return ( !backgroundPaused && !ocrInFlight && !importInFlight && !migrationState && !ocrPendingQuery && !enrichPendingQuery && !transcribePendingQuery && ocrQueue.length > 0 && Date.now() - lastOcrStart >= OCR_GAP_MS ); } // Fork the OCR worker if it isn't alive. Returns the worker object. function ensureOcrWorker() { if (ocrWorker && !ocrWorkerDead) return ocrWorker; ocrWorkerDead = false; const worker = { process: null, dead: false, }; // Language set comes from settings.json (eng always on + CJK toggles); // each *.traineddata downloads once into ocr-data/ and is reused offline. let ocrLangs = "eng+chi_sim+chi_tra+jpn+kor"; try { ocrLangs = resolveOcrLangString(); } catch { /* default above */ } worker.process = utilityProcess.fork( path.join(__dirname, "indexer", "ocr-worker.js"), [], { env: { ...process.env, // *.traineddata (~17MB for the full default set, downloaded // once from the tesseract CDN on the first OCR ever) lands // here and is reused offline. OCR_DATA_DIR: ocrDataDir(), OCR_LANGS: ocrLangs, }, serviceName: "ocr-worker", }, ); worker.process.on("message", (event) => { const msg = event && event.data !== undefined ? event.data : event; if (!msg || !msg.id) return; const req = ocrPending.get(msg.id); if (!req) return; ocrPending.delete(msg.id); clearTimeout(req.timer); if (msg.ok) req.resolve(msg); else req.reject(new Error(msg.error || "OCR failed")); }); worker.process.on("exit", () => { worker.dead = true; ocrWorkerDead = true; ocrWorker = null; for (const [, req] of ocrPending) { clearTimeout(req.timer); req.reject(new Error("OCR worker exited")); } ocrPending = new Map(); }); ocrWorker = worker; return worker; } function askOcr(filename) { const worker = ensureOcrWorker(); return new Promise((resolve, reject) => { const id = `o${++ocrRequestId}`; const timer = setTimeout(() => { ocrPending.delete(id); reject(new Error("OCR request timed out")); }, 120000); ocrPending.set(id, { resolve, reject, timer }); worker.process.postMessage({ type: "ocr-photo", id, path: path.join(PHOTOS_DIR, filename), }); }); } async function pumpOcr() { if (!ocrAllowed()) { if (ocrQueue.length > 0 && !backgroundPaused) scheduleOcr(); return; } const filename = ocrQueue[0]; // Row deleted while queued — drop without a tesseract run or a failed // entry in the drain summary. if (!libraryHasRow(filename)) { console.log( `[memories] OCR ${filename}: row deleted while queued — dropping job`, ); ocrQueue.shift(); broadcastOcrEvent({ phase: ocrQueue.length === 0 ? "idle" : "ocr", filename: lastOcrFile, done: ocrDoneCount, total: ocrTotalCount, pending: ocrQueue.length, failed: ocrFailedCount, }); scheduleOcr(); return; } ocrInFlight = true; ocrInFlightEpoch = purgeEpoch; const ocrGen = ocrWorkerGeneration; lastOcrStart = Date.now(); lastOcrFile = filename; try { const reply = await askOcr(filename); // Rows can shift while OCR runs (a delete, import, or purge can // land mid-photo); resolve by filename so the text lands on the // right entry — a vanished, purged, or tombstoned file is skipped. // The tombstone is consumed unconditionally (never inside the // commit condition) so a leaked entry can't discard a future // re-queued job for the same filename. const cancelled = isJobDead(filename); const l = loadLibrary(); const idx = l.filenames.indexOf(filename); if ( idx !== -1 && reply.ok && typeof reply.text === "string" && ocrInFlightEpoch === purgeEpoch && !cancelled ) { if (!l.ocr) l.ocr = []; if (!l.ocrWords) l.ocrWords = []; l.ocr[idx] = reply.text; l.ocrWords[idx] = Array.isArray(reply.words) ? reply.words : []; l.ocrRevision = (l.ocrRevision || 0) + 1; // Adopt the language stamp on first OCR (fresh installs have no // stamp yet — backfillOcr stamps at launch, the set-langs IPC // stamps on change; this covers rows OCR'd before either ran). if (!l.ocrLangs) { try { l.ocrLangs = resolveOcrLangString(); } catch { /* stamp optional — backfill covers it */ } } ocrDoneCount++; console.log( `[memories] OCR ${filename}: ${reply.text.length > 0 ? JSON.stringify(reply.text.slice(0, 60)) + "…" : "(no text)"}`, ); // Incremental checkpoint: persist every N photos so a crash // doesn't lose all OCR progress. if (++ocrSinceCheckpoint >= OCR_CHECKPOINT_INTERVAL) { ocrSinceCheckpoint = 0; await saveLibrary(); // Progressive refresh: a full backfill (first launch on an // existing library, or a big import batch) can run for tens of // minutes, and this broadcast used to wait for the ENTIRE queue // to drain — the whole time, tiles showed no query highlights // and the OCR tab found nothing, so the feature read as broken. // Each checkpoint bumps ocrRevision, which is part of the // renderer's index fingerprint, so it reloads the refreshed // metadata without touching the unchanged embedding bins. for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "library-updated" }); } } } else if (cancelled || ocrInFlightEpoch !== purgeEpoch) { // Purged or deleted mid-photo: the reply belongs to cancelled // work, not to a vanished row (that case stays silent as // before). The tail shift below drops the job either way. console.log( `[memories] OCR ${filename}: purged or deleted mid-flight — discarding reply`, ); } } catch (err) { // A failed OCR leaves a never-OCR'd slot null so a later backfill // retries it; a previously-healthy row keeps its prior text. Either // way the failure counts toward the drain summary and blocks the // upgrade marker, so failed rows retry on the next launch instead of // being stamped as migrated. OCR failure never touches imports or // search. A job dropped by purge/delete/reset is none of that — // it is no longer queued, so skip it quietly instead of polluting // the drain summary with a file that is gone or was cancelled. if (!ocrQueue.includes(filename)) { console.log( `[memories] OCR ${filename}: job dropped while running — skipping`, ); } else if (ocrGen !== ocrWorkerGeneration) { // Deliberate kill (language-change restart, drain kill) landed // mid-photo — not a recognition failure. The re-queued attempt // carries on, so don't pollute the drain summary. console.log( `[memories] OCR ${filename}: worker restarted mid-photo — skipping failure count`, ); } else { ocrFailedCount++; if (ocrFailedFiles.length < 5) ocrFailedFiles.push(filename); console.warn(`[memories] OCR failed for ${filename}: ${err.message}`); } } finally { ocrInFlight = false; } ocrQueue.shift(); // Tray: this photo finished — the next job's progress takes over, or the // queue drained into "idle". broadcastOcrEvent({ phase: ocrQueue.length === 0 ? "idle" : "ocr", filename: lastOcrFile, done: ocrDoneCount, total: ocrTotalCount, pending: ocrQueue.length, failed: ocrFailedCount, }); if (ocrQueue.length === 0) { // Queue drained: persist the OCR text (saveLibrary writes the whole // index trio, so the renderer's next /memories-index.json fetch sees // it), free the tesseract process, and tell the renderer to reload. await saveLibrary(); killOcrWorker(); if (ocrFailedCount > 0) { const names = ocrFailedFiles.join(", "); const more = ocrFailedCount > ocrFailedFiles.length ? `, +${ocrFailedCount - ocrFailedFiles.length} more` : ""; console.warn( `[memories] OCR drain finished: ${ocrDoneCount} recognized, ${ocrFailedCount} failed (${names}${more}) — failed rows keep prior text and retry on next launch (first CJK run downloads ~17MB of traineddata: check network if every row fails)`, ); } else { console.log( `[memories] OCR drain finished: ${ocrDoneCount} recognized, 0 failed`, ); // A zero-failure drain proves every row is current-language: record // the upgrade marker so later launches skip the migration. Failed // drains deliberately leave it unset (see readOcrLangsMigrated). try { if (!readOcrLangsMigrated()) writeSettings({ ocrLangsMigrated: true }); } catch (err) { console.warn( `[memories] OCR migration marker write skipped: ${err.message}`, ); } } ocrDoneCount = 0; ocrTotalCount = 0; ocrFailedCount = 0; ocrFailedFiles = []; ocrSinceCheckpoint = 0; for (const win of BrowserWindow.getAllWindows()) { win.webContents.send("memories:status", { type: "library-updated" }); } return; } scheduleOcr(); } function killOcrWorker() { if (ocrWorker && !ocrWorker.dead) { try { ocrWorker.process.kill(); } catch { /* already gone */ } } ocrWorker = null; ocrWorkerDead = true; ocrWorkerGeneration++; } // --------------------------------------------------------------------------- // Photo thumbnails: the grid renders a ~480px JPEG instead of the original — // camera originals average 2MB+ and GIFs can be hundreds of MB, so decoding // those per tile was the app's biggest memory/jank source. Generated lazily // on the first /images/thumbs/* request (covers legacy libraries without a // backfill pass) and cached on disk forever, with per-filename single-flight // plus a small global cap so a first browse can't fan out one sharp decode // per tile. // --------------------------------------------------------------------------- const THUMB_WIDTH = 480; const THUMB_CONCURRENCY = 2; const thumbJobs = new Map(); // filename → Promise const thumbSlots = Array.from({ length: THUMB_CONCURRENCY }, () => Promise.resolve(), ); let thumbSlotIndex = 0; async function ensureThumb(filename) { const out = thumbFor(filename); try { await fs.promises.access(out); return out; } catch { /* not generated yet */ } const existing = thumbJobs.get(filename); if (existing) return existing; const slot = thumbSlotIndex++ % THUMB_CONCURRENCY; const job = thumbSlots[slot] .then(async () => { try { await fs.promises.access(out); return out; } catch { /* generate below */ } fs.mkdirSync(THUMBS_DIR, { recursive: true }); const sharp = (await import("sharp")).default; // First page only (animated GIFs collapse to their first frame), // EXIF-rotated, downscaled, quality-tuned JPEG. await sharp(path.join(PHOTOS_DIR, filename)) .rotate() .resize({ width: THUMB_WIDTH, withoutEnlargement: true }) .jpeg({ quality: 82 }) .toFile(out); return out; }) .finally(() => { thumbJobs.delete(filename); }); thumbSlots[slot] = job.catch(() => {}); thumbJobs.set(filename, job); return job; } // --------------------------------------------------------------------------- // Protocol: renderer bundle + library files, all under app:// // --------------------------------------------------------------------------- async function resolveRenderer(p) { if (p === "/" || p === "") p = "/index.html"; const filePath = path.normalize(path.join(RENDERER_DIR, p)); if ( !filePath.startsWith(RENDERER_DIR + path.sep) && filePath !== RENDERER_DIR ) return null; try { const stat = await fs.promises.stat(filePath); if (stat.isFile()) return filePath; } catch { /* fall through */ } if (!path.extname(p)) { const withHtml = filePath + ".html"; try { if ((await fs.promises.stat(withHtml)).isFile()) return withHtml; } catch { /* 404 below */ } } return null; } // A web ReadableStream over a byte window of a file — the streaming // equivalent of readFile().subarray(). The file is OPENED first so a // vanished file is a clean ENOENT (→ 404 from the caller's catch) instead // of an aborted response after headers were already sent. The handle is // released when the stream ends or errors. function openFileStreamRange(filePath, start, end) { return fs.promises.open(filePath, "r").then((handle) => { const nodeStream = fs.createReadStream(null, { fd: handle.fd, start, end, autoClose: false, }); const release = () => handle.close().catch(() => {}); nodeStream.on("error", release); nodeStream.on("end", release); nodeStream.on("close", release); return Readable.toWeb(nodeStream); }); } async function handleAppRequest(request) { const urlPath = safeDecode(new URL(request.url).pathname); const headers = new Headers(); const respond = (status, body, contentType, extra = {}) => { if (contentType) headers.set("Content-Type", contentType); for (const [k, v] of Object.entries(extra)) headers.set(k, v); return new Response(body, { status, headers }); }; // Library files (served from memory so imports are atomic) if (urlPath === "/memories-index.json") { const idx = libraryIndex(); const l = loadLibrary(); const etag = `"idx-${l.filenames.length}-${l.dim}-${l.modelId}"`; if (request.headers.get("if-none-match") === etag) { return respond(304, null, null); } return respond( 200, JSON.stringify(idx), "application/json; charset=utf-8", { "Cache-Control": "no-store", ETag: etag, }, ); } if (urlPath === "/memories-models.json") { return respond( 200, JSON.stringify(modelsManifest()), "application/json; charset=utf-8", { "Cache-Control": "no-store", }, ); } if ( urlPath === "/memory-embeddings.bin" || urlPath === "/memory-phrase-embeddings.bin" ) { const bins = cachedLibraryBins(); const body = urlPath.endsWith("phrase-embeddings.bin") ? bins.phrases : bins.embeddings; // ETag from the bin cache key (modelId|count|dim|phraseDim): the // renderer caches parsed Float32Arrays keyed on this, so a 304 // means "same data — don't re-parse thousands of Float32Array views". const etag = `"${bins.key}"`; if (request.headers.get("if-none-match") === etag) { return respond(304, null, null); } return respond(200, body, "application/octet-stream", { "Cache-Control": "no-store", ETag: etag, }); } // Scene-segment sidecars (Phase 1): ok:false means "no scene data for the // active model" — the renderer treats that as a SUCCESS (file-level // search is unchanged), never an error. if (urlPath === "/memory-segments.json") { const l = loadLibrary(); const c = loadSegments(l.modelId); if (!c.loaded) { return respond( 200, JSON.stringify({ ok: false }), "application/json; charset=utf-8", { "Cache-Control": "no-store", }, ); } const videos = [...c.videos.entries()].map(([filename, segments]) => ({ filename, segments, })); return respond( 200, JSON.stringify({ ok: true, version: 1, modelId: c.modelId, dim: c.dim, total: c.rows.length, videos, }), "application/json; charset=utf-8", { "Cache-Control": "no-store" }, ); } if (urlPath === "/memory-segment-embeddings.bin") { const c = loadSegments(loadLibrary().modelId); if (!c.loaded) return respond(404, "Not found", "text/plain"); return respond(200, cachedSegmentsBin().buf, "application/octet-stream", { "Cache-Control": "no-store", }); } // Transcript sidecars (speech index): ok:false means "no transcript data // for the active model" — renderer treats as SUCCESS (visual-only search // unchanged), never an error. Same header validation as segments. if (urlPath === "/memory-transcripts.json") { const l = loadLibrary(); const c = loadTranscripts(l.modelId); if (!c.loaded) { return respond( 200, JSON.stringify({ ok: false }), "application/json; charset=utf-8", { "Cache-Control": "no-store", }, ); } const videos = [...c.videos.entries()].map(([filename, chunks]) => ({ filename, chunks, })); return respond( 200, JSON.stringify({ ok: true, version: 1, modelId: c.modelId, dim: c.dim, total: c.rows.length, videos, }), "application/json; charset=utf-8", { "Cache-Control": "no-store" }, ); } if (urlPath === "/memory-transcript-embeddings.bin") { const c = loadTranscripts(loadLibrary().modelId); if (!c.loaded) return respond(404, "Not found", "text/plain"); return respond( 200, cachedTranscriptsBin().buf, "application/octet-stream", { "Cache-Control": "no-store", }, ); } // Photo thumbnails (lazy-generated, immutable — a thumbnail for a given // filename never changes). On a generation failure the request falls // through to the original-file route below so a tile never breaks (the // no-store fallback keeps the browser honest until the thumb exists). if (urlPath.startsWith("/images/thumbs/")) { const filename = path.basename(urlPath); if ( filename && IMAGE_EXTENSIONS.has(path.extname(filename).toLowerCase()) ) { try { const body = await fs.promises.readFile(await ensureThumb(filename)); return respond(200, body, "image/jpeg", { "Cache-Control": "public, max-age=31536000, immutable", }); } catch (err) { console.warn( `[memories] thumbnail failed for ${filename}: ${err.message}`, ); /* fall through to the original-file route below */ } } } // Posters (generated video thumbnails; never part of the index) if (urlPath.startsWith("/images/posters/")) { const filePath = posterFor(path.basename(urlPath)); // never allow traversal try { const body = await fs.promises.readFile(filePath); return respond(200, body, "image/jpeg", { "Cache-Control": "public, max-age=31536000, immutable", }); } catch { return respond(404, "Not found", "text/plain"); } } // In-app preview proxies for searchable-only videos (mkv/hevc/...): a // short mp4/h264 clip transcoded on demand around ?t= (the matched scene // timestamp). Deterministic cache name per (file, t) so repeat views // never re-transcode; failures are 404 so the lightbox falls back to the // poster. Range support mirrors /images/projects/ so