//! Persistent vector store for native memory (real embeddings, m2). //! //! Each native_memory entry that is indexed gets a normalized vector persisted //! under `~/.nur/native-memory//vectors.json` keyed by entry id. Search is //! cosine top-k over the L2-normalized vectors (from `embed::embed`). //! //! The vector is a *denormalized index complement*: the text lives in the //! memory entry; the vector makes semantic top-k retrieval fast/precise. When a //! memory is retired or deleted we drop its vector (keep the archive row). use crate::agent::embed; use crate::config::{atomic_write, nur_home}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::path::PathBuf; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct VectorDoc { pub id: String, pub vec: Vec, /// Which embedding source produced this (api|local) — for honesty/telemetry. pub source: String, } pub struct VectorStore { scope: String, docs: BTreeMap, } fn store_path(scope: &str) -> PathBuf { let safe: String = scope .chars() .map(|c| { if c.is_alphanumeric() || c == '-' || c == '_' { c } else { '_' } }) .take(80) .collect(); nur_home() .join("native-memory") .join(safe) .join("vectors.json") } impl VectorStore { /// Open the store for a scope, loading persisted vectors. pub fn open(scope: &str) -> Self { let docs = std::fs::read_to_string(store_path(scope)) .ok() .and_then(|t| serde_json::from_str::>(&t).ok()) .unwrap_or_default(); Self { scope: scope.to_string(), docs, } } #[cfg(test)] fn len(&self) -> usize { self.docs.len() } pub fn is_empty(&self) -> bool { self.docs.is_empty() } /// Fetch the persisted vector for a memory id. Helix mirroring reuses this /// exact vector so the local and remote indexes never drift dimension or /// embedding source. pub fn get(&self, id: &str) -> Option { self.docs.get(id).cloned() } /// Index (embed + persist) a memory entry's text under its id. pub fn index(&mut self, id: &str, text: &str) -> String { // Auto mode writes a local vector first, then upgrades it remotely in a // detached task when a provider key and an unbudgeted session allow it. // A durable memory write therefore never waits on an embedding network // round trip. Explicit `api` mode keeps its documented synchronous path. let (vec, source, upgrade) = if crate::agent::embed::remote_upgrade_enabled() { ( crate::agent::embed::embed_local_with_telemetry(text, "local-pending-remote"), "local-pending-remote".to_string(), true, ) } else { let (vec, source) = crate::agent::embed::embed_with_source(text); (vec, source.to_string(), false) }; let source_for_return = source.clone(); self.docs.insert( id.to_string(), VectorDoc { id: id.to_string(), vec, source, }, ); let _ = self.save(); if upgrade { schedule_remote_upgrade(self.scope.clone(), id.to_string(), text.to_string()); } source_for_return } /// Remove a vector by entry id (memory retired/deleted). pub fn remove(&mut self, id: &str) -> bool { let removed = self.docs.remove(id).is_some(); if removed { let _ = self.save(); } removed } /// Top-k semantic neighbors of `query` via cosine. Returns (id, score). pub fn search(&self, query: &str, k: usize) -> Vec<(String, f32)> { let q = embed::embed(query); self.search_with_embedding(&q, k) } /// Top-k semantic neighbors using a query embedding computed by the caller. /// This lets the memory router share one embedding across every resident. pub fn search_with_embedding(&self, query_embedding: &[f32], k: usize) -> Vec<(String, f32)> { let mut scored: Vec<(String, f32)> = self .docs .values() .map(|d| (d.id.clone(), embed::cosine(query_embedding, &d.vec))) .collect(); scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); scored.truncate(k.max(1)); scored } /// The ids whose similarity to `query` exceeds `threshold` (for routing). pub fn above(&self, query: &str, threshold: f32, k: usize) -> Vec<(String, f32)> { self.search(query, k) .into_iter() .filter(|(_, s)| *s >= threshold) .collect() } pub fn source_distribution(&self) -> BTreeMap { let mut m: BTreeMap = Default::default(); for d in self.docs.values() { *m.entry(d.source.clone()).or_default() += 1; } m } fn save(&self) -> Result<(), String> { let p = store_path(&self.scope); if let Some(parent) = p.parent() { std::fs::create_dir_all(parent).map_err(|e| e.to_string())?; } let body = serde_json::to_string_pretty(&self.docs).map_err(|e| e.to_string())?; atomic_write(&p, body.as_bytes()).map_err(|e| e.to_string()) } } fn schedule_remote_upgrade(scope: String, id: String, text: String) { let _ = std::thread::Builder::new() .name("nur-memory-embed".into()) .spawn(move || { let Ok(vec) = crate::agent::embed::embed_api(&text) else { return; }; let mut store = VectorStore::open(&scope); // Do not resurrect entries retired between the local write and the // remote reply. Updating an existing id is safe and idempotent. if let Some(doc) = store.docs.get_mut(&id) { doc.vec = vec; doc.source = "api-background".into(); let _ = store.save(); if let Some(entry) = crate::agent::native_memory::get_by_id(&scope, &id) { if !entry.retired { crate::agent::helix_memory::enqueue(&scope, &entry); } } } }); } #[cfg(test)] mod tests { use super::*; fn scope() -> String { format!("vec-test-{}", uuid::Uuid::new_v4().simple()) } #[test] fn index_search_and_remove() { let s = scope(); let mut vs = VectorStore::open(&s); let src_a = vs.index("mem-a", "I prefer grep over bash find for search"); let _ = vs.index("mem-b", "deploy pipeline runs on kubernetes"); assert_eq!(vs.len(), 2); // Semantic-ish search should rank mem-a near "which search tool do I prefer". let hits = vs.search("how should I search files grep or bash", 2); assert_eq!(hits.len(), 2); assert_eq!(hits[0].0, "mem-a", "expected mem-a most similar: {hits:?}"); assert!( hits[0].1 > hits[1].1, "mem-a should outrank mem-b: {hits:?}" ); // Remove assert!(vs.remove("mem-b")); assert_eq!(vs.len(), 1); let _ = std::fs::remove_dir_all(crate::config::nur_home().join("native-memory").join(&s)); let _ = src_a; } #[test] fn open_reloads_persisted() { let s = scope(); { let mut vs = VectorStore::open(&s); vs.index("x", "remember this fact about tokyo weather"); } let vs = VectorStore::open(&s); assert_eq!(vs.len(), 1); let hits = vs.search("what is the weather in tokyo", 1); assert_eq!(hits[0].0, "x"); let _ = std::fs::remove_dir_all(crate::config::nur_home().join("native-memory").join(&s)); } }