#pragma once #if !defined( RIPWIRE_INGEST_TU ) #error "ingest_cache.h is a SECTION of src/ingest.cpp's translation unit - include it only from ingest.cpp (see the ingest-family split note there)" #endif #include "infra/emit.h" // rw::emitTo / emitRaw / formatTo — THE emitter and its siblings #include "pathguard.h" // CWE-59/367 round 5: rw::pathguard::createExclTempFile — saveCache's temp is created exclusively, never through a link #include // %.*s (precision, pointer) collapses to one view #include // std::endian — the cache blob's arch tag, without a compiler-specific macro #include // std::unique_ptr — CacheRecords holds its open frame #include // std::optional — cacheBlobIdentity: "this blob's identity" vs "there is no blob to identify" #include // std::numeric_limits — saturatedTo // ingest_cache.h — the raw-facts model + incremental cache, moved VERBATIM from ingest.cpp in the // 2026-08-29 split: RawDef (the pre-id-assignment definition record), the extraction identity // (kCacheVersion + kParserVer + parserVerFor), content/blob hashing, FileFacts, the ByteW/ByteR // codecs with every def/ref/bind/ffi/route record writer/reader and their record-minima tripwire, // and loadCache/saveCache themselves. The one place the on-disk cache blob's shape is spelled out. // Same contract as every ingest_*.h: reopens `namespace rw` and the unnamed namespace inside it — // one TU, one unnamed namespace, internal linkage unchanged, zero new API surface — under the // RIPWIRE_INGEST_TU guard. namespace rw { namespace { // A walk count stored in a narrower RawDef field: clamped to the field's maximum (every such count is past every bar there). template inline constexpr Narrow saturatedTo( std::uint32_t v ) noexcept { return Narrow( std::min( v, std::numeric_limits::max() ) ); } // ---- a raw definition pulled from one query match (pre-id-assignment) ---- struct RawDef { std::uint32_t fileId = 0; std::uint32_t line = 0; // 1-based std::uint32_t startByte = 0; // span start of the @definition node std::uint32_t endByte = 0; // span end of the @definition node std::uint32_t nameByte = 0; // start byte of the @name identifier (dedup identity) std::uint32_t bodyByte = 0; // start of the def's "body" field = signature end (0 ⇒ none → endByte) std::uint32_t cx = 0; // cyclomatic complexity (1 + decision points); functions/methods only std::uint32_t ccx = 0; // cognitive complexity (nesting-weighted); functions/methods only std::uint32_t loc = 0; // Q4: physical line span of the def (end line − start line + 1) std::uint32_t locals = 0; // Phase 1 (local-variable-indexing, docs/LOCALS_INDEXING.md): local-decl // count from cc_walk; C/C++ only (see model.h localsCountedLang), 0 elsewhere std::uint16_t ppAlt = 0; // preproc alternative branches (#else/#elif) inside the def (see model.h Symbol::ppAlt) std::uint16_t humps = 0; // nesting profile: regions reaching quality::kNestBar (see model.h Symbol::humps) std::uint16_t deepLoc = 0; // nesting profile: lines inside them, a FLOOR (see model.h Symbol::deepLoc) std::uint16_t ev = 0; // essential complexity, a FLOOR (see model.h Symbol::ev); 0 outside evCountedLang std::array evWhy{}; // per-tag contributing-jump counts (model.h kEvWhyTagTable) std::uint16_t params = 0; // Q4: parameter count (from the def's parameter-list child); fns/methods std::uint16_t paramsMin = 0; // B2.2: the fewest arguments a call can pass (params − defaulted); 0 = no lower bound known std::uint8_t maxNest = 0; // Q4: max control-structure nesting depth inside the def (from cc_walk) std::uint8_t steer = 0; // QD-TANGLE steering locals (model.h Symbol::steer; from cc_walk, saturating at 255) std::uint16_t deepDispatch = 0; // QD-TANGLE: deepLoc lines inside short flat-dispatch arms (model.h Symbol::dispatchShare; saturating) std::uint8_t nestArmCut = 0; // QD-TANGLE: maxNest less the deepest nesting outside those arms (model.h Symbol::dispatchShare) std::uint8_t arityExact = 0; // B2.2: 1 ⇒ params is a fixed call-comparable arity (no variadic/default, not implicit-self) std::uint8_t testScope = 0; // L8: 1 ⇒ an IN-FILE test convention encloses this def (see inFileTestScope) std::uint8_t recovered = 0; // extent honesty (extentsuspect.h kRecovered*): the parse RECOVERED this def's container // (a class whose body holds an error, inside an ERROR region) or its kind (a scopeless // C++ method inside one); 0 ⇒ no recovery claim. Feeds the `error` reason at load. std::uint8_t internalLinkage = 0; // C/C++: anonymous-namespace or namespace-scope `static` def (model.h Symbol::internalLinkage) std::uint8_t scopeRootsStd = 0; // #150: 1 ⇒ this def's full enclosing-namespace chain roots at std (model.h Symbol::scopeRootsStd) std::uint8_t rankFlags = 0; // kRankFlag* (model.h Symbol::rankFlags): set by the doc post-pass on every load, never in the cache record std::uint32_t fnScopeStart = 0; // a FUNCTION-LOCAL def (ingest_names.h enclosingFunctionScope): the byte span of the std::uint32_t fnScopeEnd = 0; // function whose body binds its name; End 0 ⇒ not function-local (model.h fnLocalScopes) SymKind kind = SymKind::Other; Lang lang = Lang::Unknown; std::string name; std::string scope; // enclosing class/namespace name (C++), for canonical scope::name std::uint64_t cloneKey = 0; // CLONES-PERSIST (ingest_parsepool.h rawDefCloneKey): Function/Method body's exact-clone key; 0 = none RawDefLex lex; // B0.2: doc+body weighted subtoken stats (rich ingests only; rides the // cache record so a warm --for never re-reads/re-tokenizes the file) }; struct RawRef : RefColdFields // fieldName, composeRel, memberPath, memberCtor, memberVia: model.h RefColdFields { std::uint32_t fileId = 0; std::uint32_t startByte = 0; // span start of the @reference node (for enclosing lookup) std::uint32_t line = 0; // 1-based line of the use site (ABS-3: --uses p="file:line") Lang lang = Lang::Unknown; bool isInherit = false; // true ⇒ base-class/implements edge (derived → base), not a call bool isDocLink = false; // true ⇒ doc→code mention (markdown `backtick` identifier), not a call bool isCompose = false; // true ⇒ HAS-A member-variable type edge (S5-E); stays OUT of call graph RefRole role = RefRole::Call; // ABS-3 use-site role (call/read/write/import/extends); see RefRole RecvKind recv = RecvKind::None; // call-site receiver shape (P2-D narrowing) std::uint16_t argCount = 0; // B2.2: call-site positional arg count when countable; 0 otherwise bool argCountKnown = false; // B2.2: true ⇒ argCount is reliable (no spread/splat/apply) bool viaArrow = false; // a C++/ObjC call: the member access was written `->`. A compose ref: `name` is the POINTEE // of a std smart pointer member, which only `->` reaches (see Reference::viaArrow) bool qualifierRootsStd = false; // #150: the FULL written qualifier chain is rooted at namespace std (model.h Reference::qualifierRootsStd) bool memberCall = false; // FE-A: a Go/JS/TS/Rust call written on a receiver (`x.f()`), whose recv stays None (model.h Reference::memberCall) rw::InternedText name; // MEM-REF-SHRINK step 2: name / qualifier / recvVar are 4 B pool ids (infra/internedtext.h) rw::InternedText qualifier; // explicit scope at a call site (`A` in `A::b()`); C++; "" if bare/method rw::InternedText recvVar; // receiver variable when recv==NamedVar/FieldOfVar (`x` in `x->m()`); Rule 2 fuel rw::InternedText memberRoot; // FE-A: the receiver chain's ROOT identifier when memberCall (`JSON` in `JSON.parse()`); see model.h bool memberFactory = false; // FE-B on #373: memberCtor names a FactoryBot factory, not a class bool forwarded = false; // lane fcall-forward fix 1: a JS/TS `f.call/f.apply` site standing for `f( … )` (model.h Reference::forwarded) bool trailingLambda = false; // #155: a Kotlin call whose lambda is written after its argument list (model.h Reference::trailingLambda) std::uint32_t baseBind = 0; // lane lego-transitive fix 1, an inherit ref only: 1 + the name byte of the same-file // declaration the base binds to (ingest_relations.h localBaseBinding); 0 none, ~0u unknown std::uint32_t litAt = 0; // MEM-R1a: a registrar Value ref's 1 + object-literal start byte (model.h Reference::litAt) }; // MEM-REF-SHRINK (model.h, beside Reference's own budget): Reference minus fromSymbol, in the same 64 B on every ABI (MEM-R1a: // memberRoot a pool id; step 2 80 B on libc++, step 1 136, before it 248). Every worker's vector and the merged one hold // these rows at the parse-pool peak. static_assert( sizeof( RawRef ) == 64, "RawRef is one 64 B row: see the MEM-REF-SHRINK note in model.h" ); // P2-D Rule 2 raw local-variable type binding (pre-attribution). startByte sits inside the enclosing // function body so the enclosing-def attribution (the same byte-span scan as RawRef) assigns the binding's // scope. `var : typeName` — e.g. `Foo x;` → { "x", "Foo" }. typeName is the written type's final segment. struct RawBind { std::uint32_t fileId = 0; std::uint32_t startByte = 0; // position inside the enclosing function (for enclosing-def attribution) Lang lang = Lang::Unknown; LocalBindKind kind = LocalBindKind::Type; // Type = Rule 2 var→type; FnDecl/FnAssign = L3 var→function bool isFromAssignment = false; // kind==Type: an ASSIGNMENT's callee name, not a declaration's (model.h Binding) std::uint8_t resultIndex = kWholeResult; // kind==CallResult/ReturnType: the result position (model.h LocalBindKind) std::uint8_t unwrapDepth = 0; // kind==CallResult/ReturnType: unwrap operators / layers (model.h LocalBindKind) std::uint32_t spanStart = 0; // lexical visibility or declaration span; see model.h Binding/LocalBindKind std::uint32_t spanEnd = 0; rw::InternedText var; // the declared variable identifier (`x`). MEM-B1: pool ids, as model.h Binding's rw::InternedText importedName; // ES export identity or Elixir callable/import fact; see LocalBindKind. rw::InternedText typeName; // kind==Type: the written type's final segment (`Foo`); // kind==FnDecl/FnAssign: the bound function name (or an L3 sentinel: a fixed text) }; // MEM-B1: four u32 positions, five u8 facts in one padded u32 pair, three pool ids: 36 B on every ABI (no std::string member). static_assert( sizeof( RawBind ) == 9 * sizeof( std::uint32_t ), "RawBind grew: see the MEM-B1 note at model.h Binding" ); // B6.3 raw client-side HTTP-route USE (pre-attribution). startByte sits at the call-expression's own // start so the enclosing-def byte-span scan (same DefSweep as RawRef/RawBind) assigns fromSymbol. The // server-side RouteDef needs no such raw/final split (its handler is resolved by NAME, see graph.h), so // it rides model.h's own RouteDef struct straight through the cache — same posture as BindingAlias. struct RawRouteUse { std::uint32_t fileId = 0; std::uint32_t startByte = 0; // call-expression start (for enclosing-def attribution) std::uint32_t line = 0; // 1-based HttpMethod method = HttpMethod::Unknown; std::string path; }; // ---- incremental cache (--cache): per-file content hash + raw facts so a re-run re-parses ONLY // changed files. A parserVer bump invalidates the whole cache on any extraction change. // Node ids are NOT cached — they're reassigned each run by the existing deterministic sort, // so the cache stores fileId-free facts and re-stamps fileId on load (R3 id-positionality). -- constexpr std::uint32_t kCacheMagic = 0x4b505443; // "CTPK" // 2: whole-blob FNV checksum trailer (corrupt-cache detection). // 3 (T5): the file-path KEY stored per record switched from the verbatim absolute/as-typed ingest // path (`/`, root-spelling-dependent) to a ROOT-RELATIVE key via relForHash — the // same lexical root-prefix strip S2 uses for the baseline sidecars. This makes the cache PORTABLE // (committable): a cache built at one root spelling/absolute path is re-absolutized against the // CURRENT invocation's root on load, so `repo.ripwirecache` built in CI or by a teammate at a // different checkout path still warm-hits. Old (v2) caches store the pre-T5 key shape; bumping // this field makes loadCache's version guard reject them outright (magic/version/parserVer must // all match) rather than silently re-absolutizing a key that was never root-relative to begin // with — a v2 cache simply misses on every lookup that survives the guard, which is exactly the // self-healing full-reparse path already used for any other corrupt/stale cache. // 43 (train 36, #155 Kotlin scope functions; the PR's placeholder was 41): the `trailingLambda` u8 in writeRef/readRef changes // the on-disk ref record, so an older cache must be rejected. 43 sits above every live value: main / train 35 40, #438's 41, // lane rust-degraded's placeholder 42. // 44 (train 37b, lane rust-degraded; its placeholder was 42): the FILE record gains a sixth FileHealth u32, useBoundsBlanked // (the Rust precise-capture re-parse, src/rustusebounds.h), so an older cache must be rejected. 44 is above train 36's 43. constexpr std::uint32_t kCacheVersion = 54; // 54: lane qd-tangle (above 53, held by lane qual-ns-call-bind / train-1117d2 / import-proved-splits / rust-self-chain) — a def record gains the steering-locals byte, the dispatch-deep u16 and the arm-nesting cut byte (RawDef::steer, ::deepDispatch, ::nestArmCut, after maxNest). // 52: lane failure-default — the construct hits gain the failure-default shape tag // (lintrules.h kConstructShapeTags, appended); an older record lacks those hits. // 51: lane qd-constructs — the FILE record gains a ninth family, the --quality-delta // construct hits (ConstructHit: startByte, line, tag byte), with its head count. // 50: lane clones-persist — a def record ends with a clone-key presence u8 (+ the u64 key). // 49: lane warm-direct — the FILE record's head gains the eight family counts // (FileFactCounts, right after FileHealth) so a warm run can size and plan every // record from its head alone and decode the body in the parse worker. // 48: train 42b — lane alias-import's LocalBindKind ReExport (appended; readBind's kind bound // moves; its placeholder 47), strictly above main 46 and that placeholder; no other // train-42b lane changed a record format. // 46: train 40 — lane js-def-naming's LocalBindKind JsObjectLiteral (appended; its placeholder 45), // strictly above main 44 and that placeholder; no other train-40 lane changed a record format. // 44: train 37b — rust-degraded FileHealth::useBoundsBlanked (above 43 / placeholder 42) // 43: train 36 — #155 Kotlin trailingLambda ref byte (above 40 / 41 / 42) // 40: train 34 — strictly above main 39 and every live value (train-33 refs, #432 39; #427 / #429 38): the // fc3-js-rebind record change (LocalBindKind JsRebind, appended) lands; no other train-34 lane changed a format. // 39: train 33 — strictly above main 36 and every live value (lane l6-iface 38, #427 / #429 38): the // l6-iface record change (LocalBindKind DispatchType + ImplBlock, appended) lands; no other train-33 lane changed a format. // 38: train 32 lane l6-iface fix round 1 (train renumbers) — LocalBindKind gained ImplBlock // (appended): an older binary would reject the byte, a newer must not read a blob that predates it as complete. // 37: train 32 lane l6-iface (train renumbers) — LocalBindKind gained DispatchType // (appended): an older binary would reject the byte, a newer must not read a blob that predates it as complete. // 36: train 30 — strictly above every number a train-30 build used: lane call-result-typing // 33 (RawBind resultIndex + unwrapDepth), lane fcall-forward 33, lane name-template-dispatch 35 // (RefRole Pattern 11), main 34; so no blob a branch build wrote is read here. // 35: train 30 lane name-template-dispatch (train renumbers) — RefRole gained // Pattern (11, appended): an older binary would reject the byte, a newer must not read a blob that // predates it as complete. // 34: train 29 — lane arity-min's paramsMin def field (its branch used 33, as did lane // call-result-typing's; 34 sits above both, so no blob a branch build wrote is read here). // 33: train 29 lane arity-min — a def record appends `paramsMin` (u32 after params: // the fewest arguments a call can pass) — a FORMAT change → reject v32 blobs. The train renumbers. // 33: lane call-result-typing (train 30; the train renumbers) — RawBind gains two u8, // resultIndex and unwrapDepth (LocalBindKind::CallResult / ReturnType), written after // isFromAssignment: a record FORMAT change → reject v32 blobs. kParserVer moves with it. // 33: train 30 (lane fcall-forward fix 1; train renumbers) — a RawRef record appends // `forwarded` (u8) after memberFactory; the lane's 52d773b3 builds wrote parserVer-188 // blobs at 32 without it, so 32 must not read as this shape. // 32: train 28 — the RefRole byte a record holds gained two appended values: lane // ruby-pair's Subscript (9) and PR #380's Inert (10; 9 on the PR's branch, which predates // Subscript). readRef validates the byte against kRefRoleCount, so no record changes // shape, but a branch build's blob would read its role 9 as the other role here; 32 // sits above every live value (31 on main and every lane). // 31: train 27 — lane lego-transitive's record format below (the lane numbered it 30; // 31 is above every live value: the lane's branch builds wrote c30 blobs at 162). // 30: lane lego-transitive fix 1 — an INHERIT RawRef appends `baseBind` (u32: the // same-file declaration its base binds to, ingest_relations.h localBaseBinding); // every other ref record is unchanged in size. A FORMAT change: a v29 blob lacks it. // The train renumbers if another lane takes 30. // 29: FE-B (test/receiverevidencecheck.sh) — RawRef gains `memberPath` (str) and // `memberCtor` (str) after `memberRoot` (ref record 46 -> 54 bytes lean; // on #373 also `memberVia` (str) + `memberFactory` (u8), 54 -> 59: the Ruby // typed receiver moved out of recvVar — same version, one FE-B format), // and RawBind gains four APPENDED kinds (RecvType, MemberType, // MethodAlias, NameAlias). A FORMAT change: the version guard rejects a v28 blob. // 28: FE-A (test/falseedgecheck.sh) — RawRef gains `memberCall` (u8) and // `memberRoot` (str) after `qualifierRootsStd` (ref record 41 -> 46 // bytes lean). A FORMAT change: a v27 blob lacks the bytes, so the // version guard rejects it (self-healing full reparse). // 27: same record layout as 26, new VALUES — the lane's second // round changed which defs record fnScopeStart/End (the // scope search descends to the OUTER def node, and // kEscapingNestedBindings keeps globally-binding nested // functions out). A 26 blob would serve the old spans // (every top-level C/C++ function "nested in itself", // PHP/Lua globals yielding) until its files changed. // 26: a FUNCTION-LOCAL def records the span of the function // whose body binds its name — RawDef gains `fnScopeStart` // and `fnScopeEnd` (two u32 after `scopeRootsStd`, def // record 80 -> 88 bytes lean), read by graph.h's // reachableByName (test/fnliteralcheck.sh §6). A FORMAT // change: a v25 blob lacks the bytes, so the version // guard rejects it (self-healing full reparse). // kParserVer stays 128 (unreleased, same lane). // 25: #150 AND #157 (train 18 carries both bumps as ONE 24 -> 25: // neither lane's 25 reached main or a release). // #150 — nested std::-namespace resolution. RawDef gains // `scopeRootsStd` (a u8 after `internalLinkage`, def record // 79 -> 80 bytes lean) and RawRef gains `qualifierRootsStd` // (a u8 after `viaArrow`, ref record 40 -> 41 bytes) — the // call/def-side facts graph.h::keepStdQualifiedCandidates // needs to tell a NESTED std call (`std::ranges::move`) from // a same-shaped user namespace (`mylib::ranges::move`), // which the pre-existing IMMEDIATE-qualifier-only fields // cannot (see model.h Reference::qualifierRootsStd / // Symbol::scopeRootsStd). A FORMAT change: a v24 blob ends // both records one byte early and would read the next // record's own first byte as the new bit → reject them // outright. kParserVer moves with it (119 -> 120). // #157 (the same 25): NO record field changed shape — bumped anyway // because a v24-or-older cache can hold a NORMAL (real-hash, // zero-fact) record for a json/yaml/markdown file the nesting // guard refused. Pre-fix, only Kotlin's refusal was forgotten // before the save (forgetNestRefusalsForCache); the other // three warm-hit that record forever and would stay invisible // even after this fix ships, on any cache built before it. // Bumping rejects every pre-fix blob outright (T5's v2 // precedent, two comments up) so every refused file is // re-scanned, re-refused and re-rowed once, cold, instead of // staying silently hidden. quality.h's // kIngestCacheVersionMirror moves in the SAME commit. // 24: RawRef gains `viaArrow` (parser version 113, a u8 after // `argCountKnown` in the ref record, kMinRefRecordBytes 39 -> 40) // — a call's member access was written `->`, and a compose // ref's type is the pointee of a std smart pointer member // (test/fieldnarrowcheck.sh arm p). A FORMAT change: v23 blobs // end the ref record one byte early and would read the next // record's start byte as the bit → reject them. PR #282 // declared 22 -> 23; integration/train-5 assigns 24 over // #278's 23. // 23: RawBind gains `isFromAssignment` (parser version 106, a u8 after // `kind` in the bind record, 26 -> 27 bytes lean) — a C++ // assignment's callee-read type is kept only when it names a // class (test/narrowcheck.sh arms 44-51). A FORMAT change: v22 // blobs read the new byte as spanStart's low byte → reject them. // 22: RawDef gains `internalLinkage` (parser version 96, a u8 after // `recovered` in the def record, 78 -> 79 bytes lean) — an // anonymous-namespace or namespace-scope `static` C/C++ def // is visible to its own TU alone, and graph.h's decl-to-def // widening stops gathering it for another file's declaration // (test/decltodefcheck.sh arm B2). A FORMAT change: v21 blobs // read the new byte as `kind` → reject them outright. // 21: Include gains `bool isValueUse` (parser version 93, a fourth // u8 after isSymbolic) — the constant-argument / rescue-class // origin bit the call narrow skips. A FORMAT change → reject // v20 blobs. kParserVer moves with it. // PR #81 (Elixir, parser version 95) changes NO record shape: its // RecvKind / LocalBindKind enumerators are APPENDED and ride the // existing u8, so the format stays at #139's 21. // 20: the member-macro re-parse (test/macroreparsecheck.sh) — each // FILE record gains FileHealth::macroBlanked, a fifth health // u32 after wsBytes — a FORMAT change → reject v19 blobs. // kParserVer moves with it. // 19: extent honesty (test/extentcheck.sh) — each def record gains // the `recovered` u8 (RawDef::recovered, after testScope) — a // FORMAT change → reject v18 blobs. kParserVer moves with it. // 18: #62 — call refs inside a preprocessor-DECIDED-dead region // (`#if 0`, the `#else` of `#if 1`) are no longer captured. The // record SHAPE is unchanged, but a v17 blob holds refs this build // would not produce, and replaying them warm would resurrect the // over-count on exactly the corpora most likely to be cached → // reject v17 blobs. (A CONTENT-only bump, the 38/39 precedent's // mirror image: shape-stable but semantically stale.) // 17: Include gains `bool isSymbolic` + `u32 byte` (Ruby constant // directives, parser version 82) and the per-file record gains a // ConstOpen family (Ruby class/module opens, model.h) after // routeUses — a FORMAT change → reject v16 blobs. // 16: RawBind gains importedName for ES named imports. // 15 (offset-table blob): the blob gains a RECORD OFFSET // TABLE and a 24-byte trailer, so a run deserialises only // the records for the files it actually crawled, and a // save CARRIES OVER — byte for byte — the records for // files the blob holds but this run did not crawl. This is // the registered retry of "The auto-cache key ignores // --exclude" (docs/EVALS.md): the auto-cache key stays // exclude-INDEPENDENT (the per-configuration key was // measured and REVERTED as a NEGATIVE) and a subset // configuration is made cheap by SHAPE instead. Layout, // offsets and what each checksum covers: the v15 frame // block below. A FORMAT change → v14 blobs are rejected // and DISCLOSED (the self-healing full reparse rewrites a // v15 blob on the same run). // 14 (card A3 follow-up): each FILE record gains ctimeNs, // a THIRD stat-gate discriminator written right after // mtimeNs. (size, mtime) alone let a byte-length- // preserving edit with a restored mtime be trusted and // served STALE — reproduced on argv in docs/EVALS.md and // gated by statgatecheck (b2) / freshnesscheck arms 6-7. // st_ctime moves on any write AND on the utimes() that // performs the restore, and POSIX gives an unprivileged // process no way to set it back; ::stat already returns // it, so the fix costs no syscall and no file read. A // FORMAT change → reject v13 blobs (self-healing full // reparse writes the field on the next run). // 13 (§L1 parse health): each FILE record gains the // four FileHealth u32s (errNodes, errBytes, fileBytes, // wsBytes) right after the stat-gate pair. The auto-cache // is the DEFAULT path, so a health measurement that did // not round-trip it would evaporate on the second run and // report "nothing degraded" on a corpus it had never // re-read — the exact zero-means-none-exists defect the // lane exists to kill. A v12 blob has no such bytes, so // the version guard rejects it (self-healing full reparse // repopulates health). // 12 (B6.3): FILE records gain two new record arrays — // RouteDef (server-side route registrations) and // RawRouteUse (client-side HTTP calls). A v11 (or older) // blob has neither, so the version guard rejects it // outright (self-healing full reparse repopulates routes). // 11 (B2 graph precision): RawDef gains arityExact, RawRef gains // argCount+argCountKnown (call-site arity for B2.2) — a v10 // (or older) blob lacks those bytes, so the version guard // rejects it (self-healing full reparse rebuilds with arity). // 10 (B0 r2, H3): the rich def records' postings shrink — // each FILE record gains a sorted per-file subtoken // DICTIONARY (u64 hashes) and each def row stores narrow // dict INDICES + narrowest-exact-width tfs instead of raw // u64 hash + u32 tf pairs (the v9 shape grew the rich // blob +38-56% and dragged index/warm p95). Lossless by // construction (widths from maxima, ASSUME'd) — a FORMAT // change → reject v9 blobs (v9 never shipped; local v9 // blobs self-heal via the same full-reparse path). // 9 (B0.2 postings): each RICH-family def record gains the // persisted doc/body subtoken statistics (dlWeighted + // sorted (hash,tf) pair arrays; lexindex.h) — a FORMAT // change → reject v8 blobs (self-healing full reparse // rebuilds them with stats). Lean records are unchanged // in shape but share the header version, so lean v8 // blobs self-heal identically (one version, one guard). // 8 (A1 team-index): header gains a kArtifactArch tag byte // (endian + pointer width) after parserVer — a native-endian // blob is only same-arch-consumable, so a foreign-arch blob // must self-heal to a cold parse. A HEADER change → reject // v7 blobs (they lack the byte → the guard mismatches). // 7 (A4-R5): each file record gains a BindingAlias (FFI) // stream after binds — a FORMAT change → reject v6 blobs. // 6 (A4-P7): header gains a u64 blob-write timestamp; each // file record gains (sizeBytes,mtimeNs) for the warm-run // stat-gate + racy rule — a FORMAT change → reject v5 blobs. // 5: non-C import targets now store the CLEAN specifier // (Py `pkg.mod`, TS `./x`, Rust `crate::a::b`/`mod:x`) — // a target FORMAT change → old caches must be rejected. // 4: Include gained a `bool isAngle` (quote/angle) field constexpr std::uint32_t kParserVer = 331; // bump on any grammar/.scm/extraction change // 331 = 2026-11-18 (train 1117d1; above main's 311/312, #688's 313/314 and the members' placeholders // 317/321/323/327/329): rich file tag 332. kCacheVersion unchanged. The members' extraction changes: // go-cpp-edges (placeholder 317): a Go result type written as a generic instantiation (`Set[T]`, `*pkg.Set[string]`) records its // ReturnType as the instantiated class (ingest_binds.h goResultClass); it recorded none. // kCacheVersion unchanged: no record changes shape. // cpp-ptr-alias (placeholder 321): a C++ Type / ParamType record whose declarator names the variable or a reference to it // carries model.h kDirectDeclaratorBit in unwrapDepth (ingest_binds.h cppDirectDeclarator). // overload-split (placeholder 329): a Go type written as a qualified generic instantiation (`addrs.Set[addrs.Module]`) keeps its // package qualifier on the field / parameter / local record (ingest_binds.h goQualifiedWhole). // kCacheVersion unchanged: no record changes shape. // small-edges-1118 (placeholder 323): a Go method expression `(*T).m` in a value position is a Value row naming m with // memberRoot T (ingest_valuerefs.h methodValueAt, GO-METHOD-EXPR); it recorded none. kCacheVersion unchanged. // edge-strays-1118 (placeholder 327): a Rust call written through a bracketed type (`::run( .. )`, `::f( .. )`) is qualified by // the type it names, or by the trait when the type names none (ingest_names.h rustBracketedQualifier). // Rich file tag 328. kCacheVersion unchanged. // 313 = 2026-11-17 (TRAIN-1117b: ONE number for its two parser members, strictly above main c565a3fad's 311/312 // (#420, landed by TRAIN-1117a-r #686)). Rich file tag 314. kCacheVersion unchanged. // - #446 round 2, @SeddonShen, landed as lane land-606 (its placeholder 316): Java base clauses read as // @reference.base captures (queries/java/tags.scm, LangFacts::basesAsCaptures on the Java row). // - #355, PR #426 by @conka8, landed as lane/land-426 (its placeholders 216, then 311): the tags pass evaluates // tags.scm's string predicates (#eq?, #not-eq?, #any-of?, #not-any-of?; src/infra/tspredicates.h), and Elixir's // definition keywords and metadata attribute names move into queries/elixir/tags.scm. // - #688 review (unreleased, same train, so 313 stays): a type-annotated Java superclass (`extends @M Base`) // is captured inside annotated_type (CodeRabbit 4614203119). // A string predicate tests every node bound to its capture (CodeRabbit 4614203164). // 311 = 2026-11-17 (#358 C# 1a, @LaimaWu; lands #420, rebased; was 198 on the PR, above main's 309/310): using // directives move to shared @import.path captures; import-free deep containers no longer disclose a cut or // lose whole-cache eligibility. Alias/edge facts unchanged; rich tag 312. kCacheVersion unchanged. // 309 = 2026-11-13 (lane macro-p0, PLACEHOLDER: the train renumbers; above main's 307/308): a C-family macro // invocation read as a typeless declarator (`REGISTER_FACTORY( F, B ){ "n" };`, a class-member `MOCK_METHOD( … );`) // mints no function def; the site is a use of the macro (ingest_names.h cFamilyMacroInvocationDecl). Rich file // tag 310. kCacheVersion unchanged. MACRO-P1 (stacked on macro-p0, same placeholder): a trailing annotation // macro (`T f() ABSL_LOCKS_EXCLUDED( mu ) { body }`) is a use, and f's prototype adopts the body // (ingest_names.h annotationHostDeclarator / annotatedDefinitionAfter). // 307 = 2026-11-11 (lane rust-macro-path, PLACEHOLDER: the train renumbers; above main's 305/306): a Rust path // macro invocation (`crate::m!`, `serde_json::json!`) is captured with its written path, and a transcriber's // `$crate::m!` is a Macro reference of its `macro_rules!`. Rich file tag 308. kCacheVersion unchanged. // 305 = 2026-11-10 (#446 round 1, @SeddonShen; above main's 303 and the PR head's 304): C++ base clauses read as @reference.base tags captures // (queries/cpp/tags.scm, LangFacts::basesAsCaptures). Rich file tag 306. kCacheVersion unchanged. // 303 = 2026-11-06 (lane proto-index, PLACEHOLDER: the train renumbers; above main's 301 and open #538's 302; // rich file tag 304): .proto files are indexed (Lang::Proto, queries/proto/tags.scm). kCacheVersion stays 49. // 301 = 2026-11-04 (#478 gap 2, PLACEHOLDER: the train renumbers; above main's 299/300, rich file tag 302; // test/pyimportprecisecheck.sh): `import a.b as c` records the Include target `a.b`, not the aliased_import span // `a.b as c` no resolver could find. kCacheVersion stays 48: no record changes shape. // 299 = 2026-11-03 (lane index-alias-const main merge): both sides bumped (lane 293/294, main 297/298 after #515), // so the merged value goes above both. Rich file tag 300. kCacheVersion stays 48. // 297 = 2026-11-03 (lane edge-prelude-veto fix): a Rust `macro_rules!` without #[macro_export] sets RawDef::internalLinkage // (textual scope, graph.h rustMacroScopeOf). Rich file tag 298. kCacheVersion unchanged. // 295 = 2026-11-03 (lane edge-prelude-veto): a Rust macro invocation is captured @reference.macro (role Macro, // queries/rust/tags.scm), no longer @reference.call. Rich file tag 296. kCacheVersion unchanged. // 293 = 2026-11-03 (lane index-alias-const): C++ `using X = Y` / `namespace L = lib` / function-type and pointer // typedefs are definitions (src/aliasdef.h). kCacheVersion stays 48: no record changes shape (the alias target // rides RawDef::bodyByte). Rich file tag 294. // 291 = 2026-11-03 (lane docs-rest #506 re-bump): both sides bumped (lane 287/288, main 289/290), so the merged value // goes above both; caches built on main at 289 re-index and pick up the reST .txt docs. Rich file tag 292. kCacheVersion stays 48. // 289 = 2026-11-02 (lane barrel-short-key): the TS/TSX/JS tags queries capture `export * as ns from 'm'` // (ReExport var=ns importedName="*", a namespace re-export). Placeholder above RULE3 #504 (285/286, merged) and // DOCS-REST #506 (moving to 287/288 in its next round); rich file tag 290; the train renumbers. kCacheVersion stays 48: no record changes shape. // 285 = 2026-11-02 (lane rule3-member-guard fix 4; PLACEHOLDER, the train renumbers): a Rust path call with a // qualifier of more than one segment keeps it whole in Reference::fieldName (ingest_names.h rustWrittenQualifier). // Rich file tag 286. kCacheVersion stays 48: no record changes shape. // 287 = 2026-11-02 (lane docs-rest, #409 reST tier, after #504's 285/286): reST/Sphinx units, and the crawl // admits declared .txt. Rich file tag 288. kCacheVersion stays 48. Merged over main's 289 (barrel-short-key) // 2026-11-03: the higher value 289 stands for both changes; the train renumbers. // 283 = 2026-10-31 (train 43b: lanes registrar (placeholders 273..281), string-import (274, 275) and their extraction: Python registrar facts, // TS/JS provider-record literal marks, Python dotted-string and import-sink facts). Strictly above 281 (rich 282) and every lane value. Rich file tag 284. // kCacheVersion stays 48: no record changes shape. // 281 = 2026-10-30 (lane registrar fix round 4, PLACEHOLDER; 280 is 279's rich tag): a ProviderRecord literal is marked only when it is an element of an // array (call argument / const / `providers` property) and holds only plain unique pairs (no spread, method, accessor, computed or duplicate key) // 279 = 2026-10-30 (lane registrar fix round 3, PLACEHOLDER: the train renumbers; 278 is 277's rich tag): a TS/JS Value row // read from a ProviderRecord object literal carries that literal's start byte (qualifier "@") when the // literal is a direct call argument or a `const` initialiser; a Python registrar fact is refused unless the method's other // parameters are provably beside the point (a defaulted one stored into `self` refuses; a subscript key of the store is allowed). // 277 = 2026-10-30 (lane registrar fix round 2, PLACEHOLDER: the train renumbers; 276 is 275's rich tag): a Python // lambda parameter / comprehension target files a VarDecl `~name` (kPythonLocalScopePrefix), and a method whose body also // uses another parameter written without a default records no registrar fact. // 275 = 2026-10-30 (lane registrar fix round 1, PLACEHOLDER: the train renumbers; 274 is 273's rich tag): a Python // parameter written with a default (`sender=None`) records no registrar fact. // 273 = 2026-10-30 (lane registrar, PLACEHOLDER: the train renumbers): Python files record // registrar facts (model.h kValueRefRegParamScope: a method parameter stored into a container a // field of self holds). Strictly above train-42b's 271 (rich 272). Rich file tag 274. kCacheVersion // stays 46: no record changes shape (a new scope char in an existing field). // 271 = 2026-10-30 (train 42b: lanes iface-peel (placeholders 259/260: a C++ smart-pointer alias record carries its // pointee, a Rust `-> &dyn T` / `Box` / `impl T` result records a ReturnType with kDispatchResultBit, a C++ // member whose written type is an enclosing template parameter carries kTemplateParamMemberType) and alias-import // (261/262: TS/TSX/JS re-export specifiers captured as LocalBindKind::ReExport records)). Strictly above train // 41b's 269 (rich 270) and every live lane placeholder (259, 260, 261, 263, 265, 267). Rich file tag 272. // kCacheVersion 48 (ReExport appended). // 269 = 2026-10-29 (train 41b: lane fn-slot-s2 (placeholders 257/258, 259 and 260 over its fix rounds): C++ `&Cls::m` // value refs (memberRoot = Cls; memberVia = written qualifier / enclosing scope / `using namespace` directives), // `( o.*x.f )( … )` slot calls, C nested designated paths and macro-cast slot calls + their "()M" alias facts, a class // body fact's scope path, and a C++ base that is a template parameter of the deriving class (baseBind 0xFFFFFFFE)). // Strictly above train 41's 259 (rich 260) and every live lane placeholder (iface-peel 259, alias-import 261, // the string/literal lanes 263/265/267; train 42 goes above 269). Rich file tag 270. kCacheVersion stays 46: // no record changes shape. // 259 = 2026-10-28 (train 41: lanes fn-slot-2 (placeholder 249/250; Rust files record the slot join's records: // function-path arguments of path calls, field stores, registrar parameter facts, `(x.f)( … )` slot calls) and // py-split-marker (255/256; a Python module-scope def/class inside an if/try/with/loop files a second VarDecl // `?name`)). Strictly above train 40's 251 (rich 252), every lane placeholder (249/250, 255/256) and // fn-slot-s2's 257/258 (train 41b takes the next). Rich file tag 260. kCacheVersion stays 46: no record changes shape. // 251 = 2026-10-27 (train 40: lanes js-def-naming (placeholder 238; JS `module.exports = ` named // by its file stem, module-scope `NAME = { … }` binding names), string-keyed (241; Pattern qualifier "t" at a // hasattr test / JS-TS subscript store) and fn-slot (247; Go function-valued slot records)). Strictly above // train 39's 239 (rich 240) and every lane placeholder (238, 241/242, 247/248, fn-slot-2's 249/250). Rich file // tag 252. kCacheVersion 46 (js-def-naming's LocalBindKind JsObjectLiteral record change). // 239 = 2026-10-27 (train 39: lane receiver-fixes, placeholder 237 / rich 238): TS `#private` field declarations and // `this.#f = …` record MemberType; Python `self.f: T` / `T | None` / `Optional[ T ]` annotations and `mod.X( … )` // constructors type fields and locals (a RecvType / MemberType with its qualifier), and `x = mod.X( … )` no longer // writes the base Type tombstone. Strictly above train 38's 235 (rich 236) and every lane placeholder (237/238). Rich file // tag 240. No record format changed: kCacheVersion stays 44. // 235 = 2026-10-27 (train 38: lane queue-hop, placeholder 228 / rich 229): a Python queue call (put/put_nowait/get/get_nowait) // carrying keyword arguments records them on its call reference's composeRel (model.h pyKwTag), so QueueHops reads // `put( v, timeout=1 )` as one item. Strictly above train 37b's 233 (rich 234). Rich file tag 236. No record format // changed (the tag rides the existing composeRel string of writeRef/readRef): kCacheVersion stays 44. // 233 = 2026-10-26 (train 37b: lane rust-degraded, placeholder 229 / rich 230): a Rust file whose // first parse holds error bytes is re-parsed with its precise-capturing use<..> bounds blanked // (src/rustusebounds.h) — different defs. Strictly above train 36's 231 (rich 232). Rich file tag 234. // 231 = 2026-10-25 (train 36: #438 Kotlin trailing-lambda capture, #437, #431): strictly above every // live number — main / train 35 226 (rich 227), #438's placeholder 228, lane rust-degraded's placeholder // 229 (rich 230). Rich file tag 232. kCacheVersion 43 moves with it (#438's trailingLambda ref byte). // 226 = 2026-10-24 (train 34: lean2b, fc2-lego-extparent, honest-lex, ind4b-k5-tighten-k5c, hop1, fc3-js-rebind): // strictly above every live number — main 224 (rich 225), #432 224, lane ind4b's placeholder 224 (it collided with // main's), lane ind5-glob-imports 222 (223, reserved), lane fc3-js-rebind 218, #426 216, #420 214. Rich file tag 227. // Mandatory: fc3-js-rebind adds LocalBindKind::JsRebind records (kCacheVersion 40); ind4b emits VarDecl records for a // Python match capture, a PEP 695 type alias / type parameter, and "from:x" in an aliased import's importedName. // 224 = 2026-10-23 (train 33: ls2-size-targets, fc1-std-qualified-name, ix5-classnew, mf1-nx, qn1-qualname, l6-iface, // changelog-fragments): strictly above every live number — main 220 (rich 221), lane ind5-glob-imports 222 (223, reserved), // #420 214 (215), #426 / lane fc1 216 (217), lane ind4b / fc3-js-rebind 218 (219), lane l6-iface and #429 212 (213). // Rich file tag 225. Mandatory: l6-iface adds DispatchType / ImplBlock records (kCacheVersion 39) and fc1 changes what a // Rust qualified call result resolves to, so a 220 cache would replay the older facts. // 220 = 2026-10-22 (train 32b: ix1-slot + cppqual-alarm): strictly above every live number — main 212 (rich 213), // lane ix1-slot 204 (205), #420 214 (215), #426 / lane fc1 216 (217), lane ind4b 218 (219), lane l6-iface 212 (213). // Rich file tag 221. Mandatory for IX-1: its fix rounds changed slot extraction under the one 204 reservation, so a // stale 204 cache would bring back the typedef-vs-tag and two-bodies joins. kCacheVersion stays 36: IX-1 rides // memberPath/memberCtor of the existing value/through records (no record format changed); cppqual-alarm is test-only. // 212 = 2026-10-21 (train 32a: member-names + rust-receivers): strictly above every live number — main 206 (rich 207), // lane rust-receivers 202 (203), lane ix1-slot 204 (205), lane l6-iface 210 (211); #420 reserves 214 (215), so train // 32b can go above. Rich file tag 213. kCacheVersion stays 36: neither lane changed a record format (rust-receivers // writes ReturnType isFromAssignment in an existing byte; member-names is output only). // 212 = 2026-10-21 (lane l6-iface fix round 1, rich 213; the train renumbers): an ImplBlock record // (model.h) for every Rust `impl T for X` block (its span), a Rust trait under a mock attribute, a Java // anonymous class, a Kotlin object expression and a Kotlin `fun interface` (ingest_relations.h // rustImplVisitNode, ingest_binds.h anonymousImplementorOf). // 210 = 2026-10-19 (train 32 lane l6-iface, reservation 210 / rich 211; the train renumbers): // Rust records a DispatchType (model.h) for a parameter, `let` or struct field typed `&dyn T`, // `Box`, `impl T` or a generic bounded by T (ingest_binds.h rustDispatchTrait) in place of // its tombstone. // Merge of main cd11226a: strictly above main 206 (rich 207) and lane rust-receivers 202 (203). kCacheVersion 37 is // above main's 36 (main appended AugAssign / OperatorResult; DispatchType follows them). // 206 = 2026-10-18 (#409, PR #414, renumbered again at merge with main 939dbb69): strictly above every // live number — main 196 (rich 197), train 31 198 (199), lane rust-receivers 202 (203), lane ix1-slot // 204 (205). Rich file tag 207. kCacheVersion stays 36 (no record format change). // 198 = 2026-10-18 (train 31): strictly above every number a live build used — main 196 (rich 197), // lane h1-annot 194 (195); body-fold, hrr, callers-band2, dangling-warnings and ix2-slice kept // their base value 184. Rich file tag 199. kCacheVersion stays 36: no lane changed a record format // (h1-annot appends two LocalBindKind values; no lane touched a cache serializer). // 185 = 2026-10-18 (#409): a cross-file markdown [text](other.md#slug) keeps the link as // written (RawRef::memberPath, unbound) so graph.h pins the exact file + heading. // 185, not 184: 184 is train 29. Rich file tag 186. kCacheVersion stays 34. // 202 = 2026-10-18 (train 32 lane rust-receivers, reservation 202 / rich 203; the train renumbers): // Rust records a typed RecvType for a parameter / `let x: T` / `let x = T { .. }` / `T::default()` // written or constructed as a type, and a MemberType for a struct's named field (ingest_binds.h // captureRustEvidence); a written or declared `Self` and a generic's head name their class. // Fix round 1 changes the CONTENT under the same 202 (lane dev caches only; the train renumbers): a Rust // CallResult's importedName is the call's WRITTEN qualifier path, and a ReturnType's isFromAssignment // marks a result written `Self` — a cache from a pre-fix-round 202 build reads every result as not-Self. // kCacheVersion unchanged (no record format changes). // 204 = 2026-10-18 (train 32 lane ix1-slot, IX-1 slot-type join; train renumbers): C/C++ value rows // whose slot is a struct field carry memberPath/memberCtor, calls through a struct field are // Through records (qualifier 's'), and struct field types / typedef aliases are slot facts // (Value, qualifier 't'). Same record layout: kCacheVersion unchanged. Rich file tag 205. // 196 = 2026-10-18 (train 30): strictly above every number a live build used — main 184 (rich 185), // lane call-result-typing 180 (rich 181), lane fcall-forward 188 (189), lane go-local-var 190 (191), // lane name-template-dispatch 192 (193), lane h1-annot (train 31) 194 (195); tail-sym and // path-owners kept 176. Rich file tag 197. kCacheVersion 36 (see its note). // 194 = 2026-10-18 (lane h1-annot, train 31; the train renumbers — reservation 194/195): // a Python / TS parameter, annotated local or TS field written `mod.T` / `ns.T<…>` keeps the // qualified text in its RecvType / MemberType importedName; Python records LocalBindKind:: // AugAssign for `x op= v` and OperatorResult in an operator method (in-place or binary) whose // result is provably its receiver's class ("self" on every path, or `-> Self` / `-> C`); a JS // `x op= v` records constructedBy( rhs ) like a plain reassignment (`x ||= new B()` names B). // Rich file tag 195. kCacheVersion unchanged. // 192 = 2026-10-17 (train 30 lane name-template-dispatch, IX-3; train renumbers): RefRole::Pattern // records at Python getattr/hasattr/__dict__.get and JS/TS computed-member name templates. Rich 193. // 190 = 2026-10-17 (train 30, lane go-local-var): Go var/const, Rust const/static and GDScript const // captures are anchored to their item/package scope, so a function-local declaration is no // longer a symbol; Go package-level `var ( … )` groups are captured. Train-30 reservation // 190 / rich file tag 191; the train renumbers. kCacheVersion unchanged. // 188 = 2026-10-17 (train 30 reservation, lane fcall-forward; train renumbers): JS/TS // call forwarders (model.h kCallForwarders) — `f.call/f.apply` adds a call reference to a // proven f, `f.bind` a value reference. Rich file tag 189. // 184 = 2026-10-17 (train 29): strictly above every number a train-29 binary used — lane arity-min 178 // (rich 179), lane call-result-typing 180 (rich 181; carried to train 30), lane go-scope 182 (rich 183); // the other train-29 lanes kept 176. Rich file tag 185. kCacheVersion 34 (see its note). // 182 = 2026-10-16 (train 29 reservation L5 go-scope, #403; the train renumbers): Go methods // carry their receiver type as scope and interface members their interface's name // (ingest_names.h kScopeOwners), so Go canonical ids change. Rich file tag 183. // 180 = 2026-10-16 (lane call-result-typing, carried to train 30; the train renumbers — reservation 180/181): // Go, Rust, C++, Java, Kotlin and TS (never JS) record LocalBindKind::CallResult for a // local bound to a call's result and LocalBindKind::ReturnType for each declared result // (ingest_binds.h pushCallResult, pushReturnType); Java, Kotlin and Rust record a // TypeImport per imported name, and a Go field keeps its `pkg.T` qualifier; Rust records a // RecvType tombstone for every `let` name and member calls carry FE-B's receiver path. // Rich file tag 181. kCacheVersion 33. // 178 = train 29 lane arity-min (reservation 178, rich 179; the train renumbers): defs extract // paramsMin (cc_paramShape + kParamMinLangs) for the B2.2 minimum-arity exclusion. // 176 = 2026-10-15 (train 28): strictly above every number a train-28 binary used — main 173 // (slice 4a, rich 174), lane grep-carrier 173 (value refs through ?? || && / or and, rich 174), // lane ruby-pair 174 (subscript references + Ruby main-object mixins, rich 175), PR #380 145 (Elixir inert // regions, RefRole::Inert), the other lanes 171. Rich file tag 177. // 145 = PR #380 (issue #357, @LaimaWu; branch based on 143, 144 then main's rich tag): Elixir quote // live holes and counted inert call candidates (RefRole::Inert, appended after ruby-pair's // Subscript at the train, so its byte is 10, not the branch's 9). // 173 = 2026-10-15 (slice 4a, #400): the Ruby mixin directives (`include` / `extend` / `prepend`) // move off the captureIncludes walk onto the shared base-relations path — // queries/ruby/tags.scm captures the directive as @reference.mixin and captureTagsFacts // emits each constant's inherit reference through emitBaseRef (captureRubyMixinBases retired). // Output byte-identical; the extraction path changed, so the number moves. 173, not 172: 172 is // 171's rich file tag. Rich file tag 174. kCacheVersion stays 31. // 173 = 2026-10-14 (train 28 lane grep-carrier): value-reference capture walks the // value-preserving operators (JS ?? || &&, Python or/and, JS ??= ||= &&=), so the // Value rows a cache holds change. 173, not 172: a lean 172 would read 171's rich // blobs, and a concurrent lane may take 172 — the train renumbers. Rich file tag 174. // 174 = lane ruby-pair (train 28, 2026-10-14; the train renumbers): Ruby and Python element // accesses record RefRole::Subscript references (tags.scm @reference.subscript, ingest_binds.h // subscriptOperatorRef; a NEW role byte, which readRef validates against kRefRoleCount), the // Ruby typing walk types their receivers, and a Ruby file's top-level `include M` / `extend M` // records LocalBindKind::RubyMainMixin (an appended kind). 172/173 skipped: rich file tag 172 // is 171's. Rich file tag 175. kCacheVersion stays 31. // 171 = 2026-10-13 (PR #349 lands, @tag-und-nacht): Clojure grammar, tags.scm and extraction // (ingest_clojure.h) and the qualified-call namespace in Reference::qualifier. The branch built // 156, 158 and 169 (rich file tag 170); main is 167 (rich file tag 168). 171, not 168: a lean 168 // would read main's rich 167 blobs as its own; not 169/170: the branch's own 169 build and its // rich tag. Rich file tag 172. kCacheVersion stays 31. // 167 = 2026-10-11 (train 27 release): the fix27-cpp-subscript extraction below, renumbered above the // lane build's 166 (rich file tag 167 there), so no cache a lane build wrote is read as this release's. // Rich file tag 168. No other extraction change; kCacheVersion stays 31. // 166 = lane fix27-cpp-subscript (2026-10-11): a C++ member call on ONE element access of a // one-hop container (`m[ k ].empty()`, `v.at( i ).size()`) records the container's shape in // memberPath (ingest_binds.h cppElementContainer); a type alias of a std element container and // a member field written as one record its whole text (ingest_relations.h stdElementContainerText; // the field as a new composeRel "elem" record). 166, not 165: 165 is 164's rich file tag. No // record layout change (kCacheVersion stays 31). Renumbered to 167 by the train. // 164 = 2026-10-11 (train 27: the signed lanes merged onto train 26c's 158): cache-key hygiene. One merged lane // changed extraction under lane-local numbers: lego-transitive (157, 160, then 162; the 160 and 162 notes are below). Live // branch builds have used numbers up to 162 (rich file tag 163). 164 (rich file tag 165) is above every one // of them, so no cache another build wrote is read as this build's. kCacheVersion moves with it (31). // 162 = lane lego-transitive fix 1 (2026-10-10): an inherit ref records its // same-file base binding; Python dotted / subscript / keyword bases and JS // member / call heritage read as one base or none; a JS class factory needs a // same-file class receiver. 162, not 161: 161 is 160's rich file tag. The train renumbers. // 160 = lane lego-transitive (2026-10-10): a JS/TS class factory // `const Sub = Base.extend('Sub', ...)` is a class definition extending // its receiver (tags.scm @definition.classfactory). 160, not 157: 157 is // 156's rich file tag, and train 26c holds 158 (rich 159). The train renumbers. // 158 = 2026-10-09 (train 26c: PR #339, Rails db/schema.rb columns, merged onto main's 156): // a Ruby file whose tree holds a `create_table "x", … do |t| … end` call is a rendered // db/schema.rb BY CONTENT; each `t. "name"` / `t. :name` in the table block // mints a SymKind::Section def at Lang::Ruby (plus the implicit / `id:` / `primary_key:` // id rule and `t.timestamps`). DEFINITIONS ONLY: buildGraph's byName skips Section-Ruby, // so a column takes no call edge. The branch carried 122 -> 126 -> 144 -> 149 across its // merges with main; 158, not 157: 156's rich file tag is 157. No record layout change // (kCacheVersion 29, kQSnapCacheScheme 19 stay). // 156 = 2026-10-08 (train 26b: signed lanes merged onto main's 148): cache-key hygiene. // Two merged lanes changed extraction under lane-local numbers, both 145: // honesty-small-068 (Go named-type kinds, `type A = B` aliases) and // train25-cr2-followup (prototype-parameter scope, the depth-cut record, // vrClean key slots); their notes are below. Branch builds in flight have // used numbers up to 154 (rich file tag 155). 156 (rich file tag 157) is // above every one of them, so no cache another build wrote is read as this // build's. kCacheVersion 29 (FE-B); kQSnapCacheScheme 18 (cr2-followup). // FE-B (receiver evidence, notes below: lane-local 145-154) merged in // phase B: 156 stays above its 154 / rich tag 155. // 154 = lane FE-B fix round 2 (review B4): every binding form records its name with // the bytes it is visible in (RecvType spans; an unknown class is a tombstone), and a // written type that names a type parameter names no class. 154, not 153: 152's rich // file tag is 153; the train renumbers if it collides. // 152 = lane FE-B fix round 1 (review B3): Java/C#/Kotlin/Swift typed parameters, locals, // fields and constructed declarations record RecvType/MemberType. 152, not 151: 150's // rich file tag is 151; the train renumbers if it collides. // 150 = lane FE-B (receiver evidence) merged onto train 26a's 148: the receiver chain // (memberPath/memberCtor), the RecvType/MemberType/MethodAlias/NameAlias/StaticMember // bindings, appended after #373's Ruby kinds. 150, not 149: 148's RICH file tag is 149; // the lane's own builds used 145-147 (lean) and their tags; the train renumbers if it collides. // 146 = lane FE-B fix round: JS/TS `static` members record a StaticMember // binding (the class side of the lookup); the lane's 145 builds lack it. // 145 = lane FE-B (receiver evidence): member calls record memberPath/memberCtor and // Java/Kotlin/C#/Swift member calls are memberCall; typed parameters/locals, field types // method aliases and aliased class imports are RecvType/MemberType/MethodAlias/NameAlias // bindings. Above train 25's 143/144 (rich); // the train renumbers. // 148 = 2026-10-04 (train 26a: PR #373, Ruby method lookup, merged onto main's 143): // cache-key hygiene. The branch's Ruby extraction steps ran as 130–137, // 142 and 145 (its notes below); main used 130–143 for other lanes, and // lanes in flight hold 145–147. 148 (rich file tag 149) is above every one // of them, so no cache another build wrote is read as this build's. // kCacheVersion stays main's 28 (the branch said 27: its appended // LocalBindKind values follow ModuleAlias here, and the parser number // already keys every blob); kQSnapCacheScheme stays main's 17. // 145 = lane honesty-small-068 (a lane-local number; the train renumbers): // Go named types get their kind from the type_spec's form (t="type" / // "functype"), and `type A = B` is now indexed (t="alias") — new defs and // new kind bytes. 145, not 144: 143's RICH file tag is 144. // 145 = lane train25-cr2-followup (lane-local; the train renumbers): three extraction changes — // a parameter of a prototype, function-pointer typedef, Go func type or TS // function type no longer declares its name in the enclosing scope; a file // whose value walk stops at kVrMaxDepth gains ONE depth-cut Value record; // the remaining written key slots (a C `[expr]` designator, a numeric or // identifier key) go through vrClean. 145, not 144: 143's RICH file tag is // 144 (parserVerFor below adds 1). The train renumbers if another lane // takes 145/146. // 143 = 2026-10-04 (train 25 review fixes): two extraction changes — // a declaration named like the global object (`var self = this`, a // parameter `window`) is now a JsShadow binding, and a value-reference // slot text (into=/through=) is cut on a UTF-8 boundary, gets "…" only // when really cut, and a JS string key is capped too. 143, not 142: // 141's RICH file tag was 142 (parserVerFor below adds 1), and a lean // 142 would have read those blobs as its own. // 141 = 2026-10-04 (train 25): cache-key hygiene above every branch build's number. // Two merged lanes changed extraction under their own numbers: FE-A 134/135 // (member-call shape, ModuleAlias bindings, Rust path calls; kCacheVersion 28) // and refval-edges 140 (reference-as-value rows; kQSnapCacheScheme 17); main // was 133. 141 is above all of them, so no cache a branch build wrote is ever // read as this build's. kCacheVersion stays FE-A's 28: every ingest key also // carries kParserVer (lean 141 / rich 142 file tags, and a refval-edges rich // blob was c27p141 — a different format number, so never this build's c28p141). // 140 = lane refval-edges (reference-as-value round): RefRole::Value / // RefRole::Through rows from ingest_valuerefs.h, in BOTH families. // A lane-local number above train 24's 133 — the train renumbers. // 135 = 2026-10-03 (lane FE-A fix round): Rust path calls (`::f()`) are member // calls — an extraction change; this lane's own earlier binaries ran 134 with the old // extraction, so their caches must never be read as this build's. // 134 = 2026-10-03 (lane FE-A, test/falseedgecheck.sh): Go/JS/TS/Rust member calls // record memberCall/memberRoot; JS/TS `require`, namespace-import and // global-destructure aliases and Go import specs are ModuleAlias bindings; JsShadow // also covers declarations spelled like a JS global (externalnames.h tables — a // table edit is an extraction change). 133 is train 24's; the train renumbers. // --- PR #373 branch numbers (lane-local; 130–137 there are NOT main's 130–137) --- // 145 = 2026-10-04 (test/rubyclassrecvcheck.sh, PR #373 review): a // Ruby mixin the class object's lookup reaches — an `extend`, an // `include` in `class << self` — mints a LocalBindKind:: // RubyClassMixin binding at its constant (an appended kind), and a // RubySingletonDef binding carries importedName "included" when // its def is each includer's class method (a singleton def in a // concern's `included do`, a def in `class_methods do`, which now // mints one). Above main's 143 and 144. Same layout: kCacheVersion // stays 27, kQSnapCacheScheme stays 16. A Ruby cache written at 142 // lacks them and must re-parse. // 142 = 2026-10-04 (the Ruby method-lookup branch rebased onto main 2720d1c5): // cache-key hygiene, not an extraction change. The branch's eight // Ruby extraction steps carried 130–137 (the notes below keep // them); main took 130–133 for other lanes, and train 25 uses // 134, 135, 140 and 141. 142 is above all of them, so no cache // another build wrote is read as this build's. Same layout: // kCacheVersion stays 27, kQSnapCacheScheme stays 16. // 137 = 2026-10-02 (test/rubyclassrecvcheck.sh): a Ruby call whose // receiver is a constant written with a path keeps the path in // RawRef::fieldName (`Billing::Invoice.issue` → "Billing:: // Invoice"; recvVar keeps "Invoice"), and a singleton method — // `def self.m`, a def or an accessor in `class << self` — mints a // LocalBindKind::RubySingletonDef binding (an appended kind), both // read by graph.h's class-object lookup. Same layout: kCacheVersion // unchanged (27). A Ruby cache written at 136 lacks them and must // re-parse. // 136 = 2026-10-02 (test/rubyrspectargetcheck.sh): RSpec's matcher // builders (`receive`, `change`, … model.h kRspecTargets) type // their receivers in an example group, and a call on a chain // rooted at one is typed as the root (ingest_binds.h // rubyMatcherChainType). Same layout: kCacheVersion unchanged // (27). A Ruby cache written at 135 holds the untyped chains. // 135 = 2026-10-02 (test/rubyrakejbuildercheck.sh): `.rake` and // `.jbuilder` files index as Ruby; a Jbuilder template's `json` // is a local typed JbuilderTemplate, and `helper_method :m` at // class-body position mints a LocalBindKind::RubyHelperMethod // binding (an appended kind). New records, same layout: // kCacheVersion unchanged (27). A Ruby cache written at 134 // lacks them and must re-parse. // 134 = 2026-10-02 (test/rubydeclrefcheck.sh): a Rails callback, // validation condition or rescue_from handler named by SYMBOL // mints a Ruby call RawRef at the symbol, and `send( :m )` and // its kin a renamed copy of their own call RawRef // (ingest_binds.h noteDeclaredCall, sendTargets). New records, // same layout: kCacheVersion unchanged (27). A Ruby cache // written at 133 lacks them and must re-parse. // 133 = 2026-10-02 (test/rubyrspectargetcheck.sh): inside an // RSpec example group, a receiver built by `expect`, `allow`, // `expect_any_instance_of`, `allow_any_instance_of` or a bare // `is_expected` carries RSpec's target class as its type // (model.h kRspecTargets). Same layout: kCacheVersion // unchanged (27). A Ruby cache written at 132 holds those // receivers untyped and must re-parse. // 132 = 2026-10-02 (test/rubytypedrecvcheck.sh): a Ruby call // receiver the file BUILDS (`Client.new`, a finder, a FactoryBot // build, a let or local holding one) carries its type in the // call RawRef's recvVar (FE-B on #373 moved it to memberCtor/memberVia), and each // FactoryBot `factory` mints a RawBind of the APPENDED kind // LocalBindKind::RubyFactory (ingest_ruby.h). Same layout: // kCacheVersion unchanged (27). A Ruby cache written at 131 // holds untyped receivers and no factories and must re-parse. // 131 = 2026-10-02 (test/rubyreachcheck.sh): a Ruby MIXIN is an // ancestor — `include`/`extend`/`prepend` at class-DSL // position mints one inherit RawRef per constant, at the // constant's byte — ingest_relations.h captureRubyMixinBases // then; since 173 the @reference.mixin capture, read by // ingest_ruby.h emitCapturedRubyMixinBase. // New records, same layout: kCacheVersion unchanged (27). A // Ruby cache written at 130 holds none of them and must re-parse. // 130 = 2026-10-01 (test/rubybarecallcheck.sh): Ruby's BARE-WORD call — an // identifier with no receiver, arguments or parentheses, which // tree-sitter-ruby parses as the same (identifier) a local read // is — mints a receiver-less call RawRef wherever Ruby's own // lexical rule says it is not a local (ingest_binds.h // captureRubyBareCalls). New records, same layout: // kCacheVersion unchanged (27). A Ruby cache written at 129 // holds none of them and must re-parse. // --- end of PR #373 branch numbers; main's history continues --- // 133 = 2026-10-02 (train 24): cache-key hygiene above every branch build's number. Two // merged lanes changed extraction under their own numbers: 130 (a body-less C/C++ // enum/struct/union/class specifier in a function signature keeps its own span, so // record VALUES change) and 131 (TS/TSX `await f(x)` and unary `!f(x)` are call // references: new records); 132 was train 23's. 133 is above all of them, so no cache a // branch build wrote is ever read as this build's. Same layout: kCacheVersion stays 27, // kQSnapCacheScheme stays 16. // 132 = 2026-10-02 (train 23): cache-key hygiene, not an extraction change. Every // branch build of this train's work ran at 129, and unmerged branch builds already // ran at 130 and 131 with different extraction; 132 is above all of them, so no // cache one of those builds wrote is ever read as this build's. Same layout: // kCacheVersion stays 27, kQSnapCacheScheme stays 16. // 129 = 2026-09-30 (train 22): ONE bump past fn-literal's 128 for the two community // PRs that merge after it — #338 (RSpec described_class, its 125 note below) and #325 // (Ruby inheritance edges, its note below). Each note keeps the number its PR carried. // Record values change for Ruby only; kCacheVersion stays 27. // 129 (#325, train 22) = 2026-09-30 (Ruby inheritance edges, test/rubyinheritcheck.sh; carried as // 98 and 99 on the PR): isBaseTypeNodeIn gains a Ruby arm ((constant)/(scope_resolution)), so // `class Child < Parent` emits an inherit RawRef (role Extends) where it emitted none, and // captureBases does not descend into a computed superclass (`Struct.new( :a )` mints no ref to // its receiver). New records, same layout: kCacheVersion unchanged (27; NOT the PR's 22/24); // kQSnapCacheScheme unchanged. A Ruby cache written before this holds no inheritance refs. // 128 = 2026-09-29 (test/fnliteralcheck.sh): a name bound // to a function literal (JS/TS `const f = (x) => …`, Lua // `M.f = function … end`, a Python class-body lambda) owns // the LITERAL's body (ingest_relations.h kFnLiteralBinding): // bodyByte/sigEnd, params/cx/nest from the literal, and a // multi-name binding's span narrows to its own binding. // 123/125/126/127 stay reserved for community PRs. // Same lane, still 128: a function-local def's scope // span (ingest_names.h enclosingFunctionScope) — a record // layout change, so kCacheVersion 25 -> 26 rejects every // older blob, including one this lane's first build wrote. // 125 = 2026-09-27 (#338, test/rubydescribedclasscheck.sh): RSpec's // `described_class` receiver is classified as the constant // the innermost constant-described example group names // (ingest_binds.h::rspecDescribedClass), so RawRef::recv / // recvVar change VALUE for those call sites — NamedVar // "Calc" where they were NamedVar "described_class". Carried // as 121 on the PR; renumbered 125 after #310 (121), #320 // (122) and #220 (124), with 123 reserved for #325. No record // layout change: kCacheVersion stays 25 (NOT 24); a Ruby cache // written at 124 holds the old receiver and must re-parse. // 124 = 2026-09-26 (#220 part 2, test/depsprecisecheck.sh // P2-O): a TS/JS RE-EXPORT (`export … from './y'`) is an // Include like an import (ingest_relations.h // directiveTargetOf's export_statement branch), so a // barrel's edges exist. 123 stays reserved for #325. // No record layout change: kCacheVersion stays 25 (NOT 24); // kQSnapCacheScheme stays 14. // 122 = 2026-09-25 (#320/#67, test/astrocheck.sh): .astro joins // kLangTable on the TypeScript grammar, parsed through ONE // included range over its `---` frontmatter; blank lines // before the opening fence are skipped. Carried as 120 on // the PR; renumbered 122 after #150 (120) and #310 (121). // No record layout change: kCacheVersion stays 25 (NOT 24); // kQSnapCacheScheme stays 14. // 121 = 2026-09-24 (#310, Ruby class-level attribute DSL, // test/rubyattrscheck.sh — the attr_* floor reversal): a // class-body-level receiver-less attr_reader/attr_writer/ // attr_accessor/attribute/attributes call defines one Var // per simple_symbol argument (plus the `=` setter for // writer-side macros), so setter CALLS (`record.x = v`) // now bind; the extracted def SET grows on every Ruby // corpus. Landed at 115 on the pre-train-6 base, carried // as 120 on the PR, renumbered 121 when merged after #150 // took 120. No record layout change: kCacheVersion stays // 25 (NOT 24); kQSnapCacheScheme stays 14. Folded in at 121 // (unreleased) in the train-19 review round: a comment before // `attribute`'s first argument is skipped, `class << X` for // X != self and `module_function attr_*` define nothing. A // 121 cache written before that round differs only there. // 120 = 2026-09-23 (#150): two new per-record extraction // facts — RawRef::qualifierRootsStd (a C++ call's FULL // written qualifier chain is rooted at namespace std, at // any nesting depth) and RawDef::scopeRootsStd (a C++ // def's full enclosing-namespace chain roots at std) — see // ingest_names.h cppQualifiedChainRootsStd / // cppEnclosingChainRootsStd / cppDefinitionRootsStd, and // graph.h::keepStdQualifiedCandidates, which reads both to // stop `std::ranges::move`/`std::chrono::duration_cast` // from binding an unrelated in-repo `move`/`duration_cast` // the way #134's IMMEDIATE-qualifier-only guard could not // tell from a user's own nested `mylib::ranges::`/ // `vendorlib::chrono::`. A cache written before this // carries neither bit, always reading as "not std-rooted" — // the SAFE direction (a stale-cache read degrades toward // #134's old, narrower behaviour, never toward a new false // bind) — but the format itself changed (RawDef/RawRef both // grew a byte, kCacheVersion 24 -> 25), so a v24 blob is // rejected outright by the version guard before this even // matters. // SAME-LANE CORRECTNESS FIX, folded into this same 120 // (2026-09-24, adversarial review, F1/F2 — never a // separate shipped value: 120 never reached main or a // release before this landed, and this codebase's caches // are per-worktree, so no reader anywhere could have a // cache written under the brief-lived intermediate 121 // this fix once used; collapsing back to 120 is honest, // not a hidden bump). cppDefinitionRootsStd // (ingest_names.h) trusted ts_node_parent(nameNode) to // always be the OUTERMOST qualified_identifier of a // definition's written chain; for a 3+-segment OUT-OF-LINE // definition (`std::detail::f(){}`, `std::hash::mix`) // it is only the INNERMOST link (ingest.cpp re-seats a // definition's @name there), so the fix ADDS a climb to // the true outermost node before reading the root, and OR's // in the enclosing-namespace walk (fixes a second shape, // `namespace std { int detail::innerHelper(){} }`, whose // PARTIAL written qualifier never reached that walk at // all). Also graph.h::keepStdQualifiedCandidates: the "has // a body" test now applies to function-like kinds only, so // a std-rooted VARIABLE (a niebloid) is no longer refused // for having no body. See test/stdqualcheck.sh §14. // SECOND same-train fix, folded in on the same terms // (2026-09-24, CodeRabbit on #331; 120 still unmerged): // cppDefinitionRootsStd no longer marks a qualified // `std::…` def written inside a NAMED namespace // (`namespace vendor { void std::ranges::f(){} }` defines // vendor::std::ranges::f) as std-rooted. A 120 cache from // before it can only over-mark that one perverse shape. // See test/stdqualcheck.sh §15. // #310/#325/#320 (open PRs that also bump kParserVer): take // the next free value above whatever lands after this on // integration; do not reuse 120. // 119 = 2026-09-20 (T13/fix3): queries/java/tags.scm // and queries/kotlin/tags.scm's import captures were // @reference.call — an import is a dependency edge, // not a call, and since #60 gave top-level statements // a owner, an import line got a REAL // caller edge: --callers=/--impact= fan-in on a // Java/Kotlin symbol was inflated by one phantom // "caller" per importing file. Both now spell // @reference.import (RefRole::Import), the same // mechanism C++'s `using ns::name;` already uses — // still a role="import" --uses site, never a call- // graph edge (graph.h isResolvableCallReference is // Call+Macro only). A cache written before this // double-counts every JVM import as a caller. // 118 = 2026-09-19 (CodeRabbit follow-up, thread // 4053600599: isJsxIntrinsicTagIdentifier // (src/ingest_names.h) tested `!isUppercase`, which kept // any non-uppercase-first-letter tag as intrinsic — so // `<_Widget/>`/`<$Widget/>` were wrongly treated as // intrinsic tags and lost their call edge, exactly like // `
`. Fixed to test ASCII-lowercase directly: only // `a`-`z` is intrinsic now, so `_Widget`/`$Widget` are // kept as components. A cache written before this // under-counts their callers. // 117 = 2026-09-19 (train 7: one bump past main's 116 for // both members; each lane had bumped 114 -> 115) // (a) 2026-09-19 lane/disclose-sink-form (degrade disclosure, test/skipreasoncheck.sh // arm 10): a PARTIALLY extracted file (an ExtractShortfall — a // nesting bound, an unavailable tags query, a throw part-way) is // itemized why="extract-partial" and written to the cache as // UNKNOWN. A cache written before this stored such a file's // partial facts under its real hash, which a warm run would // serve as whole: every such cache must be re-extracted. // (b) 2026-09-18 lane/jsx-element-calls (#285 JSX element invocation: // queries/typescript/tags.scm's ``/`` // patterns moved to a NEW queries/tsx/tags.scm — plain // .ts's grammar has no jsx_*_element node types, so a // shared query with them fails to compile AT ALL for // .ts — kLangTable's .tsx row now uses querySub "tsx", // not "typescript"; queries/javascript/tags.scm gained // the same patterns in place, since .js/.jsx/.mjs/.cjs // already share one grammar that has the nodes. New // isJsxIntrinsicTagIdentifier (src/ingest_names.h) drops // an intrinsic tag (`
`) at capture time. // 116 = 2026-09-18 (issue #287 round 2, review // rv-p6.md HIGH: capturePythonRebindShadowDecls // emits a LocalBindKind::VarDecl RawBind for every // Python name-rebinding form — plain/augmented // assignment, for-target, with/except-as, walrus, // del, a nested def/class name, global/nonlocal — // so Rule 2d's alias-receiver narrow can no longer // mistake a rebound name for the still-live import) // 115 = 2026-09-18 (issue #287: capturePythonImportBinds // now sets RawBind::importedName="module" iff the bound // name IS the imported module (`import a.b`/`import // a.b as c`), empty for `from m import x [as y]` — the // module-alias receiver narrow reads this bit) // 114 = 2026-09-17 (reference-returning definitions, // test/narrowcheck.sh arms 61-63, test/shadowcheck.sh arms am // and q8): a C++/ObjC function definition returning `T&` or // `T&&` records its parameters (VarDecl + ParamType; it recorded // none — the declarator walk stopped at reference_declarator), // and an attributed declarator records its VarDecl. No record // layout changes (kCacheVersion stays 24). The lane declared 112 // over #282's 104; integration/train-5 assigns 114 after #282's 113. // 113 = 2026-09-17 (smart-pointer members, PR #282, // test/fieldnarrowcheck.sh arm p): a C++ member written // `std::unique_ptr` / `std::shared_ptr` records T with // viaArrow set (it recorded nothing), and a call ref records // whether its member access was `->`. The ref record grows by // one u8 (kCacheVersion 23 -> 24 in the same commit). The PR // declared 104 over main's 103; integration/train-5 assigns // 113 over main's 112. // 112 = 2026-09-17 (type aliases, PR #280, test/fieldnarrowcheck.sh // arm t): a C/C++/ObjC `typedef` / `using` alias of a named class // emits a compose-shaped RawRef (composeRel "alias", empty // fieldName) that the base walk follows to its target. Format // unchanged (kCacheVersion stays 23); an older blob holds no such // records and would dead-end the walk at the alias on a warm run: // content change, bump required. The PR declared 105 over main's // 99; re-declared 111 over main's 110 when train 3 merged; // integration/train-4 assigns 112 after irbuilder-local's 111. // 111 = 2026-09-17 (block-scope direct-initialized locals, // test/narrowcheck.sh arms 52-60): a C++ function_declarator // inside a function/lambda body that an argument list also // produces (`IRBuilder<> Builder(Rem);`) mints no function // symbol (ingest_names.h cppBlockScopeDirectInit), so the // local's type binding is attributed to the enclosing function. // Symbols and fact attribution change; no record layout changes // (kCacheVersion unchanged). Claimed 111; integration/train-4 keeps it over main 110. // 110 = 2026-09-17 (Java catch/enhanced-for/resource shadows, PR #235 // follow-up from CodeRabbit on #281, test/javamethodrefcheck.sh): // a catch parameter, an enhanced-for variable and a try-with- // resources resource now emit Java VarDecl shadow binds, so a // `Widget` declared there vetoes `Widget::m` inside that scope. // The extracted bind SET changes; no record changes shape. // 109 = 2026-09-17 (Ruby constant receivers narrow calls, PR #267, // test/rubyrecvnarrowcheck.sh): classifyReceiver classifies a // Ruby (constant)/(scope_resolution) receiver as NamedVar with // the FINAL constant segment, so `Calc.add(…)` / // `Outer::Engine.run(…)` reach resolve.h's Rule 2c instead of // the §2a name spray. RECORD LAYOUT unchanged — recv/recvVar // are fields RawRef already had (kCacheVersion stays 23) — but // their VALUES change, so old Ruby extraction facts must be // re-parsed. quality.h's kIngestParserVerMirror bumped in the // SAME commit. The PR declared 96 -> 97 over main; assigned // 109 on integration/train-3 after #233's 108. // 108 = 2026-09-17 (GDScript, PR #233, test/gdscriptcheck.sh): a // new grammar (third_party/deps/gdscript) and queries/gdscript/ // tags.scm, `.gd` a kLangTable row, and NodeField::Op appended: // a tree with `.gd` files yields new files, symbols and edges. // The PR declared 96 -> 98 over main; assigned 108 on // integration/train-3 after #235's 107. kCacheVersion stays 23. // 107 = 2026-09-17 (Java Type::method, issue #74, PR #235): the PR's // two steps below, declared 97 and 98 over main's 96, land as one; // assigned 107 on integration/train-3 after #278's 106. // RecvKind::JavaTypeCandidate is appended after train 2b's Lit* // kinds (kRecvKindCount follows it); kCacheVersion stays 23. // PR step 98, 2026-09-15 (Java Type::method review, // test/javamethodrefcheck.sh): Java shadow binds carry // lexical spans (block / lambda / method body) and inferred // lambda parameters (`Widget ->`, `(Widget) ->`) are captured. // Extracted bind SET and span values change, so a v97 blob // must be rejected. kCacheVersion stays 22 — Binding already // has spans; RecvKind and record shapes are unchanged. // PR step 97, 2026-09-15 (Java Type::method candidates, // test/javamethodrefcheck.sh): method_reference member names // after `::` plus declaration-aware resolver gating. The query // cannot distinguish a type identifier from a value identifier. // #216 already spent 96 on internalLinkage, so this RE-BUMPS // (never-reuse / collision). kCacheVersion stays 22 — RecvKind // is appended, record shapes are unchanged. // 106 = 2026-09-17 (assignment types, PR #278, test/narrowcheck.sh arms // 44-51): a C++ ASSIGNMENT's Type RawBind is marked // isFromAssignment, and buildGraph keeps its callee-read name // only when a class of that name exists (`t = llvm::cast( y )` // recorded `cast` and tombstoned `T* t`). The bind record grows // one u8 (kCacheVersion 22 -> 23 in the same commit). The PR // declared 104 over main's 99; assigned 106 on integration/ // train-3 after small-fixes' 105. // 105 = 2026-09-17 (A4, found-items 2026-09-17, // test/filerootcheck.sh arm 4): `.hxx` gained a kLangTable row // (Lang::Cpp, same as `.hpp`/`.hh`) — the crawl previously // skipped every `.hxx` file outright (unindexed), so a repo // that spells its headers `.hxx` now yields NEW files, // symbols and edges a pre-bump cache never saw: content // change, bump required. The lane declared 100 over main's // 99; assigned 105 on integration/train-3 after #276's 104. // 104 = 2026-09-17 (template arguments in a receiver's written type, // test/narrowcheck.sh arms 39-43): a C++ declaration's Type/ParamType // record takes its type's LAST NAME through the grammar's fields. // An unqualified template-id (`Vec& v`, `Vec v;`) // recorded nothing and now records `Vec`; `Outer::Inner` // recorded `Outer` (finalSegment cut at the first `<`) and now // records `Inner`, written or constructed (`Outer::Inner()`, // and `Foo::create()` records `create`, not `Foo`); an // unqualified `Vec()` stays unread (ingest_binds.h // ctorNameNode's floor); `Vec` no longer records a // qualified text for its argument's `::`. No record changes shape // (kCacheVersion stays 22). The PR declared 103 over main's 99; // assigned 104 on integration/train-3 over train 1b's 103. // quality.h's kIngestParserVerMirror moves in the SAME commit. // 103 = 2026-09-17 (TS/JS signed numeric literal receivers, train 1b // #277): `(-1).toFixed()` is a Number receiver, not an unrelated // `toFixed`. An extraction change on #244's literal receivers; // no record changes shape. // 102 = 2026-09-17 (C++ template scopes, test/cpptmplscopecheck.sh, // PR #256): a primary template's out-of-line member keys the bare // template name (`void Box::grow()` joins `Box::grow`); a // specialization keeps its canonical template-id; a reference // keeps the template-id it writes (3+ segments too); and a class // specialization HEADER's base clause is captured as inherit // refs (tags.scm @definition.specialization). Symbol scopes, // RawRef::qualifier and the extracted refs change; no record // layout changes (kCacheVersion stays 22). The PR declared 100; // assigned 102 on integration/train-2b after #243's 101. // 101 = 2026-09-17 (member template calls, test/cppqualcheck.sh // §12, PR #243): a C++ member call with explicit template arguments // (`r.f()`, `p->f()`, `x.template f()`) mints a // call reference with its receiver, where it minted none; // a `template` disambiguator no longer leaks into a qualified // call's name (`template f`) or qualifier (`template Rebind`). // The extracted SET and names change; no record changes shape // (kCacheVersion stays 22). The PR declared 99; assigned 101 on // integration/train-2b in merge order after #244's 100. // quality.h's kIngestParserVerMirror moves in the SAME commit. // 100 = 2026-09-17 (TS/JS literal receivers, issue #163, PR #244): RecvKind // gains LitString/LitArray/LitRegex/LitNumber/LitBoolean // (appended u8, no RawRef field, kCacheVersion stays 22). // A `"x".replace()` call no longer takes the bare-name ladder. // The PR declared 97; assigned 100 on integration/train-2b in // merge order over train 2's 99. // 99 = 2026-09-16 (std-typed member fields, test/fieldnarrowcheck.sh // arm q): a C++ field's compose RawRef records the namespace its // type was written in as `qualifier` (`std` for `std::string // name_;`). Format unchanged; a 98 blob holds "" there and would // let Rule 2b and the HAS-A edges read `string` as an in-repo // class on a warm run: content change, bump required. // 98 = 2026-09-16 (std-qualified receivers, test/narrowcheck.sh // arm 21): a C++ ASSIGNMENT from a constructor (`x = std:: // map()`) records the constructor's qualified text in // importedName, like a declaration's record. Format unchanged; // a 97 blob holds "" there and would let Rule 2 narrow `x` to // an in-repo `map` on a warm run: content change, bump required. // 97 = 2026-09-16 (parameter receivers, test/narrowcheck.sh arm // 17): a declaration's Type/ParamType RawBind records its // written type WHOLE in importedName when the type is // QUALIFIED (`std::map`). Record FORMAT unchanged (the // field was already serialised, empty on these kinds), but a // 96 blob holds "" there and would let Rule 2's lexical lookup // narrow a qualified parameter type to an unrelated same-named // in-repo class on a warm run: content change, bump required. // (Binding::startByte, same lane, is re-derived from the cached // RawBind::startByte and needed none.) // 96 = 2026-09-13 (internal linkage, test/decltodefcheck.sh arm // B2): every C/C++ def carries a new syntactic // `internalLinkage` bit — inside an anonymous namespace at any // depth, or a namespace-scope `static`. The def record grows // by one u8 (kCacheVersion 21 -> 22 in the same commit). Next // free number over the merged tip (main at 95 after #81); // quality.h's kIngestParserVerMirror and // kIngestCacheVersionMirror bumped in the SAME commit. // 95 = 2026-09-12 (Elixir module/name/arity resolution, PR #81, // test/elixirsemanticcheck.sh): module/name/arity identities, // lexical aliases, filtered imports, default arguments, pipes, // captures, delegates, attributes and protocol/behaviour // contracts. Existing bind/ref record layouts are unchanged // (kCacheVersion stays #139's 21; the same PR's kQSnapCacheScheme // 10 -> 11 is quality.h's arity-key fold, not an extraction change); // old Elixir extraction facts must be re-parsed. The branch // carried 87; main spent 87..92 while it was open, #139 takes // 93 and #172 takes 94 in the 0.6.1 round, so this lands on // the next free number over the merged tip (the 78/80/88/91 // rule). quality.h's kIngestParserVerMirror bumped in the // SAME commit. // Three pre-merge follow-ups on the same PR move extraction // OUTPUT within 95 (test/elixirnamearitycheck.sh H, I, J): a // dotted nested defmodule binds its first segment (ref // qualifiers), an alias of __MODULE__ classifies its receiver // ElixirSelfModule (RawRef::recv), and `&_seed/0` is captured // (the extracted SET). No second bump: the never-reuse rule is // about TWO LANES shipping one in-flight number with different // sets, and 95 is this lane's alone — minted here over the // merged tip, written by no released binary. Record shapes are // unchanged (kCacheVersion stays 21). A blob a pre-follow-up // build of this branch wrote is the one exposure, and it is // local to whoever built that branch: --no-cache, or let the // stat gate re-parse the edited files. // 94 = 2026-09-11 (#62/#72 follow-up, all roles + definitions): // the decided-dead `#if 0` filter moved from captureTagsFacts' // @reference.call/@reference.import arm to a window post-pass // over every fact family a C-family file produces. FIVE more // emitters now honour it — the value-use pass (role=read/write), // the type-mention pass (role=type), captureBases (role=extends), // captureIncludes' directive site (role=import, plus the Include // FILE-dependency record the same directive minted) and the // blanked member-macro use — and a DEFINITION whose name sits in // a dead range is no longer indexed at all, so it can no longer // split resolution (overloads=/amb=/prov="split"/graph_ambiguous= // against a definition that cannot compile). The extracted SET // SHRINKS on any C-family tree carrying a literal `#if 0`/`#if 1`, // so a v93 blob replays rows this binary refuses. Record shapes // are untouched, so kCacheVersion stays #139's 21. quality.h's // kIngestParserVerMirror carries the same value (gated). Gate: // test/ppdeadrolescheck.sh, one live/dead pair per --uses role. // RENUMBERED 93 -> 94 on the merge with main 558a2e03: the // branch spent 93 while main spent it on #139 (Ruby arguments). // 93 = 2026-09-11 (Ruby argument + rescue constants, // test/rubyargcheck.sh): a constant chain that is a direct // argument of a call/super/yield (or a keyword pair's value // there) and every class in a rescue list are symbolic // Include records, deduped with receivers per (file, open, // written); a rescue class is lazy always. An older blob // holds none of them, so it is stale rather than wrong — the // header version is what rejects it. RENUMBERED 89 -> 93 on // the merge with main 40a1895b: the branch spent 89 while main // spent 89..92 (extent, Kotlin, yaml). quality.h's mirror // bumped in the SAME commit. // 92 = 2026-09-11 (yaml unsigned-char, PR #140): vendor patch // yaml/003-scan-status-enum gives tree-sitter-yaml's scan status // (SCN_SUCC 1, SCN_STOP 0, SCN_FAIL -1) a real `ScanStatus` enum // type instead of returning it through plain `char`. `char` is // UNSIGNED on aarch64 Linux — the platform the linux-arm64 release // asset is built for — and there SCN_FAIL came back as 255, which // no `case SCN_FAIL:` label matched: a malformed %-escape in a tag // or a %TAG prefix was swallowed into the token rather than ending // it, so `a: ! b` parsed as ERROR under a signed `char` // and as a clean tagged scalar under an unsigned one. The patch // makes the unsigned-`char` build parse as the signed one always // did — byte-identical on a signed-`char` host, CHANGED on an // unsigned one. kArtifactArch cannot tell an aarch64 cache blob // from an x86-64 one, so only this version can reject a blob // written by the pre-patch unsigned-`char` binary. Record shapes // are untouched, so kCacheVersion stays #135's 20. quality.h's // kIngestParserVerMirror carries the same value (gated). Live // tripwire: vendorpatchcheck arm K, which compiles the vendored // grammar -fsigned-char and -funsigned-char on any host. // 91 = 2026-09-11 (Kotlin, PR #126): a 24th grammar joins kLangTable // (.kt), so the crawl admits files a v90 blob never saw — ABSENT, // not stale, and only the header version can reject that blob. // 91 and not 89/90: #135 spent both, and main stands at 90. Also // under 91: vendor patch kotlin/002-triple-dollar-escape changes a // real parse (a triple-quoted string ending right after an escaped // `\$`, e.g. """a\$""", lost its first closing quote); // kotlin/003-dollar-run-saturate changes the parse of a run of // 65,536 or more `$`, where upstream's counter wrapped; and the // ingest nesting guard refuses a .kt file whose string templates // nest past kMaxKotlinStringNestDepth before the parse, which // changes WHICH .kt files are extracted. kotlin/001-stack-push- // no-abort alone would not need a bump: it only changes input that // used to abort (the yaml/001 and markdown/001 precedent). A // refused file's cache record is written UNKNOWN with the existing // hash-0 encoding, so record SHAPES are unchanged and kCacheVersion // stays #135's 20. The bodyless-Kotlin-type and own-JVM-language rules // live in graph.h and are recomputed every run. quality.h's // kIngestParserVerMirror carries the same value (gated). The // branch's two earlier steps, folded in under their branch numbers // so no 89/90 label collides with #135: // (branch 90, 2026-09-08, adversarial-corpus follow-up) // measureFileHealth now also validates UTF-8 in the leading // sample (a file that only trips this check needs a cold // re-measure to pick up the new degraded-parse disclosure — // a cached FileHealth from before this version predates the // check and would read as healthy); enum_class_body added to // the Kotlin positional body-fallback (an `enum class` was // read as bodyless, same collapse bug as 89's class_body // fix, just for the enum-class node shape). RE-BUMPED from // 87 on rebase: main independently spent 86 (Ruby receiver), // 87 (markdown scanner counter saturation) and 88 (Dart) // while this branch was in progress, and 86/87 collided // EXACTLY with this branch's own prior use of those numbers. // quality.h's kIngestParserVerMirror bumped in the SAME // commit. // (branch 89, 2026-09-08) grammar + queries/kotlin/tags.scm; // positional body/scope lookups (function_body, class_body and // type_identifier are children, not fields — a def read as // bodyless is deleted by graph.h's decl/def collapse); // captureBases reads delegation_specifier; when_entry/ // when_expression/do_while_statement/catch_block count as // decisions; function_value_parameters counts `parameter` // children. Originally 86, RE-BUMPED for the same // collision reason as 90 above. // 90 = 2026-09-11 (member-macro re-parse, test/macroreparsecheck.sh): // a C-family file whose first parse holds error bytes may be // extracted from a re-parse with its semicolon-less member macro // invocations blanked (src/macroreparse.h) — different defs, // scopes, complexity and health for exactly those files, plus // one role=Type use per blanked invocation on the rich family. // A v89 record cannot say which parse it came from, and the // file record grows a u32, so kCacheVersion moves 19 -> 20. // 89 = 2026-09-11 (extent honesty, test/extentcheck.sh): every // def carries RawDef::recovered — the parse recovered its // container (a class whose body holds an error, inside an // ERROR node) or its kind (a scopeless C++ method inside one), // set by ingest_names.h parseRecoveredBits. A new extracted // fact, so a v88 record cannot answer it; the record SHAPE // grows one u8 too, so kCacheVersion moves 18 -> 19 with it. // Every symbol, edge and metric is byte-identical (verified // against the pre-change binary on src/ and the test/ corpus); // only rows the new check flags gain extent_suspect=. // quality.h's kIngestParserVerMirror bumped in the SAME commit. // 88 = 2026-09-10 (Dart, test/dartcheck.sh): a 23rd grammar joins // kLangTable, so the CRAWL ADMITS FILES IT PREVIOUSLY REFUSED — // a v87 blob has no record for the `.dart` it never saw, so the // file is ABSENT rather than stale and only the header version // can reject it. The definition SPAN is also extended for Dart // only (dartFollowingBody, adopted in ingest_sidecap.h) and // formal_parameter_list joins cc_isParamList; both are // extraction identity, which is what parserVer covers, and every // other language is byte-identical (verified against the // pre-change binary on src/ and a 1 406-file multi-language // corpus). Record shapes unchanged, so kCacheVersion stays 18. // RE-BUMPED from 87 on landing: main spent 82..87 while PR #75 // was open, and 87 collided EXACTLY — the declaration line // auto-merged clean at the same wrong number while only the // comment conflicted. quality.h's kIngestParserVerMirror bumped // in the SAME commit. // 87 = 2026-09-10 (test/vendorpatchcheck.sh arm I, // third_party/patches/markdown/002-counter-saturate): // the vendored markdown scanner accumulated consumed // whitespace, and its fence delimiter count, into // uint8_t counters with a bare `+=` and WRAPPED. Both // are read by ordering tests against a FIXED threshold // (`>= 4` indented chunk; `>= 3` before a fence may // open), so wrapping did not blur them, it INVERTED // them — and only inside a narrow window. MEASURED: // N=255 correct, N=256/257 WRONG at exit 0, N=300 // correct again by luck (300-256=44, still over both // thresholds). At 256 an indented code block came back // as a heading, and a fence never opened so its body // leaked out as live markdown; ripwire minted phantom // symbols for both. Counters now saturate at 255, which // is exact for every threshold in that file. // Map output is byte-identical over 3 538 real files — // no corpus file reaches 256 columns of indent — but a // constructed 256-column line does move, so a v86 blob // can hold a phantom heading and the extraction // identity must move with it. // The sibling patches rust|lua|csharp/001-delimiter- // count-cast take an explicit CAST, not saturation: // those counters close a token by matching the opening // count, and measurement found NO extraction difference // at any width (255/256/257/300), so they contribute // nothing to this bump. // Record shapes unchanged, so kCacheVersion stays 18. // quality.h's kIngestParserVerMirror bumped in the // SAME commit. // 86 = 2026-09-09 (test/rubyrecvcheck.sh): a Ruby constant // RECEIVER's lazy bit is the AND over its occurrences — // a later load-time site clears the bit the first, // closure-written site set, so the answer no longer // depends on statement order; and rubyIsConstantChain // walks the left-nested scope_resolution ITERATIVELY // (a 5 000-segment chain overflowed a parse worker's // stack). A cached v85 blob can hold a lazy bit this // binary would compute as load-time, so the extraction // identity moves. Record shapes unchanged (Include's // four fields), so kCacheVersion stays 18. RE-BUMPED // from 85 on landing: main had already spent 85 on the // plain-text prose tier below, per the collision rule. // quality.h's kIngestParserVerMirror bumped in the // SAME commit. // 85 = 2026-09-09 (test/textdocscheck.sh): the plain-text // prose tier — .rst/.adoc/.org/.mdx join kLangTable on // Lang::Markdown and the markdown block grammar. The // CRAWL ADMITS FILES IT PREVIOUSLY REFUSED, so a v84 // blob describes a strictly smaller corpus: its file // list has no record for the .rst it never saw, and a // warm run over it would answer a document query with // the pre-lane silence. That is the one class of change // a per-file stat gate cannot self-heal — the file is // not stale, it is ABSENT — so the header version is // the only guard. Record shapes are unchanged (a // markdown file's records already existed), so // kCacheVersion stays 18. quality.h's // kIngestParserVerMirror bumped in the SAME commit. // 84 = 2026-09-08 (test/rubyrecvcheck.sh): a Ruby constant // RECEIVER (`User.find`, `App::Mailer.deliver`) is a // symbolic directive, one per (file, innermost open, // written name), lazy inside a closure; the Ruby walk // descends every node. Record shape unchanged (format 17). // Renumbered from 83 on merge: main had already spent 83 // on the JS/TS default-import facts below. // 83 = 2026-09-08 (test/tsimportprecisecheck.sh): JS/TS // DEFAULT imports and local default-export facts — // `import save from './storage.js'` now resolves by the // module's exported identity, not by the importer's // chosen local name, so an unrelated same-spelled // function is no longer evidence of that edge. New // extraction facts on an unchanged record shape, so // kCacheVersion is untouched. Renumbered from 82 on // merge: main had already spent 82 on Ruby constants. // 82 = 2026-09-07 (test/rubyconstcheck.sh): Ruby constant // references are dependencies — superclass, include/ // extend/prepend, `autoload :Name` (constant) and // `autoload :Name, "path"` (path) — resolved through the // corpus's own class/module index by Ruby's lexical rule; // class/module opens carry namespaceOnly. // 81 = 2026-09-07 (four-language imports: // test/bashsourcecheck.sh, test/luarequirecheck.sh, // test/rubyrequirecheck.sh, test/eliximportcheck.sh, // test/deplangscheck.sh): Bash `source`/`.`, Lua // `require`, Ruby `require`/`require_relative`/`load` and // Elixir `alias`/`import`/`require`/`use` become Include // records, each with a container table and a resolve.h // Step-A. Four languages that emitted NO Include record on // any tree now emit one per directive, so a v80 blob on a // tree holding any of them carries an EMPTY include list // where a real one exists → reject. Record shapes are // unchanged (Include is the same four fields), so // kCacheVersion stays 16 — the 38/39 precedent exactly. // quality.h's kIngestParserVerMirror bumped in the SAME // commit. // 78 = 2026-09-07 (Elixir, test/elixircheck.sh): a // twenty-second grammar (.ex/.exs) whose defs and call // edges are extracted through the keyword/head filters in // ingest_elixir.h. The extracted SET grows on any tree // holding Elixir, so v77 blobs must be rejected. // Landed at 78 (not the 83 the fork carried) per the // collision rule below: RE-BUMP to the next free number // over main's, never keep a fork's value. // 79 = 2026-09-07 (ES import facts, test/lib/jsimportfacts.sh): // named import aliases bind through the export table, and // `export { f }` / `export { f as g }` CLAUSE exports join it // (JsExport gains importedName = the local name); re-export // and default clauses stay deliberately absent. quality.h's // kIngestParserVerMirror bumped in the SAME commit. // NOTE: both fork branches carried a "skip N, it is the rich // family of N-1" rationale. That is WRONG and is not repeated // here: parserVerFor() derives rich as kParserVer+1, but // main.cpp's A4-P4 split gives lean and rich SEPARATE cache // FILES, so a lean-79 blob can never reach a rich-78 reader. // main's own history is 74->75->76->77, four consecutive +1. // 80 = 2026-09-07 (Ruby scope + setters, PR #47 rebased, // test/rubyscopecheck.sh, test/rubysettercheck.sh): two Ruby // extraction FACTS moved. (a) every Ruby def records its // enclosing class/module as `scope` (rubyEnclosingScopeOf), // so Ruby rows gain id= and same-named methods in different // classes stop folding into one overloads= row; (b) the // (setter) node is accepted by queries/ruby/tags.scm, so // `def name=` is indexed, and a call captured as the `left:` // of an (assignment) is renamed `name=` so a WRITE stops // edging the getter. Record shapes unchanged, kCacheVersion // stays 16; the def and ref FACTS changed for every Ruby // file → parserVer moves. The fork carried 79, which the // Elixir + ES-import bumps had already taken: RE-BUMPED to // the next free number over the merged tip, per the rule // above. quality.h's kIngestParserVerMirror moved in the // SAME commit. // 77 = 2026-09-03 (Phase 5, docs/EVALS.md): two Python // ingest FACTS — (a) a `super()` call receiver classifies // RecvKind::SuperObj (appended) instead of None, so // `super().m()` stops reading as a BARE call; (b) every // import statement records the NAMES it binds as file- // scope LocalBindKind::Import RawBinds (appended kind), // `np`→`numpy`, the fuel of the external-name veto. // Record shapes unchanged, kCacheVersion stays; Python // ref and bind FACTS changed → parserVer moves. // 75 = 2026-09-02 (member-variable round, card A3): a new // SymKind::Field — C/C++ @definition.field (non-static // field_declaration) and Python (`self.x = …` / annotated // class attribute, the latter re-kinded from Var) — plus // the value-use visitor now captures the FIELD half of a // non-call member access (`a.f`/`p->f`/`o.f`) as a // Read/Write ref carrying its receiver shape (recv/ // recvVar/fieldName, fields the record already had). // Record shape unchanged, kCacheVersion stays; the def // and ref FACTS changed for every C-family and Python // file → parserVer moves. // 74 = 2026-08-30 (objc-sniff lane): looksObjC masks // comments and string/char literals before testing for // @interface/@protocol/@implementation — a C++ .h whose // doc comments MENTION those tokens (ingest_model.h's own // collapseObjCDeclDefs contract) was rerouted wholesale to // the objc grammar and its C++ symbols (struct DefSweep + // DefSweep::find, anon namespace) shredded at extraction. // Language ROUTING changed for such headers → cached facts // extracted under the wrong grammar must be rejected. // 73 = 2026-08-25 (candhead-ugrep lane): C++ out-of-line // NESTED CLASS/STRUCT definitions (`class Outer::Inner : // Base { … };`) — queries/cpp/tags.scm gains a // qualified_identifier name: pattern for class_specifier/ // struct_specifier, mirroring the out-of-line METHOD // pattern already there. Previously dropped at extraction // with no symbol minted at all (N11/N12). // 72 = 2026-08-24 (fnbody-require lane): CommonJS `require("./x")` / // dynamic `import("./x")` captured INSIDE a function body — kJsImportContainers // grows past the 71 top-level-only set (statement_block, return_statement, // the six function-body node kinds, control-flow clauses, and the // object/pair/arguments/call_expression/parenthesized_expression chain // needed to reach a `get X() { return require(…); }` getter sitting // inside an object literal passed as a call argument — webpack's // lib/index.js lazy-getter barrel, read off a real parse). Same three // guards as top-level (bare require/import callee, one arg, string // literal) — SAME jsModuleLoadTarget, called from a deeper set of // containers. Include gains `bool isLazy` (a FORMAT change → reject v71 // blobs): true when the call sits inside a function-body container, // disclosed on --impact's import tier as `lazy=` (graph.h::impactImportTier, // graphlegend.h::kImpactImportTierLegend) — a lazy require is a real // dependency (the importer tier must still name the file) but a WEAKER // one: it only fires if and when the function runs. See // test/impactimportcheck.sh's lazy fixture arm and // test/nestedimportfix/scope_control.ts. // 70 = 2026-08-22 test-macro blocks (LB-E, r10 harvest): a known // doctest/Catch2 block-forming test macro (kTestBlockMacroNames) // invoked as `TEST_CASE( "title" ) { … }` — which tree-sitter-cpp // parses as an expression_statement with a MISSING ";" plus a // SIBLING compound_statement, so pre-70 it minted NO symbol and // every call inside the body attributed to NOTHING (measured: // five pageRankDouble sites invisible to --callers on this // repo's own test/verify_pagerank.cpp) — now extracts as a // t="fn" symbol named by its title literal, spanning through // the block, testScope=1. queries/cpp/tags.scm gains the // sibling pattern (@definition.testmacroblock). The extracted // SET grows on any C++ tree using those harnesses, so a v69 // blob misses rows and must be rejected, not served. See // test/testmacrocheck.sh. // 69 (2026-08-21 wave-2 merge): TWO INDEPENDENT extraction changes // both landed on 68 in separate branches. Per the note at 65 below, // a collision is resolved by RE-BUMPING to the next free number, never // by keeping one side's value: no released binary ever wrote a 68 blob, // and a v67 blob is missing BOTH sets of rows, so one reject covers both. // (a) PHP + Lua language port (phpcheck.sh / luacheck.sh): // TWO new grammars (tree-sitter-php v0.24.2's `php/` // sub-grammar, tree-sitter-grammars/tree-sitter-lua // v0.5.0) and TWO new .scm files, so the extracted SET // changes on any tree holding a .php/.phtml/.lua file // that previously fell out as unsupported-ext. ONE bump // for both is deliberate: they land in the same commit, // so no released version ever keyed a cache on one // without the other. Three shared-path edits ride along // and each is LANG-GATED so every existing corpus stays // byte-identical: isDecisionType/cc_isNestingControl take // a Lang and exclude Lua's `do … end` (a bare scope // block, not a loop — counting it would be a WRONG // number, not a floor) while admitting Lua // repeat/elseif and PHP `match` arms; the &&/|| // accumulator learns the word forms `and`/`or`/`xor` // behind a Php||Lua gate; captureBases learns PHP's // base_clause/class_interface_clause and captureIncludes // its namespace_use_declaration. Known, disclosed cost: // one cold re-parse everywhere. // (b) receiver-guard misfires (chainguardcheck.sh): // receiverOf classified EVERY chained receiver as // RecvKind::None, so five guard sites keyed on // `recv == None` misread `this->m_pool.run()` as a // BARE name: Rule 1's bareCish arm wrong-narrowed it // to the caller's OWN class, and shadow suppression // deleted it under a same-named local. RecvKind gains // FieldOfThis / FieldOfVar (u8, 5 of 256 used) and the // INTERMEDIATE field name rides RawRef::fieldName, // which is free on a call ref (isCompose owns it // otherwise). The wire FORMAT is unchanged — // writeRef/readRef already carry recv and fieldName — // but the extracted VALUES change and DO move edges= / // ambiguous= (recovered + split-widened calls). NO // resolve rule consumes the new kinds yet. A v67 blob // holds None where this binary expects a chain and // must be rejected rather than served. quality.h // kIngestParserVerMirror bumped in the SAME commit. // Registered + measured: docs/EVALS.md §4. // 67 (2026-08-20 RefRole::Type use-sites, typerefcheck.sh): // usesVisitNode's accept set was the single test // `strcmp( t, "identifier" ) != 0 → return`, so a // `type_identifier` node — a bare TYPE mention in a // signature, a declaration or a template argument — // was captured by NOTHING. It now emits a RawRef with // the new RefRole::Type (model.h), which joins // Read/Write/Import/Extends on the NEVER-in-the-CSR // list, so the default ranked map, PageRank, edges= // and ambiguous= are unchanged BY CONSTRUCTION — the // value-uses pass is also RICH-family only, so the // lean blob that backs the default map holds no such // ref at all. What changes is the extracted SET of // references, which the RICH per-file cache record // persists (writeRef/readRef already carry `role`, so // the wire FORMAT is unchanged — a v66 rich blob would // simply be missing every type row and must be // rejected rather than served). quality.h // kIngestParserVerMirror bumped in the SAME commit. // Registered + measured: docs/EVALS.md §4. // 66 (2026-08-19 subtoken acronym shredding, subtokencheck.sh): // the shared subtoken state machine (lexindex.h // forEachLexSubtoken/forEachLexSubtokenHashed) stopped // cutting a token at EVERY interior uppercase byte, which // had reduced an all-caps run to 1-byte fragments that the // ≥2-byte rule then dropped — an acronym was indexed as // nothing at all. A run is now one token, split only at the // last upper before a lowercase ("HTTPServer" → http|server). // lexSubtokenHash also lowercases EVERY byte now, not just // the first. Both feed the PERSISTED rich-cache record — // RawDefLex's dlWeighted/tokenHashes/tokenTfs and the // per-file 512-bit H3 signature derived from them — so v65 // blobs hold token statistics this binary would never // produce and must be rejected. quality.h // kIngestParserVerMirror bumped in the SAME commit. // Registered + measured: docs/EVALS.md §4. // 65 (2026-08-15 C++ nested out-of-line defs, cppqualcheck.sh §11): // queries/cpp/tags.scm gains a second out-of-line // definition pattern (`qualified_identifier name: // (qualified_identifier)`), so a C++ definition written // with TWO OR MORE qualifier segments // (`void nsD::OuterD::InnerD::deep3(){}`) is indexed at // last — it was dropped at extraction, silently, at every // depth past one. The extracted SET grows on any C++ tree // using that spelling (memgraph: 575 defs), so v64 blobs // must be rejected. quality.h kIngestParserVerMirror // bumped in the SAME commit. NOTE for whoever lands the // unmerged integration/lang-round, which also holds a // kParserVer bump: resolve a collision by RE-BUMPING to // the next free number, never by keeping one side's value. // 64 (2026-08-14 in-file test scope, test/testscopecheck.sh): // every def carries a new syntactic `testScope` bit // (Rust `#[cfg(test)] mod` / `#[test] fn`, Python // `class Test*` / module-level `def test_*`, JS/TS // `describe(`/`it(`/`test(` blocks, C# `[Fact]`/`[Test]`/ // `[TestMethod]`), written into the cache record — so a // v63 blob has no such field and must be rejected. // quality.h kIngestParserVerMirror bumped in the SAME commit. // 63 (2026-08-12 markdown section tier, test/mdsectioncheck.sh): // .md/.markdown now parse with the vendored tree-sitter-markdown // block grammar — headings (ATX + setext) become sections with // REAL SPANS (heading → next same-or-higher heading), parent- // heading scopes, and link/mention edges; html-block phantom // headings vanish; .markdown joins the table. The extracted SET // and the spans both change on any md-bearing tree, so v62 // blobs must be rejected. // 62 (2026-08-12 module-constant round, test/moduleconstcheck.sh): // C/C++ const-qualified module constants index CASE-BLIND — // a const/constexpr/constinit type_qualifier on an INITIALIZED // module-scope declaration keeps the binding regardless of // name case (declarationCarriesConstQualifier; previously the // r3 q10 SCREAMING-only gate dropped every k-camel constant, // which made this repo's own kParserVer unfindable by its own // --for/--uses — the 2026-08-12 census's 21.4% constant-shaped // lookup family). queries/cpp/tags.scm also gains the // class-static field_declaration capture (static + const // qualifier + in-class initializer, fieldConstantCaptureKept — // NO class-static constant was indexed before, even SCREAMING). // The extracted SET grows on any C/C++-bearing tree (this repo: // src/ +~350 rows), so v61 blobs miss rows -> reject. // Deferred with probe evidence (2026-08-12): enumerators // (corpus blow-up: >=5000 capture-cap hits on a 2 377-file private ObjC++/C++ validation tree), // TS/JS non-SCREAMING top-level consts (r3 q10 pinned policy, // constcheck arm 3), variable templates, `T* const` pointer // constants, out-of-line static-member definitions. // 61 (2026-08-11 YAML config-key tier): .yml/.yaml indexed for // the first time — mapping keys at mdepth<=2 (sequences // transparent) become t="sec" symbols via queries/yaml/ // tags.scm + the definition.yamlkey gate. A v60 blob on a // YAML-bearing tree is missing every such row -> reject. // Known, disclosed cost: one cold re-parse everywhere. // 60 (2026-08-10 language-port round): one shared bump covering // THREE hand-ported language rounds, each stranded on a branch // this tree never had. Listed separately because each changes // the extracted SET on a different corpus. // (a) PYTHON shapes: annotated class attributes, gated // enum-family members, class lambda attrs, one-guard-deep // + tuple-unpack module bindings, and .pyi routing. // RE-MEASURED at v59 on django@c334c1a8ff / // pydantic@8898b8f: every one of those shapes read 0.0% // EXCLUSIVE recall. A v59 blob on a Python-bearing tree // is missing those rows -> reject. // (b) SWIFT shapes (hand port of stranded bb78f97, which // originally landed at kParserVer 41): enum_entry / // typealias_declaration / associatedtype_declaration / // protocol_property_declaration / the builtin-operator- // token alternation in function_declaration's name: // field. RE-MEASURED at v59/v60 on Alamofire@0455bfb + // swift-nio@72973283, the 2026-08-04 corpora pinned to // the same SHAs. // (c) TYPESCRIPT #private: tags.scm gains the #private // method / field-arrow / call-ref coverage JS already // had -- a sibling-completeness gap, not a new shape. // Shared finalSegment() gains the leading-'<' carve-out (a // Swift operator name like `<` or `<+>` is not a generic // type-argument list, which the unconditional strip erased // to ""). That touches the shared C++/TS/JS/Python path, so // it changes the extracted SET only for a name legitimately // starting with '<'; every other language's bare-identifier // path is unaffected (tsshapecheck / jsshapecheck / // pyshapecheck / constcheck / langcheck stay green). // The vendored scanner.c UBSan fix that rode the same source // commit was deliberately NOT ported -- see the header of // test/swiftshapecheck.sh for the reason and its trigger. // (d) CUDA memory-space module bindings (cudacheck 7b // close-out): queries/cpp/tags.scm gained the // UNINITIALIZED qualified-declaration patterns and // ingest gained cudaMemorySpaceQualifierOf // (`__constant__` case-blind, `__device__`/ // `__managed__` behind the SCREAMING gate). Keyed on // Lang::Cpp, NOT on constCaptureNeedsScreamingGate -- // that gate also covers TS/JS/Ruby/Java/C#/C, whose // constants bind through non-C-family nodes and would // read as uninitialized and be dropped wholesale. // Verified zero removed rows over ~250K symbol rows / // ~2 200 C/C++ files; MONAI's 80 adds are an exact // multiset match to its 80 __constant__ source lines. // ONE bump for all four is deliberate: they land together, // so no released version ever keyed a cache on one without // the others. Record shape unchanged, kCacheVersion stays. // 59 (TOML config-key tier): +TOML (.toml) — a NEW // grammar and a new .scm, so the extracted SET // changes on any tree holding a .toml. Table // headers and their keys become t="sec" defs; no // references, so edges are unchanged. Key depth is // HEADER-relative, not root-relative — JSON's // "top-level + 2nd-level" cut ported literally // would capture 38.3% of keys and miss every key // under a 2-dotted table. Known, disclosed cost: // one cold re-parse everywhere. // 58 (r9 A5 iteration 6): the L3 DECLARATION arm gets // the matching value-INITIALIZATION noise gate — a // bare-identifier initializer mints a fn-pointer // binding only when the declarator spells one, the // written type is a same-file fn-pointer alias, or // the type is UNKNOWN (`auto`, template, decltype). // A CLASS type used to sail through the primitive- // only gate, so `std::string tag = zzz;` minted a // binding that vetoed shadow suppression for the // local's whole scope. // 57 (r9 A5 iteration 5): the L3 assignment arm gets // the value-assignment NOISE GATE — a bare- // identifier `x = y;` mints a fn-pointer binding // only when the file's own declarations do not // prove x a value variable (`std::string line; // line = zzz;` no longer vetoes shadow // suppression, nor tombstones a same-named // file-scope binding corpus-wide). // 56: three declarator shapes isNonValueContext // could not see stop leaking their DECLARED name // out as a read of the symbol they shadow — a // DEFAULTED parameter (`int key = 0`, parent // optional_parameter_declaration), a pack // (`Ts... key`) and an attributed declarator // (`int key [[maybe_unused]]`). // 55 (r9 A5 iteration 4): an ordinary BLOCK // declaration's shadow span starts at its // DECLARATION POINT (end of the complete // declarator, [basic.scope.pdecl]) instead of the // block's brace, so a genuine call written above // the local survives; whole-scope shapes keep // their spans, and isDeclSiteName gains the // `declaration` arm the narrowed span un-masks. // Span VALUES + reference population change → // old caches must be rejected; mirror bumped in // the SAME commit. // 54 (r9 A5 iteration 3): shadow spans stop at the // owning CONTROL STATEMENT — a for/if/while/switch // header declaration scopes to that statement's // span, no longer leaking past the loop; range-for // unified to the whole-statement span; catch // parameters captured (handler-block span). Span // VALUES land in cached bind records → extraction // output changes → old caches must be rejected. // quality.h's mirror bumped in the SAME commit. // 53 (r9 A5 fix round): shadow suppression tightened // to BLOCK spans (RawBind gains spanStart/spanEnd — // a bind-record FORMAT change, rejected via this // bump) and the capture now sees reference // declarators, structured bindings, lambda params // and capture-list names. quality.h's mirror // bumped in the SAME commit. // 52 (r9 loss bucket 2): local-shadow suppression — // captureBindings gains kind=VarDecl records (every // declared C++/ObjC variable NAME incl. primitives, // definition parameters, range-for vars) — a NEW // bind kind on the per-file record → old caches // lack the rows and must be rejected. quality.h's // kIngestParserVerMirror bumped in the SAME commit. // 51 (r9 loss bucket 1): C++ `using ns::name;` // re-export sites now mint a role="import" RawRef // (new tags.scm @reference.import pattern + // usingDeclarationIsDirective guard) — a NEW ref // kind on the per-file record → old caches lack // the rows and must be rejected. quality.h's // kIngestParserVerMirror bumped in the SAME commit. // 48 (macro-edges): function-like #define → t="macro" // symbols (C++ gains the capture; C/Rust re-kind // Function→Macro), replacement-text call scan, and // the role="macro" invocation retag. // 47 (L3, 2026-08-08 audit): `locals` now counts // DECLARATORS, not declaration statements — // cc_countLocalDeclarators sums every // `declarator`-fielded child of a countable // `declaration` node instead of the fused DFS // incrementing by one per statement. A // comma-separated local (`int a,b,c;`) moves from // locals=1 to locals=3, and a type-only local // declaration (no declarator at all) moves from 1 // to 0 — a VALUE change on the per-file RawDef // record's existing `locals` u32 (no format // change) → old caches carry the undercounted // number and must be rejected. quality.h's // kIngestParserVerMirror bumped in the SAME // commit (P0.2). // 46: integration/quality-fleet merge of TWO independent 45s // — the integrated ppalt+nestcal 45 (below) and ev(G), // which took 45 on feat/nest-profile (entry next; it had // skipped 44 to dodge exactly this trap, but the // integration line had already spent 45). The merged // extraction (ppalt + nestcal clause semantics + ev/evWhy // + Swift guard decision counting) matches neither // lineage, so neither's blobs may be served. Mirror moved // in the same commit; qschemetrip re-pinned. // 45 (feat/nest-profile numbering): essential complexity (the essential-complexity design note). // 45 and not 44: the nesting-quirk round on a sibling // branch independently took 44 for the else-clause hump // rewrite; two independent 44s would cross-hit caches at // merge (that trap already fired once between two 43s). // RawDef/Symbol gain `ev` (u16 FLOOR) + `evWhy` (8×u8 tag // counters), computed inside the fused cc_walk DFS — a // FORMAT change to the per-file def record (u32 + 8×u8 // after deepLoc) → old caches must be rejected. ALSO a // VALUE change: Swift `guard_statement` joins // isDecisionType (it is a decision point every cyclomatic // tool counts; required so ev's counting of the // guard-else exit keeps ev <= cx structural), so Swift // cx moves on guard-bearing defs. quality.h's // kIngestParserVerMirror bumped in the SAME commit (P0.2). // 43 (feat/nest-profile numbering): deepLoc line accounting fixed in cc_walk's else/elif // clause — the hump PROFILE pass now runs forward // (document order) instead of inside the backwards PUSH // loop, so the `else` token's own line is no longer // clamped away behind its block's high-water end. deep= // VALUES move on else-at-the-bar shapes, so caches // written by 42 hold numbers this build would not // produce and must be rejected. // 45: integration/quality-fleet merge of the ppalt line // (43 below) and the nestcal r1 line (44 below) — the // merged extraction (ppalt disclosure + r1 else/elif // clause semantics) matches neither lineage, so the // merge takes a fresh number and both sides' blobs are // rejected. Mirror moved in the same commit. // 44: the merge of TWO independent 43s, so neither 43's // caches may be served. One 43 was nestcal r1 (else/elif // clause bodies no longer double-deepen — no per-child // maxNest bump or hump minting; nest/humps/deep/ccx // values shift). The other 43 fixed deepLoc's clamp // order for else-clause regions (cc_noteElseRegions, // forward pass); r1's removal of clause noting subsumes // it — every surviving cc_noteHump site notes one node // pre-descent, so document order holds by construction. // 43: ppalt disclosure — RawDef/Symbol gained a `ppAlt` u16 // counting the ALTERNATIVE-introducing preprocessor nodes // (preproc_else/preproc_elif/preproc_elifdef) inside the // def, filled by the same fused cc_walk DFS: structural // metrics sum branches that never coexist at compile time // (bullet's btMatrix3x3.h::getRotation, ~2x vs any one // build), and the row now DISCLOSES it instead of anyone // guessing a branch. A FORMAT change (new u32 between // locals and params in the def record) → reject old blobs. // quality.h kIngestParserVerMirror bumped in the SAME // commit (P0.2). (Was 42 on its own branch; renumbered 43 // at integration — it collided with the independent 42 // below.) // 42: nested-closure span attribution — the body-climb in // the tags pass no longer adopts an ancestor whose body // CONTAINS the def (JS/TS named const-closures nested in // a function stole the encloser's whole span, so their // cached startByte/endByte/loc/cx/params are wrong) → // old blobs carry the bad spans and must be rejected. // 41: Phase 1 (local-variable-indexing, // docs/LOCALS_INDEXING.md): RawDef/Symbol gained a `locals` uint32_t // FLOOR field, populated inside the existing fused cc_walk // DFS (C/C++ only — model.h localsCountedLang). A FORMAT // change to the per-file RawDef cache blob (new u32 between // loc and params) → old caches must be rejected, not // misread as an off-by-one on every later field. quality.h's // kIngestParserVerMirror bumped in the SAME commit (P0.2). // 40: captureIncludes descends into import CONTAINERS instead of // scanning the file root's direct children. Two families, one // walk: (a) preprocessor conditionals — // `#if`/`#ifdef`/`#ifndef`/`#else`/`#elif`/`#elifdef` — in the // C family and C#; (b) ordinary language constructs — Python's // `if TYPE_CHECKING:` / `try…except ImportError` / any function, // method or class body, Rust's `mod x { … }` / fn / impl / trait // / block bodies, and C#'s block-scoped `namespace Foo { … }`. // None of those were ever visited before, so a v39 blob on any // tree using them carries a SHORT include list → reject. // (39: JavaScript gains four definition shapes measured missing // against real repos (webpack@957bf3a, node@427d2e1 lib/) — // field_definition bound to an arrow/function, #private // methods (+ their call references), gated CJS export // assignments, gated prototype assignments. A v38 blob on a // JS-bearing tree is missing those rows → reject.) // (38: TypeScript // gains three definition shapes measured missing against a real // repo (openclaw, 24 658 .ts files) — abstract_method_signature, // public_field_definition bound to an arrow, and a declarator // whose value is an as/satisfies cast WRAPPING the arrow. A v37 // blob on a TS-bearing tree is missing those rows, so it // describes a different graph and must be rejected; the on-disk // RECORD SHAPE did not change, so only parserVer moves, not // kCacheVersion — the CUDA (37) precedent exactly. // 37: +CUDA (.cu/.cuh) // on the vendored tree-sitter-cuda grammar (kLangTable) — the // crawl SET changed (two new extensions) and `<<<>>>` launch // sites now extract as call references, so a v36 blob on a // CUDA-bearing tree describes a different graph and must be // rejected; the on-disk RECORD SHAPE did not change, so only // parserVer moves, not kCacheVersion — the +Metal (30) // precedent exactly; 36: H4 W3 V3-verifier // fixup L-1 — a Rust CONTAINER no longer scopes ITSELF. // rustEnclosingScopeOf started its ancestor walk at the node's // parent, and a `mod util`/`trait Shape` definition node IS that // owner's own `name:` child, so the module published `util::util` // and the trait `Shape::Shape` — a self-scope in the canonical-id // space that ids are keyed on. Per-def `scope` is a CACHED field, // so a v35 blob carries the old self-scoped ids and a warm run // would serve them: extraction change, bump required. (The M-2 // file-module guard and the M-3 canonical-multi-match routing that // ship alongside are RESOLUTION-only, in graph.h, and would not // have needed one on their own.); 35: H4 W3 MERGE of two // lane bumps that each shipped an in-flight 34 with a DIFFERENT // extraction set (the never-reuse rule again) — the RUST // qualified-call widening: two new reference patterns // (`scoped_identifier name: (identifier)` — depth-unbounded because // Rust nests scoped paths LEFT — and `generic_function function: // (identifier)` for the bare turbofish), so `Widget::new()` / // `util::deep::deepfn()` / `Self::helper()` / `Vec::::new()` / // `generic::()` now extract at all, PLUS both halves of the // canonical tier for Rust: per-REF `qualifier` (the path's last // segment, turbofish-stripped, `Self` resolved to the enclosing // impl type) and per-DEF `scope` (the enclosing impl/trait/mod // owner), PLUS the Rust method-SPAN fix (the @definition.method // capture moved off the `declaration_list` wrapper onto the // `function_item`, so spans/loc/cx/ccx/params and ref attribution // for Rust all change); AND the W2b-fixup operator re-split — a // qualified call to an OPERATOR (`outer::inner::operator>`) now // re-splits on the operator-name tail instead of being handed to // the angle-depth scan, which its trailing `>` poisoned: the // per-ref `qualifier` for every `>`-family operator call changes // from the outermost scope to the immediate one. Blobs from // EITHER in-flight v34 describe a different graph and must be // rejected; 33: H4 W2b — the C++ // qualified-call widening. The 2-segment `qualified_identifier // name: (identifier)` reference pattern is REPLACED by the // depth-unbounded `name: (_)` (3+-segment calls now extract at all // depths) and a `template_function` pattern is added (explicit // template-argument calls, cast keywords excluded at capture time), // plus ingest's re-split of the widened capture into // name + immediate qualifier. Both the extracted REFERENCE SET and // per-ref `qualifier` change → every blob written by a v32 binary // describes a different graph and must be rejected; 32: H4 wave-2a MERGE of two // lane bumps that each shipped an in-flight 31 with a DIFFERENT extraction // set (the never-reuse-an-in-flight-number rule, further down this comment log) — C# // conditional-access ("?.") calls (two member_binding_expression patterns, // so `w?.Bump()` / `a?.b?.C()` / `w?.Gen()` now emit references) AND // qualified-`new` call refs for TS/JS (member_expression constructor) and // Java (scoped_type_identifier object_creation_expression), plus the ObjC // field_expression call-ref parity line with C; Go's explicit-generic- // instantiation widening was investigated and REJECTED (comment-only .scm change, extraction unaffected), // so Go is unaffected by this bump; 30: +Metal (.metal) on the // C++ grammar, C-family `#import` include edges, and the phantom-scope // guard in qualifierOf (a MISSING `::` no longer publishes the return type // as a canonical scope) — the crawl SET, the include-edge set, AND the // per-def `scope` field all changed; the on-disk RECORD SHAPE did not, so // only parserVer moves, not kCacheVersion. 29 was an in-flight value that // covered only the first two of those three; it must be rejected, because a // v29 blob carries the pre-guard phantom scopes and a warm run would serve // them (observed: `--uses` reported `Out::f` warm and `f` cold on the same // tree). RULE THIS COST US: a cached FIELD changing is an extraction change // even when no record grows — bump again, do not reuse the round's earlier // number; 28: JSON-lane ceilings — // kMaxJsonConfigBytes crawl skip + kMaxJsonNestDepth hostile-data guard; // the crawl/parse SET changed; 27: +C (.c); 26: +JSON config keys) // THE QUALITY-CACHE MIRROR, as a compile error. quality.h keys every qsnap/qbody blob on kIngestCacheVersionMirror and // kIngestParserVerMirror, because quality.h cannot see these two constants from every translation unit that includes it. // Until now only test/qextractionkeycheck.sh kept the pair equal, by parsing both files. A kParserVer bump that missed // the mirror would re-serve quality snapshots computed under the old extraction, which is the poisoned-cache defect the // mirror exists for (quality.h's r27 note). This translation unit includes both files, so the equality is a static_assert // here. The gate still runs its source-text arm; this makes the same fact fail the build first. static_assert( quality::kIngestParserVerMirror == kParserVer && quality::kIngestCacheVersionMirror == kCacheVersion, "quality.h's kIngestParserVerMirror / kIngestCacheVersionMirror must equal kParserVer / kCacheVersion — bump both in one commit" ); // A1 (team-index artifact): architecture/ABI tag for the cache-blob header. The blob is NATIVE-ENDIAN — // ByteW/ByteR memcpy raw ints (see ByteW below), no portable varint/LE re-encoding — so it is only safely // consumable on a machine with the same integer byte order AND pointer width that WROTE it. This one byte // lets loadCache's existing header guard reject a foreign-arch blob exactly like a version mismatch → the // blob is ignored → full cold reparse → correct output, just not fast. Encodes precisely the two properties // that make a native-endian blob same-arch-consumable: bit 0 = byte order // (0 little / 1 big), bits 1.. = sizeof(void*) (pointer width in bytes, the ABI word size the raw-int // widths are keyed to). "Correctness never depends on the artifact" is preserved without paying for a // portable encoding on the hot (de)serialize path that fe47139/PERF.md P2 optimized; a future big-endian // target that actually needs a portable re-encode is a separate gated decision, // not pre-paid here — the guard already makes such a target CORRECT (self-heal), just not fast. // std::endian, not __BYTE_ORDER__/__ORDER_BIG_ENDIAN__: those are GCC/Clang predefined macros that MSVC does // not define, so the whole initializer vanished there and left a constexpr object with no value ("error C2737: // constexpr object must be initialized", windows-latest CI, 2026-09-20). 's std::endian is the C++20 // spelling of exactly this question and needs no compiler condition at all. constexpr std::uint8_t kArtifactArch = static_cast( ( std::endian::native == std::endian::big ) ? 1u : 0u ) // bit 0: endianness | static_cast( sizeof( void* ) << 1 ); // bits 1..: pointer width (bytes) // The lean/rich cache FAMILY split (documented against reality per a reviewer note). // captureValueUses gets its OWN parserVer so a lean blob and a value-uses ("rich") blob can never cross-hit; // the caller (main.cpp:587) sets it via `needsValueUses = !usesSym.empty() || metrics || !forTask.empty() || // !exemplar.empty()`. So which verbs each committed artifact accelerates, precisely: // LEAN (captureValueUses=false): the DEFAULT map, all nav/read verbs (--callers/--callees/--around/ // --expand/--impact/--path/--grep/--match), --pr-context, --affected, AND --situ (it is NOT in // needsValueUses — the DESIGN's "--situ needs rich" aside is backwards; the lean artifact serves it). // RICH (captureValueUses=true): --for, --exemplar, --metrics (and its --hotspots/quality lenses), and // --uses. NB: --for is the flagship orientation verb every CLAUDE.md session opens with, and it is // RICH — so a team that commits ONLY the lean artifact gets NO warm hit on --for/--exemplar/--metrics // (those cold-parse unless the rich artifact is also committed). "Lean accelerates the common // orientation path" therefore overstates: lean speeds the default map + nav/read + --pr-context; // --for-led sessions need the rich family too. See the ingest-report to the wiring wave for the // measured lean-vs-rich blob sizes and the both-families recommendation. inline constexpr std::uint32_t parserVerFor( bool captureValueUses ) noexcept { return kParserVer + ( captureValueUses ? 1u : 0u ); // lean and full-use caches must never cross-hit } // The auto blob's NAME carries the same pair its header does (quality.h rootBlobTail): a build tag that disagreed // with the stamp would put a blob this binary refuses at the path it reads, which is the #334 thrash back again. static_assert( quality::ingestParserVerFor( false ) == parserVerFor( false ) && quality::ingestParserVerFor( true ) == parserVerFor( true ), "quality.h's ingestParserVerFor must derive both classes exactly as parserVerFor does — the build tag names what the header stamps" ); // T5: renamed from fnv1a64 to contentHash64 to avoid an ODR clash now that this file also includes // arch.h (which defines its OWN fnv1a64 for the baseline-hash path, quality.h's canonId hashing, etc. // — same FNV-1a family, DIFFERENT offset-basis constant; this is ingest's file-content-hash cache // KEY and must not change bit-for-bit, so it keeps the stored seed, hashutil::kFnv1aLegacySeed64, not arch.h's). inline std::uint64_t contentHash64( std::string_view s ) noexcept { return hashutil::fnv1a64( s, hashutil::kFnv1aLegacySeed64 ); } // BONUS (S): a whole-blob checksum for the cache trailer. The single-lane contentHash64 above is the file-content // hash (cache KEY — must never change), but a scalar byte loop over the multi-MB cache blob costs ~1 byte/ // cycle. This uses 8 INDEPENDENT FNV lanes (byte i feeds lane i&7) so the multiplies pipeline, then folds — // ~5× faster on the big blob while staying a pure, deterministic function of the bytes (a lane permutation of // FNV-1a). Purpose is integrity detection (fuzzer-found bit-flips inside cached strings), not the cache key. inline std::uint64_t blobChecksum( std::string_view s ) noexcept { std::uint64_t lane[ 8 ] = { hashutil::kFnv1aLegacySeed64, 1099511628211ull, 0x100000001b3ull, 0x9e3779b97f4a7c15ull, 0xc2b2ae3d27d4eb4full, 0x165667b19e3779f9ull, 0xff51afd7ed558ccdull, 0xc4ceb9fe1a85ec53ull }; const unsigned char* p = reinterpret_cast( s.data() ); const std::size_t n = s.size(); std::size_t i = 0; for( ; i + 8 <= n; i += 8 ) { // 8 lanes advance in lockstep — no cross-lane dependency for( int k = 0; k < 8; ++k ) { lane[ k ] ^= p[ i + k ]; lane[ k ] = hashutil::fnv1aMultiply( lane[ k ] ); } } for( int k = 0; i < n; ++i, ++k ) { lane[ k ] ^= p[ i ]; lane[ k ] = hashutil::fnv1aMultiply( lane[ k ] ); } // tail (< 8 bytes) std::uint64_t h = hashutil::kFnv1aLegacySeed64; // fold the 8 lanes into one 64-bit digest (stored: the trailer) for( int k = 0; k < 8; ++k ) { h ^= lane[ k ]; h = hashutil::fnv1aMultiply( h ); } return h; } // ───────────────────────────────────────────────────────────────────────────────────────────────── // v15 — the OFFSET-TABLE blob frame. // // WHY. `main.cpp::defaultCachePath` keys the warm-by-default blob on realpath(root) + the lean/rich // class only, and that is DELIBERATE and stays: folding the exclude set into the key was built, // measured and REVERTED (docs/EVALS.md, "The auto-cache key ignores --exclude", RUN 2026-09-03 — one // blob per configuration on a 158K-file root thrashed the cache directory's 2 GiB sweep and took the // battery from 7 min to 62). One superset blob per (root, class) is therefore the contract, and the // two costs it used to carry are paid off by SHAPE instead: // * a subset run (--exclude / --max-file-size) used to deserialise the whole superset it then threw // away — now the offset table lets it read ONLY its own records; // * a DIRTY subset run used to rewrite the shared blob with its own file set, destroying every // record for a file it did not crawl — now saveCache carries those records over verbatim, so the // blob only ever GROWS toward the union of the configurations that use it. // // LAYOUT (native-endian; the blob is host-local and arch-guarded, see kArtifactArch): // // HEADER — 25 bytes at offset 0, byte-identical in shape to v14: // [ 0: 4) u32 magic kCacheMagic "CTPK" // [ 4: 8) u32 version kCacheVersion // [ 8:12) u32 parserVer parserVerFor( captureValueUses ) (the lean/rich family split) // [12:13) u8 arch kArtifactArch (endianness | pointer width << 1) // [13:21) u64 blobWriteNs diagnostic write stamp; NOT the racy-rule reference // [21:25) u32 entryCount file records == offset-table entries; cross-checked against the trailer // // RECORD REGION — [ kCacheHeaderBytes, tableOffset ), entryCount records in ASCENDING pathHash order. // Each record is the v14 per-file record plus v20's fifth FileHealth u32: the root-relative path string, // the content hash, the (size, mtime, ctime) stat-gate triple, the five FileHealth u32s, the rich // family's subtoken dictionary, then the seven counted record arrays (defs, refs, includes, binds, // FFI aliases, route defs, route uses). Nothing inside a record moved — that is what lets a carry- // over be a raw byte copy and what will let a future content-addressed store lift a record whole. // // OFFSET TABLE — [ tableOffset, tableOffset + entryCount*32 ), ascending pathHash, 32 B per entry: // [ 0: 8) u64 pathHash contentHash64( the record's root-relative path key ) // [ 8:16) u64 recOffset absolute byte offset of the record inside this blob // [16:24) u64 contentHash the file's content hash, mirrored out of the record // [24:28) u32 recLength record length in bytes // [28:32) u32 recSum low 32 bits of blobChecksum( the record's bytes ) // // TRAILER — the last 24 bytes: // [ 0: 8) u64 tableOffset // [ 8:12) u32 entryCount must equal the header's // [12:16) u32 reserved 0 // [16:24) u64 tableSum blobChecksum( HEADER bytes || OFFSET TABLE bytes ) // // WHAT THE CHECKSUMS COVER, and how a torn write is caught. `tableSum` covers the header and the whole // offset table, and is verified on every open — so the map from "which file" to "which bytes" is never // trusted unchecked. Each entry's `recSum` covers exactly its own record's bytes and is verified for // each record ACTUALLY READ; a record nobody reads is never checksummed and never trusted. Together // that is "the trailer checksum covers the table plus the records actually read". // A torn write is caught three ways, in this order: (a) the file is shorter than header+trailer; // (b) the EXACT-FIT invariant `fileSize == tableOffset + entryCount*32 + 24` fails, which is what // catches a truncation that removes whole records or lands mid-table; (c) `tableSum` mismatches. Any // of the three rejects the WHOLE blob with a DISCLOSE and self-heals to a full reparse — // never a partial load. A record that is individually torn while the table survives is caught by its // own `recSum`: that one file is dropped (disclosed) and reparsed, and the rest of the blob stands. // saveCache still publishes by tmp-then-rename with the write byte-count and fclose both checked, so // a short write never reaches `path` in the first place; the invariants above are the second line. // ───────────────────────────────────────────────────────────────────────────────────────────────── constexpr std::size_t kCacheHeaderBytes = 25; constexpr std::size_t kCacheEntryBytes = 32; constexpr std::size_t kCacheTrailerBytes = 24; // One offset-table entry, laid out so the on-disk array IS the in-memory array (no per-entry decode). // Content-addressed-store handoff: `contentHash` is the field a CAS keys on — moving to a CAS means // splitting this row into a per-blob { pathHash → contentHash } manifest and a shared // { contentHash → (offset,length,recSum) } store, with pathHash/recOffset/recLength/recSum leaving // this struct for the store's own index. Nothing inside a RECORD has to move for that. struct CacheEntry { std::uint64_t pathHash = 0; std::uint64_t recOffset = 0; std::uint64_t contentHash = 0; std::uint32_t recLength = 0; std::uint32_t recSum = 0; }; static_assert( sizeof( CacheEntry ) == kCacheEntryBytes, "CacheEntry must be the exact 32-byte on-disk row (no padding)" ); static_assert( alignof( CacheEntry ) == 8, "CacheEntry must stay 8-byte aligned so the table is a raw array copy" ); // The size pin above does NOT prove "no padding", although its message says so. Narrow recSum from u32 to u16 and the // struct still rounds up to 32 bytes, with two indeterminate bytes in every row of a committed, checksummed blob: the // determinism contract broken, and a portable cache that differs by build. has_unique_object_representations is the // compiler's own answer to "is every byte of this type part of its value", so it refuses padding (and any float). static_assert( std::is_trivially_copyable_v && std::has_unique_object_representations_v, "CacheEntry is copied to disk as raw bytes — it must carry no padding bytes and no float" ); // ...and no rw::InternedText: a pool id has unique object representations but a timing-dependent value (infra/internedtext.h). // The per-record digest stored in the table: the low half of the same 8-lane FNV the trailer uses. // 32 bits is a detection budget, not a security one — the whole-blob guards above sit in front of it. inline std::uint32_t recordSum32( std::string_view rec ) noexcept { return std::uint32_t( blobChecksum( rec ) & 0xFFFFFFFFull ); } // A read-only descriptor open on ONE cache blob, held for the whole of a load or a save so the inode // survives another process renaming a new blob over `path` mid-read (rename(2) unlinks, it does not // truncate). Owning, non-copyable, closed on scope exit — the one place this file needs RAII. struct ReadFd { int fd = -1; ReadFd() = default; ReadFd( const ReadFd& ) = delete; ReadFd& operator=( const ReadFd& ) = delete; ReadFd( ReadFd&& other ) noexcept : fd( other.fd ) { other.fd = -1; } ~ReadFd() { if( fd >= 0 ) { os::close( fd ); } } // openOnce, not a move-assignment: the only mutation this type needs is "fill an empty guard", and // a move-assign operator here would be a byte-for-byte clone of ingest_sidecap.h's TreeGuard one // (--quality-delta's duplication kind says so). One owner, one transfer direction, no reopen. bool openOnce( const std::string& path ) noexcept { ASSUME( fd < 0 ); fd = os::open( path.c_str(), O_RDONLY | O_CLOEXEC ); return fd >= 0; } bool valid() const noexcept { return fd >= 0; } }; // WHICH guard refused a blob. `ok == false` used to be the whole answer for every reason at once — // absent, wrong shape, foreign version/parserVer/arch, torn, or a table that failed its checksum or its // bounds — which is exactly right for the READ path (they all mean "reparse") and useless to a person // holding a committed artifact that stopped working. The refusals are the SAME set the fused bool always // covered; naming them costs one byte in the frame and is what makes `--doctor`'s index-cache row possible. // // The reason= spelling beside each enumerator IS the user-facing vocabulary: `cacheRejectName` returns it, // `--doctor` prints it, and `test/cacheidentitycheck.sh` (C) reads this block to assert that every refusal // branch in openCacheFrame carries one and that doctor's legend documents every value. enum class CacheReject : std::uint8_t { Ok, // reason="ok" — the frame opened and validated Absent, // reason="absent" — nothing at that path (the ordinary cold-start miss) NotRegular, // reason="not-regular" — a directory/device/fifo, or a path that changed shape under us Unreadable, // reason="unreadable" — a regular file whose open() was refused (permissions) Truncated, // reason="truncated" — shorter than a frame, or a short read of header/trailer/table NotACache, // reason="not-a-cache" — magic mismatch: some other file entirely FormatVersion, // reason="format-version" — a ripwire blob of a different kCacheVersion ParserVersion, // reason="parser-version" — right format, different kParserVer (an extraction change, or the lean/rich family split) ArtifactArch, // reason="artifact-arch" — right format, foreign endianness or pointer width Checksum, // reason="checksum" — header+offset-table checksum mismatch (a damaged write) CorruptFrame, // reason="corrupt-frame" — structural invariant broken (exact-fit, entry bounds, table ordering) Count // enumerator count, never a value — the static_assert below binds the table to the enum }; // A parallel table rather than a switch (the symTag/pinMechName idiom): the static_assert makes adding an // enumerator without its name a COMPILE error, which a switch's default arm cannot do, and a name lookup // this small has no reason to be branchy. inline constexpr const char* kCacheRejectNames[] = { "ok", "absent", "not-regular", "unreadable", "truncated", "not-a-cache", "format-version", "parser-version", "artifact-arch", "checksum", "corrupt-frame" }; static_assert( std::size( kCacheRejectNames ) == static_cast( CacheReject::Count ), "every CacheReject enumerator needs its user-facing reason= name" ); inline const char* cacheRejectName( CacheReject r ) noexcept { return enumTableAt( kCacheRejectNames, r, "corrupt-frame" ); // infra/enumcount.h: the shared bounded table lookup } // A validated, still-open v15 blob: everything a caller needs to pull records out of it by pathHash. // `ok == false` means "there is no usable cache here"; `reason` says WHICH guard said so. struct CacheFrame { ReadFd blob; std::vector entries; // ascending pathHash; the on-disk table, verbatim std::uint64_t tableOffset = 0; // records live in [ kCacheHeaderBytes, tableOffset ) long long mtimeNs = -1;// the blob's own mtime — the warm-run racy-rule reference bool ok = false; CacheReject reason = CacheReject::Absent; // meaningful only while ok == false std::uint32_t foundStamp = 0; // FormatVersion: the blob's kCacheVersion; ParserVersion: its parserVer // The trailer's tableSum: the digest of this blob's HEADER and its WHOLE offset table, which carries every // record's own recSum. It is read and verified below on every open; `cacheBlobIdentity` publishes it as the // blob's identity, and nothing else reads it. std::uint64_t tableSum = 0; }; // pread the whole of [ off, off+n ) into `dst`. Short reads are retried (a pread on a regular file can // return less than asked for); anything else is a hard false, which every caller turns into a degrade. inline bool preadExact( int fd, void* dst, std::size_t n, std::uint64_t off ) noexcept { char* out = static_cast( dst ); while( n > 0 ) { const os::ssize_t got = os::pread( fd, out, n, os::off_t( off ) ); if( got <= 0 ) { return false; } out += std::size_t( got ); off += std::uint64_t( got ); n -= std::size_t( got ); } return true; } // Open `path` and validate its v15 frame: header guard, exact-fit invariant, table checksum, per-entry // bounds and ordering. Shared by loadCache (which reads records out of it) and saveCache (which copies // the records this run did not crawl out of it). Never throws, never partially trusts. inline CacheFrame openCacheFrame( const std::string& path, bool captureValueUses ) { PROFILE_SCOPE_DESCRIBE( "ingest/cache: open + validate blob frame (header, trailer, offset table)" ); CacheFrame frame; if( !isReadableCacheBlob( path ) ) { // The read path treats "nothing there" and "wrong shape" identically; the disclosure surface does // not, so the shape is re-asked here — on a branch that is already refusing, never on the hot path. frame.reason = ( shapeOfPath( path ) == PathShape::Absent ) ? CacheReject::Absent : CacheReject::NotRegular; return frame; // absent (silent) or an odd shape (already disclosed by isReadableCacheBlob) } if( !frame.blob.openOnce( path ) ) { frame.reason = CacheReject::Unreadable; return frame; } os::stat_t st; if( os::fstat( frame.blob.fd, &st ) != 0 || !S_ISREG( st.st_mode ) ) { frame.reason = CacheReject::NotRegular; return frame; } const std::uint64_t fileBytes = std::uint64_t( st.st_size ); if( fileBytes < kCacheHeaderBytes + kCacheTrailerBytes ) { frame.reason = CacheReject::Truncated; return frame; // too short to hold even an empty frame — an ordinary corrupt/partial file } // header — magic/version/parserVer/arch, in that order, short-circuiting like v14's guard did. char header[ kCacheHeaderBytes ]; if( !preadExact( frame.blob.fd, header, kCacheHeaderBytes, 0 ) ) { frame.reason = CacheReject::Truncated; return frame; } { std::uint32_t magic = 0, version = 0, parserVer = 0; std::memcpy( &magic, header + 0, 4 ); std::memcpy( &version, header + 4, 4 ); std::memcpy( &parserVer, header + 8, 4 ); const std::uint8_t arch = std::uint8_t( header[ 12 ] ); if( magic != kCacheMagic ) { frame.reason = CacheReject::NotACache; return frame; } if( version != kCacheVersion ) { // v14 (and every older format) is REJECTED, not misread: the records moved behind an offset // table, so a pre-v15 blob has no table to trust. The alert is DEBUG-ONLY (NDEBUG deletes it, // and every installed binary is Release) — --doctor's index-cache row is where a user of a // shipped binary reads this. Same for the two guards below, split from one fused test because // "your binary's extraction changed" and "this artifact came from another architecture" are // different things to be told about a committed artifact. DISCLOSE( Diagnostics::answerUnchanged, "a rejected cache is rebuilt from source: this run parses and answers byte-identically", "ingest: cache blob is a different format version — rejected and rebuilt (full reparse)" ); frame.reason = CacheReject::FormatVersion; frame.foundStamp = version; return frame; } if( parserVer != parserVerFor( captureValueUses ) ) { DISCLOSE( Diagnostics::answerUnchanged, "a rejected cache is rebuilt from source: this run parses and answers byte-identically", "ingest: cache blob parserVer mismatch (older binary, or the other lean/rich family) — rejected and rebuilt (full reparse)" ); frame.reason = CacheReject::ParserVersion; frame.foundStamp = parserVer; return frame; } if( arch != kArtifactArch ) { DISCLOSE( Diagnostics::answerUnchanged, "a rejected cache is rebuilt from source: this run parses and answers byte-identically", "ingest: cache blob arch mismatch (foreign endianness/pointer width) — rejected and rebuilt (full reparse)" ); frame.reason = CacheReject::ArtifactArch; return frame; } // header[13:21) is the legacy wall-clock write stamp — diagnostic only (see saveCache). } std::uint32_t headerEntryCount = 0; std::memcpy( &headerEntryCount, header + 21, 4 ); // trailer — tableOffset / entryCount / reserved / tableSum, then the EXACT-FIT invariant. char trailer[ kCacheTrailerBytes ]; if( !preadExact( frame.blob.fd, trailer, kCacheTrailerBytes, fileBytes - kCacheTrailerBytes ) ) { frame.reason = CacheReject::Truncated; return frame; } std::uint64_t tableOffset = 0, tableSum = 0; std::uint32_t entryCount = 0; std::memcpy( &tableOffset, trailer + 0, 8 ); std::memcpy( &entryCount, trailer + 8, 4 ); std::memcpy( &tableSum, trailer + 16, 8 ); // Every term below is written so it cannot wrap: the file is at least a header plus a trailer (checked above) and the // entry clause bounds the table by the bytes between them, so the right-hand side of the exact-fit compare is never // negative. The earlier `tableOffset + entries + trailer != fileBytes` wrapped for a trailer naming an offset near // 2^64 — still refused, but through a sum the G1 sanitizer build (-fsanitize=integer) aborts on // (test/cachefuzzcheck.sh mutation table_offset_near_u64_max; found by test/fuzz/readers, reader ingestframe). if( entryCount != headerEntryCount || tableOffset < kCacheHeaderBytes || entryCount > ( fileBytes - kCacheHeaderBytes - kCacheTrailerBytes ) / kCacheEntryBytes || tableOffset != fileBytes - kCacheTrailerBytes - std::uint64_t( entryCount ) * kCacheEntryBytes ) { DISCLOSE( Diagnostics::answerUnchanged, "a rejected cache is rebuilt from source: this run parses and answers byte-identically", "ingest: cache blob trailer does not describe the file (torn write) — cache treated as corrupt" ); frame.reason = CacheReject::CorruptFrame; return frame; } // the table itself, checksummed together with the header it belongs to. std::string hdrTable; hdrTable.resize( kCacheHeaderBytes + std::size_t( entryCount ) * kCacheEntryBytes ); std::memcpy( hdrTable.data(), header, kCacheHeaderBytes ); if( entryCount != 0 && !preadExact( frame.blob.fd, hdrTable.data() + kCacheHeaderBytes, std::size_t( entryCount ) * kCacheEntryBytes, tableOffset ) ) { frame.reason = CacheReject::Truncated; return frame; } if( blobChecksum( std::string_view( hdrTable.data(), hdrTable.size() ) ) != tableSum ) { DISCLOSE( Diagnostics::answerUnchanged, "a rejected cache is rebuilt from source: this run parses and answers byte-identically", "ingest: cache offset-table checksum mismatch — cache treated as corrupt (full reparse)" ); frame.reason = CacheReject::Checksum; return frame; } frame.entries.resize( entryCount ); if( entryCount != 0 ) { std::memcpy( frame.entries.data(), hdrTable.data() + kCacheHeaderBytes, std::size_t( entryCount ) * kCacheEntryBytes ); } // per-entry bounds + strict ordering: every record must lie wholly inside the record region, and the // table must be ascending so the lookup below can binary-search it. A table that fails either is // corrupt in a way the checksum cannot see (it was written wrong, not damaged) → reject the blob. for( std::size_t i = 0; i < frame.entries.size(); ++i ) { const CacheEntry& e = frame.entries[i]; if( e.recOffset < kCacheHeaderBytes || e.recLength == 0 || e.recLength > tableOffset || e.recOffset > tableOffset - e.recLength // recOffset + recLength > tableOffset, without the wrap || ( i != 0 && frame.entries[ i - 1 ].pathHash > e.pathHash ) ) { DISCLOSE( Diagnostics::answerUnchanged, "a rejected cache is rebuilt from source: this run parses and answers byte-identically", "ingest: cache offset-table entry out of bounds or out of order — cache treated as corrupt" ); frame.entries.clear(); frame.reason = CacheReject::CorruptFrame; return frame; } } frame.tableOffset = tableOffset; frame.tableSum = tableSum; // F3/X5: the racy-rule reference is THIS blob's own on-disk mtime, from the fd we already hold — the // same stat() domain and granularity every per-file mtime is compared against. frame.mtimeNs = statSizeTimes( path ).mtimeNs; frame.ok = true; frame.reason = CacheReject::Ok; return frame; } // The DISCLOSURE seam: classify an artifact WITHOUT deserialising a record out of it, so `--doctor` can // answer "can this binary read the artifact you handed it, and if not, which guard said no" for the price // of a header + offset-table read. Deliberately the SAME openCacheFrame the read path runs — a second, // simpler copy of the guard would be free to drift, and a disclosure that disagrees with the behaviour it // describes is worse than none. // // What this does NOT answer, and the row that prints it must not imply: whether the artifact's per-file // records still match the tree. That is decided per file at ingest time (the stat gate + the racy rule), // re-run on EVERY invocation — see docs/EVALS.md "card A3", which measured that and rejected a freshness // attribute on the CLI map for it. This is a FORMAT verdict about an artifact, not a freshness verdict // about an index. inline CacheReject inspectCacheArtifact( const std::string& path, bool captureValueUses ) { return openCacheFrame( path, captureValueUses ).reason; } // ── the blob's IDENTITY, for the warm snapshot that is derived from it (src/warmsnap.h) ─────────────────────────── // // WHY THE BLOB NEEDS AN IDENTITY AT ALL. The warm snapshot stores the OUTCOME of stages a warm run would otherwise // repeat — the crawl, the assembled model, the resolved graph — and it is only valid for the records it was built // from. It lives in ripwire's own cache directory, NOT beside the blob and NOT inside it (owner ruling 2026-11-15, // superseding the in-blob design on this point): writing anything beside the user's `--cache=PATH` breaks // test/cacheidentitycheck.sh arm (D3), and writing INTO that file breaks test/portablecachecheck.sh arm (a) — // a committed blob is read-shared by many checkouts and a consuming run must not mutate the artifact another // checkout owns. So the binding to the blob is made by NAME instead: the snapshot's filename carries this // identity, and a blob that is rewritten, replaced or deleted names a different artifact or none. // // WHAT THE IDENTITY IS. The trailer's `tableSum` — the digest over this blob's header and its whole offset table, // which holds every record's own `recSum`. One re-parsed file moves that record's recSum, and the header carries // the write stamp besides, so any rewrite of the blob moves the identity. It is verified on every open before this // returns it, so it is never read out of a blob this build would refuse. // // Nothing (0 of 2^64 values aside) when there is no blob at `path` this build can read — absent, another format, // another parser family, truncated or corrupt. A blob whose tableSum is exactly 0 is indistinguishable from that // and simply never carries a snapshot: the cost is one unstored run, and the answer is the same either way. inline std::optional cacheBlobIdentity( const std::string& path, bool captureValueUses ) { PROFILE_SCOPE_DESCRIBE( "ingest/cache: the blob's identity (trailer tableSum), for the warm snapshot's name" ); const CacheFrame frame = openCacheFrame( path, captureValueUses ); if( !frame.ok || frame.tableSum == 0 ) { return std::nullopt; } return frame.tableSum; } // The half-open [ first, last ) span of table entries carrying `pathHash`. A u64 FNV collision across a // corpus this size is ~1e-9, but a collision that SERVED the wrong file's facts would be a wrong answer, // so the lookup returns a range and the caller confirms identity against the record's own stored path. inline std::pair cacheEntryRange( const std::vector& entries, std::uint64_t pathHash ) noexcept { const auto byHash = []( const CacheEntry& e, std::uint64_t h ) noexcept { return e.pathHash < h; }; const auto lo = std::lower_bound( entries.begin(), entries.end(), pathHash, byHash ); auto hi = lo; while( hi != entries.end() && hi->pathHash == pathHash ) { ++hi; } return { std::size_t( lo - entries.begin() ), std::size_t( hi - entries.begin() ) }; } // sizeBytes/mtimeNs/ctimeNs (A4-P7 + card A3 follow-up): the file's stat at the run that HASHED it — the // warm-run stat-gate trusts this record (skips read+hash) only when ALL THREE still match AND mtimeNs is // not racy. ctimeNs is the one an unprivileged writer cannot restore (see ingest_crawl.h statSizeTimes), // so it is what makes a same-(mtime,size) edit visible for free. -1 ⇒ unknown (the file was unstatable at // hash time) → the gate always re-hashes, which is the safe direction. struct FileFacts { std::uint64_t hash = 0; long long sizeBytes = -1; long long mtimeNs = -1; long long ctimeNs = -1; FileHealth health; std::vector defs; std::vector refs; std::vector incs; std::vector binds; std::vector ffis; std::vector routeDefs; std::vector routeUses; std::vector constOpens; std::vector constructs; bool constructsAbsent = false; }; // Empty every family of a decode scratch, keeping each array's capacity for the next record (WARM-DIRECT reuse). inline void clearFileFactsKeepCapacity( FileFacts& f ) noexcept { f.defs.clear(); f.refs.clear(); f.incs.clear(); f.binds.clear(); f.ffis.clear(); f.routeDefs.clear(); f.routeUses.clear(); f.constOpens.clear(); f.constructs.clear(); f.constructsAbsent = false; } // v49 (WARM-DIRECT): the eight per-file fact-family counts. The record carries them in its HEAD as well as before each // family, so a reader holding only the head (the warm planner, CacheRecords below) knows every record's exact sizes: // the parse pool reserves its accumulators exactly and decides a factless record without decoding the body. struct FileFactCounts { std::uint32_t defs = 0, refs = 0, incs = 0, binds = 0, ffis = 0, routeDefs = 0, routeUses = 0, constOpens = 0; std::uint32_t constructs = 0; // v51 (QD-CONSTRUCTS-AT-INGEST): the ninth family, after constOpens bool none() const noexcept { return ( defs | refs | incs | binds | ffis | routeDefs | routeUses | constOpens | constructs ) == 0; } }; // A record's head: everything a warm run decides on before the body is decoded — the stat gate, the hit test, the // factless-record compare (WARM-REWRITE) and the exact reserve. struct CachedRecordHead { std::uint64_t hash = 0; long long sizeBytes = -1; long long mtimeNs = -1; long long ctimeNs = -1; FileHealth health; FileFactCounts counts; bool constructsAbsent = false; // v51: the record was written by a pool that ran no construct pass (kConstructsAbsent) }; // tiny native-endian binary (de)serializer (the cache is host-local, never shipped) struct ByteW { std::string b; std::uint64_t streamed = 0; // C1 SAVE-STREAM: bytes saveCache already wrote out of `b` to its file (0 for an in-memory writer) std::uint64_t offset() const noexcept { return streamed + b.size(); } // the absolute blob offset the next byte lands at void u8 ( std::uint8_t v ) { b.push_back( char( v ) ); } void u32( std::uint32_t v ) { b.append( reinterpret_cast( &v ), 4 ); } void u64( std::uint64_t v ) { b.append( reinterpret_cast( &v ), 8 ); } void str( std::string_view s ) { u32( std::uint32_t( s.size() ) ); b.append( s ); } void raw( const void* p, std::size_t n ) { if( n ) { b.append( reinterpret_cast( p ), n ); } } // B0.2: bulk array append (memcpy-speed (de)serialize) // H3 (v10): bulk row extension — one resize + raw-pointer fill instead of a bounds-checked push_back // per byte (the postings rows are ~3 B × millions of pairs — this seam is the saveCache hot loop). // The narrow ints written through it are explicit little-endian (see writeDef), so the width-packed // postings fields read back identically on either endianness the kArtifactArch guard admits. char* extend( std::size_t n ) { const std::size_t off = b.size(); b.resize( off + n ); return b.data() + off; } }; struct ByteR { const char* p; const char* end; bool ok = true; std::uint8_t u8 () { if( p >= end ) { ok = false; return 0; } return std::uint8_t( *p++ ); } std::uint32_t u32() { std::uint32_t v = 0; if( p + 4 > end ) { ok = false; return 0; } std::memcpy( &v, p, 4 ); p += 4; return v; } std::uint64_t u64() { std::uint64_t v = 0; if( p + 8 > end ) { ok = false; return 0; } std::memcpy( &v, p, 8 ); p += 8; return v; } std::string_view view() { const std::uint32_t n = u32(); if( !ok || p + n > end ) { ok = false; return {}; } std::string_view s( p, n ); p += n; return s; } std::string str () { const std::string_view s = view(); return ok ? std::string( s ) : std::string{}; } bool rawInto( void* dst, std::size_t n ) // B0.2: bulk array read — overflow-safe bound, memcpy into caller storage { if( !ok || std::size_t( end - p ) < n ) { ok = false; return false; } if( n ) { std::memcpy( dst, p, n ); p += n; } return true; } // A decoded value that must lie below `count`. The blob is external input — a committed team artifact, a copied // cache directory, a file whose digests were rebuilt around an edit — so a value at or past its bound is // corruption, never a value this binary forgot. It folds into `ok` exactly like a short read, so the record // takes readFileRecord's one refusal path: that file reparses and the rest of the blob stands. One compare on the // warm path; never an assumption, because nothing upstream makes it true. bool fitsBelow( std::uint64_t v, std::uint64_t count ) { if( !VALIDATE( v < count ) ) { DISCLOSE( Diagnostics::answerUnchanged, "a rejected cache is rebuilt from source: this run parses and answers byte-identically", "ingest: cache record carries a field past its range (an enum byte past its last enumerator, or a 16-bit field wider than 16 bits) — cache treated as corrupt" ); ok = false; return false; } return true; } // An ENUM byte. Accepting one past the count was never harmless downstream: symTag/refRoleTag serve such a value // as "other"/"read", and clones.h shifts a 32-bit language mask by the Lang (UB at 32 and up). model.h proves // each k*Count exact at compile time. template E enumU8( std::size_t enumCount ) { const std::uint8_t v = u8(); return fitsBelow( v, enumCount ) ? E( v ) : E{}; } // A FLAG byte: the writer stores a bool as exactly 0 or 1, so any other value is corruption, same as an enum byte past its // count. `u8() != 0` read a 2 as true; a doctored trailingLambda byte then dropped the Java candidates of a Kotlin call that // has no trailing lambda (CodeRabbit on PR #441; test/cachefuzzcheck.sh Part 3, flag arms). Same refusal path as fitsBelow. bool flagU8() { return enumU8( 2 ) != 0; } // A 16-bit field the writer stores in a u32 slot (writeDef's ppAlt/humps/deepLoc/ev/params, writeRef's argCount). // The writer only ever holds a uint16_t there; a plain `std::uint16_t( u32() )` kept the low bits of a wider value // and believed them (test/hazardpatterncheck.sh rule D, test/cachefuzzcheck.sh Part 3). std::uint16_t u16Of32() { std::uint32_t v = u32(); if( !fitsBelow( v, 0x10000u ) ) { v = 0; } return std::uint16_t( v ); } }; // withLex (B0.2 / v10 H3): the RICH family (captureValueUses=true) persists each def's doc/body subtoken // stats. v10 encoding: the FILE record carries a sorted per-file subtoken DICTIONARY (see saveCache), and // each def row stores dict-relative indices at the narrowest width the dict size allows + tfs at the // narrowest width this def's max tf needs — exact integers either way (widths chosen from maxima, never // lossy), ~3 B/pair instead of the raw 12 B/pair of v9. The LEAN family never computes them, so its // record shape is unchanged; the two families can never cross-hit (parserVerFor), so one flag at both // (de)serialize seams keeps the formats in lock-step. inline unsigned lexDictIndexWidth( std::size_t dictCount ) noexcept { return dictCount <= 0x100 ? 1u : dictCount <= 0x10000 ? 2u : 4u; // indices go up to dictCount-1 } // v50 (CLONES-PERSIST): the def kinds that get a clone key — the kinds findClones compares — as a bit per SymKind // (ingest_parsepool.h rawDefCloneKey). The record does not depend on it: every def record carries a presence byte. inline constexpr std::uint32_t kCloneKeyKindBits = ( 1u << unsigned( SymKind::Function ) ) | ( 1u << unsigned( SymKind::Method ) ); static_assert( kSymKindCount <= 32, "kCloneKeyKindBits holds one bit per SymKind" ); inline void writeDef( ByteW& w, const RawDef& d, bool withLex, std::size_t fileDictCount, const std::uint32_t* rowDictIndex ) { w.u32( d.line ); w.u32( d.startByte ); w.u32( d.endByte ); w.u32( d.nameByte ); w.u32( d.bodyByte ); w.u32( d.cx ); w.u32( d.ccx ); w.u32( d.loc ); w.u32( d.locals ); w.u32( d.ppAlt ); w.u32( d.humps ); w.u32( d.deepLoc ); w.u32( d.ev ); w.u32( d.params ); w.u32( d.paramsMin ); w.u8( d.maxNest ); w.u8( d.steer ); w.u32( d.deepDispatch ); w.u8( d.nestArmCut ); w.u8( d.arityExact ); w.u8( d.testScope ); w.u8( d.recovered ); w.u8( d.internalLinkage ); w.u8( d.scopeRootsStd ); w.u32( d.fnScopeStart ); w.u32( d.fnScopeEnd ); w.u8( std::uint8_t( d.kind ) ); w.u8( std::uint8_t( d.lang ) ); w.str( d.name ); w.str( d.scope ); for( const std::uint8_t tagCount : d.evWhy ) { w.u8( tagCount ); } // 8×u8, fixed order (model.h kEvWhyTagTable) w.u8( d.cloneKey != 0 ? 1 : 0 ); // v50: clone-key presence byte (0 = no key), then the key if( d.cloneKey != 0 ) { w.u64( d.cloneKey ); } if( withLex ) { ASSUME( d.lex.tokenHashes.size() == d.lex.tokenTfs.size() ); w.u32( d.lex.dlWeighted ); w.u32( std::uint32_t( d.lex.tokenHashes.size() ) ); std::uint32_t maxTf = 0; for( const std::uint32_t tf : d.lex.tokenTfs ) { if( tf > maxTf ) { maxTf = tf; } } const unsigned tfWidth = maxTf <= 0xFFu ? 1u : maxTf <= 0xFFFFu ? 2u : 4u; // exact-preserving by construction const unsigned idxWidth = lexDictIndexWidth( fileDictCount ); w.u8( std::uint8_t( tfWidth ) ); // rowDictIndex: this def's precomputed dict indices (saveCache assigns them during the k-way // dict merge — no per-hash search here), ascending because the row is sorted. One bulk extend + // specialized fill loops per width — no per-byte push_back on this multi-million-pair seam. const std::size_t count = d.lex.tokenTfs.size(); char* p = w.extend( count * ( idxWidth + tfWidth ) ); ASSUME( count == 0 || rowDictIndex != nullptr ); // null only with an empty row: verifyCacheRecordMinimaTripwire's probe if( idxWidth == 1 ) { for( std::size_t k = 0; k < count; ++k ) { *p++ = char( rowDictIndex[k] ); } } else if( idxWidth == 2 ) { for( std::size_t k = 0; k < count; ++k ) { const std::uint32_t v = rowDictIndex[k]; p[0] = char( v ); p[1] = char( v >> 8 ); p += 2; } } else { for( std::size_t k = 0; k < count; ++k ) { const std::uint32_t v = rowDictIndex[k]; p[0] = char( v ); p[1] = char( v >> 8 ); p[2] = char( v >> 16 ); p[3] = char( v >> 24 ); p += 4; } } const std::uint32_t* tfs = d.lex.tokenTfs.data(); if( tfWidth == 1 ) { for( std::size_t k = 0; k < count; ++k ) { *p++ = char( tfs[k] ); } } else if( tfWidth == 2 ) { for( std::size_t k = 0; k < count; ++k ) { const std::uint32_t v = tfs[k]; p[0] = char( v ); p[1] = char( v >> 8 ); p += 2; } } else { for( std::size_t k = 0; k < count; ++k ) { const std::uint32_t v = tfs[k]; p[0] = char( v ); p[1] = char( v >> 8 ); p[2] = char( v >> 16 ); p[3] = char( v >> 24 ); p += 4; } } } } // MEM-R1a: a registrar Value ref's object-literal offset is the row field litAt, but the record still spells it as the qualifier's // "@" tail (the format every blob since the registrar lane holds), so the blob is byte-identical and no kParserVer moves. // Only a Value ref's qualifier is ever split (its text is "", two characters, from emitValueRef): a call's written // qualifier keeps any '@' it has. inline void writeRefQualifier( ByteW& w, const RawRef& r ) { if( r.litAt == 0 ) { w.str( r.qualifier ); return; } DASSERT( r.role == RefRole::Value && r.qualifier.size() == 2 ); w.str( std::string( r.qualifier ) + "@" + std::to_string( r.litAt ) ); } // The reader's half. The blob is external input: a tail that is not a nonzero u32 in decimal is corruption (the record takes // readFileRecord's one refusal path), never a text to keep. inline void readRefQualifier( ByteR& r, std::string_view qualifier, RawRef& x ) { if( x.role != RefRole::Value || qualifier.size() <= 2 || qualifier[ 2 ] != '@' ) { x.qualifier = qualifier; return; } const std::string_view digits = qualifier.substr( 3 ); std::uint64_t litAt = 0; // length and every digit are checked BEFORE any arithmetic (CodeRabbit 4545944847): a 20-nines tail or a `1/` would // otherwise wrap the accumulator, which -fsanitize=integer aborts on before VALIDATE can refuse the record bool valid = !digits.empty() && digits.size() <= 10 && std::all_of( digits.begin(), digits.end(), []( char c ) { return c >= '0' && c <= '9'; } ); for( std::size_t k = 0; valid && k < digits.size(); ++k ) { litAt = litAt * 10 + std::uint64_t( digits[ k ] - '0' ); // <= 10 digits: at most 9 999 999 999, far under 2^64 } if( !VALIDATE( valid && litAt != 0 && litAt <= 0xFFFFFFFFu ) ) { DISCLOSE( Diagnostics::answerUnchanged, "a rejected cache is rebuilt from source: this run parses and answers byte-identically", "ingest: cache ref record carries a registrar qualifier whose literal offset is not a number — cache treated as corrupt" ); r.ok = false; return; } x.qualifier = qualifier.substr( 0, 2 ); x.litAt = std::uint32_t( litAt ); } inline void writeRef( ByteW& w, const RawRef& r ) { w.u32( r.startByte ); w.u8( std::uint8_t( r.lang ) ); w.str( r.name ); w.u8( r.isInherit ? 1 : 0 ); w.u8( r.isDocLink ? 1 : 0 ); writeRefQualifier( w, r ); w.u8( std::uint8_t( r.recv ) ); w.str( r.recvVar ); w.u8( r.isCompose ? 1 : 0 ); w.str( r.fieldName() ); w.str( r.composeRel() ); w.u8( std::uint8_t( r.role ) ); w.u32( r.line ); w.u32( r.argCount ); w.u8( r.argCountKnown ? 1 : 0 ); w.u8( r.viaArrow ? 1 : 0 ); w.u8( r.qualifierRootsStd ? 1 : 0 ); w.u8( r.memberCall ? 1 : 0 ); w.str( r.memberRoot ); w.str( r.memberPath() ); w.str( r.memberCtor() ); w.str( r.memberVia() ); w.u8( r.memberFactory ? 1 : 0 ); w.u8( r.forwarded ? 1 : 0 ); w.u8( r.trailingLambda ? 1 : 0 ); if( r.isInherit ) { w.u32( r.baseBind ); } } // loadCache's countFits() bounds a corrupt on-disk record COUNT against remaining bytes / // minRecordBytes BEFORE reserve() — the guard that keeps a hostile blob's 0xFFFFFFFF count from reaching // an allocator. The minima below are named + pinned here (not hand-recounted inline at the call site) so // they sit next to the writer functions whose field list they must match. A LEAN def record is 14 u32 + // 13 u8 + 2 empty str(len u32) fields = 14*4 + 13*1 + 2*4 = 77 bytes (Phase 1, local-variable-indexing, // docs/LOCALS_INDEXING.md: `locals` u32 joined the run — 9 -> 10; the ppalt disclosure added `ppAlt`, // written as a u32 — 10 -> 11; the nesting profile then added `humps` and `deepLoc`, written as u32 each — // 11 -> 13; essential complexity then added `ev` as a u32 in the run plus the 8×u8 evWhy tag counters // after the strings — 13 -> 14 u32 and 4 -> 12 u8, so 56 + 12 + 8 = 76; L8's in-file `testScope` then // added one u8 in the run — 12 -> 13 u8, so 56 + 13 + 8 = 77; the extent-honesty `recovered` bit then added // one more u8 in the run — 13 -> 14 u8, so 56 + 14 + 8 = 78; the internal-linkage bit then added one more u8 in // the run — 14 -> 15 u8, so 56 + 15 + 8 = 79; #150's `scopeRootsStd` bit then added one more u8 in the run — // 15 -> 16 u8, so 56 + 16 + 8 = 80; the function-local scope span then added two u32 after the u8 run — 14 -> 16 // u32, so 64 + 16 + 8 = 88); the RICH (withLex) extra is // dlWeighted u32 + tokenCount u32 + tfWidth u8 = 9 bytes. A ref record is 3 u32 + 8 u8 + 5 empty // str(len u32) fields = 3*4 + 8*1 + 5*4 = 40 bytes (39 until the `viaArrow` u8, kCacheVersion 23; #150's // `qualifierRootsStd` u8 then added a 9th u8 — 40 -> 41, kCacheVersion 25; FE-A's `memberCall` u8 and `memberRoot` // str then added a 10th u8 and a 6th str — 41 -> 46, kCacheVersion 28; FE-B's `memberPath` and `memberCtor` strs then // added a 7th and 8th str — 46 -> 54, kCacheVersion 29; fcall-forward fix 1's `forwarded` u8 then added a 12th u8 — // 59 -> 60, kCacheVersion 33; #155's `trailingLambda` u8 then added a 13th u8 — 60 -> 61, kCacheVersion 41). // verifyCacheRecordMinimaTripwire() below derives these // same numbers from the REAL writer functions at runtime so the next field added to writeDef/writeRef // can't silently stale them. inline constexpr std::size_t kMinDefRecordBytesLean = 99; // 18×u32 + 19×u8 + 2×str(len u32, empty) (paramsMin u32: 88 -> 92, kCacheVersion 33; clone-key presence u8: 92 -> 93, v50; QD-TANGLE steer u8 + deepDispatch u32 + nestArmCut u8: 93 -> 99, v54) inline constexpr std::size_t kMinDefRecordBytesRichExtra = 9; // v10 rich withLex extra: dlWeighted u32 + tokenCount u32 + tfWidth u8 inline constexpr std::size_t kMinRefRecordBytes = 61; // 3×u32 + 13×u8 + 9×str(len u32, empty) inline std::size_t minDefRecordBytes( bool captureValueUses ) noexcept { return kMinDefRecordBytesLean + ( captureValueUses ? kMinDefRecordBytesRichExtra : 0 ); } // runtime tripwire: serialize a DEFAULT-CONSTRUCTED (all-empty) RawDef/RawRef through the real // writer functions and ASSUME the byte count matches the hand-pinned constants above. This is the runtime // equivalent of the house `static_assert( sizeof(X) == N )` layout tripwire — ByteW's size is only known at // runtime (strings, not a POD struct), so it can't be a compile-time static_assert. ASSUME is a no-op // optimizer hint in release (never costs a shipped run anything); in any debug/ASan build or CI it fires the // moment a field is added to writeDef/writeRef without updating the matching constant above. inline void verifyCacheRecordMinimaTripwire() noexcept { ByteW probe; writeDef( probe, RawDef{}, false, 0, nullptr ); ASSUME( probe.b.size() == kMinDefRecordBytesLean ); probe.b.clear(); writeDef( probe, RawDef{}, true, 0, nullptr ); ASSUME( probe.b.size() == kMinDefRecordBytesLean + kMinDefRecordBytesRichExtra ); probe.b.clear(); writeRef( probe, RawRef{} ); ASSUME( probe.b.size() == kMinRefRecordBytes ); } inline RawDef readDef( ByteR& r, bool withLex, const std::vector& fileDict ) { RawDef d; d.line = r.u32(); d.startByte = r.u32(); d.endByte = r.u32(); d.nameByte = r.u32(); d.bodyByte = r.u32(); d.cx = r.u32(); d.ccx = r.u32(); d.loc = r.u32(); d.locals = r.u32(); d.ppAlt = r.u16Of32(); d.humps = r.u16Of32(); d.deepLoc = r.u16Of32(); d.ev = r.u16Of32(); d.params = r.u16Of32(); d.paramsMin = r.u16Of32(); d.maxNest = r.u8(); d.steer = r.u8(); d.deepDispatch = r.u16Of32(); d.nestArmCut = r.u8(); d.arityExact = r.u8(); d.testScope = r.u8(); d.recovered = r.u8(); d.internalLinkage = r.u8(); d.scopeRootsStd = r.u8(); d.fnScopeStart = r.u32(); d.fnScopeEnd = r.u32(); d.kind = r.enumU8( kSymKindCount ); d.lang = r.enumU8( kLangCount ); d.name = r.str(); d.scope = r.str(); // B2.2: the extractor (ingest_sidecap.h) stores paramsMin only when it is <= params. A record past that rule is // corruption, and a too-high minimum is the UNSAFE direction (the resolver drops a candidate for too few arguments), // so it takes the same one refusal path as a short read: that file reparses and the rest of the blob stands. if( r.ok && !VALIDATE( d.paramsMin <= d.params ) ) { DISCLOSE( Diagnostics::answerUnchanged, "a rejected cache is rebuilt from source: this run parses and answers byte-identically", "ingest: cache def record carries paramsMin above params — cache treated as corrupt" ); r.ok = false; } for( std::uint8_t& tagCount : d.evWhy ) { tagCount = r.u8(); } // mirrors writeDef's fixed 8×u8 order // v50: mirrors writeDef. The presence byte is 0 or 1; any other value is corruption and takes the one refusal path. Any // u64 is a valid key: findClones re-reads and compares every shared key, so a wrong key costs a read, never an answer. const std::uint8_t hasCloneKey = r.u8(); if( r.ok && !VALIDATE( hasCloneKey <= 1 ) ) { DISCLOSE( Diagnostics::answerUnchanged, "a rejected cache is rebuilt from source: this run parses and answers byte-identically", "ingest: cache def record carries a clone-key presence byte other than 0/1 — cache treated as corrupt" ); r.ok = false; } if( r.ok && hasCloneKey == 1 ) { d.cloneKey = r.u64(); } if( withLex && r.ok ) { d.lex.dlWeighted = r.u32(); const std::uint32_t tokenCount = r.u32(); const std::uint8_t tfWidth = r.u8(); // a corrupt on-disk count/width must never reach resize() (the throw would escape ingest()'s // never-throw contract) — every honest count is bounded by remaining bytes / min pair bytes if( !r.ok || ( tfWidth != 1 && tfWidth != 2 && tfWidth != 4 ) ) { r.ok = false; return d; } const unsigned idxWidth = lexDictIndexWidth( fileDict.size() ); const std::size_t needBytes = std::size_t( tokenCount ) * ( idxWidth + tfWidth ); if( std::size_t( r.end - r.p ) < needBytes ) { r.ok = false; return d; } d.lex.tokenHashes.resize( tokenCount ); d.lex.tokenTfs.resize( tokenCount ); // decode indices → hashes through the file dict; the dict is strictly ascending (validated at // load) and honest rows store strictly-ascending indices, so the decoded row is sorted — the // invariant the query-time binary search relies on. Violations ⇒ corrupt ⇒ self-healing reparse. // The whole row was bounds-checked ONCE above, so these are tight raw-pointer loops (this is the // warm-path deserialize hot spot — ~1M pairs on a jax-class rich blob). const unsigned char* q = reinterpret_cast( r.p ); const std::uint64_t* dict = fileDict.data(); const std::uint32_t dictCount = std::uint32_t( fileDict.size() ); std::uint64_t* hashOut = d.lex.tokenHashes.data(); std::uint32_t prevIndex = 0; bool rowOk = true; const auto decodeIndexRun = [ & ]( auto readOne ) { for( std::uint32_t k = 0; k < tokenCount; ++k ) { const std::uint32_t dictIndex = readOne(); if( dictIndex >= dictCount || ( k > 0 && dictIndex <= prevIndex ) ) { rowOk = false; return; } hashOut[k] = dict[ dictIndex ]; prevIndex = dictIndex; } }; if( idxWidth == 1 ) { decodeIndexRun( [ & ]() noexcept { return std::uint32_t( *q++ ); } ); } else if( idxWidth == 2 ) { decodeIndexRun( [ & ]() noexcept { const std::uint32_t v = std::uint32_t( q[0] ) | std::uint32_t( q[1] ) << 8; q += 2; return v; } ); } else { decodeIndexRun( [ & ]() noexcept { const std::uint32_t v = std::uint32_t( q[0] ) | std::uint32_t( q[1] ) << 8 | std::uint32_t( q[2] ) << 16 | std::uint32_t( q[3] ) << 24; q += 4; return v; } ); } if( !rowOk ) { r.ok = false; return d; } std::uint32_t* tfOut = d.lex.tokenTfs.data(); if( tfWidth == 1 ) { for( std::uint32_t k = 0; k < tokenCount; ++k ) { tfOut[k] = *q++; } } else if( tfWidth == 2 ) { for( std::uint32_t k = 0; k < tokenCount; ++k ) { tfOut[k] = std::uint32_t( q[0] ) | std::uint32_t( q[1] ) << 8; q += 2; } } else { for( std::uint32_t k = 0; k < tokenCount; ++k ) { tfOut[k] = std::uint32_t( q[0] ) | std::uint32_t( q[1] ) << 8 | std::uint32_t( q[2] ) << 16 | std::uint32_t( q[3] ) << 24; q += 4; } } r.p += needBytes; } return d; } inline RawRef readRef ( ByteR& r ) { RawRef x; x.startByte = r.u32(); x.lang = r.enumU8( kLangCount ); x.name = r.view(); x.isInherit = r.flagU8(); x.isDocLink = r.flagU8(); const std::string_view qualifier = r.view(); x.recv = r.enumU8( kRecvKindCount ); x.recvVar = r.view(); x.isCompose = r.flagU8(); x.setFieldName( r.view() ); x.setComposeRel( r.view() ); x.role = r.enumU8( kRefRoleCount ); readRefQualifier( r, qualifier, x ); x.line = r.u32(); x.argCount = r.u16Of32(); x.argCountKnown = r.flagU8(); x.viaArrow = r.flagU8(); x.qualifierRootsStd = r.flagU8(); x.memberCall = r.flagU8(); x.memberRoot = r.view(); x.setMemberPath( r.view() ); x.setMemberCtor( r.view() ); x.setMemberVia( r.view() ); x.memberFactory = r.flagU8(); x.forwarded = r.flagU8(); x.trailingLambda = r.flagU8(); if( x.isInherit ) { x.baseBind = r.u32(); } return x; } inline void writeBind( ByteW& w, const RawBind& b ) { w.u32( b.startByte ); w.u8( std::uint8_t( b.lang ) ); w.u8( std::uint8_t( b.kind ) ); w.u8( b.isFromAssignment ? 1 : 0 ); w.u8( b.resultIndex ); w.u8( b.unwrapDepth ); w.u32( b.spanStart ); w.u32( b.spanEnd ); w.str( b.var ); w.str( b.typeName ); w.str( b.importedName ); } inline RawBind readBind( ByteR& r ) { RawBind b; b.startByte = r.u32(); b.lang = r.enumU8( kLangCount ); b.kind = r.enumU8( kLocalBindKindCount ); b.isFromAssignment = r.flagU8(); b.resultIndex = r.u8(); b.unwrapDepth = r.u8(); b.spanStart = r.u32(); b.spanEnd = r.u32(); b.var = r.view(); b.typeName = r.view(); b.importedName = r.view(); return b; } inline void writeFfi( ByteW& w, const BindingAlias& a ) { w.u8( std::uint8_t( a.kind ) ); w.u8( a.lowConf ? 1 : 0 ); w.str( a.aliasName ); w.str( a.targetName ); w.str( a.targetScope ); } inline BindingAlias readFfi( ByteR& r ) { BindingAlias a; a.kind = r.enumU8( kBindKindCount ); a.lowConf = r.flagU8(); a.aliasName = r.str(); a.targetName = r.str(); a.targetScope = r.str(); return a; } // B6.3: RouteDef needs no startByte (its handler is resolved by NAME in buildGraph); RawRouteUse mirrors // RawBind (startByte for the enclosing-def byte-span attribution done in the ingest() model-build below). inline void writeRouteDef( ByteW& w, const RouteDef& d ) { w.u32( d.line ); w.u8( std::uint8_t( d.method ) ); w.str( d.path ); w.str( d.handlerName ); } inline RouteDef readRouteDef( ByteR& r ) { RouteDef d; d.line = r.u32(); d.method = r.enumU8( kHttpMethodCount ); d.path = r.str(); d.handlerName = r.str(); return d; } inline void writeRouteUse( ByteW& w, const RawRouteUse& u ) { w.u32( u.startByte ); w.u32( u.line ); w.u8( std::uint8_t( u.method ) ); w.str( u.path ); } inline RawRouteUse readRouteUse( ByteR& r ) { RawRouteUse u; u.startByte = r.u32(); u.line = r.u32(); u.method = r.enumU8( kHttpMethodCount ); u.path = r.str(); return u; } // Minimum serialized bytes of one row per family, beside kMinRefRecordBytes and minDefRecordBytes: an honest count is // bounded by the bytes left / its row minimum (readFileRecord's countFits, the warm planner's headCountsFit). inline constexpr std::size_t kMinIncRecordBytes = 12; // 4×u8 (isAngle,isLazy,isSymbolic,isValueUse) + 1×u32 (byte) + 1×str(len u32, empty) inline constexpr std::size_t kMinBindRecordBytes = 29; // 3×u32 + 5×u8 (lang,kind,isFromAssignment,resultIndex,unwrapDepth) + 3×str(len u32, empty) inline constexpr std::size_t kMinFfiRecordBytes = 14; // 2×u8 (kind,lowConf) + 3×str(len u32, empty) inline constexpr std::size_t kMinRouteDefRecordBytes = 13; // B6.3: 1×u32 (line) + 1×u8 (method) + 2×str(len u32, empty) inline constexpr std::size_t kMinRouteUseRecordBytes = 13; // B6.3: 2×u32 (startByte,line) + 1×u8 (method) + 1×str(len u32, empty) inline constexpr std::size_t kMinConstOpenRecordBytes = 13; // parser version 82: 2×u32 (startByte,endByte) + 1×u8 (namespaceOnly) + 1×str(len u32, empty) inline constexpr std::size_t kConstructRecordBytes = 9; // v51: 2×u32 (startByte,line) + 1×u8 (tag) — fixed size // v51: the HEAD's construct count when the record's pool ran no construct pass (ingest.h WantConstructHits): the hits are // unknown, not zero. The body then holds a zero count; a --quality-delta ingest treats the record as a miss. inline constexpr std::uint32_t kConstructsAbsent = 0xFFFFFFFFu; // Read a record's HEAD: its root-relative key (a view into the reader's bytes), hash, stat triple, health and the v49 // family counts. readFileRecord starts with it; the warm planner (loadCache) reads only this much of each record. inline bool readRecordHead( ByteR& r, std::string_view& relOut, CachedRecordHead& head ) { relOut = r.view(); head.hash = r.u64(); head.sizeBytes = (long long)r.u64(); // A4-P7 stat-gate discriminator head.mtimeNs = (long long)r.u64(); // A4-P7 stat-gate discriminator + racy-rule input head.ctimeNs = (long long)r.u64(); // v14 stat-gate discriminator: the one a restore cannot forge head.health.errNodes = r.u32(); // §L1 parse health (v13) — fileBytes==0 keeps its head.health.errBytes = r.u32(); // NOT-MEASURED meaning across the round trip head.health.fileBytes = r.u32(); head.health.wsBytes = r.u32(); head.health.macroBlanked = r.u32(); // v20: member-macro re-parse — rides the record so the disclosure survives a warm run head.health.useBoundsBlanked = r.u32(); // v44: its Rust layer (use<..> bounds blanked) — same reason FileFactCounts& c = head.counts; // v49: the eight family counts, in the body's family order c.defs = r.u32(); c.refs = r.u32(); c.incs = r.u32(); c.binds = r.u32(); c.ffis = r.u32(); c.routeDefs = r.u32(); c.routeUses = r.u32(); c.constOpens = r.u32(); c.constructs = r.u32(); // v51; kConstructsAbsent = never computed for this record head.constructsAbsent = c.constructs == kConstructsAbsent; if( head.constructsAbsent ) { c.constructs = 0; } return r.ok; } // A head's counts fit the record they came from: every row of every family needs at least its minimum bytes, so the sum // bounds what an honest record can claim. The warm planner reserves from these counts before the body (and its checksum) // is read, so this is what keeps a damaged head from asking reserve() for more than the blob could ever hold. inline bool headCountsFit( const FileFactCounts& c, std::uint64_t recLength, bool captureValueUses ) noexcept { const std::uint64_t need = std::uint64_t( c.defs ) * minDefRecordBytes( captureValueUses ) + std::uint64_t( c.refs ) * kMinRefRecordBytes + std::uint64_t( c.incs ) * kMinIncRecordBytes + std::uint64_t( c.binds ) * kMinBindRecordBytes + std::uint64_t( c.ffis ) * kMinFfiRecordBytes + std::uint64_t( c.routeDefs ) * kMinRouteDefRecordBytes + std::uint64_t( c.routeUses ) * kMinRouteUseRecordBytes + std::uint64_t( c.constOpens ) * kMinConstOpenRecordBytes + std::uint64_t( c.constructs ) * kConstructRecordBytes; return need <= recLength; } // Deserialise ONE file record out of a reader bounded to exactly that record's bytes. Lifted verbatim // out of loadCache's old per-file loop when v15 made records individually addressable — same field // order, same guards, same self-healing posture; the only change is that `r` now ends at the record's // own last byte, so a corrupt count can never walk into the next record. // Returns false on any corruption; `relOut` is the record's own stored root-relative path key, which // the caller compares against the key it looked up (the pathHash-collision guard). inline bool readFileRecord( ByteR& r, bool captureValueUses, std::vector& fileDict, std::string& relOut, FileFacts& ffOut ) { // count validation: a corrupt on-disk count (e.g. 0xFFFFFFFF) must never reach reserve() — the throw // (length_error/bad_alloc) would escape ingest(), violating its never-throw contract. Every honest // count is bounded by remaining bytes / the record's MINIMUM serialized size; past that → corrupt → // the same self-healing reparse path as a truncated record. // (B0.2) a RICH def record additionally carries at least dlWeighted + tokenCount (2×u32) — the pair // arrays themselves are bounded per record inside readDef. const std::size_t kMinDefRecordBytes = minDefRecordBytes( captureValueUses ); // F8: named + tripwire-pinned above // v49: a FAMILY count must also equal the head's copy, which the warm planner reserved from before this body was // read. The dictionary count has no head copy (it passes kNoHeadCount). One lambda, so every count is bounded by the bytes // left at the line that names it, before any reserve() or resize() sizes from it. constexpr std::int64_t kNoHeadCount = -1; const auto countFits = [ &r ]( std::uint32_t recordCount, std::size_t minRecordBytes, std::int64_t headCount ) noexcept { if( headCount != kNoHeadCount && std::int64_t( recordCount ) != headCount ) { DISCLOSE( Diagnostics::answerUnchanged, "a rejected cache is rebuilt from source: this run parses and answers byte-identically", "ingest: cache record family count differs from its head — cache treated as corrupt" ); r.ok = false; return false; } if( recordCount <= std::size_t( r.end - r.p ) / minRecordBytes ) { return true; } DISCLOSE( Diagnostics::answerUnchanged, "a rejected cache is rebuilt from source: this run parses and answers byte-identically", "ingest: cache record count exceeds remaining bytes — cache treated as corrupt" ); r.ok = false; return false; }; std::string_view rel; CachedRecordHead head; if( !readRecordHead( r, rel, head ) ) { return false; } relOut.assign( rel ); ffOut.hash = head.hash; ffOut.sizeBytes = head.sizeBytes; ffOut.mtimeNs = head.mtimeNs; ffOut.ctimeNs = head.ctimeNs; ffOut.health = head.health; ffOut.constructsAbsent = head.constructsAbsent; if( captureValueUses ) { // H3 (v10): the file's subtoken dictionary — def rows below index into it. Must be strictly // ascending (readDef's sorted-row invariant hangs on it); anything else is corrupt. const std::uint32_t dictCount = r.u32(); if( !countFits( dictCount, sizeof( std::uint64_t ), kNoHeadCount ) ) { return false; } fileDict.resize( dictCount ); if( !r.rawInto( fileDict.data(), std::size_t( dictCount ) * sizeof( std::uint64_t ) ) ) { return false; } for( std::size_t k = 1; k < fileDict.size(); ++k ) { if( fileDict[k] <= fileDict[ k - 1 ] ) { DISCLOSE( Diagnostics::answerUnchanged, "a rejected cache is rebuilt from source: this run parses and answers byte-identically", "ingest: cache file dictionary not strictly ascending — cache treated as corrupt" ); r.ok = false; return false; } } } const std::uint32_t nd = r.u32(); if( !countFits( nd, kMinDefRecordBytes, head.counts.defs ) ) { return false; } ffOut.defs.reserve( nd ); for( std::uint32_t j = 0; j < nd && r.ok; ++j ) { ffOut.defs.push_back( readDef( r, captureValueUses, fileDict ) ); } const std::uint32_t nr = r.u32(); if( !countFits( nr, kMinRefRecordBytes, head.counts.refs ) ) { return false; } ffOut.refs.reserve( nr ); for( std::uint32_t j = 0; j < nr && r.ok; ++j ) { ffOut.refs.push_back( readRef( r ) ); } const std::uint32_t ni = r.u32(); if( !countFits( ni, kMinIncRecordBytes, head.counts.incs ) ) { return false; } ffOut.incs.reserve( ni ); for( std::uint32_t j = 0; j < ni && r.ok; ++j ) { const bool isAngle = r.flagU8(); const bool isLazy = r.flagU8(); // kParserVer 72: TS/JS function-body require/import marker; parser version 82: Ruby autoload const bool isSymbolic = r.flagU8(); // parser version 82: a Ruby constant target, resolved by index, never by path const std::uint32_t byte = r.u32(); // parser version 82: the directive's start byte (lexical nesting recovery) const bool isValueUse = r.flagU8(); // parser version 93 / format 21: argument or rescue origin — not narrow evidence ffOut.incs.push_back( Include { 0, isAngle, isLazy, isSymbolic, byte, isValueUse, r.str() } ); } const std::uint32_t nb = r.u32(); if( !countFits( nb, kMinBindRecordBytes, head.counts.binds ) ) { return false; } ffOut.binds.reserve( nb ); for( std::uint32_t j = 0; j < nb && r.ok; ++j ) { ffOut.binds.push_back( readBind( r ) ); } const std::uint32_t na = r.u32(); if( !countFits( na, kMinFfiRecordBytes, head.counts.ffis ) ) { return false; } ffOut.ffis.reserve( na ); for( std::uint32_t j = 0; j < na && r.ok; ++j ) { ffOut.ffis.push_back( readFfi( r ) ); } const std::uint32_t nrd = r.u32(); if( !countFits( nrd, kMinRouteDefRecordBytes, head.counts.routeDefs ) ) { return false; } ffOut.routeDefs.reserve( nrd ); for( std::uint32_t j = 0; j < nrd && r.ok; ++j ) { ffOut.routeDefs.push_back( readRouteDef( r ) ); // B6.3 } const std::uint32_t nru = r.u32(); if( !countFits( nru, kMinRouteUseRecordBytes, head.counts.routeUses ) ) { return false; } ffOut.routeUses.reserve( nru ); for( std::uint32_t j = 0; j < nru && r.ok; ++j ) { ffOut.routeUses.push_back( readRouteUse( r ) ); // B6.3 } const std::uint32_t nco = r.u32(); if( !countFits( nco, kMinConstOpenRecordBytes, head.counts.constOpens ) ) { return false; } ffOut.constOpens.reserve( nco ); for( std::uint32_t j = 0; j < nco && r.ok; ++j ) { ConstOpen co; // parser version 82 (fileId re-labelled by the caller) co.startByte = r.u32(); co.endByte = r.u32(); co.namespaceOnly = r.flagU8(); co.written = r.str(); ffOut.constOpens.push_back( std::move( co ) ); } const std::uint32_t nch = r.u32(); if( !countFits( nch, kConstructRecordBytes, head.counts.constructs ) ) { return false; } ffOut.constructs.reserve( nch ); for( std::uint32_t j = 0; j < nch && r.ok; ++j ) { ConstructHit ch; // v51 (fileId re-labelled by the caller) ch.startByte = r.u32(); ch.line = r.u32(); ch.tag = r.enumU8( kConstructTagCount ); // a tag past the table is a corrupt record ffOut.constructs.push_back( ch ); } return r.ok; } // What a load learned about the blob, beyond the facts themselves. Three numbers that used to be one // out-param: the racy-rule reference, plus the two the RIPWIRE_CACHE_STATS line reports so "a subset // configuration reads only its own records" is an EXECUTABLE fact and not a wall-clock claim // (test/cacheoffsetcheck.sh band (2)). struct CacheLoadStats { long long blobWriteNs = -1; // the blob's own on-disk mtime; -1 ⇒ no usable cache (every stat-gate check re-hashes) std::size_t blobEntries = 0; // offset-table entries: how many files the blob holds std::size_t recordsRead = 0; // records actually deserialised: how many this crawl asked for and got }; // The stderr notice for a blob loadCache refused. 2026-09-06 stranger audit: every reject self-healed to a full // reparse with NO signal a Release binary keeps (the debug-only alert compiles out under NDEBUG) — a torn blob, an older // binary's blob, a directory passed as --cache: all byte-identical to a healthy run, just slower, every time. The // ordinary cold-start miss (absent) stays silent; anything else says what it found, once per run. // // #334: a Windows tester alternating 0.6.2 and 0.6.3 on one tree saw `format-version — not used` on every run and read // it as "the CLI never reuses its cache". The auto path was keyed by root and verb class only, so two builds of // different formats refused and rewrote each other's blob every time. The auto path now carries the build tag // (quality.h rootBlobTail), so this notice on an AUTO blob means a hand-copied or foreign file; on a --cache file // the user named, it still means another build (or the other verb class) wrote that one file. A version refusal // names the number it found and the one this binary reads, so the cause is on the line. It is APPENDED: the line up // to "rewrites it" is unchanged, and gates grep that prefix (test/localscountcheck.sh, test/cachefuzzcheck.sh). // // WHO WROTE IT (the #334 review, M1). A parser stamp equal to this build's OTHER verb class is what one --cache file // shared by a lean and a rich verb holds, and it used to be blamed on "another ripwire build". The stamp alone // cannot tell that apart from an older build whose parserVer was one lower or higher (a rich class of kParserVer-1 // stamps exactly this build's lean number), so that case names both. The advice then depends on WHO named the file: // a --cache file the user named is fixed by giving each class its own file; an automatic file carries this build's // own name (quality.h isThisBuildRootBlobName), which no build writes with another stamp, so it was copied in by // hand — and "two builds alternating on one cache file" cannot happen to it either. inline std::string_view cacheRejectCause( const CacheFrame& frame, bool captureValueUses, bool automaticPath ) noexcept { if( frame.reason == CacheReject::ParserVersion && frame.foundStamp == parserVerFor( !captureValueUses ) ) { if( automaticPath ) { return captureValueUses ? "this build's lean verb class writes that number, or another ripwire build wrote it; an automatic cache file holds it only when copied in by hand" : "this build's rich verb class writes that number, or another ripwire build wrote it; an automatic cache file holds it only when copied in by hand"; } return captureValueUses ? "this build's lean verb class writes that number, or another ripwire build wrote it; give each verb class its own --cache file" : "this build's rich verb class writes that number, or another ripwire build wrote it; give each verb class its own --cache file"; } return automaticPath ? "another ripwire build wrote it; an automatic cache file holds that only when copied in by hand" : "another ripwire build wrote it; two builds alternating on one cache file re-parse every run"; } inline void noteCacheReject( const std::string& path, const CacheFrame& frame, bool captureValueUses ) { if( frame.reason == CacheReject::Absent ) { return; } const bool format = frame.reason == CacheReject::FormatVersion; const bool version = format || frame.reason == CacheReject::ParserVersion; char detail[ 256 ] = ""; if( version ) { const std::size_t slash = path.find_last_of( "/\\" ); const bool automatic = quality::isThisBuildRootBlobName( std::string_view( path ).substr( slash == std::string::npos ? 0 : slash + 1 ), quality::ownBuildBlobTails() ); rw::formatTo( detail, sizeof( detail ), " (blob {} {}, this binary {}: {})", format ? "format" : "parser", frame.foundStamp, format ? kCacheVersion : parserVerFor( captureValueUses ), cacheRejectCause( frame, captureValueUses, automatic ) ); } rw::emitTo( stderr, "ripwire: cache {}: {} — not used; this run parses from source and rewrites it{}\n", path.c_str(), cacheRejectName( frame.reason ), rw::cstr( detail ) ); } // WARM-DIRECT (reports/graph-efficiency-ideas.md #4): a warm run's view of the blob. Until v49 loadCache decoded every // wanted record into a HashMap, the parse workers moved those facts into their per-thread arrays, and // the merge moved them again — three sets of fact arrays at the parse-pool join (warm peak +40 to +49 % over cold on // envoy, cockroach and vscode). Now the load reads only each record's HEAD (readRecordHead: key, hash, stat triple, // health, the v49 family counts) into a per-fileId table, and keeps the blob open; a worker that finds a hash-verified // hit reads THAT record (pread), checks its recordSum32 and its path key, and decodes it straight into its own arrays. // The fileId space is this crawl's (result.files), so no path-keyed map is built and no lookup hashes a string. // // A record that fails in the worker (a short read, its checksum, its decode, its key) is a LATE MISS: runParsePool // parses those files after the join, once their grammars are installed, so the answer is the cold one. struct CacheRecords { std::unique_ptr frame; // the open, validated blob; null when there is no usable cache std::vector entryOf; // per crawled fileId: index into frame->entries, or kNoCacheEntry std::vector head; // per crawled fileId; meaningful only where entryOf names an entry std::string rootDir; // the crawl's root: a decoded key is compared root-relative std::size_t records = 0; // crawled files with a usable record head bool captureValueUses = false; static constexpr std::uint32_t kNoCacheEntry = 0xFFFFFFFFu; bool empty() const noexcept { return records == 0; } const CachedRecordHead* headOf( std::size_t fileId ) const noexcept { return fileId < entryOf.size() && entryOf[ fileId ] != kNoCacheEntry ? &head[ fileId ] : nullptr; } // Read, verify and decode one crawled file's record into `out` (its families cleared first; capacity is kept, so a // worker's scratch stops allocating once it has seen its largest record). `recBuf` and `dict` are the caller's // per-thread scratch. False on any failure, disclosed; the caller then treats the file as a late miss. bool decode( std::size_t fileId, const std::string& crawledPath, std::string& recBuf, std::vector& dict, FileFacts& out ) const { const CachedRecordHead* h = headOf( fileId ); if( h == nullptr || frame == nullptr ) { return false; } const CacheEntry& e = frame->entries[ entryOf[ fileId ] ]; recBuf.resize( e.recLength ); bool read = false; { PROFILE_SCOPE_DESCRIBE( "ingest/parse-pool: pread one cache record" ); read = preadExact( frame->blob.fd, recBuf.data(), recBuf.size(), e.recOffset ); } if( !read ) { DISCLOSE( Diagnostics::answerUnchanged, "the affected files are reparsed from source: the answer is byte-identical, only slower", "ingest: cache blob read failed mid-load — the unread records are reparsed" ); return false; } bool recordSumOk = false; { PROFILE_SCOPE_DESCRIBE( "ingest/parse-pool: record digest (recordSum32)" ); recordSumOk = recordSum32( std::string_view( recBuf.data(), recBuf.size() ) ) == e.recSum; } if( !recordSumOk ) { // A record torn on its own while the table survived: drop THIS file (it reparses) and keep the rest of the // blob. The table is what must be trusted whole, not each record. DISCLOSE( Diagnostics::answerUnchanged, "the affected files are reparsed from source: the answer is byte-identical, only slower", "ingest: cache record checksum mismatch — that file is reparsed, the rest of the blob stands" ); return false; } clearFileFactsKeepCapacity( out ); ByteR r{ recBuf.data(), recBuf.data() + recBuf.size() }; std::string rel; bool decoded = false; { PROFILE_SCOPE_DESCRIBE( "ingest/parse-pool: decode one cache record (readFileRecord)" ); decoded = readFileRecord( r, captureValueUses, dict, rel, out ); } if( !decoded ) { return false; // corrupt record → that file reparses (readFileRecord already disclosed) } if( rel != relForHash( crawledPath, rootDir ) ) { // The planner matched this record's head key, so a different key in the body is a record that changed // under us: serving it would be a wrong answer. DISCLOSE( Diagnostics::answerUnchanged, "the affected files are reparsed from source: the answer is byte-identical, only slower", "ingest: cache path-hash collision — the colliding file is reparsed" ); return false; } return true; } // Close the blob and free the per-file table: called once the parse pool (and its late-miss pass) is done. void release() noexcept { frame.reset(); std::vector().swap( entryOf ); std::vector().swap( head ); records = 0; } }; // Plan a warm run from the blob at `path`: the per-fileId record heads of THIS crawl's files (result.files, // absolute-as-crawled; the on-disk key is root-relative, T5 portability — see kCacheVersion=3 above). Empty on missing / // corrupt / version-or-parserVer mismatch. // // v15: only the records for this crawl's files are touched — a blob written by a wider configuration is not read past // its offset table and those records' heads (docs/EVALS.md band (2)). Heads are read in ascending offset order, a small // prefix each, so the reads run forward through the file. // // stats.blobWriteNs (supersedes A4-P7): the racy-rule reference a warm run's stat-gate compares every cached file's mtime // against. STAMPED FROM A FRESH stat() OF THE CACHE FILE ITSELF, NOT from the ns-precision wall-clock the header still // carries for legacy/diagnostic reasons. Same clock+granularity domain (stat()) as the per-file mtimes it is compared // against — on a coarse-mtime filesystem (HFS+, many network mounts) the old wall-clock-vs-stat comparison was a // tautology (a floored mtime is always < an unfloored LATER timestamp), so a same-granule post-hash edit could slip // through undetected. Left at -1 on any miss/corrupt/version-mismatch/unstatable-path (the result is empty then too), // which makes every stat-gate check see a racy entry and re-hash — the safe default. inline CacheRecords loadCache( const std::string& path, std::string_view rootDir, bool captureValueUses, const std::vector& crawledFiles, CacheLoadStats& stats ) { PROFILE_SCOPE_DESCRIBE( "ingest: loadCache (offset table + record heads)" ); CacheRecords out; stats = CacheLoadStats{}; auto frame = std::make_unique( openCacheFrame( path, captureValueUses ) ); if( !frame->ok ) { noteCacheReject( path, *frame, captureValueUses ); return out; } stats.blobWriteNs = frame->mtimeNs; stats.blobEntries = frame->entries.size(); // Which of this crawl's files the blob actually holds. `wanted` rides in TABLE order (== ascending pathHash), then is // re-sorted by record offset so the reads below run forward through the file. struct Wanted { std::uint32_t entryIndex; std::uint32_t fileIndex; }; std::vector wanted; { PROFILE_SCOPE_DESCRIBE( "ingest/loadCache: offset-table lookup" ); wanted.reserve( std::min( crawledFiles.size(), frame->entries.size() ) ); for( std::size_t f = 0; f < crawledFiles.size(); ++f ) { const std::string_view rel = relForHash( crawledFiles[f], rootDir ); const auto [ lo, hi ] = cacheEntryRange( frame->entries, contentHash64( rel ) ); for( std::size_t e = lo; e < hi; ++e ) { wanted.push_back( Wanted{ std::uint32_t( e ), std::uint32_t( f ) } ); // identity is confirmed below } } std::sort( wanted.begin(), wanted.end(), [ & ]( const Wanted& a, const Wanted& b ) noexcept { return frame->entries[ a.entryIndex ].recOffset < frame->entries[ b.entryIndex ].recOffset; } ); } if( wanted.empty() ) { return out; } out.entryOf.assign( crawledFiles.size(), CacheRecords::kNoCacheEntry ); out.head.resize( crawledFiles.size() ); // The head is the record's first bytes: its key (u32 length + bytes) then 4 u64 + 15 u32 (6 health fields and the 9 // family counts readRecordHead reads, v51's construct count the ninth). One prefix read covers every key up to 416 // bytes; a longer key takes a second read of exactly what the head needs (test/statgatecheck.sh (f)). constexpr std::size_t kHeadFixedBytes = 4 + 4 * 8 + 15 * 4; constexpr std::size_t kHeadProbeBytes = 512; std::string headBuf; { PROFILE_SCOPE_DESCRIBE( "ingest/loadCache: read record heads" ); for( const Wanted& w : wanted ) { const CacheEntry& e = frame->entries[ w.entryIndex ]; const std::uint32_t file = w.fileIndex; if( out.entryOf[ file ] != CacheRecords::kNoCacheEntry ) { continue; // an earlier record already matched this file's key } const std::string_view expectRel = relForHash( crawledFiles[ file ], rootDir ); std::size_t want = std::size_t( std::min( e.recLength, kHeadProbeBytes ) ); headBuf.resize( want ); if( !preadExact( frame->blob.fd, headBuf.data(), want, e.recOffset ) ) { DISCLOSE( Diagnostics::answerUnchanged, "the affected files are reparsed from source: the answer is byte-identical, only slower", "ingest: cache blob read failed mid-load — the unread records are reparsed" ); break; } std::uint32_t keyBytes = 0; if( want >= 4 ) { std::memcpy( &keyBytes, headBuf.data(), 4 ); if( !VALIDATE( keyBytes == expectRel.size() ) ) { // The second read below is sized by this stored length. A key of any other length is not this file's // record (a hash collision or a damaged length), so refuse it before reading that many more bytes. DISCLOSE( Diagnostics::answerUnchanged, "the affected files are reparsed from source: the answer is byte-identical, only slower", "ingest: cache path-hash collision — the colliding file is reparsed" ); continue; } } const std::uint64_t headBytes = std::uint64_t( keyBytes ) + kHeadFixedBytes; if( headBytes > want && headBytes <= e.recLength ) { want = std::size_t( headBytes ); headBuf.resize( want ); if( !preadExact( frame->blob.fd, headBuf.data(), want, e.recOffset ) ) { DISCLOSE( Diagnostics::answerUnchanged, "the affected files are reparsed from source: the answer is byte-identical, only slower", "ingest: cache blob read failed mid-load — the unread records are reparsed" ); break; } } ByteR r{ headBuf.data(), headBuf.data() + want }; std::string_view rel; CachedRecordHead h; if( !readRecordHead( r, rel, h ) || !headCountsFit( h.counts, e.recLength, captureValueUses ) ) { // A head that does not parse, or claims more rows than its record can hold, is a corrupt record: that // file reparses. Its checksum (checked in the worker) would refuse it too; this keeps reserve() honest. DISCLOSE( Diagnostics::answerUnchanged, "a rejected cache is rebuilt from source: this run parses and answers byte-identically", "ingest: cache record head is corrupt — that file is reparsed" ); continue; } if( rel != expectRel ) { // The pathHash matched but the record is for a DIFFERENT file — a 64-bit collision. Serving it would be a // wrong answer, so the file reparses instead. DISCLOSE( Diagnostics::answerUnchanged, "the affected files are reparsed from source: the answer is byte-identical, only slower", "ingest: cache path-hash collision — the colliding file is reparsed" ); continue; } out.entryOf[ file ] = w.entryIndex; out.head[ file ] = h; ++out.records; } } stats.recordsRead = out.records; out.frame = std::move( frame ); out.rootDir = std::string( rootDir ); out.captureValueUses = captureValueUses; return out; } // The cache-write side's per-file record indexes: the seven flat fact arrays grouped back by fileId, so // each file's record serializes in one pass. Split by MEASURED shape, not by symmetry: // defIndex — one entry per DEFINITION; same distribution as model.h's SymbolsByFile (mean 8.4/18.2 per // file, p90 18/37), so it takes that index's measured N=8 knee. // incIndex/ffiIndex/routeDefIndex/routeUseIndex — includes, FFI aliases and the two route tables. N=2 is // FREE (rw::svector's inline array shares storage with the heap pointer, so and // are both 16 B — a THIRD smaller than the std::vector header it replaces) and covers // 85.4%/59.1%, 99.7%/100%, 99.9%/100% and 99.9%/100% of files across the two census corpora. // refIndex/bindIndex — deliberately LEFT as std::vector. Means of 155/248 and 28/59 references and // bindings per file with 17%/40% of files empty and no early knee: an N that covered them would // have to be in the hundreds. They are CSR candidates, a separate wave, not small-vector material. struct CacheFileIndexes { std::vector> defIndex; std::vector> refIndex, bindIndex; std::vector> incIndex, ffiIndex, routeDefIndex, routeUseIndex, constOpenIndex, constructIndex; }; inline CacheFileIndexes buildCacheFileIndexes( std::size_t fileCount, const std::vector& defs, const std::vector& refs, const std::vector& incs, const std::vector& binds, const std::vector& ffis, const std::vector& routeDefs, const std::vector& routeUses, const std::vector& constOpens, const std::vector& constructs ) { CacheFileIndexes ix; ix.defIndex.resize( fileCount ); ix.refIndex.resize( fileCount ); ix.bindIndex.resize( fileCount ); ix.incIndex.resize( fileCount ); ix.ffiIndex.resize( fileCount ); ix.routeDefIndex.resize( fileCount ); ix.routeUseIndex.resize( fileCount ); ix.constOpenIndex.resize( fileCount ); ix.constructIndex.resize( fileCount ); // One generic grouping pass per family — an out-of-range fileId is DROPPED, never clamped, exactly as // the seven hand-written loops this replaces did. const auto group = [ fileCount ]( const auto& facts, auto& index ) { for( std::uint32_t i = 0; i < facts.size(); ++i ) { if( facts[i].fileId < fileCount ) { index[ facts[i].fileId ].push_back( i ); } } }; group( defs, ix.defIndex ); group( refs, ix.refIndex ); group( incs, ix.incIndex ); group( binds, ix.bindIndex ); group( ffis, ix.ffiIndex ); group( routeDefs, ix.routeDefIndex ); // B6.3 group( routeUses, ix.routeUseIndex ); // B6.3 group( constOpens, ix.constOpenIndex ); // parser version 82 group( constructs, ix.constructIndex ); // v51 return ix; } // The per-file write KEYS: each crawled file's root-relative cache key, its pathHash, and the fileId order // the v15 record region is written in — ascending (pathHash, rel). Deterministic, and the reason the offset // table comes out sorted and a full crawl's read coalesces into one contiguous span. struct CachePathKeys { std::vector rels; std::vector pathHashes; std::vector order; }; inline CachePathKeys buildCachePathKeys( const std::vector& files, std::string_view rootDir ) { CachePathKeys keys; const std::size_t F = files.size(); keys.rels.resize( F ); keys.pathHashes.resize( F ); keys.order.resize( F ); for( std::uint32_t f = 0; f < F; ++f ) { keys.rels[f] = relForHash( files[f], rootDir ); keys.pathHashes[f] = contentHash64( keys.rels[f] ); keys.order[f] = f; } std::sort( keys.order.begin(), keys.order.end(), [ & ]( std::uint32_t a, std::uint32_t b ) noexcept { return keys.pathHashes[a] != keys.pathHashes[b] ? keys.pathHashes[a] < keys.pathHashes[b] : keys.rels[a] < keys.rels[b]; } ); return keys; } // One planned output row of a v15 write: either a file THIS run crawled (fileIndex valid) or a record // carried over verbatim from the previous blob (carryIndex valid). kNoCacheIndex marks the unused half. constexpr std::uint32_t kNoCacheIndex = 0xFFFFFFFFu; struct CacheWriteRow { std::uint64_t pathHash; std::uint32_t fileIndex; std::uint32_t carryIndex; }; // The SUPERSET plan for one save. `prevEntries` is the previous blob's offset table (empty when there is // no usable previous blob); every entry whose pathHash no file in THIS crawl owns becomes a CARRY row, so // an --exclude run can never truncate the shared blob to its own file set. Both input streams are already // ascending by pathHash, so the merge is one linear pass and the record region it describes comes out // ascending too — which is what makes the offset table binary-searchable and a full crawl's read one // contiguous span. `orderIn` is the crawled fileIds in ascending (pathHash, rel) order. inline std::vector buildCacheWritePlan( const std::vector& orderIn, const std::vector& pathHashes, const std::vector& prevEntries, std::vector& carryOut ) { ASSUME_NO_ALIAS( prevEntries, carryOut ); carryOut.clear(); carryOut.reserve( prevEntries.size() ); { std::size_t f = 0; // walks `orderIn` in lockstep with the (ascending) previous table for( const CacheEntry& e : prevEntries ) { while( f < orderIn.size() && pathHashes[ orderIn[f] ] < e.pathHash ) { ++f; } if( f >= orderIn.size() || pathHashes[ orderIn[f] ] != e.pathHash ) { carryOut.push_back( e ); // no file in THIS crawl owns that path — keep the record } } } std::vector plan; plan.reserve( orderIn.size() + carryOut.size() ); std::size_t a = 0, b = 0; while( a < orderIn.size() || b < carryOut.size() ) { const bool takeFresh = b >= carryOut.size() || ( a < orderIn.size() && pathHashes[ orderIn[a] ] <= carryOut[b].pathHash ); if( takeFresh ) { plan.push_back( CacheWriteRow{ pathHashes[ orderIn[a] ], orderIn[a], kNoCacheIndex } ); ++a; } else { plan.push_back( CacheWriteRow{ carryOut[b].pathHash, kNoCacheIndex, std::uint32_t( b ) } ); ++b; } } return plan; } // Copy one carry-over record out of the previous blob into `w`, VERIFYING it against that blob's own // per-record checksum first — a torn record must never be laundered into a freshly-checksummed blob. On // any failure the record is dropped (disclosed) and the file simply reparses the next time a // configuration crawls it; the rest of the blob is unaffected. inline bool appendCarryRecord( ByteW& w, int fd, const CacheEntry& src, std::string& scratch, CacheEntry& entryOut ) { const std::uint64_t recOffset = w.offset(); scratch.resize( src.recLength ); if( !preadExact( fd, scratch.data(), scratch.size(), src.recOffset ) || recordSum32( std::string_view( scratch.data(), scratch.size() ) ) != src.recSum ) { DISCLOSE( Diagnostics::answerUnchanged, "a carry-over record for a file this run did not crawl: this answer never reads it", "ingest: cache carry-over record failed its checksum — dropped (that file reparses)" ); return false; } w.raw( scratch.data(), scratch.size() ); entryOut = CacheEntry{ src.pathHash, recOffset, src.contentHash, std::uint32_t( scratch.size() ), src.recSum }; return true; } // Close a v15 blob: append the offset table, then the 24-byte trailer whose digest covers HEADER || TABLE. Each record carries // its own digest in its table entry and is checked only when that record is actually read, so this is the whole of "the // checksum covers the table plus the records actually read". The exact-fit invariant the trailer encodes — fileSize == // tableOffset + entryCount*32 + 24 — is what makes a truncation that removes whole records detectable at all, so it is asserted // here at the one place that can still be wrong about it. C1 SAVE-STREAM: the header's bytes left `w` with the first window, so // they ride in `index` beside the table, entryCount patched in here (carry-over drops only settle it now); saveCache patches the // FILE's copy once the stream is out. struct CacheBlobIndex { std::string header; // the blob's first kCacheHeaderBytes, as written (entryCount 0 until finishCacheBlob) std::vector table; // one entry per record, in record order }; inline void finishCacheBlob( ByteW& w, CacheBlobIndex& index ) { const std::vector& table = index.table; ASSUME( index.header.size() == kCacheHeaderBytes ); const std::uint32_t entryCount = std::uint32_t( table.size() ); std::memcpy( index.header.data() + 21, &entryCount, 4 ); const std::uint64_t tableOffset = w.offset(); // The digest's input is header || table, which are not adjacent in the blob, so it is assembled once // here and the table half is then appended to the blob out of the same buffer. std::string hdrTable; hdrTable.resize( kCacheHeaderBytes + std::size_t( entryCount ) * kCacheEntryBytes ); std::memcpy( hdrTable.data(), index.header.data(), kCacheHeaderBytes ); if( entryCount != 0 ) { std::memcpy( hdrTable.data() + kCacheHeaderBytes, table.data(), std::size_t( entryCount ) * kCacheEntryBytes ); } w.raw( hdrTable.data() + kCacheHeaderBytes, std::size_t( entryCount ) * kCacheEntryBytes ); w.u64( tableOffset ); w.u32( entryCount ); w.u32( 0 ); // reserved w.u64( blobChecksum( std::string_view( hdrTable.data(), hdrTable.size() ) ) ); ASSUME( w.offset() == tableOffset + std::uint64_t( entryCount ) * kCacheEntryBytes + kCacheTrailerBytes ); } // write the cache atomically (path.tmp → rename); groups the merged raw facts back by file. // T5: `rootDir` is the CURRENT invocation's ingest root — every file key is stored root-relative // (relForHash) rather than verbatim, so the cache blob is committable/portable (see kCacheVersion=3). // v51: the construct hits saveCache writes, and per fileId whether they are KNOWN (0 = the record says kConstructsAbsent). struct ConstructHitsToSave { const std::vector& hits; const std::vector& known; }; inline void saveCache( const std::string& path, std::string_view rootDir, const std::vector& files, const std::vector& fileHash, const std::vector& fileSize, const std::vector& fileMtime, const std::vector& fileCtime, // v14: the third stat-gate discriminator, per fileId const std::vector& fileHealth, // §L1 (v13): parse health, per fileId const std::vector& defs, const std::vector& refs, const std::vector& incs, const std::vector& binds, const std::vector& ffis, const std::vector& routeDefs, const std::vector& routeUses, // B6.3 const std::vector& constOpens, // parser version 82 const ConstructHitsToSave& constructHits, // v51 bool captureValueUses ) { PROFILE_SCOPE_DESCRIBE( "ingest: saveCache (serialize + write)" ); // L1 (write half): never publish over a non-regular file, and decide it BEFORE the serialize so a // directory at `path` costs no wasted pass and temp write before rename(tmp,dir) fails EISDIR. if( shapeOfPath( path ) == PathShape::Other ) { DISCLOSE( Diagnostics::answerUnchanged, "this answer is already computed in memory: only the next run starts cold", "ingest: cache path is not a regular file (directory/device/fifo) — cache not written" ); return; } const std::size_t F = files.size(); const CacheFileIndexes ix = buildCacheFileIndexes( F, defs, refs, incs, binds, ffis, routeDefs, routeUses, constOpens, constructHits.hits ); // This header field is no longer the racy-rule reference — loadCache now derives that from // a fresh stat() of the cache file itself (same clock+granularity domain as the per-file mtimes it's // compared against; see loadCache's blobWriteNsOut comment for why the wall-clock-vs-stat comparison // this used to feed was a tautology on coarse-mtime filesystems). Kept written, at the same wire offset, // purely as a diagnostic "when was this blob generated" stamp — changing/removing it would bump // kCacheVersion for no behavioral gain. const long long blobWriteNs = wallClockNs(); // v15 — the SUPERSET contract. This run crawled `files`; the blob at `path` may hold records for // files this configuration excluded (or size-skipped), and those records must SURVIVE. So the write // plan is a merge of two streams, both in ascending pathHash order: // FRESH — one record per crawled file, serialized from this run's facts; // CARRY — the previous blob's records for pathHashes this run did not crawl, copied VERBATIM out // of that blob (each verified against its own recSum before it is trusted forward). // The previous blob is opened once, through the same validating frame reader loadCache used, and is // held open for the whole copy: rename(2) unlinks rather than truncates, so a concurrent writer // publishing a new blob cannot pull the bytes out from under this one. If the frame does not // validate — absent, foreign version/parserVer/arch, torn — CARRY is simply empty and this run // writes its own file set, which is exactly v14's behaviour and self-heals on the next wider run. const CachePathKeys keys = buildCachePathKeys( files, rootDir ); const CacheFrame prev = openCacheFrame( path, captureValueUses ); std::vector carry; const std::vector plan = buildCacheWritePlan( keys.order, keys.pathHashes, prev.entries, carry ); // unique per-process temp so two concurrent runs (this repo runs ~20 parallel sessions) don't // interleave writes into ONE shared "path.tmp" and rename a torn file. rename(2) is atomic; the // per-pid temp makes each writer's bytes whole → last-writer-wins, never a corrupt cache. // // A3-F9: fwrite/fclose were never checked, so an ENOSPC (or any short write) produced a truncated // temp file that still got rename()'d over the previous GOOD cache — silently destroying it (the // checksum trailer self-heals on next load via a full reparse, so this was perf-only, but a good // cache should never be clobbered without a peep). Mirrors mcpedit::atomicWrite's discipline // (src/mcp.h): check the write byte-count AND fclose's return, and on any failure unlink the temp // and leave the prior on-disk cache (if any) untouched. // Round 5 (rw::pathguard): the temp is created EXCLUSIVELY and WITHOUT following a link, under an // unpredictable name beside the cache blob, refusing an existing entry at that name. The name keeps its // `.tmp` tail (a *.tmp residue glob still // matches) and 0666 preserves the fopen("wb") default mode. The RAII holder owns the temp NAME and removes // it on every early return below; fdopen adopts the descriptor so fwrite/fclose keep their bookkeeping. rw::pathguard::ExclTempFile temp = rw::pathguard::createExclTempFile( path + ".", ".tmp", 0666 ); const int rawFd = temp.ok() ? temp.releaseFd() : -1; std::FILE* fp = rawFd >= 0 ? os::fdopen( rawFd, "wb" ) : nullptr; // A throw between here and the checked close below (table.reserve, a window's growth) must not leak the stream and its // descriptor (CodeRabbit 4545944852): this owner closes it on unwind; the normal path takes the stream back first. struct StreamOwner { std::FILE*& stream; ~StreamOwner() { if( stream != nullptr ) { std::fclose( stream ); } } } closeOnUnwind{ fp }; if( !fp ) { const int openErr = errno; if( rawFd >= 0 ) { os::close( rawFd ); // fdopen did not adopt the descriptor; the holder still removes the temp name } DISCLOSE( Diagnostics::answerUnchanged, "this answer is already computed in memory: only the next run starts cold", "ingest: saveCache could not open temp file for write — cache left unchanged" ); rw::emitTo( stderr, "ripwire: cache {}: cannot write ({}) — every run parses from source until this is fixed\n", path.c_str(), std::strerror( openErr ) ); // 2026-09-06: Release kept no signal for this return; } // C1 SAVE-STREAM (reports/mem-audit-fable-1111 §C1): the blob is no longer built whole in memory. `w` is a WINDOW: once it // holds kSaveWindowBytes at a record boundary it is written to the temp and cleared, `w.streamed` carrying the absolute offset // every recOffset and the trailer's tableOffset are read from. The one header field that settles only at the end, entryCount, // is patched in the file with one seek-and-write before the rename; the bytes are the ones the whole-blob write produced. // Before this the doubling std::string set the cold peak on envoy and vscode (+1.4-1.5 x the blob over the pool join, M2 11-11). // The stream owns the descriptor since releaseFd(), so it closes on every path: a write failure only marks `wErr`, and the // close is evaluated on its own, never short-circuited into one expression. constexpr std::size_t kSaveWindowBytes = std::size_t( 4 ) << 20; bool wErr = false; ByteW w; const auto drain = [ & ] { if( !wErr && !w.b.empty() && std::fwrite( w.b.data(), 1, w.b.size(), fp ) != w.b.size() ) { wErr = true; } w.streamed += w.b.size(); w.b.clear(); }; CacheBlobIndex index; std::vector& table = index.table; table.reserve( plan.size() ); // A1 (team-index): kArtifactArch byte sits in the header right after parserVer so loadCache's guard // rejects a foreign-arch (endian/pointer-width) blob before trusting any raw-int record bytes. w.u32( kCacheMagic ); w.u32( kCacheVersion ); w.u32( parserVerFor( captureValueUses ) ); w.u8( kArtifactArch ); w.u64( (std::uint64_t)blobWriteNs ); w.u32( 0 ); // entryCount — patched in place below, once the carry-over verification has settled it ASSUME( w.b.size() == kCacheHeaderBytes ); index.header = w.b; // finishCacheBlob's digest input: the first window takes these bytes to the file { PROFILE_SCOPE_DESCRIBE( "ingest/saveCache: serialize records" ); std::vector fileDict; // per-file subtoken dictionary, reused across files std::vector mergeA, mergeB; // ping-pong buffers of the balanced run-merge, reused std::vector runOffsets, nextRunOffsets; // sorted-run bounds inside the ping-pong buffer std::vector pairDictIndex; // pair slot → dict index, in def-row order std::string carryBuf; // one carry-over record, verified before it is appended for( const CacheWriteRow& row : plan ) { if( w.b.size() >= kSaveWindowBytes ) // a record boundary: no record straddles a window, so its bytes are whole in `w` { drain(); } if( wErr ) { break; // the temp is torn already: nothing below can publish it } if( row.fileIndex == kNoCacheIndex ) { CacheEntry copied; if( appendCarryRecord( w, prev.blob.fd, carry[ row.carryIndex ], carryBuf, copied ) ) { table.push_back( copied ); } continue; } const std::size_t recStart = w.b.size(); // the record's first byte in the window const std::uint64_t recOffset = w.offset(); // …and in the blob const std::uint32_t f = row.fileIndex; w.str( keys.rels[f] ); w.u64( f < fileHash.size() ? fileHash[f] : 0 ); w.u64( f < fileSize.size() ? (std::uint64_t)fileSize[f] : (std::uint64_t)-1 ); // A4-P7 stat-gate: size at hash time (-1 ⇒ unknown → gate re-hashes) w.u64( f < fileMtime.size() ? (std::uint64_t)fileMtime[f] : (std::uint64_t)-1 ); // A4-P7 stat-gate: mtimeNs at hash time (-1 ⇒ unknown) w.u64( f < fileCtime.size() ? (std::uint64_t)fileCtime[f] : (std::uint64_t)-1 ); // v14 stat-gate: ctimeNs at hash time (-1 ⇒ unknown → gate re-hashes) { // §L1 (v13): parse health, at a FIXED wire offset right after the stat-gate pair. An // out-of-range f writes the default (fileBytes 0), which the reader already means as // NOT MEASURED — no sentinel of its own, and no way to mistake it for "clean". const FileHealth fh = f < fileHealth.size() ? fileHealth[f] : FileHealth{}; w.u32( fh.errNodes ); w.u32( fh.errBytes ); w.u32( fh.fileBytes ); w.u32( fh.wsBytes ); w.u32( fh.macroBlanked ); // v20: the member-macro re-parse's per-file count (0 ⇒ the first parse was kept) w.u32( fh.useBoundsBlanked ); // v44: the Rust precise-capture re-parse's per-file count (0 ⇒ first parse kept) } // v49: the head copy of the eight family counts (readRecordHead), in the body's family order for( const std::size_t familyCount : { std::size_t( ix.defIndex[f].size() ), ix.refIndex[f].size(), std::size_t( ix.incIndex[f].size() ), ix.bindIndex[f].size(), std::size_t( ix.ffiIndex[f].size() ), std::size_t( ix.routeDefIndex[f].size() ), std::size_t( ix.routeUseIndex[f].size() ), std::size_t( ix.constOpenIndex[f].size() ) } ) { w.u32( std::uint32_t( familyCount ) ); } const bool constructsKnownHere = f >= constructHits.known.size() || constructHits.known[f] != 0; w.u32( constructsKnownHere ? std::uint32_t( ix.constructIndex[f].size() ) : kConstructsAbsent ); // v51 if( captureValueUses ) { // H3 (v10): the sorted union of this file's def rows — subtokens repeat heavily across a // file's defs (nested spans re-tokenize the same text), so hoisting each distinct hash into // ONE per-file dictionary and storing narrow indices per row shrinks the rich blob's // postings from 12 B/pair to ~dictShare×8 + idxWidth + tfWidth bytes. Every row is ALREADY // sorted (lexindex.h buildDefLexStats), so the dict is a BALANCED PAIRWISE MERGE over the // rows (P·log2(rows) sequential std::merge steps — measured cheaper than both a flat // O(P log P) sort and a per-element k-way heap), and the final merged (hash, slot) order // assigns every row position's dict index in one walk. Deterministic: equal hashes all land // on the same dict entry regardless of slot order. This is the --index-out / prime hot path. fileDict.clear(); mergeA.clear(); runOffsets.clear(); runOffsets.push_back( 0 ); std::uint32_t slotCount = 0; for( const std::uint32_t i : ix.defIndex[f] ) { const std::vector& row = defs[i].lex.tokenHashes; for( const std::uint64_t hash : row ) { mergeA.push_back( LexPair{ hash, slotCount++ } ); // braced, not emplace_back( a, b ): aggregate emplace needs P0960, absent in Clang < 20 (CI's Xcode 15.4) } if( !row.empty() ) { runOffsets.push_back( mergeA.size() ); } } std::vector* src = &mergeA; std::vector* dst = &mergeB; while( runOffsets.size() > 2 ) // > 1 run left → one merge pass { dst->resize( src->size() ); // exact pass size — merges write via raw pointers, nextRunOffsets.clear(); // no per-element back_inserter capacity branch nextRunOffsets.push_back( 0 ); LexPair* writeCursor = dst->data(); for( std::size_t runIndex = 0; runIndex + 1 < runOffsets.size(); runIndex += 2 ) { const std::size_t lo = runOffsets[runIndex]; const std::size_t mid = runOffsets[runIndex + 1]; if( runIndex + 2 < runOffsets.size() ) // a full pair of runs → merge them { const std::size_t hi = runOffsets[runIndex + 2]; writeCursor = std::merge( src->data() + lo, src->data() + mid, src->data() + mid, src->data() + hi, writeCursor ); } else // odd tail run: carry over { std::memcpy( writeCursor, src->data() + lo, ( mid - lo ) * sizeof( LexPair ) ); writeCursor += mid - lo; } nextRunOffsets.push_back( std::size_t( writeCursor - dst->data() ) ); } runOffsets.swap( nextRunOffsets ); std::swap( src, dst ); } pairDictIndex.resize( slotCount ); for( const auto& [hash, slot] : *src ) { if( fileDict.empty() || fileDict.back() != hash ) { fileDict.push_back( hash ); } pairDictIndex[slot] = std::uint32_t( fileDict.size() - 1 ); } w.u32( std::uint32_t( fileDict.size() ) ); w.raw( fileDict.data(), fileDict.size() * sizeof( std::uint64_t ) ); } w.u32( std::uint32_t( ix.defIndex[f].size() ) ); { std::size_t rowOffset = 0; // running slot offset into pairDictIndex for( std::uint32_t i : ix.defIndex[f] ) { writeDef( w, defs[i], captureValueUses, fileDict.size(), pairDictIndex.data() + rowOffset ); if( captureValueUses ) { rowOffset += defs[i].lex.tokenHashes.size(); } } } w.u32( std::uint32_t( ix.refIndex[f].size() ) ); for( std::uint32_t i : ix.refIndex[f] ) { writeRef( w, refs[i] ); } w.u32( std::uint32_t( ix.incIndex[f].size() ) ); for( std::uint32_t i : ix.incIndex[f] ) { w.u8( incs[i].isAngle ? 1 : 0 ); w.u8( incs[i].isLazy ? 1 : 0 ); w.u8( incs[i].isSymbolic ? 1 : 0 ); w.u32( incs[i].byte ); w.u8( incs[i].isValueUse ? 1 : 0 ); w.str( incs[i].target ); } w.u32( std::uint32_t( ix.bindIndex[f].size() ) ); for( std::uint32_t i : ix.bindIndex[f] ) { writeBind( w, binds[i] ); } w.u32( std::uint32_t( ix.ffiIndex[f].size() ) ); for( std::uint32_t i : ix.ffiIndex[f] ) { writeFfi( w, ffis[i] ); } w.u32( std::uint32_t( ix.routeDefIndex[f].size() ) ); for( std::uint32_t i : ix.routeDefIndex[f] ) { writeRouteDef( w, routeDefs[i] ); // B6.3 } w.u32( std::uint32_t( ix.routeUseIndex[f].size() ) ); for( std::uint32_t i : ix.routeUseIndex[f] ) { writeRouteUse( w, routeUses[i] ); // B6.3 } w.u32( std::uint32_t( ix.constOpenIndex[f].size() ) ); for( std::uint32_t i : ix.constOpenIndex[f] ) { // parser version 82: span + own-body bit + written name (fileId is the record's) w.u32( constOpens[i].startByte ); w.u32( constOpens[i].endByte ); w.u8( constOpens[i].namespaceOnly ? 1 : 0 ); w.str( constOpens[i].written ); } w.u32( std::uint32_t( ix.constructIndex[f].size() ) ); for( std::uint32_t i : ix.constructIndex[f] ) { // v51: span start + line + tag byte (fileId is the record's) w.u32( constructHits.hits[i].startByte ); w.u32( constructHits.hits[i].line ); w.u8( constructHits.hits[i].tag ); } table.push_back( CacheEntry{ row.pathHash, recOffset, f < fileHash.size() ? fileHash[f] : 0, std::uint32_t( w.b.size() - recStart ), recordSum32( std::string_view( w.b.data() + recStart, w.b.size() - recStart ) ) } ); } } // symmetric bare scope: serialize-records profiling span { PROFILE_SCOPE_DESCRIBE( "ingest/saveCache: offset table + trailer" ); const std::uint32_t entryCount = std::uint32_t( table.size() ); finishCacheBlob( w, index ); drain(); // entryCount into the file's header: one seek-and-write after the stream, before the close and the rename if( !wErr && ( std::fseek( fp, 21, SEEK_SET ) != 0 || std::fwrite( &entryCount, 1, 4, fp ) != 4 ) ) { wErr = true; } } PROFILE_SCOPE_DESCRIBE( "ingest/saveCache: write + rename" ); const bool closed = std::fclose( std::exchange( fp, nullptr ) ) == 0; // taken back from closeOnUnwind: closed once, checked wErr = wErr || !closed; if( wErr ) { // never rename a short/torn write over a good cache — the holder removes the temp on return DISCLOSE( Diagnostics::answerUnchanged, "this answer is already computed in memory: only the next run starts cold", "ingest: saveCache write failed (short write or fclose error) — old cache preserved" ); rw::emitTo( stderr, "ripwire: cache {}: write failed (short write; disk full?) — old cache kept, this run was parsed from source\n", path.c_str() ); return; } if( !temp.commit( path ) ) { // the holder removes the temp on return DISCLOSE( Diagnostics::answerUnchanged, "this answer is already computed in memory: only the next run starts cold", "ingest: saveCache rename(tmp -> cache) failed — old cache preserved" ); rw::emitTo( stderr, "ripwire: cache {}: cannot replace ({}) — old cache kept, this run was parsed from source\n", path.c_str(), std::strerror( errno ) ); return; } // A5 (cache-dir hygiene): --doctor measured ~11,914 ripwire-* blobs / 2.4 GB accumulating in the cache-ladder // dir because only the qsnap/qheadsnap families ever evicted — this main parse-cache family (this very // `path`) never did. Best-effort, silent, at most once per process (see sweepStaleCacheBlobsOnce for the // policy + the concurrency-safety argument). `path` is the blob we just rename()'d into place, so it is // always the retained "keepPath" even if it happens to be the oldest survivor by mtime granularity. quality::sweepStaleCacheBlobsOnce( quality::cacheDirLadder(), path ); } } // namespace — ingest_cache.h section of ingest.cpp } // namespace rw #include // errno — the cache save-failure notices (2026-09-06) #include // std::strerror — same