#pragma once #include "infra/emit.h" // rw::emitTo / emitRaw / formatTo — THE emitter and its siblings #include "gitcmd.h" // rw::gitCmd — every git child starts with --no-optional-locks -c core.fsmonitor=false // crossref.h — the CROSS-BRANCH CONTENT INDEX: --whereis=SYM and --stray-content. // Evidence: a completed, soak-verified canyon fix sat UNMERGED on 1 of // 30 branches for two days while a ledger claim said "ported" — `git cherry` answers commit ANCESTRY, and // every other ripwire verb indexes ONE worktree, so nothing could answer "where does this CONTENT live?". // // SCOPE, once, for both verbs (§B12.2): "every ref" here has always meant `refs/heads` — every LOCAL branch, // worktree branches included — and never `refs/remotes/*`; see enumerateRefs for why (they mirror the local // ones in the usual checkout and would double every row). That is a real limit, not a detail: on a FRESH // CLONE, the standard CI/agent shape, the work is all on `refs/remotes/origin/*` and only the checked-out // branch has a local head, so both verbs cover ~nothing. Only `--help` used to say "local"; both payload // legends now carry the clause, with refs_scanned=/refs= named as the number that shows it. // // The two questions: // --whereis=SYM — every local ref whose tree DEFINES or MENTIONS SYM, and whether HEAD has it. Scans each // ref's FULL tree (that is the question), so a symbol a branch merely inherited is // still found. Trees only, though: content that NO ref still carries is invisible to // a tree scan however loudly history remembers it, which is why --with-history adds // the gitoracle.h lane — one `git log` pass that says whether HEAD's own history ever // removed the name, and in which commit. hits="0" then separates "never existed here" // from "deleted in commit X on DATE", which is the difference between a typo and rot. // HEAD's side is the CHECKOUT: a path the working tree changed is read from disk // (scanWorktree), so an uncommitted edit is never answered from the commit it left. // --stray-content — per ref: the content the ref's own divergent work AUTHORED that the live line does // NOT have, plus a verdict (merged / superseded / unmerged). // // ── the cost model: per-BLOB, keyed by blob sha ─────────────────────────────────────────────────────────── // Git is content-addressed, so 30 branches off one trunk share ~99% of their blobs. EVERY byte-level fact // this module needs (a blob's line multiset, its shingle sketch, whether SYM occurs in it) is a pure // function of the blob's CONTENT — therefore of its sha — so it is computed ONCE per distinct blob and // reused by every ref that references it. A blob sha is immutable by construction, so the memo can never go // stale (the same reasoning quality.h's per-sha ingest caches rest on). Blobs are streamed through ONE // `git cat-file --batch` for the whole run (not one process per file), and reduced to fixed-size facts as // they arrive — raw bytes are never retained, so peak memory is bounded by the facts, not by the trees. // Net effect: indexing all N refs costs barely more than indexing one. // // ── what "stray content" means, precisely ───────────────────────────────────────────────────────────────── // For ref R with base B = merge-base(R, HEAD): // authored(R,P) = lines ADDED by R vs B in path P (R's OWN work — not what it inherited) // stray(R,P) = authored(R,P) lines absent from HEAD's blob at P // Restricting to AUTHORED lines is load-bearing and was the bug in the hand-rolled sweep this verb replaces: // a plain "lines R has that HEAD lacks" also counts BASE content that HEAD itself later deleted — the live // line's own decision, not the branch's work — which buried 194 real stray lines under ~800 phantom ones on // the motivating repo. Deletions on either side are handled by MULTISET arithmetic over line hashes, so // removing one of two identical lines registers (a set would silently drop it). // // ── supersession: the case `git cherry` structurally cannot see ─────────────────────────────────────────── // A branch fix that the live line RE-IMPLEMENTED differently is still an unmerged commit to `git cherry`, // forever. The evidence that separates it is the DELETION SITE: both sides diff against the SAME base, so // "R deleted base line L" and "HEAD deleted base line L" are exactly comparable with no fuzzy matching at // all. When the live line removed the very base code the branch removed, it has demonstrably revisited that // site on its own — the branch's copy is a variant, not a missing feature: // redoDel(R,P) = |del(B→R,P) ∩ del(B→HEAD,P)| / |del(B→R,P)| (multiset) // A file with redoDel ≥ kRedoDelMin is SUPERSEDED; a file whose stray content is a PURE ADDITION (nothing // deleted, so no site to compare) can only be superseded through the minhash lane below. The ref-level // verdict is the stray-LINE-weighted majority, so one coincidentally-shared deleted line can never vouch for // a few hundred added ones. // // The minhash lane (bottom-k sketch over normalized token shingles) covers the pure-addition case: content // the branch added that the live line holds in some rewritten/relocated form. It is deliberately held to a // HIGH bar (kSimSupersede, and a minimum stray size) because containment alone is a weak signal — measured // on the motivating repo it ran HIGHER for genuinely-unmerged branches than for superseded ones, so it // corroborates the deletion-site evidence rather than competing with it. Every file row reports its raw // del=/redo=/sim= numbers so the verdict is auditable, never a black box. // // ── ANCHORING, per verb, stated rather than assumed (r26 merge-base audit) ──────────────────────────────── // --stray-content is a HYBRID, deliberately, and the split is the whole point: // SCOPE (which lines are even considered) — BASE-anchored. authored(R,P) is `diff base..R`, never // `diff HEAD..R`. This is the §"what stray content means" rule above; getting it wrong is what buried // 194 real stray lines under ~800 phantom ones, and it is the same correction --abi later needed. // ABSENCE (the test each authored line then faces) — HEAD-anchored, ON PURPOSE. "Does the live line have // this content TODAY" is literally the user's question ("is my fix in?"), and it is only answerable // against live HEAD; anchoring the absence test at the merge base would compare the branch to a tree // neither side has cared about for weeks and would call every landed fix "stray". A HEAD-anchored // comparison is correct here for the same reason it is wrong for scope: the question is about now. // --whereis has NO diff and therefore no anchor at all. It is a TREE SCAN of every ref (plus the gitoracle // history lane), which is what makes it able to find content a branch merely INHERITED — precisely // what a base-anchored diff would exclude. Nothing here can fire "because HEAD moved"; the only // HEAD-relative fact it reports is on-head=, which is a statement about HEAD by construction. // // Read-only, always: `git cat-file`, `git diff --raw`, `git merge-base` and `git for-each-ref` only. This // module never checks out, never writes a ref, never touches the working tree. Determinism: refs are sorted // by name, files by path, every hash is FNV-1a over trimmed bytes, and no wall clock is ever consulted (ref // dates come from git's committer clock) — a fixed repo state gives byte-identical output. #include "model.h" #include "quality.h" // gitOneLine / gitHeadSha / gitRepoHasHistory / popenTrimmed #include "pathguard.h" // readRegularFileNoFollow — the worktree overlay reads a changed path regular-file-only, no-follow #include "gitstamp.h" // stampAt — the at="[+dirty]" anchor (M10: --stray-content had head= with no dirty bit) #include "gitoracle.h" // the SHARED name-history oracle — the deleted-from-every-tree lane for whereis #include "arch.h" // fnv1a64 #include "infra/hashutil.h" // fnv1aMultiply — the sanitizer-safe wrapping multiply (G1 runs -fsanitize=integer) #include "infra/jsonesc.h" // shSingleQuote #include "serialize.h" // escapeXml #include "pageview.h" // §P8: pageWindow / pageDisclosure — the shared --limit/--offset contract #include "nextverb.h" // nextAttrXml / nextFlag — the follow-up of the default listing #include "compactlegend.h" // compactDeliveredBytesOrWritten — the default listing is chosen on the bytes the compact dialect delivers #include "workspace.h" // wsdetail::segmentsOf #include "filter.h" // §P11.5: rw::pathTierOf — the shared source/test/doc ORDERING tier #include "infra/Diagnostics.h" // ASSUME / DISCLOSE #include "infra/sortutil.h" // svLess — the rename probe orders names in byte order, never via string_view operator< #include "ingest.h" // definitionLinesInBlobs — the tags path over another ref's blob (labelBranchRowsByParse) #include "btree.hpp" // gtl::btree_map — sorted iteration (house rule: never std::map) #include #include // the git-spawn pool's work index #include #include // enumerateRefs: the committer date's unix seconds (the ref window's order) #include #include #include // the worktree overlay: symlink_status / read_symlink / file_size on a changed path #include // blobsUnreadCause — the one refusal sentence #include // streamBlobs: a stream with no piece reader keeps the whole-or-skip rule #include "infra/pieceread.h" // LARGE-PARTIAL-1: a lexical reader reads a blob of any size, in line-aligned pieces #include "limitwarn.h" // LARGE-PARTIAL-1: a ref window or parse guard that cuts says so on stderr too #include #include // parseCatFileHeader: the size field never overflows #include // labelBranchRowsByParse holds quality::headSnapshotIngestMutex around the blob parse #include #include #include #include #include // the git-spawn pool (fork/exec is the cost, not compute) #include "infra/os.h" // rw::os::getpid / unlink / popen — the blob-batch temp list and its git reader #include #include namespace rw { namespace crossref { // ── tuning constants (every threshold the verdict rests on, in one place) ──────────────────────────────── constexpr std::size_t kShingleTokens = 5; // tokens per shingle for the minhash sketch constexpr std::size_t kSketchSize = 128; // bottom-k minhash sketch width (fixed per blob) constexpr std::size_t kMaxBlobBytes = 2u << 20; // 2 MB — above this a blob is data, not content worth diffing constexpr std::size_t kBinaryProbe = 8192; // NUL within this prefix ⇒ binary, skipped constexpr float kRedoDelMin = 0.60f; // deletion-site overlap that marks a file superseded constexpr float kSimSupersede = 0.90f; // minhash containment bar for the PURE-ADDITION lane constexpr std::uint32_t kSimMinStray = 8; // …and its minimum stray size (tiny bodies match by luck) constexpr float kRefSupersedeMin = 0.60f; // stray-line-weighted share that marks a whole ref superseded // The REF WINDOW: a sweep reads at most kMaxRefs refs (HEAD included, for --whereis). Past it the NEWEST by committer date // are read and the rest counted on the answer's root (refs_omitted=), never refused and never dropped unsaid (windowNewestRefs). // Measured (CROSSREF-MAXREFS, synthetic one-commit-per-branch repos and a 524-branch, 5,239-file worktree): --stray-content // pays ~2 git spawns per ref (77 s at 511 refs, 278 s at 2,048 on an M2, peak flat at 13-19 MB, ~240 B of answer per ref); // --whereis holds every ref's tree listing at once (~0.1 MB per ref on a 200-file tree, 849 MB at 8,192 refs; 0.7-1.0 GB at // 524 refs of a 5,239-file tree). Linear, but not small on a large repo, so the window stays and only its refusal went. // // REF-WINDOW-COST (LARGE-PARTIAL-1): the per-ref git spawns are gone (one rev-list walk for the merge bases, batched // `diff-tree --stdin`), so the window bounds WORK now, sized by measurement on an M2 under load: --stray-content 4.3 s and // --abi 2.3 s at 602 branches of a one-file repo (86 s and 125 s before), 8.0 s and 5.4 s at 602 branches of cockroach // (20,517 files). 1,024 refs of cockroach would pass 10 s, so 512 stays. --whereis is bounded by SITES instead (one per // tree entry per ref, kWhereisSiteBudget): it holds every listed site at once, ~4.5 MB per cockroach ref (2.8 GB and 15.5 s // at 512 refs). --detail lifts both, and every cut prints one stderr warning naming the limit (rw::warnLimitReached). constexpr std::uint32_t kMaxRefs = 512; // ref window — past it the newest refs are read, refs_omitted= counts the rest constexpr std::size_t kWhereisSiteBudget = std::size_t( 2 ) << 20; // whereis: refs × HEAD tree entries it reads at most constexpr std::size_t kWhereisMinRefs = 16; // …but never fewer refs than this (HEAD included), however large the tree // Display caps (--detail lifts both): the ranked head is the answer, and a 30-branch sweep would otherwise // bury it under thousands of rows. What is dropped is always COUNTED in a element, never silently. constexpr std::size_t kStrayFilesPerRef = 12; constexpr std::size_t kWhereisHits = 60; // --whereis: a row's t= is a PREVIEW of a LONG source line: a line over kWhereisTextWholeMaxBytes bytes prints its first // bytes, cut on a UTF-8 boundary, plus an ellipsis (truncateUtf8WithEllipsis) — kWhereisTextPreviewCapBytes at most, the // ellipsis included — and the row says the whole line's size (t_bytes=); any shorter line prints whole. Picked from the data: // over 978 t= values of the 26c command sweep (9 repos, every whereis listing) the median line is 54 B and the 95th // percentile 108 B, so the cap binds only on the tail; on ripwire's own --whereis=lego 38 of 60 rows were 206 B - 7.3 KB // (a Python dict, markdown paragraphs, a legend literal) and held 121 KB of the 129 KB answer. The 100-byte gap between the // two (the MIN DROP) keeps a cut worth its disclosure: a 202-byte line cut to 200 saved 2 bytes and its page paid ~330 B of t_bytes=, and their // readings (ripwire's --whereis=ripwire, 16,084 -> 16,415 B), so a cut always drops at least 100 bytes of the line. Even so a // page whose only long lines sit just past the threshold can pay more for its disclosure than its cuts save (review C2: one // 395-byte line, +158 B), so a page prints its previews only when that page is SHORTER than the same page whole // (writeWhereisPage measures both). --whereis-full-text (MCP full_text) prints every line whole and lifts nothing else: the // element names that replay of the same page; --detail lifts it with the row cap and the parse guard. constexpr std::size_t kWhereisTextPreviewCapBytes = 200; // the printed preview, the ellipsis included constexpr std::size_t kWhereisTextWholeMaxBytes = 300; // a line up to this prints whole, so a cut drops 100+ bytes constexpr std::size_t kUtf8EllipsisBytes = 3; // U+2026, as truncateUtf8WithEllipsis appends it static_assert( kWhereisTextPreviewCapBytes > kUtf8EllipsisBytes && kWhereisTextWholeMaxBytes > kWhereisTextPreviewCapBytes, "the preview leaves room for its ellipsis and is shorter than every line it previews" ); // --whereis: the RUNAWAY GUARD on the tags parse of other refs' blobs (labelBranchRowsByParse; --detail=N lifts it). Only a // blob holding a parse-worthy row (parseWorthyLine) is parsed, one parse per distinct (blob, path): ripwire's own most-called helper // across 162 refs needs 104 parses and 43 MB (~1.7 s at load 20 on an M2 Pro); a typical name, a handful. Hitting // either bound is disclosed on the answer (), and the blobs it leaves confirm no definition. constexpr std::size_t kWhereisParseMaxBlobs = 256; constexpr std::size_t kWhereisParseMaxBytes = std::size_t( 64 ) << 20; // WHICH ROWS a --whereis answer lists (--whereis-listing=, MCP `listing`). A where-defined question is answered by // the kind="def" rows; the kind="ref" rows (8-59 per answer on the comparison-table cells, ~8 KB) supplied no gold // item there. So the default lists the definitions and COUNTS the references in one element whose // next= lists exactly those rows (Refs). All is the whole hit list, the pre-listing answer and the uncapped variant // a lean answer is graded against (owner rule: a capped variant is graded against the uncapped one). // Defs every kind="def" row; the kind="ref" rows counted. Degrades to All (no listing=, no ) when the hit // list holds no ref row (nothing to elide) or no def row (the mentions ARE the answer then). // Refs the kind="ref" rows only, in the hit list's order: the page. // All every row. // ShorterOfDefsAll the DEFAULT (no flag, no MCP `listing`): complete answers first. The Defs page when it SHOWS MORE // definitions than the All page under the shared row cap (a capped All page can list fewer: HEAD's references fill // the cap before the branch definitions arrive; review D1), whatever its bytes; when both pages show the same // definitions, the Defs page only when it is STRICTLY shorter in bytes (review B1: on a symbol with one or two // refs the Defs page pays more for its element, listing= and their legend readings than the ref rows it // elides), else the All page. Measured, never guessed: whereisServedListing renders both. An explicit defs is // served as asked. enum class WhereisListing : std::uint8_t { Defs, Refs, All, ShorterOfDefsAll }; // ── blob facts (the per-sha memo payload) ──────────────────────────────────────────────────────────────── // Everything this module needs to know about one blob, reduced from its bytes as they stream past and kept // instead of them. `lineHashes` is SORTED so the multiset ops below are linear merges. struct BlobFacts { std::vector lineHashes; // FNV-1a of each trimmed, non-blank line — sorted, DUPLICATES KEPT std::vector sketch; // bottom-k minhash of kShingleTokens-token shingles — sorted, unique std::uint32_t lineCount = 0; bool isText = true; // false ⇒ binary/oversized: facts are empty, rows say so }; // ── content-preserving normalization (deliberately NOT clones.h's) ─────────────────────────────────────── // clones.h maps every identifier to `$I` so two DIFFERENT functions with the same shape collide — exactly // right for clone detection and exactly wrong here, where the identifiers ARE the signal that two spellings // of a fix are the same fix. So: identifiers kept verbatim, numbers → $N and strings → $S (a re-implementation // that changes a literal is still the same work), comments dropped, punctuation kept. inline bool isIdentByte( unsigned char c ) noexcept { return std::isalnum( c ) || c == '_'; } // Append the next normalized token starting at src[i] to `out`, returning the index just past it. A comment // or whitespace run emits nothing (out unchanged) — the caller loops until the end of the buffer. inline std::size_t nextNormToken( std::string_view src, std::size_t i, std::vector& out ) { static constexpr std::string_view kNum = "$N", kStr = "$S"; const std::size_t n = src.size(); if( std::isspace( (unsigned char)src[i] ) ) { return i + 1; } if( src[i] == '/' && i + 1 < n && src[i + 1] == '/' ) { i += 2; while( i < n && src[i] != '\n' ) { ++i; } return i; } if( src[i] == '/' && i + 1 < n && src[i + 1] == '*' ) { i += 2; while( i + 1 < n && !( src[i] == '*' && src[i + 1] == '/' ) ) { ++i; } return std::min( n, i + 2 ); } if( src[i] == '"' || src[i] == '\'' ) { const char q = src[i]; std::size_t j = i + 1; while( j < n && src[j] != q ) { if( src[j] == '\\' ) { ++j; } ++j; } out.push_back( kStr ); return std::min( n, j + 1 ); } if( std::isdigit( (unsigned char)src[i] ) ) { std::size_t j = i; while( j < n && ( isIdentByte( (unsigned char)src[j] ) || src[j] == '.' ) ) { ++j; } out.push_back( kNum ); return j; } if( std::isalpha( (unsigned char)src[i] ) || src[i] == '_' ) { std::size_t j = i; while( j < n && isIdentByte( (unsigned char)src[j] ) ) { ++j; } out.push_back( src.substr( i, j - i ) ); // identifier KEPT — the semantic fingerprint return j; } out.push_back( src.substr( i, 1 ) ); // operator / punctuation return i + 1; } inline std::vector normalizeTokens( std::string_view src ) { std::vector out; out.reserve( src.size() / 4 + 1 ); for( std::size_t i = 0; i < src.size(); ) { i = nextNormToken( src, i, out ); } return out; } // Bottom-k minhash over kShingleTokens-token shingles of the normalized stream. Bottom-k (the k smallest // distinct shingle hashes) is a fixed-width, order-free, deterministic sketch: containment between two // sketches estimates containment between the full shingle sets, and two identical bodies always produce the // identical sketch. A body shorter than one shingle degrades to its bare token hashes (still comparable). inline std::vector minhashSketch( std::string_view src ) { const std::vector tok = normalizeTokens( src ); std::vector all; if( tok.size() < kShingleTokens ) { all.reserve( tok.size() ); for( std::string_view t : tok ) { all.push_back( fnv1a64( t ) ); } } else { all.reserve( tok.size() - kShingleTokens + 1 ); for( std::size_t i = 0; i + kShingleTokens <= tok.size(); ++i ) { // FNV-1a over the shingle's per-token hashes. The multiply MUST wrap — that is the algorithm — // so it goes through hashutil::fnv1aMultiply (a widened multiply masked back to 64 bits) rather // than a bare `*`: the G1 stack runs -fsanitize=integer -fno-sanitize-recover=all, under which a // plain unsigned overflow aborts the process. (Caught by that gate, not by inspection.) std::uint64_t h = hashutil::kFnv1aOffset64; for( std::size_t k = 0; k < kShingleTokens; ++k ) { h ^= fnv1a64( tok[ i + k ] ); h = hashutil::fnv1aMultiply( h ); } all.push_back( h ); } } sortutil::sortUnique( all ); if( all.size() > kSketchSize ) { all.resize( kSketchSize ); // bottom-k } return all; } // ── sorted-sequence set algebra (shared by the sketch lane and the line-multiset lane) ────────────────── // |a ∩ b| over two SORTED sequences. Duplicates are consumed pairwise, so this is a MULTISET intersection // on the line lane (where a line repeated twice on one side and once on the other must count once) and a // plain set intersection on the sketch lane (whose sketches are unique by construction). inline std::size_t sortedIntersectSize( const std::vector& a, const std::vector& b ) { // SORTEDNESS is this whole family's precondition — an unsorted input does not fail loudly, it silently // under-counts the intersection and quietly shifts a verdict. Free in release (__builtin_assume). ASSUME( std::is_sorted( a.begin(), a.end() ) ); ASSUME( std::is_sorted( b.begin(), b.end() ) ); std::size_t hit = 0, i = 0, j = 0; while( i < a.size() && j < b.size() ) { if( a[i] < b[j] ) { ++i; } else if( b[j] < a[i] ) { ++j; } else { ++hit; ++i; ++j; } } return hit; } // Containment of `a` in `b`: |a ∩ b| / |a| over two SORTED unique sketches. Containment, not Jaccard — // HEAD's file is typically far larger than a branch's added region, and Jaccard would punish that size gap // for no reason. Empty `a` ⇒ 0 (nothing to be contained; never a divide-by-zero). inline float sketchContainment( const std::vector& a, const std::vector& b ) { if( a.empty() || b.empty() ) { return 0.0f; } return float( sortedIntersectSize( a, b ) ) / float( a.size() ); } // ── line multisets ─────────────────────────────────────────────────────────────────────────────────────── // Hash every trimmed, non-blank line of `src` into a SORTED vector (duplicates kept — this is a multiset). // Blank/whitespace-only lines carry no content and would otherwise dominate every intersection. inline std::vector lineMultiset( std::string_view src ) { std::vector out; std::size_t i = 0; while( i < src.size() ) { std::size_t e = src.find( '\n', i ); if( e == std::string_view::npos ) { e = src.size(); } std::size_t a = i, b = e; while( a < b && std::isspace( (unsigned char)src[a] ) ) { ++a; } while( b > a && std::isspace( (unsigned char)src[b - 1] ) ) { --b; } if( b > a ) { out.push_back( fnv1a64( src.substr( a, b - a ) ) ); } i = e + 1; } std::sort( out.begin(), out.end() ); return out; } // Multiset difference a \ b over two SORTED multisets (an element present twice in `a` and once in `b` // survives once). The linear merge that makes "what did this side ADD / REMOVE" exact under duplicates. inline std::vector msetDiff( const std::vector& a, const std::vector& b ) { ASSUME( std::is_sorted( a.begin(), a.end() ) ); ASSUME( std::is_sorted( b.begin(), b.end() ) ); std::vector out; std::size_t i = 0, j = 0; while( i < a.size() ) { if( j >= b.size() || a[i] < b[j] ) { out.push_back( a[i++] ); } else if( b[j] < a[i] ) { ++j; } else { ++i; ++j; } } return out; } // Count members of the SORTED multiset `a` that are absent from the SORTED multiset `b` (b treated as a SET: // "does the live line have this line ANYWHERE in the file"). This is the stray-line count — a line the branch // authored that simply does not occur in HEAD's version, wherever it might have moved to. inline std::size_t countAbsent( const std::vector& a, const std::vector& b ) { std::size_t miss = 0; for( std::uint64_t h : a ) { if( !std::binary_search( b.begin(), b.end(), h ) ) { ++miss; } } return miss; } // ── the streaming blob reader (ONE `git cat-file --batch` for the whole run) ───────────────────────────── // A blob sha is 40 hex bytes (SHA-1) or 64 (SHA-256). Anything else came from a malformed diff line and is // refused before it reaches the batch, so a corrupt line can never be spliced into the command. // Byte-for-byte the same predicate as quality::isBareCommitSha — an object name is an object name, blob or // commit — so it delegates rather than keeping a second copy that could drift when SHA-256 rules change. inline bool isBlobSha( std::string_view s ) noexcept { return quality::isBareCommitSha( s ); } // An all-zero sha is git's "this side does not exist" marker in `diff --raw` (an add or a delete). inline bool isNullSha( std::string_view s ) noexcept { return !s.empty() && s.find_first_not_of( '0' ) == std::string_view::npos; } // Is `s` safe to hand to git as a REVISION argument? Same 40/64-hex shape as a blob name, but the property // being asserted is different and worth its own name: a resolved object name cannot be mistaken for an OPTION // word, so it can never become `--output=FILE`. // // That is not hypothetical. `git diff` — the verb diffRaw uses — honours `--output=FILE`, and the round that // found this bug demonstrated a file OUTSIDE the repo being truncated and overwritten at exit 0 by a sibling // verb that passed an unvalidated ref through. Every revision token here is git's OWN output today (an // objectname from for-each-ref, a sha from merge-base), so the exposure is latent rather than live — but // "the input happens to be trustworthy" is an invariant nothing was enforcing, and it is one careless caller // from being false. Enforce it at the seam instead, where it costs a comparison. // // NOTE for anyone hardening a sibling: the resolve/validate IS the defense. A `--` placed BEFORE the revision // makes it a pathspec and breaks the command; the separator belongs at the END, after the revisions. inline bool isRevisionToken( std::string_view s ) noexcept { return isBlobSha( s ); } // Stream every sha in `shas` through one `git cat-file --batch`, invoking `onBlob( sha, bytes )` per blob in // request order. Oversized/binary blobs invoke the callback with an EMPTY view and `isText=false` so the // caller can record the honest "not diffable" fact rather than silently omitting the path. // // The sha list goes in through a TEMP FILE rather than the command line: 30 refs' worth of changed blobs is // tens of thousands of shas, far past ARG_MAX, and popen is read-only so stdin cannot be fed directly. The // temp file lives under the same hardened cache dir the rest of the tool uses and is unlinked on every exit // path (including the early returns below). // // T1 (completeness claims): `stats`, when supplied, records the degrade shapes this streamer previously // swallowed — a caller that wants to CLAIM its scan was exhaustive (whereis' complete=) needs to know // whether any blob went unread. `binary` is counted separately from the failure shapes on purpose: a // binary blob cannot carry a text symbol, so it does not defeat a text-scoped claim, while an OVERSIZED // text blob genuinely could and must. struct StreamBlobStats { std::uint32_t missing = 0; // " missing" / malformed header — the object could not be read std::vector missingShas; // those objects' names, in the order asked: a partial clone's blobs (git will not download them) std::uint32_t nonBlob = 0; // wrong object type or negative size — never scanned std::uint32_t oversized = 0; // > kMaxBlobBytes, consumed but DISCARDED unread (a stream with no piece reader) std::uint32_t streamed = 0; // > kMaxBlobBytes and READ, in pieces, by the caller's piece reader (LARGE-PARTIAL-1) std::uint32_t binary = 0; // NUL in the probe window — outside a text-scoped claim, not a failure bool endedEarly = false; // the batch pipe died before serving every sha bool startFailed = false; // the batch never started (list file / popen failure) // The DISCLOSE sink for the batch's own failures: both flags withhold every completeness claim built on the stream. enum class DisclosureWhy : std::uint8_t { ListUnwritable, // the blob-batch list could not be written BatchNotStarted, // git cat-file --batch did not start StreamEndedMidBlob, // the pipe died part-way through a blob }; void disclose( DisclosureWhy why ) noexcept { switch( why ) { case DisclosureWhy::ListUnwritable: case DisclosureWhy::BatchNotStarted: startFailed = true; break; case DisclosureWhy::StreamEndedMidBlob: endedEarly = true; break; } } bool exhaustiveOverText() const noexcept { return !startFailed && !endedEarly && missing == 0 && nonBlob == 0 && oversized == 0; } }; // LARGE-PARTIAL-1: a LEXICAL reader (whereis' scan, stray-content's line and shingle facts) passes `onPiece`, and a text blob // over kMaxBlobBytes is then READ, in line-aligned pieces of at most rw::kPieceBytes (readInPieces), instead of being skipped: // onPiece( sha, piece ) runs once per piece and onBlob is NOT called for that blob, unless it turns out binary or the stream // ends inside it (then onBlob( sha, {}, false ) as before). A stream without one (NoPieces: --abi, which parses whole files) // keeps the old rule: over the cap is consumed unread and counted in `oversized`. struct NoPieces { }; // One payload of `size` bytes off a cat-file stream, consumed unread (a blob over the cap on a stream with no piece reader): // the bytes consumed, short when the stream ended inside it. inline std::size_t discardPayload( std::FILE* pipe, std::size_t size ) { char sink[ 65536 ]; std::size_t got = 0; while( got < size ) { const std::size_t n = std::fread( sink, 1, std::min( size - got, sizeof( sink ) ), pipe ); if( n == 0 ) { break; } got += n; } return got; } // One payload of `size` bytes off a cat-file stream, read whole into `body`: the bytes read. inline std::size_t readPayload( std::FILE* pipe, std::size_t size, std::vector& body ) { body.resize( size ); return size > 0 ? std::fread( body.data(), 1, size, pipe ) : 0; } // One blob over kMaxBlobBytes read in pieces (readInPieces) and its framing LF consumed. False when the stream ended inside it // (disclosed; the caller stops the batch). A binary one is reported through onBlob as unreadable text, as a whole blob is. template inline bool streamOversizeInPieces( std::FILE* pipe, const std::string& sha, std::size_t size, std::string& window, StreamBlobStats& st, OnBlob& onBlob, OnPiece& onPiece ) { const PieceRead read = readInPieces( size, [ & ]( char* dst, std::size_t n ) { return std::fread( dst, 1, n, pipe ); }, window, [ & ]( const BlobPiece& piece ) { onPiece( sha, piece ); } ); (void)std::fgetc( pipe ); // the framing LF after the payload if( read == PieceRead::Short ) { DISCLOSE( st, StreamBlobStats::DisclosureWhy::StreamEndedMidBlob, "crossref: git cat-file stream ended mid-blob — stopping the batch rather than risk misattributing content" ); onBlob( sha, std::string_view{}, false ); return false; } if( read == PieceRead::Binary ) { ++st.binary; onBlob( sha, std::string_view{}, false ); return true; } ++st.streamed; return true; } template inline void streamBlobs( const std::string& root, const std::vector& shas, OnBlob onBlob, StreamBlobStats* stats = nullptr, OnPiece onPiece = {} ) { constexpr bool kReadsPieces = !std::is_same_v; // Null-object: count unconditionally into a local sink when the caller did not ask, so the body below // carries no per-site null test (the branch count stays what the streaming logic needs, nothing more). StreamBlobStats localStats; StreamBlobStats& st = stats != nullptr ? *stats : localStats; if( shas.empty() ) { return; } static std::atomic listSeq{ 0 }; // concurrent batches (streamBlobsInSlices) each own their list file const std::string listPath = quality::cacheDirLadder() + "/ripwire-crossref-" + std::to_string( os::getpid() ) + "-" + std::to_string( listSeq.fetch_add( 1, std::memory_order_relaxed ) ) + ".shas"; { std::FILE* lf = std::fopen( listPath.c_str(), "wb" ); if( !lf ) { DISCLOSE( st, StreamBlobStats::DisclosureWhy::ListUnwritable, "crossref: cannot write the blob-batch list — cross-branch content unavailable" ); return; } for( const std::string& s : shas ) { rw::emitTo( lf, "{}\n", s.c_str() ); } std::fclose( lf ); } const std::string cmd = gitCmd( " -c core.quotepath=false -C " ) + shSingleQuote( root ) + " cat-file --batch < " + shSingleQuote( listPath ) + " 2>/dev/null"; std::FILE* pipe = os::popen( cmd.c_str(), "r" ); if( !pipe ) { os::unlink( listPath.c_str() ); DISCLOSE( st, StreamBlobStats::DisclosureWhy::BatchNotStarted, "crossref: git cat-file --batch failed to start — cross-branch content unavailable" ); return; } PROFILE_SCOPE_DESCRIBE( "crossref: git cat-file --batch stream (every blob, callback included)" ); std::vector body; std::string header; std::string window; // readInPieces' window: one per stream, whatever the blob's size for( std::size_t served = 0; served < shas.size(); ++served ) { // header: " \n", or " missing\n" for an unreadable object header.clear(); for( int c = std::fgetc( pipe ); c != EOF && c != '\n'; c = std::fgetc( pipe ) ) { header.push_back( char( c ) ); } if( header.empty() ) { st.endedEarly = true; break; // pipe ended early — degrade quietly } const std::size_t sp2 = header.rfind( ' ' ); const std::size_t sp1 = ( sp2 == std::string::npos ) ? std::string::npos : header.rfind( ' ', sp2 - 1 ); if( sp1 == std::string::npos ) { ++st.missing; st.missingShas.push_back( shas[ served ] ); onBlob( shas[ served ], std::string_view{}, false ); continue; } const std::string_view type( header.data() + sp1 + 1, sp2 - sp1 - 1 ); const long long size = std::strtoll( header.c_str() + sp2 + 1, nullptr, 10 ); if( type != "blob" || size < 0 ) { ++st.nonBlob; onBlob( shas[ served ], std::string_view{}, false ); continue; } // The payload must be CONSUMED whatever its size — the stream is framed, so skipping bytes would // desync every subsequent header and start attributing content to the wrong sha. But an oversized // blob is DISCARDED as it is consumed rather than buffered: a repo can hold a 500 MB object, and // resizing to it just to throw it away would spike RSS by that much for a blob we never look at. const bool tooBig = std::size_t( size ) > kMaxBlobBytes; if constexpr( kReadsPieces ) { if( tooBig ) { if( !streamOversizeInPieces( pipe, shas[ served ], std::size_t( size ), window, st, onBlob, onPiece ) ) { break; } continue; } } const std::size_t got = tooBig ? discardPayload( pipe, std::size_t( size ) ) : readPayload( pipe, std::size_t( size ), body ); (void)std::fgetc( pipe ); // the framing LF after the payload // A SHORT read means the pipe died mid-payload (git killed, disk error). The stream can no longer be // trusted to be framed, and continuing would pair later headers with the wrong sha — a silently // WRONG answer, which is worse than a short one. Report this blob as unreadable and stop. if( got != std::size_t( size ) ) { DISCLOSE( st, StreamBlobStats::DisclosureWhy::StreamEndedMidBlob, "crossref: git cat-file stream ended mid-blob — stopping the batch rather than risk misattributing content" ); onBlob( shas[ served ], std::string_view{}, false ); break; } const std::string_view bytes( body.data(), tooBig ? 0 : got ); const bool binary = bytes.substr( 0, std::min( bytes.size(), kBinaryProbe ) ).find( '\0' ) != std::string_view::npos; if( tooBig || binary ) { if( tooBig ) { ++st.oversized; } else { ++st.binary; } onBlob( shas[served], std::string_view {}, false ); } else { onBlob( shas[served], bytes, true ); } } os::pclose( pipe ); os::unlink( listPath.c_str() ); } // The per-sha memo. `want()` registers a sha; `fill()` streams every not-yet-known sha through ONE batch and // reduces it to BlobFacts. Calling fill() again after more want()s only fetches the NEW shas — the whole // point of content-addressing: a blob shared by 25 refs is read and reduced exactly once. class BlobStore { public: explicit BlobStore( std::string root ) : root_( std::move( root ) ) {} void want( const std::string& sha ) { if( sha.empty() || isNullSha( sha ) || !isBlobSha( sha ) ) { return; } if( facts_.find( sha ) != facts_.end() ) { return; } pending_.push_back( sha ); } void fill() { if( pending_.empty() ) { return; } sortutil::sortUnique( pending_ ); // determinism: batch order is sha order std::vector todo; todo.reserve( pending_.size() ); for( const std::string& s : pending_ ) { if( facts_.find( s ) == facts_.end() ) { todo.push_back( s ); } } pending_.clear(); streamBlobs( root_, todo, [ & ]( const std::string& sha, std::string_view bytes, bool isText ) { pieceFacts_ = BlobFacts{}; // a blob whose pieces stopped short (or turned out binary) keeps none of them BlobFacts f; f.isText = isText; if( isText ) { f.lineHashes = lineMultiset( bytes ); f.sketch = minhashSketch( bytes ); f.lineCount = std::uint32_t( f.lineHashes.size() ); } facts_.emplace( sha, std::move( f ) ); ++fetched_; }, &stats_, [ & ]( const std::string& sha, const BlobPiece& piece ) { // LARGE-PARTIAL-1: a blob over kMaxBlobBytes, read in pieces (it used to be "not diffable", its lines in no count). // The line multiset is exact over whole lines; a line longer than half a piece is hashed as its pieces (the carried // overlap skipped). The sketch is the bottom-k of the union of the pieces' shingles (a set, so the overlap's repeats // are harmless); only the few shingles that span a newline-aligned cut are not formed. const std::vector lines = lineMultiset( piece.bytes.substr( piece.carried ) ); const std::vector sketch = minhashSketch( piece.bytes ); pieceFacts_.lineHashes.insert( pieceFacts_.lineHashes.end(), lines.begin(), lines.end() ); pieceFacts_.sketch.insert( pieceFacts_.sketch.end(), sketch.begin(), sketch.end() ); sortutil::sortUnique( pieceFacts_.sketch ); if( pieceFacts_.sketch.size() > kSketchSize ) { pieceFacts_.sketch.resize( kSketchSize ); } if( piece.last ) { std::sort( pieceFacts_.lineHashes.begin(), pieceFacts_.lineHashes.end() ); pieceFacts_.lineCount = std::uint32_t( pieceFacts_.lineHashes.size() ); facts_.emplace( sha, std::move( pieceFacts_ ) ); pieceFacts_ = BlobFacts{}; ++fetched_; } } ); missing_.insert( stats_.missingShas.begin(), stats_.missingShas.end() ); } // An object git could not read (a partial clone's blob: ripwire never downloads one, gitcmd.h NEVER DOWNLOAD). Its facts are the // shared empty record, which is "this side contributed no lines" - a measurement it is not, so a verdict that rests on it is Unknown. bool absent( const std::string& sha ) const noexcept { return missing_.contains( sha ); } // K75: a batch that never started (the sha list could not be written under the cache dir, git did not spawn) or died // before serving every blob leaves the shas it never reached ABSENT from facts_, and get() answers an absent sha with a // shared empty record — "this side contributed no lines". A caller that then reports verdicts from it prints a measured- // looking answer made of nothing (a branch with unmerged work reading merged), so it asks this first and refuses. bool unread() const noexcept { return stats_.startFailed || stats_.endedEarly; } // The facts for `sha`, or a shared empty (non-text) record for an absent side — so a caller can treat // "this path did not exist on that side" as "it contributed no lines" without a null check. const BlobFacts& get( const std::string& sha ) const { static const BlobFacts kEmpty{}; const auto it = facts_.find( sha ); return ( it == facts_.end() ) ? kEmpty : it->second; } std::size_t distinctBlobs() const noexcept { return facts_.size(); } std::size_t fetched() const noexcept { return fetched_; } private: std::string root_; gtl::btree_map facts_; gtl::btree_set missing_; // the shas git could not read (stats_.missingShas, deduplicated) std::vector pending_; std::size_t fetched_ = 0; StreamBlobStats stats_; BlobFacts pieceFacts_; // the blob being read in pieces (fill's piece reader), until its last piece }; // K75: the words every host of a blob-reading verb refuses in when the one `git cat-file --batch` behind it did not // serve every blob. The cause is named WHERE the reader looks (stderr + exit 1, or an MCP error), never as a zero. // ONE renderer for both blob readers (the prconverge.h one-renderer rule): `batch` names which reader's blobs were not // read — the cross-branch sweep's (--stray-content, --plan, --abi, --eval-stray) or --whereis's — and every other word is // shared, so the CLI and MCP hosts of both verbs name the same cause, the cache directory and TMPDIR. enum class UnreadBlobBatch : std::uint8_t { Branches, // the cross-branch sweep's batch: every ref's side of every path it compares Trees, // --whereis's batch: the trees it searches for the name }; inline std::string blobsUnreadCause( UnreadBlobBatch batch ) { const std::string_view unread = batch == UnreadBlobBatch::Trees ? "the trees' blobs" : "the branches' blobs"; return std::format( "could not read {} (the one `git cat-file --batch` did not start or stopped early; its sha list is written under {} - is TMPDIR " "a writable directory?)", unread, quality::cacheDirLadder() ); } // ── git plumbing (read-only) ───────────────────────────────────────────────────────────────────────────── // Capture a git command's FULL multi-line stdout ("" on failure). This is exactly quality::gitOneLine's // read-only `git -C ` shape (popenTrimmed keeps internal newlines, trimming only trailing // whitespace), so it delegates rather than re-spelling the same hardened prefix and popen-and-trim call. inline std::string gitCapture( const std::string& root, const std::string& tail ) { return quality::gitOneLine( root, tail ); } struct GitListing { std::vector lines; bool ok = false; }; inline GitListing gitListChecked( const std::string& root, const std::string& tail ) { GitCommandLines res = gitCommandLines( gitCmd( " -c core.quotepath=false -C " ) + shSingleQuote( root ) + " " + tail + " 2>/dev/null" ); return GitListing{ std::move( res.lines ), res.isStarted && WIFEXITED( res.status ) && WEXITSTATUS( res.status ) == 0 }; } inline std::vector splitLines( std::string_view s ) { return wsdetail::segmentsOf( s, '\n' ); } // One ref in the sweep: its name, tip sha, and committer date (git's clock — never the wall clock). struct RefInfo { std::string name; std::string tip; std::string date; // YYYY-MM-DD std::int64_t when = 0; // committer date in unix seconds: the ref window's order (windowNewestRefs), never printed }; // Every local branch plus every worktree branch, sorted by name (determinism), excluding the ref HEAD is // currently on (a branch cannot be stray from itself). `filter` (from --stray-content=SUBSTR) keeps only // refs whose name contains it. Remote-tracking refs are deliberately EXCLUDED: they mirror local ones and // would double every row. // `filterNameHits` (optional out): how many refs/heads names CONTAIN the filter, counted before the // "a branch cannot be stray from itself" exclusion of HEAD's own ref. H7 needs that number and not // out.size(): a filter matching only the checked-out branch has SELECTED something (the answer is "nothing // but the ref you are on"), while a filter matching no branch name at all has selected nothing and must // refuse rather than report refs="0" — which reads as "no branch carries stray work". // // `enumeration` (optional out) is the DISCLOSE sink for a ref DROPPED here (a tip that is not an object name): it is // in no count the caller prints, so a caller that claims completeness must read refsDropped — --whereis withholds // complete= on it. A caller passing none has no such claim to withhold. It also records a ref KEPT with an unreadable // committer date (a commit object whose committer line has none — git prints an empty field): the ref window ranks it as // the oldest, so past kMaxRefs it is the first left out, and refs_omitted= counts it like any other. struct RefEnumeration { enum class DisclosureWhy : std::uint8_t { TipNotObjectName, DateUnreadable, }; std::uint32_t refsDropped = 0; std::uint32_t datesUnread = 0; void disclose( DisclosureWhy why ) noexcept { ++( why == DisclosureWhy::DateUnreadable ? datesUnread : refsDropped ); } }; // One for-each-ref line, split from the RIGHT, not left-to-right: '|' is a LEGAL byte in a git ref name (git // forbids space, ~ ^ : ? * [ \ and control bytes — not the pipe), so a branch named `feat|x` split // into four fields and handed `x` back as the tip sha. That ref then failed merge-base and landed in // the degraded path, i.e. a legally-named branch silently became unanalysable. The three TRAILING // fields are git-generated and pipe-free by construction (an object name is hex, a short committer // date is YYYY-MM-DD, a unix committer date is decimal), so everything before the third-to-last '|' is the name. struct RefLineFields { std::string_view name; std::string_view tip; std::string_view date; // %(committerdate:short) std::string_view unixDate; // %(committerdate:unix) }; inline std::optional splitRefLine( std::string_view line ) noexcept { const std::size_t unixPipe = line.rfind( '|' ); if( unixPipe == std::string_view::npos || unixPipe == 0 ) { return std::nullopt; } const std::size_t lastPipe = line.rfind( '|', unixPipe - 1 ); if( lastPipe == std::string_view::npos || lastPipe == 0 ) { return std::nullopt; } const std::size_t firstPipe = line.rfind( '|', lastPipe - 1 ); if( firstPipe == std::string_view::npos || firstPipe >= lastPipe ) { return std::nullopt; } return RefLineFields{ line.substr( 0, firstPipe ), line.substr( firstPipe + 1, lastPipe - firstPipe - 1 ), line.substr( lastPipe + 1, unixPipe - lastPipe - 1 ), line.substr( unixPipe + 1 ) }; } // A ref's committer time for the ref window. An unreadable %(committerdate:unix) (git prints it empty for a // commit whose committer line carries no date) sorts as the oldest: the window may omit it, and the sink counts it. inline std::int64_t refCommitTime( std::string_view unixDate, RefEnumeration& sink ) { std::int64_t when = 0; const char* const dateEnd = unixDate.data() + unixDate.size(); const std::from_chars_result parsed = std::from_chars( unixDate.data(), dateEnd, when ); if( !VALIDATE( parsed.ec == std::errc() && parsed.ptr == dateEnd ) ) { DISCLOSE( sink, RefEnumeration::DisclosureWhy::DateUnreadable, "crossref: for-each-ref yielded a ref with no readable committer date — the ref window ranks it as the oldest" ); return 0; } return when; } inline std::vector enumerateRefs( const std::string& root, std::string_view filter, const std::string& headSha, std::size_t* filterNameHits = nullptr, RefEnumeration* enumeration = nullptr ) { RefEnumeration unread; // the sink when the caller keeps none RefEnumeration& dropSink = enumeration != nullptr ? *enumeration : unread; const std::string raw = gitCapture( root, "for-each-ref --sort=refname --format='%(refname:short)|%(objectname)|%(committerdate:short)|%(committerdate:unix)' refs/heads 2>/dev/null" ); std::vector out; for( std::string_view line : splitLines( raw ) ) { const std::optional fields = splitRefLine( line ); if( !fields ) { continue; } const std::string_view name = fields->name; const std::string_view tip = fields->tip; const std::int64_t when = refCommitTime( fields->unixDate, dropSink ); if( filterNameHits != nullptr && !name.empty() && ( filter.empty() || name.find( filter ) != std::string_view::npos ) ) { ++*filterNameHits; } if( name.empty() || tip == headSha ) { continue; // the ref HEAD is on } if( !isRevisionToken( tip ) ) { // %(objectname) is always a full object name, so this cannot fire on well-formed output — which // is exactly why it is checked HERE, at the one place ref tips enter the module. Every git // command downstream takes this value as a revision argument. DISCLOSE( dropSink, RefEnumeration::DisclosureWhy::TipNotObjectName, "crossref: for-each-ref yielded a ref whose tip is not an object name — skipping it" ); continue; } if( !filter.empty() && name.find( filter ) == std::string_view::npos ) { continue; } out.push_back( RefInfo{ std::string( name ), std::string( tip ), std::string( fields->date ), when } ); } return out; } // The ref window (kMaxRefs): keeps `keep` of `refs` and returns the names of the rest, sorted, for the root's refs_omitted=. // Kept first: the `pinned` names (sorted; --eval-stray's labels, so a labelled ref is scored, not windowed out), then the // newest by committer date, ties by name. The kept refs stay in name order, so a sweep under the window is unchanged. inline std::vector windowNewestRefs( std::vector& refs, std::size_t keep, std::span pinned = {} ) { if( refs.size() <= keep ) { return {}; } const auto newestFirst = []( const RefInfo& a, const RefInfo& b ) { return a.when != b.when ? a.when > b.when : a.name < b.name; }; const auto pinnedEnd = std::partition( refs.begin(), refs.end(), [ & ]( const RefInfo& r ) { return std::binary_search( pinned.begin(), pinned.end(), r.name ); } ); std::sort( refs.begin(), pinnedEnd, newestFirst ); std::sort( pinnedEnd, refs.end(), newestFirst ); std::vector omitted; omitted.reserve( refs.size() - keep ); for( std::size_t i = keep; i < refs.size(); ++i ) { omitted.push_back( std::move( refs[ i ].name ) ); } refs.resize( keep ); const auto byName = []( const RefInfo& a, const RefInfo& b ) { return a.name < b.name; }; std::sort( refs.begin(), refs.end(), byName ); std::sort( omitted.begin(), omitted.end() ); ENSURES( refs.size() == keep && !omitted.empty(), "the window keeps exactly `keep` refs and names every other one" ); return omitted; } // The root's refs_omitted= (present only when the window left refs out): every sweep verb spells it the same way. inline std::string refsOmittedAttr( const std::vector& omitted ) { return omitted.empty() ? std::string() : ( " refs_omitted=\"" + std::to_string( omitted.size() ) + "\"" ); } // The person's half of the same disclosure (owner 2026-11-14): one stderr line when the window cut, never in the answer. inline void warnRefWindow( std::string_view verb, std::size_t read, const std::vector& omitted, std::string_view limit ) { if( omitted.empty() ) { return; } warnLimitReached( std::format( "{} read the newest {} of {} local branches; {} older were not read (limit: {})", verb, read, read + omitted.size(), omitted.size(), limit ), "pass --detail=1 to read every branch, or narrow the branches with a ref-name filter" ); } // One `diff --raw` row: both sides' blob shas for one path. `git diff --raw` is the ONE call that yields // the changed path set AND both blob shas together — no follow-up ls-tree per file. struct RawRow { std::string path; std::string aSha; // base side ("000…" ⇒ added) std::string bSha; // ref side ("000…" ⇒ deleted) }; // Parse `git diff --raw --no-renames A B`: ": \t" // // --no-abbrev is load-bearing, not cosmetic: `diff --raw` abbreviates object names to ~8 hex by default, and // an abbreviated name is NOT a key — it cannot be handed to cat-file as a stable identity, and two blobs can // share a prefix. The full name is what makes the per-sha memo sound. // One raw row (": \t") into `out`; any other line, or a malformed row, adds nothing. inline void appendRawRow( std::string_view line, std::vector& out ) { if( line.empty() || line[0] != ':' ) { return; } const std::size_t tab = line.find( '\t' ); if( tab == std::string_view::npos ) { return; } const std::vector f = wsdetail::segmentsOf( line.substr( 1, tab - 1 ), ' ' ); if( f.size() < 4 || !isBlobSha( f[2] ) || !isBlobSha( f[3] ) ) { return; // malformed row — skip, never splice } out.push_back( RawRow{ std::string( line.substr( tab + 1 ) ), std::string( f[2] ), std::string( f[3] ) } ); } inline std::vector diffRaw( const std::string& root, const std::string& a, const std::string& b ) { // Both sides must be RESOLVED object names before they reach `git diff`, which honours --output=FILE. // Degrade to an empty diff rather than spawn a command whose arguments were never checked. if( !isRevisionToken( a ) || !isRevisionToken( b ) ) { DISCLOSE( "crossref: refusing a diff whose revision arguments are not resolved object names" ); return {}; } const std::string raw = gitCapture( root, "diff --no-ext-diff --no-textconv --ignore-submodules=dirty --raw --no-abbrev --no-renames " + shSingleQuote( a ) + " " + shSingleQuote( b ) + " -- 2>/dev/null" ); std::vector out; for( std::string_view line : splitLines( raw ) ) { appendRawRow( line, out ); } std::sort( out.begin(), out.end(), []( const RawRow& x, const RawRow& y ) { return x.path < y.path; } ); return out; } // ── the git-spawn pool, and the distinct-diff table it feeds ───────────────────────────────────────────── // // Each `git` invocation here is a PROCESS (~8 ms on the measured repo), and a 35-ref sweep asked for 128 of // them strictly in sequence — ~1.1 s of pure fork/exec on a 1.65 s wall. Every one of those calls is // read-only and independent of the others, so the fix is two-sided and needs no new git knowledge at all: // ask for FEWER (skip the provably-empty ones, and never ask the same question twice), and ask in PARALLEL. constexpr std::size_t kMaxGitWorkers = 12; // matches the ingest pool's measured ~12-way; these are // processes, so more workers stops paying well before this constexpr std::uint32_t kNoPair = 0xFFFFFFFFu; // "this ref needs no diff" (see RefPlumbing::isAncestor) constexpr std::size_t kDiffTreeSlices = 4; // `git diff-tree --stdin` processes one DiffPairTable::run shares its pairs across // Run `body( i )` for every i in [0,count), across a small pool. DETERMINISM: every body writes only to the // slot its OWN index owns and reads nothing another body writes, so the result is identical to the serial // order by construction — the parallelism is in the fork/exec wait, never in the answer. // // A worker that throws abandons the indices it had not reached, and a slot nobody wrote holds its DEFAULT — which for a // RefPlumbing reads ok=true, base="" and rendered as Merged. So the pool reports which indices finished through // `sweep`, the DISCLOSE sink of its one degrade: each caller marks every unfinished slot as a failed analysis // (ok="0" v="unknown" in the document) instead of reading a default as an answer. struct ParallelSweep { enum class DisclosureWhy : std::uint8_t { WorkerThrew, // a worker's loop threw: the indices it had not reached were never run }; std::vector done; // done[i] = 1 once body( i ) returned; one writer per slot std::atomic isIncomplete{ false }; void disclose( DisclosureWhy ) noexcept // every reason records the same fact { isIncomplete.store( true, std::memory_order_relaxed ); } bool isDone( std::size_t i ) const noexcept { return !isIncomplete.load( std::memory_order_relaxed ) || ( i < done.size() && done[ i ] != 0 ); } }; template inline void parallelIndexed( std::size_t count, Body body, ParallelSweep& sweep ) { sweep.done.assign( count, 0 ); if( count == 0 ) { return; } std::size_t hwThreadCount = std::thread::hardware_concurrency(); if( hwThreadCount == 0 ) { hwThreadCount = 1; } const std::size_t workerCount = std::min( { hwThreadCount, count, kMaxGitWorkers } ); if( workerCount <= 1 ) { for( std::size_t i = 0; i < count; ++i ) { body( i ); sweep.done[ i ] = 1; } return; } std::atomic nextIndex{ 0 }; const auto worker = [ & ]() noexcept { // A throw escaping a std::thread entry is std::terminate — degrade to partial coverage instead. Only // the allocation seam can throw here (popen/parse), and a short answer beats killing the process. try { for( std::size_t i = nextIndex.fetch_add( 1 ); i < count; i = nextIndex.fetch_add( 1 ) ) { body( i ); sweep.done[ i ] = 1; } } catch( ... ) { DISCLOSE( sweep, ParallelSweep::DisclosureWhy::WorkerThrew, "crossref: a git worker threw — this shard of the sweep is incomplete" ); } }; { // symmetric bare scope: the workers live exactly as long as the pass they serve std::vector workers; workers.reserve( workerCount ); for( std::size_t w = 0; w < workerCount; ++w ) { workers.emplace_back( worker ); } for( std::thread& worker_ : workers ) { worker_.join(); } } } // ── one blob stream through several git processes (streamBlobsInSlices) ────────────────────────────────────── // streamBlobs reads every blob through ONE `git cat-file --batch` pipe. On ripwire's 162 refs that is ~900 MB, and under // load the pipe's ping-pong, not git's decompression, was the larger cost: 1.9 s for git writing to /dev/null against // 5.9 s through `| cat` at load 28, and ~9 s of a --whereis at load 20-25 spent waiting in fread. This reads `slices` // contiguous slices of the sha list through as many processes at once (parallelIndexed's pool). `onBlob( index, sha, // bytes, isText )` runs on the slice's worker thread, once per served sha, so it may write only slot `index` of its own // storage. The stats merge as one batch's would: counts add, and a slice that failed, was cut short or was never reached // withholds every completeness claim. constexpr std::size_t kWhereisStreamSlices = 4; // `onPiece( index, sha, piece )` (optional): the piece reader streamBlobs hands a blob over kMaxBlobBytes to; a served sha // reaches exactly one of onBlob or its pieces (the last one carrying piece.last). template inline void streamBlobsInSlices( const std::string& root, const std::vector& shas, std::size_t slices, OnBlob onBlob, StreamBlobStats& stats, OnPiece onPiece = {} ) { slices = std::max( 1, std::min( slices, shas.size() ) ); std::vector sliceStats( slices ); ParallelSweep sweep; parallelIndexed( slices, [ & ]( std::size_t t ) { const std::size_t begin = shas.size() * t / slices; const std::size_t end = shas.size() * ( t + 1 ) / slices; const std::vector part( shas.begin() + std::ptrdiff_t( begin ), shas.begin() + std::ptrdiff_t( end ) ); std::size_t next = begin; // streamBlobs serves its list in order, one call per served sha const auto blobAt = [ & ]( const std::string& sha, std::string_view bytes, bool isText ) { onBlob( next++, sha, bytes, isText ); }; if constexpr( std::is_same_v ) { streamBlobs( root, part, blobAt, &sliceStats[ t ] ); } else { streamBlobs( root, part, blobAt, &sliceStats[ t ], [ & ]( const std::string& sha, const BlobPiece& piece ) { onPiece( next, sha, piece ); next += piece.last ? 1 : 0; } ); } }, sweep ); for( std::size_t t = 0; t < slices; ++t ) { const StreamBlobStats& st = sliceStats[ t ]; stats.missing += st.missing; stats.nonBlob += st.nonBlob; stats.oversized += st.oversized; stats.streamed += st.streamed; stats.binary += st.binary; stats.endedEarly = stats.endedEarly || st.endedEarly; stats.startFailed = stats.startFailed || st.startFailed; if( !sweep.isDone( t ) ) { DISCLOSE( stats, StreamBlobStats::DisclosureWhy::BatchNotStarted, "crossref: a blob-stream slice was never read — completeness is withheld" ); } } } // A pair of revisions to diff, a -> b (DiffPairTable's registered pairs). struct RevisionPair { std::string a, b; }; // Each pair's raw rows, and whether a diff answered it (DiffPairTable::run, REF-WINDOW-COST). struct DiffPairRows { std::vector> rows; // one per pair, sorted by path once answered std::vector unfinished; // 1 => no diff answered that pair (a thrown worker never reached it) }; // Pairs [begin, end) through one `git diff-tree --stdin`: each input line " " diffs a against b (diff-tree reads // the second name as the first one's parent), and --always prints the header line (the first name) even for an empty // diff, so the output is one header per input line, in order, each followed by that pair's raw rows. A pair is whole // when the process exited 0, or when a later pair's header follows it; the rest stay unfinished for diffPairsBatched's fallback. inline void diffTreeSlice( const std::string& root, std::span pairs, std::size_t begin, std::size_t end, DiffPairRows& out ) { static std::atomic listSeq{ 0 }; const std::string listPath = quality::cacheDirLadder() + "/ripwire-crossref-pairs-" + std::to_string( os::getpid() ) + "-" + std::to_string( listSeq.fetch_add( 1, std::memory_order_relaxed ) ) + ".txt"; std::vector asked; // the pairs written to the list, in order { std::FILE* lf = std::fopen( listPath.c_str(), "wb" ); if( lf == nullptr ) { return; // every pair stays unfinished: diffPairsBatched diffs each on its own } for( std::size_t i = begin; i < end; ++i ) { if( isRevisionToken( pairs[ i ].a ) && isRevisionToken( pairs[ i ].b ) ) { rw::emitTo( lf, "{} {}\n", pairs[ i ].b, pairs[ i ].a ); asked.push_back( i ); } } std::fclose( lf ); } const GitListing listing = gitListChecked( root, "diff-tree --stdin -r --raw --no-abbrev --no-renames --always < " + shSingleQuote( listPath ) ); os::unlink( listPath.c_str() ); std::size_t seen = 0; // headers read so far; rows belong to asked[ seen - 1 ] for( const std::string& line : listing.lines ) { if( line.empty() ) { continue; } if( line[ 0 ] != ':' ) { if( seen == asked.size() ) { break; // more headers than pairs: the output is not the shape asked for, so stop reading it } ++seen; continue; } if( seen > 0 ) { appendRawRow( line, out.rows[ asked[ seen - 1 ] ] ); } } const std::size_t whole = ( listing.ok && seen == asked.size() ) ? seen : ( seen > 0 ? seen - 1 : 0 ); for( std::size_t k = 0; k < whole; ++k ) { std::vector& rows = out.rows[ asked[ k ] ]; std::sort( rows.begin(), rows.end(), []( const RawRow& x, const RawRow& y ) { return x.path < y.path; } ); out.unfinished[ asked[ k ] ] = 0; } for( std::size_t k = whole; k < asked.size(); ++k ) { out.rows[ asked[ k ] ].clear(); // a partial pair's rows are discarded; diffPairsBatched diffs it again on its own } } // Every DISTINCT registered pair, diffed through kDiffTreeSlices `git diff-tree --stdin` processes (REF-WINDOW-COST, // LARGE-PARTIAL-1). One process per pair cost a spawn each, ~0.2 s under load on an M2 (the /usr/bin/git shim), so a // 512-ref sweep spent most of its 86 s starting processes; one `diff-tree --stdin` answers a whole slice. A pair the // batch did not answer whole (a slice that failed, a revision it could not read) falls back to its own diffRaw, so // the rows are the ones the per-pair path gives. inline DiffPairRows diffPairsBatched( const std::string& root, std::span pairs ) { DiffPairRows out; out.rows.assign( pairs.size(), {} ); out.unfinished.assign( pairs.size(), 1 ); const std::size_t slices = std::min( kDiffTreeSlices, pairs.size() ); ParallelSweep sliceSweep; parallelIndexed( slices, [ & ]( std::size_t t ) { diffTreeSlice( root, pairs, pairs.size() * t / slices, pairs.size() * ( t + 1 ) / slices, out ); }, sliceSweep ); std::vector rest; for( std::size_t i = 0; i < pairs.size(); ++i ) { if( out.unfinished[ i ] != 0 ) { rest.push_back( i ); } } ParallelSweep sweep; parallelIndexed( rest.size(), [ & ]( std::size_t k ) { out.rows[ rest[ k ] ] = diffRaw( root, pairs[ rest[ k ] ].a, pairs[ rest[ k ] ].b ); }, sweep ); for( std::size_t k = 0; k < rest.size(); ++k ) { out.unfinished[ rest[ k ] ] = sweep.isDone( k ) ? 0 : 1; } return out; } // The DISTINCT (a,b) object-name pairs the sweep needs diffed, and their rows. Same reasoning the BlobStore // rests on: a diff between two IMMUTABLE object names is a pure function of that pair, so asking twice can // only ever spend a process for an answer already held. It pays because the sweep genuinely repeats itself — // diff(base, HEAD) is re-derived once per ref and refs sharing a merge-base (N branches off one trunk commit) // share that pair verbatim. Measured on a 35-ref repo: 72 diffRaw calls for only 66 distinct pairs. // // Registration (`want`) is SERIAL by contract and `run()` is the only parallel part, so there is no lock // anywhere: pairs are addressed by a 32-bit index into storage that is fully sized before any worker starts, // and each worker writes exactly one slot. class DiffPairTable { public: // Register a pair, returning its stable index. Serial by contract — planning pass only. std::uint32_t want( const std::string& a, const std::string& b ) { const std::string key = a + ' ' + b; const auto it = index_.find( key ); if( it != index_.end() ) { ++reuseCount_; return it->second; } const std::uint32_t pairIndex = std::uint32_t( pairs_.size() ); pairs_.push_back( RevisionPair{ a, b } ); index_.emplace( key, pairIndex ); return pairIndex; } // Every DISTINCT registered pair, diffed in batches (diffPairsBatched, REF-WINDOW-COST LARGE-PARTIAL-1). void run( const std::string& root ) { DiffPairRows got = diffPairsBatched( root, pairs_ ); rows_ = std::move( got.rows ); unfinished_ = std::move( got.unfinished ); } // Was this pair's diff actually RUN? An empty row list is also what a diff nobody ran holds, and it reads as "no // change"; a pair a thrown worker never reached is not an answer. kNoPair (no diff needed) is finished. bool isFinished( std::uint32_t pairIndex ) const noexcept { return pairIndex == kNoPair || std::size_t( pairIndex ) >= unfinished_.size() || unfinished_[ pairIndex ] == 0; } const std::vector& rows( std::uint32_t pairIndex ) const { static const std::vector kEmpty{}; if( pairIndex == kNoPair || std::size_t( pairIndex ) >= rows_.size() ) { return kEmpty; } return rows_[ pairIndex ]; } std::size_t pairCount() const noexcept { return pairs_.size(); } std::size_t reuseCount() const noexcept { return reuseCount_; } private: std::vector pairs_; std::vector> rows_; // sized once in run(), one owner per slot std::vector unfinished_; // 1 ⇒ a thrown worker never ran that pair's diff gtl::btree_map index_; std::size_t reuseCount_ = 0; }; // ── the stray-content analysis ─────────────────────────────────────────────────────────────────────────── // `Unknown` is the verdict for a ref whose ANALYSIS FAILED (no merge-base: a shallow clone, or genuinely // unrelated histories) — deliberately its own value rather than a fallback onto `Merged`. A failed analysis // must never render as the reassuring answer: "merged" on a ref nobody could analyse reads as "this branch // holds nothing you need", which is exactly the claim the evidence does NOT support. It sits LAST so the // three real verdicts keep their existing numeric values. enum class Verdict : std::uint8_t { Merged = 0, Superseded, Unmerged, Unknown }; constexpr std::size_t kVerdictCount = 4; static_assert( enumCountIsExact(), "kVerdictCount must equal the Verdict count — raise it with the append" ); inline const char* verdictTag( Verdict v ) noexcept { // The extent is DEDUCED: the old `kTag[ kVerdictCount ]` made the size assert below restate its own declaration, and // a missing tag would have zero-filled into a null pointer instead of failing. static constexpr const char* kTag[] = { "merged", "superseded", "unmerged", "unknown" }; static_assert( std::size( kTag ) == kVerdictCount, "verdictTag table must cover every Verdict" ); ASSUME( std::size_t( v ) < kVerdictCount ); return kTag[ std::size_t( v ) ]; } // One path's answer within one ref. struct FileRow { std::string path; std::uint32_t strayLines = 0; // lines the ref AUTHORED that HEAD does not have std::uint32_t authored = 0; // lines the ref authored vs its base (the denominator for context) std::uint32_t deleted = 0; // base lines the ref removed (0 ⇒ pure addition, no deletion site) std::uint32_t redone = 0; // …of which HEAD removed too (the supersession evidence) float sim = 0.0f; // minhash containment of the ref's blob in HEAD's blob bool headTouched = false; // did the live line change this path at all since the base? bool diffable = true; // false ⇒ binary/oversized on some side; counts are not meaningful Verdict verdict = Verdict::Merged; }; struct RefRow { RefInfo ref; std::string base; // merge-base(ref, HEAD) bool ok = true; // false ⇒ no merge-base (shallow clone / unrelated history) — reported, // never dropped, and verdict is forced to Unknown, never Merged std::uint32_t strayLines = 0; std::uint32_t strayFiles = 0; std::uint32_t supersededLines = 0; Verdict verdict = Verdict::Merged; std::vector files; // stray/superseded files only, sorted by strayLines desc then path }; struct StrayResult { bool ok = true; bool nonGitRoot = false; // H7 (capture-audit 2026-09-04): the --stray-content=SUBSTR filter named no local ref at all. refs="0" // unknown="0" at exit 0 reads as "no branch carries stray work" — the most reassuring possible answer, // from a sweep that never happened. ok=false, so the caller refuses in the --doc-drift/--dead-code words. bool filterMatchedNothing = false; // K75: the blob batch behind every verdict never started or died part-way (an unwritable cache dir, TMPDIR naming a missing // directory, git failing): each ref's lines would read as absent and its verdict as merged. ok=false; the caller refuses. bool blobsUnread = false; std::string filter; // H14/M6: the --stray-content=SUBSTR this sweep was narrowed by ("" = none) std::string headSha; std::string headRef; std::string atStamp; // M10: gitstamp::stampAt(root) — head= above stays a bare 9-hex sha // (gitstampcheck.sh's --abi/landing-plan arms pin that spelling); // at= is the new attribute carrying the dirty bit. std::size_t distinctBlobs = 0; std::size_t refsScanned = 0; std::uint32_t mergedRefs = 0; // scanned, found fully present on the live line, and OMITTED below // refs for-each-ref listed that enumerateRefs DROPPED (a tip that is not an object name): in none of the counts // above, so while it is non-zero refs= and the four buckets describe a sweep that skipped them — refs_dropped= says so. std::uint32_t refsDropped = 0; std::vector refsOmitted; // the ref window's leftovers (windowNewestRefs), sorted: refs_omitted= counts them std::vector refs; // only refs with stray content — merged ones are noise in a 30-ref sweep }; // Per-file verdict from the two evidences. Split out of analyzeRef so the rule is one readable expression // that a test can pin directly, and so the ordering of the lanes (deletion-site FIRST, minhash only as the // pure-addition fallback) is visible rather than buried in a chain of ifs. inline Verdict classifyFile( const FileRow& r ) { // `redone` counts a SUBSET of `deleted` (base lines this ref removed that HEAD removed too), so a // redone > deleted would mean the intersection out-counted one of its own operands — a corrupt // multiset walk, and it would push redoDel above 1.0 and silently mark the file superseded. ASSUME( r.redone <= r.deleted ); if( r.strayLines == 0 ) { return Verdict::Merged; } if( r.deleted > 0 ) { const float redoDel = float( r.redone ) / float( r.deleted ); return ( redoDel >= kRedoDelMin ) ? Verdict::Superseded : Verdict::Unmerged; } // Pure addition: no deletion site to compare, so the only remaining evidence that the live line already // holds this content is a HIGH minhash containment over a body big enough for that to mean something. if( r.headTouched && r.sim >= kSimSupersede && r.strayLines >= kSimMinStray ) { return Verdict::Superseded; } return Verdict::Unmerged; } // Everything one ref's analysis needs FROM GIT, gathered before any blob is reduced. Splitting this out of // the analysis proper is what lets the expensive, perfectly-parallel part (fork/exec) and the part that must // stay deterministic and serial (the multiset arithmetic) stop interleaving — and it is what collapses the // per-ref `git cat-file --batch` into ONE batch for the whole sweep. struct RefPlumbing { std::string base; // merge-base(ref, HEAD); empty ⇒ analysis failed bool ok = true; bool isAncestor = false; // ref.tip == base: the ref is fully on the live line already std::uint32_t refDiffPair = kNoPair; // DiffPairTable index for diff(base, ref.tip) std::uint32_t headDiffPair = kNoPair; // DiffPairTable index for diff(base, HEAD) gtl::btree_map headBlobAt; // path → HEAD's blob, for the paths HEAD changed // The DISCLOSE sink for the probe's degrades: a failed probe renders ok="0" v="unknown", never a verdict. enum class DisclosureWhy : std::uint8_t { RevisionNotObjectName, // the ref tip or HEAD is not a resolved object name: not probed MergeBaseNotObjectName, // git's merge-base answer is not an object name: discarded, read as no merge-base NoMergeBase, // shallow clone or unrelated history }; void disclose( DisclosureWhy why ) noexcept { switch( why ) { case DisclosureWhy::RevisionNotObjectName: case DisclosureWhy::NoMergeBase: ok = false; break; case DisclosureWhy::MergeBaseNotObjectName: base.clear(); break; } } }; // The merge-base probe — the one git call that must happen before the diff pairs are even known, and the // only one in this phase. Runs on the pool: refs are independent and each writes its own slot. inline RefPlumbing probeRefBase( const std::string& root, const RefInfo& ref, const std::string& headSha ) { RefPlumbing plumb; if( !isRevisionToken( ref.tip ) || !isRevisionToken( headSha ) ) { DISCLOSE( plumb, RefPlumbing::DisclosureWhy::RevisionNotObjectName, "crossref: ref tip or HEAD is not a resolved object name — refusing to probe, verdict is unknown" ); return plumb; } plumb.base = quality::gitOneLine( root, "merge-base " + shSingleQuote( ref.tip ) + " " + shSingleQuote( headSha ) + " -- 2>/dev/null" ); // The ANSWER is validated too, not just the question. It is git's own output today and therefore // well-formed — but that is an assumption, and this sha goes straight back out as the revision argument // to two more git commands. Anything that is not an object name is treated exactly like no merge-base. if( !plumb.base.empty() && !isRevisionToken( plumb.base ) ) { DISCLOSE( plumb, RefPlumbing::DisclosureWhy::MergeBaseNotObjectName, "crossref: merge-base returned something that is not an object name — discarding it" ); } if( plumb.base.empty() ) { // No merge-base: a SHALLOW clone (actions/checkout is shallow by default, so this is the CI default, // not an exotic case) or genuinely unrelated histories. Degrade, never crash — and the verdict this // produces is Unknown, never Merged: see writeStrayRef and Verdict's own comment. DISCLOSE( plumb, RefPlumbing::DisclosureWhy::NoMergeBase, "crossref: no merge-base for ref (shallow clone or unrelated history?) — verdict is unknown, not merged" ); return plumb; } // ref.tip == base ⇒ the ref is an ANCESTOR of HEAD, so diff(base, ref.tip) is a diff of a tree against // itself: provably empty, hence nothing authored, hence merged. Both of this ref's diffs are skipped — // 20 of the measured repo's 72 `diff --raw` calls were exactly this `diff A A`. plumb.isAncestor = ( ref.tip == plumb.base ); return plumb; } // ── every ref's merge base from ONE history walk (REF-WINDOW-COST, LARGE-PARTIAL-1) ───────────────────────────── // // probeRefBase is one `git merge-base` process per ref: ~0.2 s each under load on an M2 (the spawn, not git's work: the // command itself takes ~1 ms), so 512 refs paid ~86 s in --stray-content and 125 s in --abi on a one-file repo. One // `git rev-list --parents --boundary ^HEAD` lists every commit some ref holds and HEAD does not, with its parents. // For one ref, the parents its exclusive commits reach outside that set are its candidates: every common ancestor of the // ref and HEAD lies at or behind one of them. No exclusive commit at all: the tip is an ancestor of HEAD, and is the base. // Exactly one candidate: it is the unique best common ancestor, which is what `git merge-base` prints. No candidate: the // walk ended at a root or a shallow boundary, and there is no merge base (git prints none either). Two or more (a branch // that merged the live line, a criss-cross) are left to probeRefBase, which asks git, so no answer rests on a choice this // walk would have to make. Refs whose tip is not a resolved object name, or a walk that failed, are left to it too. struct BatchedBases { std::vector base; // per ref: its merge base, "" for none std::vector known; // per ref: 1 when the walk decided it; 0 → probeRefBase asks git }; inline BatchedBases mergeBasesByOneWalk( const std::string& root, const std::vector& refs, const std::string& headSha ) { BatchedBases out; out.base.assign( refs.size(), std::string() ); out.known.assign( refs.size(), 0 ); if( refs.empty() || !isRevisionToken( headSha ) ) { return out; } static std::atomic listSeq{ 0 }; const std::string listPath = quality::cacheDirLadder() + "/ripwire-crossref-tips-" + std::to_string( os::getpid() ) + "-" + std::to_string( listSeq.fetch_add( 1, std::memory_order_relaxed ) ) + ".txt"; { std::FILE* lf = std::fopen( listPath.c_str(), "wb" ); if( lf == nullptr ) { return out; } for( const RefInfo& r : refs ) { if( isRevisionToken( r.tip ) ) { rw::emitTo( lf, "{}\n", r.tip ); } } rw::emitTo( lf, "^{}\n", headSha ); std::fclose( lf ); } const GitListing listing = gitListChecked( root, "rev-list --parents --boundary --stdin < " + shSingleQuote( listPath ) ); os::unlink( listPath.c_str() ); if( !listing.ok ) { return out; } // The exclusive commits, sorted by sha: (sha, its parents). Boundary lines ('-' first) are HEAD-side parents, implied. std::vector>> exclusive; for( const std::string& line : listing.lines ) { if( line.empty() || line[ 0 ] == '-' ) { continue; } const std::vector f = wsdetail::segmentsOf( line, ' ' ); if( f.empty() || !isRevisionToken( f[ 0 ] ) ) { return out; // not the shape asked for: decide nothing } std::vector parents; for( std::size_t k = 1; k < f.size(); ++k ) { parents.emplace_back( f[ k ] ); } exclusive.emplace_back( std::string( f[ 0 ] ), std::move( parents ) ); } std::sort( exclusive.begin(), exclusive.end(), []( const auto& a, const auto& b ) { return a.first < b.first; } ); const auto indexOf = [ & ]( std::string_view sha ) -> std::size_t { const auto at = std::lower_bound( exclusive.begin(), exclusive.end(), sha, []( const auto& e, std::string_view v ) { return sortutil::svLess( e.first, v ); } ); return ( at != exclusive.end() && at->first == sha ) ? std::size_t( at - exclusive.begin() ) : SIZE_MAX; }; std::vector visitedBy( exclusive.size(), UINT32_MAX ); // the ref index whose walk last reached it std::vector stack; for( std::size_t i = 0; i < refs.size(); ++i ) { if( !isRevisionToken( refs[ i ].tip ) ) { continue; } const std::size_t tipAt = indexOf( refs[ i ].tip ); if( tipAt == SIZE_MAX ) { out.base[ i ] = refs[ i ].tip; // the tip is HEAD's ancestor: it is its own merge base out.known[ i ] = 1; continue; } std::vector candidates; stack.assign( 1, tipAt ); visitedBy[ tipAt ] = std::uint32_t( i ); while( !stack.empty() && candidates.size() < 2 ) { const std::size_t at = stack.back(); stack.pop_back(); for( const std::string& parent : exclusive[ at ].second ) { const std::size_t pAt = indexOf( parent ); if( pAt == SIZE_MAX ) { if( std::find( candidates.begin(), candidates.end(), parent ) == candidates.end() ) { candidates.push_back( parent ); } } else if( visitedBy[ pAt ] != std::uint32_t( i ) ) { visitedBy[ pAt ] = std::uint32_t( i ); stack.push_back( pAt ); } } } if( candidates.size() < 2 ) { out.base[ i ] = candidates.empty() ? std::string() : std::string( candidates.front() ); out.known[ i ] = 1; } } return out; } // Every ref's probe: the one-walk bases where the walk decided, probeRefBase (one git process each, across the pool) for // the rest. Each ref's plumbing is what probeRefBase alone would give it. inline std::vector probeRefBases( const std::string& root, const std::vector& refs, const std::string& headSha ) { std::vector plumbing( refs.size() ); const BatchedBases walked = mergeBasesByOneWalk( root, refs, headSha ); std::vector asked; for( std::size_t i = 0; i < refs.size(); ++i ) { if( walked.known[ i ] == 0 ) { asked.push_back( i ); continue; } plumbing[ i ].base = walked.base[ i ]; if( plumbing[ i ].base.empty() ) { DISCLOSE( plumbing[ i ], RefPlumbing::DisclosureWhy::NoMergeBase, "crossref: no merge-base for ref (shallow clone or unrelated history?) — verdict is unknown, not merged" ); continue; } plumbing[ i ].isAncestor = ( refs[ i ].tip == plumbing[ i ].base ); } ParallelSweep probeSweep; parallelIndexed( asked.size(), [ & ]( std::size_t k ) { plumbing[ asked[ k ] ] = probeRefBase( root, refs[ asked[ k ] ], headSha ); }, probeSweep ); for( std::size_t k = 0; k < asked.size(); ++k ) { plumbing[ asked[ k ] ].ok = plumbing[ asked[ k ] ].ok && probeSweep.isDone( k ); // a probe never run is a failed analysis, not a default } return plumbing; } // One ref's whole answer, computed from ALREADY-FETCHED plumbing and an ALREADY-FILLED blob store. Pure: // no git call, no I/O, no shared mutable state — so the verdict is a function of the evidence and nothing // else. `plumb.headBlobAt` is diff(base, HEAD) indexed by path — the live line's own change over the SAME // base, which is what makes the deletion sets directly comparable. inline RefRow analyzeRef( const RefInfo& ref, const RefPlumbing& plumb, const std::vector& refDiff, const BlobStore& blobs ) { RefRow row; row.ref = ref; row.base = plumb.base; if( !plumb.ok ) { row.ok = false; row.verdict = Verdict::Unknown; // NEVER Merged — a failed analysis is not a reassuring answer return row; } const gtl::btree_map& headBlobAt = plumb.headBlobAt; for( const RawRow& r : refDiff ) { const auto hIt = headBlobAt.find( r.path ); const bool headTouched = hIt != headBlobAt.end(); const std::string headSide = headTouched ? hIt->second : r.aSha; // unchanged by HEAD ⇒ still the base blob if( r.bSha == headSide ) { continue; // byte-identical on the live line } if( blobs.absent( r.aSha ) || blobs.absent( r.bSha ) || blobs.absent( headSide ) ) { // A side git could not read reads as an EMPTY side, and an empty side makes added work look like it was never there // ("merged"). Not known, so Unknown, never Merged (Verdict's own comment). row.ok = false; row.verdict = Verdict::Unknown; row.files.clear(); row.strayLines = row.strayFiles = row.supersededLines = 0; return row; } const BlobFacts& base = blobs.get( r.aSha ); const BlobFacts& mine = blobs.get( r.bSha ); const BlobFacts& live = blobs.get( headSide ); FileRow f; f.path = r.path; f.headTouched = headTouched; f.diffable = mine.isText && base.isText && live.isText; if( !f.diffable ) { // A binary/oversized side cannot be line-diffed. Report the path with counts zeroed and // diffable="0" rather than dropping it — an unreported path reads as "nothing here". f.verdict = isNullSha( headSide ) ? Verdict::Unmerged : Verdict::Merged; if( f.verdict == Verdict::Unmerged ) { row.files.push_back( std::move( f ) ); ++row.strayFiles; } continue; } const std::vector authored = msetDiff( mine.lineHashes, base.lineHashes ); const std::vector removed = msetDiff( base.lineHashes, mine.lineHashes ); const std::vector headGone = msetDiff( base.lineHashes, live.lineHashes ); f.authored = std::uint32_t( authored.size() ); f.deleted = std::uint32_t( removed.size() ); f.redone = std::uint32_t( sortedIntersectSize( removed, headGone ) ); f.strayLines = std::uint32_t( countAbsent( authored, live.lineHashes ) ); f.sim = sketchContainment( mine.sketch, live.sketch ); f.verdict = classifyFile( f ); if( f.verdict == Verdict::Merged ) { continue; // the live line has this work } row.strayLines += f.strayLines; if( f.verdict == Verdict::Superseded ) { row.supersededLines += f.strayLines; } ++row.strayFiles; row.files.push_back( std::move( f ) ); } std::sort( row.files.begin(), row.files.end(), []( const FileRow& a, const FileRow& b ) { return a.strayLines != b.strayLines ? a.strayLines > b.strayLines : a.path < b.path; } ); // Ref verdict: the stray-LINE-weighted share that the live line has demonstrably revisited. Weighting by // lines (not by file count) is what stops one coincidentally-shared deleted line in a mostly-additive // file from vouching for the hundreds of added lines around it. if( row.strayLines == 0 ) { row.verdict = Verdict::Merged; } else if( float( row.supersededLines ) / float( row.strayLines ) >= kRefSupersedeMin ) { row.verdict = Verdict::Superseded; } else { row.verdict = Verdict::Unmerged; } return row; } // Phases 2-3 of the sweep: plan the DISTINCT diffs it needs, then run them across the pool. A provably-empty // diff is never asked for (isAncestor) and a pair two refs share is asked for once, so the process count is // the number of distinct QUESTIONS, not the number of refs times two. inline DiffPairTable gatherRefDiffs( const std::string& root, const std::vector& refs, const std::string& headSha, std::vector& plumbing ) { ASSUME( plumbing.size() == refs.size() ); DiffPairTable diffs; for( std::size_t i = 0; i < refs.size(); ++i ) { if( !plumbing[i].ok || plumbing[i].isAncestor ) { continue; } plumbing[i].refDiffPair = diffs.want( plumbing[i].base, refs[i].tip ); plumbing[i].headDiffPair = diffs.want( plumbing[i].base, headSha ); } diffs.run( root ); return diffs; } // Phase 4: index HEAD's side per ref, then register every blob the WHOLE sweep needs so they all ride ONE // `git cat-file --batch`. This is the batching the BlobStore was built for and the old per-ref loop quietly // defeated — it called fill() inside each ref's analysis, so a 35-ref sweep paid 16 separate batch processes. inline void registerSweepBlobs( const DiffPairTable& diffs, std::vector& plumbing, BlobStore& blobs ) { for( RefPlumbing& plumb : plumbing ) { if( !plumb.ok || plumb.isAncestor ) { continue; } for( const RawRow& h : diffs.rows( plumb.headDiffPair ) ) { plumb.headBlobAt.emplace( h.path, h.bSha ); } for( const RawRow& r : diffs.rows( plumb.refDiffPair ) ) { blobs.want( r.aSha ); blobs.want( r.bSha ); const auto h = plumb.headBlobAt.find( r.path ); blobs.want( ( h != plumb.headBlobAt.end() ) ? h->second : r.aSha ); // HEAD's blob == base's unless HEAD touched it } } blobs.fill(); } // The ref window computeStrayContent reads: at most `maxRefs` local branches (SIZE_MAX under --detail), the `pinned` names // (sorted ref names; --eval-stray passes its labels) kept first, the newest by committer date after them. struct StrayRefWindow { std::span pinned = {}; std::size_t maxRefs = kMaxRefs; }; inline StrayResult computeStrayContent( const std::string& root, std::string_view filter, StrayRefWindow window = {} ) { StrayResult result; result.filter = std::string( filter ); // H14/M6: echoed on the root — a count under a filter is not a repo total if( !quality::gitRepoHasHistory( root ) ) { result.ok = false; result.nonGitRoot = true; return result; } result.headSha = quality::gitHeadSha( root ); result.atStamp = gitstamp::stampAt( root ); // M10: same anchor as every other repo-reading root, dirty bit included result.headRef = quality::gitOneLine( root, "rev-parse --abbrev-ref HEAD 2>/dev/null" ); std::size_t filterNameHits = 0; RefEnumeration enumeration; // a dropped ref is in no count below, so the root discloses it (refs_dropped=) std::vector refs = enumerateRefs( root, filter, result.headSha, &filterNameHits, &enumeration ); result.refsDropped = enumeration.refsDropped; result.filterMatchedNothing = !filter.empty() && filterNameHits == 0; if( result.filterMatchedNothing ) { result.ok = false; return result; } result.refsOmitted = windowNewestRefs( refs, window.maxRefs, window.pinned ); // past the window: the newest, the rest counted warnRefWindow( "--stray-content", refs.size(), result.refsOmitted, std::format( "{} refs", kMaxRefs ) ); // ── phase 1: the merge-base probe, one git call per ref, ACROSS THE POOL ──────────────────────────── // Refs are independent and each worker writes only its own slot, so this is the serial answer computed // in parallel — not a different answer. std::vector plumbing = probeRefBases( root, refs, result.headSha ); // REF-WINDOW-COST: one walk, git for the rest // ── phases 2-4: the distinct diffs, then ONE batched blob read for the whole sweep ─────────────────── const DiffPairTable diffs = gatherRefDiffs( root, refs, result.headSha, plumbing ); for( RefPlumbing& plumb : plumbing ) { plumb.ok = plumb.ok && diffs.isFinished( plumb.refDiffPair ) && diffs.isFinished( plumb.headDiffPair ); } BlobStore blobs( root ); registerSweepBlobs( diffs, plumbing, blobs ); if( blobs.unread() ) { result.ok = false; // refused, not answered: the caller names blobsUnreadCause() on stderr / as an MCP error result.blobsUnread = true; return result; } // ── phase 5: the analysis proper — pure, serial, deterministic ────────────────────────────────────── // A ref whose work is entirely on the live line is the boring, common case — in a 30-branch sweep it is // most of them, and printing a row per ref would bury the handful that matter. Omit them from the body // but COUNT them in the header (merged=) so the sweep's coverage is still stated, never implied. A ref // whose analysis FAILED is never omitted: it is not known to be merged, which is the whole point. for( std::size_t i = 0; i < refs.size(); ++i ) { RefRow row = analyzeRef( refs[i], plumbing[i], diffs.rows( plumbing[i].refDiffPair ), blobs ); if( row.ok && row.verdict == Verdict::Merged ) { ++result.mergedRefs; continue; } result.refs.push_back( std::move( row ) ); } result.refsScanned = refs.size(); result.distinctBlobs = blobs.distinctBlobs(); // Report order: most stray content first (that is the queue the owner works), ties by ref name. std::sort( result.refs.begin(), result.refs.end(), []( const RefRow& a, const RefRow& b ) { return a.strayLines != b.strayLines ? a.strayLines > b.strayLines : a.ref.name < b.ref.name; } ); return result; } // ── --whereis=SYM ──────────────────────────────────────────────────────────────────────────────────────── // One occurrence of SYM in one ref's tree. struct WhereHit { std::string ref; std::string tip; std::string date; std::string path; std::uint32_t line = 0; bool isDef = false; // kind="def": HEAD's from the index (or, head_labels="lexical", definitionShaped); another // ref's from the tags parse of its blob (labelBranchRowsByParse) std::string text; // the trimmed source line (evidence, so the caller can judge) bool fromWorktree = false; // read from the WORKING-TREE copy of a path that differs from HEAD (ref="worktree") bool testLocal = false; // a TEST-LOCAL definition, demoted below the production ones (test_local="1") bool unconfirmed = false; // kind="text": another ref's PARSE-WORTHY row whose blob was not parsed (the // runaway guard, or a parse that could not finish) — never kind="def", never a ref bool parseWorthy = false; // parseWorthyLine(): this line makes its blob worth a tags parse (another ref's rows) // foldSharedBranchDefs: a kind="def" row outside the checkout whose (path, line text) other refs hold too is ONE // definition held by N refs. The first row of such a group (lowest ref name) is its representative and carries // defRefs = N (1 = this ref alone); every later row of the group is foldedDef and the definitions page skips it // (listing=all still prints it). 0 = not a branch definition row. std::uint32_t defRefs = 0; bool foldedDef = false; // The CHECKOUT is HEAD's tree overlaid with the working copy of every path that differs from it: those rows // sort, label and count as one group, ahead of the other refs. Keyed on the flag, never on the ref name, so a // local branch that happens to be NAMED "worktree" is still an ordinary ref. bool inCheckout() const noexcept { return fromWorktree || ref == "HEAD"; } }; // §A7: where the parsed INDEX says SYM is defined — supplied by the caller (both surfaces hold an IngestResult // and pass whereisIndexDefSites below). `path` is spelled ROOT-RELATIVE, the way git // spells a tree entry; `line` is the index's 1-based def line. struct IndexDefSite { std::string path; std::uint32_t line = 0; bool testLocal = false; // filter.h isTestSymbol: in a test file, or in a test scope of any file }; // The optional EVIDENCE a caller can hand the tree scan — both members are "extra knowledge this surface // happens to hold", both default to "not supplied", and both only ever ADD a lane, so they travel as one // parameter rather than growing computeWhereis' argument list once per lane. struct WhereisEvidence { const gitoracle::HistoryIndex* history = nullptr; // --with-history: the name-history oracle (nullptr ⇒ not asked for) std::span indexDefs = {}; // §A7: where the parsed index defines SYM (empty ⇒ label HEAD lexically) }; // §A7: every place the parsed index defines `name`, keyed the way git spells a tree entry (model.h::relForHash // — the SAME root-relative join --abi uses to match ing.files against git paths). This is what lets whereis // stop GUESSING on HEAD rows: the tree scan reads committed blobs, the index knows where the definitions are, // and the join is a single pass over the symbol table with no extra I/O. ONE helper for both surfaces: the MCP // twin used to pass none, and labelled a wrapped call site kind="def" where the CLI said "ref" (0.6.6 sweep). inline std::vector whereisIndexDefSites( const IngestResult& ing, std::string_view name, const std::string& root ) { std::vector sites; for( const Symbol& s : ing.symbols ) { if( s.name == name ) { sites.push_back( IndexDefSite{ std::string( relForHash( ing.files[ s.fileId ], root ) ), s.line, isTestSymbol( ing, std::size_t( &s - ing.symbols.data() ) ) } ); } } return sites; } // What the scan learned about the WORKING TREE under the root (the 2026-10-01 freshness fix). The answer used // to read committed trees only and still claimed complete= on a dirty checkout, so a just-added function read // hits="0" and a just-deleted one kept its HEAD lines. Now every path that differs from HEAD is read from disk. // The ORDER is load-bearing: computeWhereis reads `state <= Read` as "every HEAD row kept is the checkout's content". // Clean nothing under the root differs from HEAD — the committed scan IS the checkout (byte-identical output) // Read every differing path was read (or is gone); its rows are ref="worktree" and replace HEAD's // Partial some differing path could not be read; ITS HEAD rows stand and may be stale — complete= is withheld // Unlisted git could not list what differs; every HEAD row may be stale — complete= is withheld enum class WorktreeOverlay : std::uint8_t { Clean, Read, Partial, Unlisted }; // How the OTHER refs' rows got their kind= (labelBranchRowsByParse), counted per distinct (blob, path) — the unit one // parse answers. `leftByGuard` and `failed` are the pairs that confirm nothing: their parse-worthy rows are the // `textRows` (kind="text"), and the page discloses all three in one element. struct BranchLabelCensus { std::size_t parsed = 0; // parsed by the tags path (definitionLinesInBlobs) std::size_t mirrored = 0; // HEAD's own blob at the same path: HEAD's index labels, line for line std::size_t leftByGuard = 0; // parse-worthy, but past kWhereisParseMaxBlobs / kWhereisParseMaxBytes std::size_t failed = 0; // parse-worthy, but the parser could not finish (BlobDefsStatus::Failed) std::size_t textRows = 0; // rows of those two that read kind="text" }; struct WhereResult { bool ok = true; bool nonGitRoot = false; std::string sym; std::string headSha; bool onHead = false; // SYM occurs in HEAD's tree bool headLabelsFromIndex = false; // §A7: HEAD rows' kind= came from the index, not the shape test bool indexKnowsNoDef = false; // honest-lex: the parsed index defines no symbol of this name (head_defs="0") BranchLabelCensus branchLabels; // how the OTHER refs' rows got kind= (labelBranchRowsByParse; on the page) std::size_t refsScanned = 0; std::vector refsOmitted; // the ref window's leftovers (windowNewestRefs): refs_omitted=, and no complete= std::size_t distinctBlobs = 0; std::string filter; // H14/M6: the --stray-filter SUBSTR this scan was narrowed by ("" = none) std::vector hits; WorktreeOverlay worktree = WorktreeOverlay::Clean; // what the overlay did (worktree= on the root, +dirty on at=) bool headHolds = false; // HEAD's COMMITTED tree holds SYM (on-head= reads the CHECKOUT on a dirty tree) std::string dottedRetry; // a Class.method spelling whose method the INDEX defines: the bare name (caller-set) std::string renamedTo; // the working tree renamed SYM to this definition (caller-set, on a zero) bool textWhole = false; // every t= prints whole (caller-set: --detail, --whereis-full-text, MCP full_text) // T1 (completeness claims): true iff the scan PROVABLY covered every text blob of every scanned ref's // full tree — no missing/oversized/short-read blob, the batch served every sha, and no ref's tree // listing came back empty (an empty listing from a non-empty repo is indistinguishable from a failed // `git ls-tree`, so it forfeits the claim rather than risk a false one). Binary blobs do NOT forfeit // it: the claim is text-scoped, and a text symbol cannot occur in one. The writer ANDs this with // "every hit printed" before emitting complete= on the root. bool scanExhaustive = false; // H7 / lens 6 F5 — the two SELECTOR facts this verb used to swallow, both filled by the caller (they are // index questions, and this module is deliberately index-free: it reads git trees, not symbols). // seedSpec the raw @FILE:LINE spelling, when the selector was a line seed. `--whereis=@src/graph.h:2500` // used to be searched as the LITERAL string "@src/graph.h:2500" across every blob, giving a // true and useless hits="0" shaped exactly like a name this repo never had — while // --owners/--mentions/--edit-check resolve that same grammar. The seed is now resolved to the // enclosing definition's name BEFORE the scan; this echoes what was typed so the answer can // still be tied to the command. // nearMiss the nearest indexed name, when the scan found nothing and the index knows one. The lexical // zero stays a measurement (a name this repo never had is a real answer); the near-miss is // what tells the reader which of the two zeros they are holding. std::string seedSpec; std::string nearMiss; // The blob stream behind hits= never started or ended before serving every blob (an unwritable batch list, a git // that did not start or died): no row count, zero included, is a measurement then, so both hosts REFUSE the answer // (exit 1 / an MCP error naming the cause) instead of printing hits= — the honesty bug of cli-floor §4. bool blobsUnread = false; // The --with-history lane: a non-owning view of the caller's oracle index (nullptr ⇒ not asked for) plus // THIS symbol's verdict, resolved once at compute time so emission stays a pure print. Views at the seam: // the caller owns the index and outlives both calls. const gitoracle::HistoryIndex* history = nullptr; gitoracle::NameFate fate; }; // Whole-word occurrence: SYM not flanked by identifier bytes (so `foo` never matches `foobar`/`myfoo`). inline bool wholeWordAt( std::string_view hay, std::size_t at, std::size_t len ) noexcept { if( at > 0 && isIdentByte( (unsigned char)hay[at - 1] ) ) { return false; } if( at + len < hay.size() && isIdentByte( (unsigned char)hay[at + len] ) ) { return false; } return true; } // The declarator END of a definition line, with trailing SPECIFIERS stripped: `noexcept`, cv/ref qualifiers, // the virtual-override words, a pure/defaulted/deleted tail, and a trailing-return arrow. Returns the index one // past the last byte that the terminator test below should look at. // // §A7 — this is not cosmetic. The house style of this very repo ends nearly every definition in `noexcept`, // and the terminator test accepted only `{ } ) :`, so `inline Config parseArgs( … ) noexcept` — the real // definition — read as kind="ref" while 3385 mention-rows outranked it. A shape test that rejects the // dominant shape of the corpus it runs on is worse than no test. inline std::size_t declaratorEnd( std::string_view line ) noexcept { static constexpr std::string_view kTrailingSpecifiers[] = { "noexcept", "const", "override", "final", "mutable", "volatile", "&&", "&", "= 0", "=0", "= default", "= delete", }; std::size_t e = line.size(); while( e > 0 && std::isspace( (unsigned char)line[e - 1] ) ) { --e; } for( bool stripped = true; stripped; ) // several may stack: `) const noexcept override` { stripped = false; for( std::string_view sp : kTrailingSpecifiers ) { if( e < sp.size() || line.compare( e - sp.size(), sp.size(), sp ) != 0 ) { continue; } const std::size_t before = e - sp.size(); // a whole WORD only, so `myconst` / `isFinal` never lose their tail (the operator forms are // punctuation and need no boundary). if( isIdentByte( (unsigned char)sp.front() ) && before > 0 && isIdentByte( (unsigned char)line[before - 1] ) ) { continue; } e = before; while( e > 0 && std::isspace( (unsigned char)line[e - 1] ) ) { --e; } stripped = true; } } // a trailing return type (`) -> Result`): cut at the arrow when what precedes it closes the parameter list. if( const std::size_t arrow = line.rfind( "->", e ); arrow != std::string_view::npos ) { std::size_t beforeArrow = arrow; while( beforeArrow > 0 && std::isspace( (unsigned char)line[beforeArrow - 1] ) ) { --beforeArrow; } if( beforeArrow > 0 && line[beforeArrow - 1] == ')' ) { e = beforeArrow; } } return e; } // Is the occurrence at `at` nested inside a parenthesised ARGUMENT LIST? A definition's own name is always at // paren depth 0 on its line (`inline T f( … )`); `w.write( escapeXml( s, esc ) );` puts the name at depth 1, // which is the false-positive class that gave a defs="1" symbol seventeen kind="def" rows. inline bool insideArgumentList( std::string_view line, std::size_t at ) noexcept { int depth = 0; for( std::size_t i = 0; i < at; ++i ) { if( line[i] == '(' ) { ++depth; } else if( line[i] == ')' && depth > 0 ) { --depth; } } return depth > 0; } // ── the lexical definition-shape tests (HEAD's head_labels="lexical" fallback, and the parse CHOOSER below) ────── // The declaration marker words, one table for both readings: definitionShaped wants one DIRECTLY before the name // (declMarkerBefore); parseWorthyLine's loose superset takes one anywhere earlier on the line (declMarkerEarlier). inline constexpr std::string_view kDeclMarkers[] = { "class ", "struct ", "enum ", "union ", "interface ", "namespace ", "trait ", "impl ", "protocol ", "def ", "func ", "fn ", "function ", "type ", "typedef ", "using ", "template", "extension ", }; // The old reading, kept for the parse chooser alone (a wrong "yes" there costs one parse): ANY marker substring // anywhere before the name. It is not a label: `// a class published its name`, `"struct (like --lego)"` and every // `struct T *data` local read definition-shaped under it (the honest-lex Stage 0: 241 false kind="def" HEAD rows). inline bool declMarkerEarlier( std::string_view line, std::size_t at ) noexcept { return std::ranges::any_of( kDeclMarkers, [ & ]( std::string_view m ) { const std::size_t f = line.find( m ); return f != std::string_view::npos && f < at; } ); } // The C-family definition: the name is immediately followed by '(', something precedes it on the line (the return // type / qualified scope), it is not a member call on a receiver, it is not itself an ARGUMENT (§A7), and the line // does NOT end in ';' — a trailing ';' is a prototype or a call statement, both of which are references, not // definitions. The accepted terminators are '{' (body opens here), '}' (whole one-line body), ')' (signature wraps // to the next line) and ':' (a constructor's init list follows), tested AFTER declaratorEnd() has stripped any // trailing specifier tail. inline bool cFamilyDefShaped( std::string_view line, std::string_view sym, std::size_t at ) { const std::size_t after = at + sym.size(); if( after >= line.size() || line[ after ] != '(' ) { return false; } const std::size_t e = declaratorEnd( line ); const char last = ( e > 0 ) ? line[ e - 1 ] : ';'; const bool endsDecl = last == '{' || last == '}' || last == ')' || last == ':'; const bool isCall = at >= 1 && ( line[ at - 1 ] == '.' || ( at >= 2 && line[ at - 2 ] == '-' && line[ at - 1 ] == '>' ) ); return endsDecl && !isCall && at > 0 && !insideArgumentList( line, at ); } inline bool isBlankByte( char c ) noexcept { return c == ' ' || c == '\t'; } // Does the line's first non-blank text open a COMMENT (or a docstring) in one of the indexed languages? Such a line // never defines anything, whatever words follow. `//`, `/*`, a block comment's ` * ` continuation or `*/`, `" ); // labels_omitted= (present only when non-zero): labelled refs past the sweep's ref window, read by no one and scored nowhere. const std::string labelsOmittedAttr = rep.omittedLabels.empty() ? std::string() : ( " labels_omitted=\"" + std::to_string( rep.omittedLabels.size() ) + "\"" ); rw::emitTo( out, "", n, rep.correct, rep.unknownCount, acc, labelsOmittedAttr.c_str() ); for( const EvalCase& c : rep.cases ) { rw::emitTo( out, "", ex( c.ref ).c_str(), verdictTag( c.expected ), verdictTag( c.got ), c.got == c.expected ? 1 : 0, c.found ? 1 : 0 ); } rw::emitRaw( out, "" ); } // ── XML emission (G4: minified, xmllint-clean; no `\n` outside CDATA) ──────────────────────────────────── using XmlEscaper = std::function; inline void writeStrayFile( std::FILE* out, const FileRow& f, const XmlEscaper& ex ) { rw::emitTo( out, "", ex( f.path ).c_str(), verdictTag( f.verdict ), f.strayLines, f.authored, f.deleted, f.redone, double( f.sim ), f.headTouched ? 1 : 0, f.diffable ? "" : " diffable=\"0\"" ); } inline void writeStrayRef( std::FILE* out, const RefRow& r, const XmlEscaper& ex, std::size_t maxFiles ) { // ok="0" ALWAYS renders v="unknown". computeStrayContent already sets Unknown at the degrade site, so // this is a belt-and-braces coercion at the one place the claim is actually made to the reader: no future // degrade path can reintroduce "the analysis failed, so let us print the reassuring answer". const Verdict shownVerdict = r.ok ? r.verdict : Verdict::Unknown; ASSUME( r.ok || shownVerdict == Verdict::Unknown ); rw::emitTo( out, "", ex( r.ref.name ).c_str(), r.ref.tip.c_str(), ex( r.ref.date ).c_str(), r.base.c_str(), r.ok ? 1 : 0, verdictTag( shownVerdict ), r.strayLines, r.strayFiles, r.supersededLines ); // The contract: shown + dropped == the total, always. Count against the CAP (maxFiles), never // against a loop variable the break has already incremented past it — that off-by-one is what made // whereis claim 20 dropped where 21 were, and it vanished entirely at exactly cap+1. std::size_t shownCount = 0; for( const FileRow& f : r.files ) { if( shownCount >= maxFiles ) { break; } ++shownCount; writeStrayFile( out, f, ex ); } ASSUME( shownCount == std::min( r.files.size(), maxFiles ) ); if( r.files.size() > maxFiles ) { rw::emitTo( out, "", r.files.size() - maxFiles ); } rw::emitRaw( out, "" ); } // §P15/§P16: res.refs is already deterministic (strayLines desc, then ref.name asc — computeStrayContent's // own sort) and used to print every stray ref unconditionally, with no historic display cap on the OUTER // listing (only the per-ref children were ever capped, at maxFiles, which stays untouched — a second, // independent listing per rule 6). Paging mirrors writeWhereisPage: pageLimit/pageOffset default to 0 (no // paging, the un-paginated three-argument writeStrayContent below keeps every existing caller byte-identical). inline void writeStrayContentPage( std::FILE* out, const StrayResult& res, std::size_t maxFiles, int pageLimit, int pageOffset ) { std::vector esc; const XmlEscaper ex = [ & ]( std::string_view s ) { return std::string( escapeXml( s, esc ) ); }; // Every scanned ref lands in EXACTLY ONE bucket, so unmerged + superseded + merged + unknown == refs=. // Before the unknown bucket existed, a ref whose analysis failed landed in none of them and the header // silently failed to add up — the counters and refs= disagreeing was the first visible symptom. std::uint32_t unmerged = 0, superseded = 0, unknown = 0; for( const RefRow& r : res.refs ) { if( !r.ok || r.verdict == Verdict::Unknown ) { ++unknown; } else if( r.verdict == Verdict::Unmerged ) { ++unmerged; } else if( r.verdict == Verdict::Superseded ) { ++superseded; } } ASSUME( std::size_t( unmerged ) + superseded + unknown + res.mergedRefs == res.refsScanned ); // G4: an XML comment may not contain a double hyphen, so this text (and writeWhereis's) names flags and // git subcommands WITHOUT their leading dashes. Keep it that way when editing. rw::emitRaw( out, "" ); // §P8: shipped as `head-ref=` while its own --abi sibling (abicheck.h's ``, over the SAME // field, reached by the SAME command line) shipped `head_ref=` — the tool's only kebab/snake pair, so a // parser written against one half read nothing from the other. Unified onto snake_case: it is the // majority (41 vs 8) and the spelling README.md documents. Kebab survives only in docs/captures/*. const PageWindow refPage = pageWindow( res.refs.size(), pageLimit, pageOffset ); char srab[ kPageDisclosureCap ]; // M10: head= stays a bare 9-hex sha (gitstampcheck.sh's existing arm pins that spelling); at= is the // new attribute, carrying the dirty bit this document never disclosed before. const std::string atAttrStr = res.atStamp.empty() ? std::string() : ( " at=\"" + res.atStamp + "\"" ); // H14/M6: refs="2" under a ref-name filter reads as "this repo has two branches" unless the filter is // named. --doc-drift already echoed its own filter=; this is the same attribute on a sibling that did not. const std::string filterAttr = res.filter.empty() ? std::string() : ( " filter=\"" + ex( res.filter ) + "\"" ); const std::string droppedAttr = ( res.refsDropped == 0 ? std::string() : ( " refs_dropped=\"" + std::to_string( res.refsDropped ) + "\"" ) ) + refsOmittedAttr( res.refsOmitted ); rw::emitTo( out, "", res.headSha.c_str(), ex( res.headRef ).c_str(), res.refsScanned, res.distinctBlobs, unmerged, superseded, res.mergedRefs, unknown, filterAttr.c_str(), droppedAttr.c_str(), pageDisclosure( srab, sizeof( srab ), refPage.end - refPage.begin, res.refs.size(), refPage.end, pageLimit, pageOffset, false ), atAttrStr.c_str() ); for( std::size_t refIndex = refPage.begin; refIndex < refPage.end; ++refIndex ) { writeStrayRef( out, res.refs[refIndex], ex, maxFiles ); } rw::emitRaw( out, "" ); } inline void writeStrayContent( std::FILE* out, const StrayResult& res, std::size_t maxFiles ) { writeStrayContentPage( out, res, maxFiles, 0, 0 ); } // ── §B11.2: the qualified spelling this verb has no selector for ───────────────────────────────────────── // --whereis matches a symbol NAME against ref-tree text. Handed a `file:name` spelling — the grammar nine // other verbs DO accept, and the exact thing an agent pastes out of a `p="file:line"` row or a --uses retry // example — it searched for that literal string, found it in no tree, and answered hits="0": true, useless, // and BYTE-IDENTICAL to the answer for a name this repo never had. That is the V2-1 guard's uncovered arm // (its `uses` twin was closed in §B6 M6). Every other zero this verb prints is a measurement; this one is a // spelling error wearing a measurement's clothes. // // The test is SYNTACTIC and therefore index-free, which is what lets it hold on both surfaces: the MCP // `whereis` verb deliberately never calls getIndex() (no rebuild, no staleness coupling), so a guard that // needed an IngestResult could only ever have covered the CLI — the one-arm asymmetry this whole class is. // Because it lives in the writer, both surfaces inherit it with no call-site change at all. // // Deliberately NOT a refusal: hits="0" here is a true statement, unlike `uses`' external="1" (a false claim // about the indexed tree), so the proportionate fix is to make the zero legible, not to change an exit code. // A `::` spelling is left alone — that is a canonical id or a C++/Ruby qualified name, not a path selector — // and so is a file half with neither a path separator nor an extension shape, which keeps every ObjC selector // (`doThing:withOther:`) and every unusual name answerable exactly as before. inline bool whereisSpecIsFileQualified( std::string_view spec ) { if( spec.find( "::" ) != std::string_view::npos ) { return false; } const std::size_t lastColon = spec.rfind( ':' ); if( lastColon == std::string_view::npos || lastColon == 0 || lastColon + 1 >= spec.size() ) { return false; } const std::string_view fileHalf = spec.substr( 0, lastColon ); if( fileHalf.find( '/' ) != std::string_view::npos ) { return true; // a path, unambiguously } // a bare file NAME with an extension ("engine.cpp:computeBudget"): the dot must sit inside the half and be // followed by a short all-alphanumeric run, so an ordinary identifier can never trip it. const std::size_t dot = fileHalf.rfind( '.' ); if( dot == std::string_view::npos || dot == 0 || dot + 1 >= fileHalf.size() ) { return false; } const std::string_view ext = fileHalf.substr( dot + 1 ); if( ext.size() > 8 ) { return false; } return std::all_of( ext.begin(), ext.end(), []( char c ) { return std::isalnum( static_cast( c ) ) != 0; } ); // explicit narrowing: an unsigned char parameter is G1's implicit sign-change on a byte >= 0x80 } // The bare-name half a caller should retype. Only meaningful when whereisSpecIsFileQualified( spec ). inline std::string_view whereisBareNameOf( std::string_view spec ) { const std::size_t lastColon = spec.rfind( ':' ); ASSUME( lastColon != std::string_view::npos && lastColon + 1 < spec.size() ); return spec.substr( lastColon + 1 ); } // A `Class.method` / `Class#method` spelling (2026-10-01, found by the edit-check lane): the way docs, Python, JS, // Java and Ruby name a method. This verb searches its selector as a LITERAL, and no tree spells a method's // definition that way (it is `def area` inside `class Shape`), so the scan answered hits="0" on-head="0" // complete="1" with no note — a zero shaped exactly like a name this repo never had. The test is the shared // resolver's dotted-scope tier condition (graph.h resolveAllByDottedScope on lane/editcheck-067: a '.' or '#', and // no ':' or '/'), narrowed to identifier segments so a quoted literal with other punctuation is left alone. inline bool whereisSpecIsDotted( std::string_view spec ) { if( spec.find_first_of( ".#" ) == std::string_view::npos || spec.find_first_of( ":/" ) != std::string_view::npos ) { return false; } std::size_t segmentLen = 0; for( const char c : spec ) { const bool separator = c == '.' || c == '#'; if( separator && segmentLen == 0 ) { return false; // an empty segment: ".x", "a..b", "a#" } if( !separator && !isIdentByte( static_cast( c ) ) ) { return false; } segmentLen = separator ? 0 : segmentLen + 1; } return segmentLen > 0; } // The bare method name a caller should retype: the last segment. Only meaningful when whereisSpecIsDotted( spec ). inline std::string_view whereisDottedNameOf( std::string_view spec ) { return spec.substr( spec.find_last_of( ".#" ) + 1 ); } // The dotted-selector retry for `spec`, or "" when the note must not fire: a dotted spelling whose LAST SEGMENT the // INDEX defines (so the offered retry finds a definition). `setup.py`, `os.path`, `README.txt` are literals a reader // may legitimately ask this "defines or mentions" verb about, and their literal scan keeps its complete= (review M3). // Once the shared dotted-scope resolver lands (graph.h resolveAllByDottedScope), this should ask it instead. inline std::string whereisDottedRetryOf( const IngestResult& ing, std::string_view spec ) { if( !whereisSpecIsDotted( spec ) ) { return {}; } const std::string_view method = whereisDottedNameOf( spec ); const bool defined = std::any_of( ing.symbols.begin(), ing.symbols.end(), [ & ]( const Symbol& s ) { return s.kind != SymKind::Section && s.name == method; } ); return defined ? std::string( method ) : std::string(); } // The legend's CONDITIONAL tail: each paragraph rides only an answer that carries what it defines, so a plain // answer pays no bytes for the with_history lane, test-local rows or the worktree overlay. Split out of // writeWhereisPage so the page writer stays a page writer. // refs= on a hit rides only a definitions page that printed one (foldSharedBranchDefs), and so does its reading: the // SHARED DEFINITIONS clause of the legend, spliced by writeWhereisListedPage beside the tail below. inline constexpr char kWhereisSharedDefsLegend[] = "SHARED DEFINITIONS: refs=\"N\" on a kind=\"def\" row outside the checkout means N scanned refs hold this same " "definition line (the same path and the same line text; on a kind=\"text\" row, the same unparsed line, never merged " "with a confirmed one); the row is printed once, for the first of those " "refs by name, whose ref=, tip=, date= and l= it carries (another of the N may hold the line at a different " "line number). It appears only under listing=\"defs\" and only when N is at least 2: a definition one ref " "alone holds carries no refs=. hits= still counts every copy, and listing=all prints each ref's own row. "; inline void writeWhereisLegendTail( std::FILE* out, const WhereResult& res ) { // §L10b: the with_history lane's own element, previously undefined on this legend — shared // verbatim with --doc-drift's copy (gitoracle.h kHistoryProbeLegend) so the two cannot drift. Only // when res.history actually made that element reachable — an unconditional splice would cost every // plain --whereis run bytes describing an absent element. if( res.history != nullptr ) { std::fputs( gitoracle::kHistoryProbeLegend, out ); } // test_local= rides only an answer that holds both kinds of definition (demoteTestLocalDefs), and so does its reading. if( std::any_of( res.hits.begin(), res.hits.end(), []( const WhereHit& h ) { return h.testLocal; } ) ) { rw::emitRaw( out, "TEST-LOCAL: test_local=\"1\" on a kind=\"def\" row marks a definition in a test scope (the index's test lens) " "or under a test, bench or fixture path (test/, bench/, benches/, fixture/, fixtures/, testdata/ and the " "test-file name patterns), on any row. It appears only when the same answer also " "holds a production definition, and those rows are ordered after the production definitions and before " "the references of their tier. Nothing is dropped. " ); } // The unparsed element's vocabulary (writeWhereisUnparsed), only on an answer that carries the element. No literal // double dash inside this XML comment: the flag is spelled by name, the next= attribute carries its real spelling. if( res.branchLabels.leftByGuard + res.branchLabels.failed > 0 ) { rw::emitTo( out, "UNPARSED: the unparsed element counts the blobs of other refs that hold a line worth parsing but that " "no parse read: blobs= of them, either past the runaway guard on the parse batch ({} blobs or {} MB, filled " "most rows first, then by path and blob) or a parse that could not finish; rows= is the kind=\"text\" rows they " "gave. Those rows are neither a confirmed definition nor a confirmed reference, and the blobs' other rows read " "kind=\"ref\". Its next= rides when the guard left some: the same question (the symbol and an explicitly named " "listing) with the detail flag at 1, which lifts the guard; a ref filter (the MCP kind argument) is not carried, " "as the CLI has no whereis ref filter. ", kWhereisParseMaxBlobs, kWhereisParseMaxBytes >> 20 ); } // The overlay's own vocabulary, only on an answer that carries it — a clean checkout pays no bytes for it. if( res.worktree != WorktreeOverlay::Clean ) { rw::emitRaw( out, "WORKTREE: worktree= on the root means the checkout under this root differs from HEAD, so HEAD's " "committed tree alone would be a stale answer. Every path that differs (modified, staged, deleted or " "untracked and not ignored) is read from the working tree instead: its rows say ref=\"worktree\" (tip= " "and date= name the HEAD commit it overlays) and REPLACE HEAD's rows for that path, so a definition the " "edit added is listed and one it deleted or renamed is not. HEAD's rows stand only for paths the working " "tree left alone. On such an answer on-head=, hits= and head_labels= read the CHECKOUT, HEAD plus those rows, not " "HEAD's commit alone: on-head=\"0\" there means the checkout lacks the name, which HEAD's commit may still " "hold. A path beyond a symbolic link to a directory is not in the checkout (git's own reading) and is never " "read through the link. worktree=read: every differing path was read, and complete= keeps its meaning over " "the checkout. worktree=partial: some differing path could not be read (permission, a " "name git had to quote, a directory such as an untracked nested repository or a submodule, whose files are " "another repository's, or more changed paths than one answer reads), so its HEAD rows stand and may be stale. worktree=unlisted: git " "could not list the changes, so any HEAD row may be stale. Under partial or unlisted complete= is never " "claimed. Other refs are always their committed trees. With the with_history lane, a name HEAD's commit holds " "but the working tree removed gets fate v=\"uncommitted\" instead of the history oracle's never or removed. " ); } } // The selector-note elements, first after the root: each says which KIND of zero (or of literal) the reader holds. inline void writeWhereisSelectorNotes( std::FILE* out, const WhereResult& res, const XmlEscaper& ex ) { // §B11.2 — a zero that is a SPELLING fact, not a repository fact, says so. Emitted first, before the // history lane, so it is the first thing after the root on the one shape where it fires: hits="0" AND a // file-qualified selector. It never appears beside a nonzero hit list, so it cannot dilute a real answer. if( res.hits.empty() && whereisSpecIsFileQualified( res.sym ) ) { rw::emitTo( out, "", ex( res.sym ).c_str(), ex( whereisBareNameOf( res.sym ) ).c_str() ); } // The dotted spelling, on EVERY answer that carries one: a nonzero list is the literal's occurrences (call sites // like `Shape.area(…)`), never the definition, so the note rides beside it too. Only when the caller found that the // INDEX defines the last segment (dottedRetry) — `setup.py` or `os.path` is a literal, not a method (review M3). if( !res.dottedRetry.empty() ) { rw::emitTo( out, "", ex( res.sym ).c_str(), ex( res.dottedRetry ).c_str() ); } // H7: the same element, two more reasons — the line seed that was RESOLVED before the scan (so sym= is a // name and not the raw @spec), and the near-miss beside a zero the index can explain. if( !res.seedSpec.empty() ) { rw::emitTo( out, "", ex( res.seedSpec ).c_str(), ex( res.sym ).c_str() ); } // Review M7: a zero for a name the WORKING TREE renamed offers the new name (worktreeRenameOf's evidence: a // definition of SYM left a changed file that now defines this one), ahead of — and instead of — a spelling neighbour. if( res.hits.empty() && !res.renamedTo.empty() ) { rw::emitTo( out, "", ex( res.sym ).c_str(), ex( res.renamedTo ).c_str() ); } else if( res.hits.empty() && !res.nearMiss.empty() ) { rw::emitTo( out, "", ex( res.sym ).c_str(), ex( res.nearMiss ).c_str() ); } } // The with_history lane, when it was asked for: what the probe did, then this symbol's own verdict (see the comment // at the call site for why an index-confirmed HEAD definition suppresses the verdict). Review M8: on a dirty checkout // a zero (on-head="0") may be a name HEAD's COMMITTED tree still holds and only the uncommitted working tree removed. // The oracle's "never" / "removed" read a zero as "HEAD does not have it", which is false there, so that case gets // its own verdict, v="uncommitted", naming what is true. inline void writeWhereisFate( std::FILE* out, const WhereResult& res, const XmlEscaper& ex ) { if( res.history == nullptr ) { return; } gitoracle::writeHistoryProbe( out, *res.history, ex ); if( !res.history->ok || ( res.onHead && res.headLabelsFromIndex ) ) { return; } if( res.headHolds && !res.onHead ) { rw::emitTo( out, "", ex( res.sym ).c_str() ); return; } gitoracle::writeNameFate( out, res.sym, res.fate, ex ); } // ── the listing and the hoist (lean-answers lane) ──────────────────────────────────────────────────────── // The rows one --whereis answer lists (see WhereisListing): indices into WhereResult::hits in the list's own order, // the root's listing= value ("" when every hit is listed), and how many kind="ref" rows the element // stands for (0 = no such element). struct ListedHits { std::vector rows; std::string_view attr; std::size_t refsElided = 0; std::size_t defsFolded = 0; // def rows of other refs a listed row's refs= stands for (folded="N" on the root) WhereisListing asked = WhereisListing::ShorterOfDefsAll; // the listing the question named (whereisServedListing) bool textPreview = false; // a long t= prints as a preview (writeWhereisPage: only when that page is shorter) }; // kind= of a row: "def" (a parser's definition), "text" (a parse-worthy line on another ref no parse confirmed, // labelBranchRowsByParse) or "ref". inline const char* whereisKindOf( const WhereHit& h ) noexcept { return h.isDef ? "def" : ( h.unconfirmed ? "text" : "ref" ); } // The ` refs="N"` a row carries, or "": it rides ONLY the definitions page (foldSharedBranchDefs), where the row stands // for N refs' copies of the same definition line. listing=all prints each copy as its own row and never carries it; a // lone copy carries none. The legend's SHARED DEFINITIONS clause rides exactly the pages where this is non-empty. inline std::string whereisSharedAttr( const ListedHits& listed, const WhereHit& h ) { return ( listed.attr == "defs" && h.defRefs >= 2 ) ? ( " refs=\"" + std::to_string( h.defRefs ) + "\"" ) : std::string(); } // A row the definitions page lists: a definition, or a parse-worthy line on another ref no parse read (kind="text"). inline bool whereisDefsPageRow( const WhereHit& h ) noexcept { return h.isDef || h.unconfirmed; } inline ListedHits listedHits( const WhereResult& res, WhereisListing listing ) { EXPECTS( listing != WhereisListing::ShorterOfDefsAll, "the default is resolved to Defs or All (whereisServedListing) before the rows are picked" ); // The definitions page lists the kind="def" rows AND the kind="text" ones (parse-worthy lines on another ref // that no parse confirmed or refuted, labelBranchRowsByParse): a possible definition is printed, labelled, never // counted away among the references. listing=refs and are the kind="ref" rows exactly. const std::size_t defs = std::size_t( std::count_if( res.hits.begin(), res.hits.end(), whereisDefsPageRow ) ); const std::size_t refs = res.hits.size() - defs; ListedHits out; // Defs with nothing to elide (no ref row) IS the whole list; Defs with no def row would list nothing but a count, // and then the mentions are the answer (a name the index does not define, a literal): both print every hit. const bool lean = listing == WhereisListing::Defs && defs > 0 && refs > 0; if( !lean && listing != WhereisListing::Refs ) { out.rows.resize( res.hits.size() ); for( std::size_t i = 0; i < res.hits.size(); ++i ) { out.rows[ i ] = i; } return out; } // The definitions page lists each shared branch definition ONCE (foldSharedBranchDefs): a folded row's line is // already on the page, on its representative, whose refs= counts it. The whole list (listing=all) keeps every row. const bool wantDefs = lean; std::size_t folded = 0; for( std::size_t i = 0; i < res.hits.size(); ++i ) { const WhereHit& h = res.hits[ i ]; if( whereisDefsPageRow( h ) != wantDefs ) { continue; } if( wantDefs && h.foldedDef ) { ++folded; continue; } out.rows.push_back( i ); } out.attr = wantDefs ? std::string_view( "defs" ) : std::string_view( "refs" ); out.refsElided = wantDefs ? refs : 0; out.defsFolded = folded; ENSURES( out.rows.size() + folded == ( wantDefs ? defs : refs ), "a listing holds exactly the rows of its kind, a folded def on its representative" ); return out; } // HEAD's committer date, read off the first row on HEAD's commit (every such row carries the same date: the scan // stamps a ref's rows with its tip's date). "" when no row is on HEAD's commit (a branch-only answer, hits="0"). inline std::string whereisHeadDate( const WhereResult& res ) { const std::string_view head = std::string_view( res.headSha ).substr( 0, 9 ); for( const WhereHit& h : res.hits ) { if( !head.empty() && std::string_view( h.tip ).substr( 0, 9 ) == head ) { return h.date; } } return {}; } // Whether a row's tip= and date= are exactly the root's at= (as printed: 9 hex, without +dirty) and head_date=, so the // row may omit both and a reader restores them losslessly. Compared on the PRINTED 9-hex spelling, the bytes a reader holds. inline bool whereisRowOnHeadCommit( const WhereHit& h, std::string_view headSha, std::string_view headDate ) noexcept { const std::string_view head = headSha.substr( 0, 9 ); return !head.empty() && !headDate.empty() && std::string_view( h.tip ).substr( 0, 9 ) == head && h.date == headDate; } // The listing a validated --whereis-listing= / MCP `listing` value names; "" (absent) is the default, defs when it // lists more definitions than all, else the shorter of the two (WhereisListing::ShorterOfDefsAll). Both surfaces refuse any other value before they get here. inline WhereisListing whereisListingOf( std::string_view v ) noexcept { return v == "refs" ? WhereisListing::Refs : v == "all" ? WhereisListing::All : v == "defs" ? WhereisListing::Defs : WhereisListing::ShorterOfDefsAll; } // The : the same symbol, listing=refs — exactly the kind="ref" rows the default listing counted. inline std::string whereisRefsNext( const WhereResult& res ) { return nextFlag( "--whereis=", res.sym ) + " --whereis-listing=refs"; } // The page a --whereis answer prints: the listed rows and the window over them (writeWhereisPage's own reading of its // arguments, handed to the two writers below so neither re-derives it). struct WhereisPageView { const ListedHits& listed; PageWindow page; int pageLimit = 0; int pageOffset = 0; }; // The open tag (writeWhereisPage). inline void writeWhereisRoot( std::FILE* out, const WhereResult& res, const WhereisPageView& v, const std::string& headDate, const XmlEscaper& ex ) { const ListedHits& listed = v.listed; const PageWindow& hitPage = v.page; const std::size_t listedCount = listed.rows.size(); const int pageLimit = v.pageLimit; const int pageOffset = v.pageOffset; char pab[ kPageDisclosureCap ]; // §A7(iii): refs_scanned=, not refs=. --stray-content and --abi both spell the MATCHED set refs=; this one // counted every branch the sweep READ (73 here, matched or not) under the same attribute name — one noun, // two meanings, across sibling verbs an agent reads together. // T1: the claim is scan-exhaustiveness AND listing-wholeness — an uncut page over a clean scan. Appended // LAST (after at=) so no existing attribute-adjacency assertion can break on it, the same placement rule // the graph verbs' floor marker follows. When either half fails, NOTHING is added: the truncation // vocabulary above already covers every partial shape, and complete-equals-zero would be noise. // A dotted selector was searched as a literal the question did not mean: exhaustive over that literal, but not // an answer to "where is Class.method", so it never claims (its selector-note below says why). // listing= (WhereisListing): the claim reads the LISTED rows — under listing=defs every def row on this page (the // ref rows are counted exactly by , never silently gone); under listing=refs every ref row. const bool completeClaim = res.scanExhaustive && hitPage.begin == 0 && hitPage.end == listedCount && res.dottedRetry.empty(); // H14/M6: refs_scanned="80" under a ref-name filter is a total for the FILTER, not for the repo (the // audit measured 80 filtered vs 189 unfiltered) — so the filter is named beside the number it bounds. const std::string whFilterAttr = ( res.filter.empty() ? std::string() : ( " filter=\"" + ex( res.filter ) + "\"" ) ) + refsOmittedAttr( res.refsOmitted ); // the ref window's leftovers, beside the refs_scanned= they bound // The overlay's two marks, both absent on a clean checkout (so its answer is byte-identical to before): +dirty // on the stamp whenever a differing path was seen, and worktree= naming what the overlay managed. static constexpr std::string_view kWorktreeAttr[] = { "", " worktree=\"read\"", " worktree=\"partial\"", " worktree=\"unlisted\"" }; const bool dirty = res.worktree == WorktreeOverlay::Read || res.worktree == WorktreeOverlay::Partial; // THE HOIST (lossless): a checkout row on HEAD's own commit repeats tip= (= at= without +dirty) and date= (HEAD's // committer date) on every row. Those rows omit both, and the root says the date ONCE (head_date=) — only when a // printed row actually omitted it, so a page of branch rows alone is unchanged. Placed before at= so the at= / // worktree= / complete= adjacency every existing assertion reads stays byte-stable. bool anyHoist = false; for( std::size_t k = hitPage.begin; k < hitPage.end; ++k ) { anyHoist = anyHoist || whereisRowOnHeadCommit( res.hits[ listed.rows[ k ] ], res.headSha, headDate ); } // folded= (foldSharedBranchDefs): the def rows of other refs that a listed row's refs= stands for, so the root's // arithmetic stays checkable in one place: shown + more + folded + refs count = hits. Only when a row folded, // so an answer with no shared branch definition is byte-identical to before. // ref_labels="parsed": the answer holds rows of OTHER refs, whose kind= a parser gave (labelBranchRowsByParse). Only // then, so an answer with no branch row — a repo with no other ref — is byte-identical to before. const bool anyBranchRow = std::any_of( res.hits.begin(), res.hits.end(), []( const WhereHit& h ) { return !h.inCheckout(); } ); const std::string listingAttr = ( listed.attr.empty() ? std::string() : ( " listing=\"" + std::string( listed.attr ) + "\"" ) ) + ( listed.defsFolded > 0 ? ( " folded=\"" + std::to_string( listed.defsFolded ) + "\"" ) : std::string() ) + ( anyBranchRow ? std::string( " ref_labels=\"parsed\"" ) : std::string() ); const std::string headDateAttr = anyHoist ? ( " head_date=\"" + ex( headDate ) + "\"" ) : std::string(); // head_defs="0" (honest-lex): the lexical labels stand AND the parsed index defines no symbol of this name, so every // HEAD kind="def" below is the text test's guess with no definition behind it in the indexed tree. Absent otherwise // (index labels, or lexical over a working tree that drifted from the index's def sites). const char* const headDefsAttr = ( !res.headLabelsFromIndex && res.indexKnowsNoDef ) ? " head_defs=\"0\"" : ""; rw::emitTo( out, "", ex( res.sym ).c_str(), res.onHead ? 1 : 0, res.refsScanned, res.distinctBlobs, res.hits.size(), res.headLabelsFromIndex ? "index" : "lexical", headDefsAttr, whFilterAttr.c_str(), pageDisclosure( pab, sizeof( pab ), hitPage.end - hitPage.begin, listedCount, hitPage.end, pageLimit, pageOffset, true ), listingAttr.c_str(), headDateAttr.c_str(), res.headSha.c_str(), dirty ? "+dirty" : "", kWorktreeAttr[ std::size_t( res.worktree ) ], completeClaim ? " complete=\"1\"" : "" ); } // The --detail=1 replay of the SAME question (the ): the symbol, the listing the question named (an explicit // one; the default stays the default) and the detail flag, which lifts every cap of this verb (the row cap, the parse guard, // the t= preview). A ref filter (the MCP `kind`) is NOT carried, as does not // carry it: the CLI has no whereis ref filter (--stray-content= is another verb, and naming it would run that verb instead), // and the legend's UNPARSED clause says so. inline std::string whereisDetailNext( const WhereResult& res, WhereisListing asked ) { std::string call = nextFlag( "--whereis=", res.sym ); switch( asked ) { case WhereisListing::Defs: call += " --whereis-listing=defs"; break; case WhereisListing::Refs: call += " --whereis-listing=refs"; break; case WhereisListing::All: call += " --whereis-listing=all"; break; case WhereisListing::ShorterOfDefsAll: break; } return nextAttrXml( call + " --detail=1" ); } // : the other refs' (blob, path) pairs that held a parse-worthy row and confirm // nothing — the runaway guard left them (WhereisParseBudget) or the parser could not finish — and the rows of theirs // that read kind="text". next= rides only when the guard left some: --detail=1 lifts it. Absent when every // parse-worthy blob was parsed (or mirrored HEAD's), so an answer the guard never touched is byte-identical. inline void writeWhereisUnparsed( std::FILE* out, const WhereResult& res, WhereisListing asked ) { const BranchLabelCensus& c = res.branchLabels; if( c.leftByGuard + c.failed == 0 ) { return; } const std::string next = c.leftByGuard > 0 ? whereisDetailNext( res, asked ) : std::string(); rw::emitTo( out, "", c.leftByGuard + c.failed, c.textRows, next.c_str() ); } // Whether a row prints its t= as a preview (and carries t_bytes=): this page prints previews (writeWhereisPage served the // cut page because it is shorter than the same page whole) and the line is over kWhereisTextWholeMaxBytes, so a cut drops at // least the 100 bytes between that and kWhereisTextPreviewCapBytes. A page that prints previews cuts EVERY such row. inline bool whereisTextCut( const ListedHits& listed, const WhereHit& h ) noexcept { if( !listed.textPreview ) { return false; // the page prints whole: its cut render was not shorter, or --detail / the full-text flag asked } return h.text.size() > kWhereisTextWholeMaxBytes; } // The rows of this page whose t= is a preview (whereisTextCut): the count and the trigger of the legend's // PREVIEW clause, so both ride exactly the pages that print a cut row and an answer of short lines is byte-identical. inline std::size_t whereisPageCutRows( const WhereResult& res, const ListedHits& listed, const PageWindow& hitPage ) { std::size_t cut = 0; for( std::size_t k = hitPage.begin; k < hitPage.end; ++k ) { cut += whereisTextCut( listed, res.hits[ listed.rows[ k ] ] ) ? 1 : 0; } return cut; } // The : THIS PAGE again with every line whole (review C1, rule 5: a cut is recovered by the page it cut, not by // a different answer). It names the listing the page SERVED (a default page replays the listing it chose, written out), the // page's own window (--limit / --offset when it was paged), --with-history when the fate rows rode, and --whereis-full-text, // which lifts the preview and nothing else: the parse guard and the row cap stay, so every kind= label, the row order and the // window are the page's own, and the replay is the same rows with whole lines (test/crossrefcheck.sh 5h, the X2 arms). // --detail=1 is NOT this call: it lifts the guard as well, which relabels a kind="text" call site kind="ref" (it leaves the // defs page), and it unpages a refs or all listing (25 MB for a hot name). The MCP `kind` filter is not carried // (whereisDetailNext says why). inline std::string whereisFullTextNext( const WhereResult& res, const WhereisPageView& v ) { std::string call = nextFlag( "--whereis=", res.sym ) + " --whereis-listing="; call += v.listed.attr.empty() ? std::string_view( "all" ) : v.listed.attr; if( v.pageLimit > 0 ) { call += " --limit=" + std::to_string( v.pageLimit ); } if( v.pageOffset > 0 ) { call += " --offset=" + std::to_string( v.pageOffset ); } if( res.history != nullptr ) { call += " --with-history"; } return nextAttrXml( call + " --whereis-full-text" ); } // : rows= rows of this page print a t= preview (each carries t_bytes=, the whole line's bytes), // and next= replays this page with every line whole (whereisFullTextNext). Absent when no row was cut. On a page the MCP // `kind` ref filter bounded, next= cannot carry the filter (the CLI has no whereis ref filter), so it replays OTHER rows: // uncarried= names the dropped filter and the PREVIEW clause names the recovery (MCP whereis with full_text:true), review // D1, rule 5 (test/crossrefcheck.sh 5h, the D1 arm). An unfiltered page carries no uncarried= and is byte-identical. inline void writeWhereisTextCut( std::FILE* out, const WhereResult& res, const WhereisPageView& v, std::size_t cutRows, const XmlEscaper& ex ) { if( cutRows > 0 ) { const std::string uncarried = res.filter.empty() ? std::string() : ( " uncarried=\"" + ex( res.filter ) + "\"" ); rw::emitTo( out, "", cutRows, whereisFullTextNext( res, v ).c_str(), uncarried.c_str() ); } } // The rows of the page, its remainder and the element (writeWhereisPage). inline void writeWhereisRows( std::FILE* out, const WhereResult& res, const ListedHits& listed, const PageWindow& hitPage, const std::string& headDate, const XmlEscaper& ex ) { const std::size_t listedCount = listed.rows.size(); // The contract, restated because it was false here: shown + dropped == the listed rows, ALWAYS. The // count must be taken against the CAP, not against a loop variable `shown++ >= cap` has already pushed to // cap+1 — that off-by-one under-reported every drop by exactly one row (81 hits, 60 shown, "20" dropped), // and at exactly cap+1 hits `size > shown` went false, so the element vanished and a row disappeared // unmarked. Nothing is dropped without a number is a headline claim; keep it arithmetically true. std::size_t shownCount = 0; for( std::size_t k = hitPage.begin; k < hitPage.end; ++k ) { const WhereHit& h = res.hits[ listed.rows[ k ] ]; ++shownCount; // t= is a PREVIEW (kWhereisTextPreviewCapBytes) on a page that prints previews: a longer line prints its first bytes, // cut on a UTF-8 boundary with the ellipsis, and t_bytes= says the whole line's size. A line that fits prints whole // and carries no t_bytes=. std::string preview; std::string cutAttr; std::string_view text = h.text; if( whereisTextCut( listed, h ) ) { // The cap counts the ellipsis: the kept prefix is 3 bytes shorter, so a cut t= is never longer than the cap // (nor than the line it previews, which a cut at the cap plus the ellipsis made of a 201-byte line). preview = h.text; truncateUtf8WithEllipsis( preview, kWhereisTextPreviewCapBytes - kUtf8EllipsisBytes ); ENSURES( preview.size() <= kWhereisTextPreviewCapBytes && preview.size() < h.text.size(), "a cut t= fits the cap, ellipsis included, and is shorter than the line" ); cutAttr = " t_bytes=\"" + std::to_string( h.text.size() ) + "\""; text = preview; } if( whereisRowOnHeadCommit( h, res.headSha, headDate ) ) { rw::emitTo( out, "", ex( h.ref ).c_str(), ex( h.path ).c_str(), h.line, whereisKindOf( h ), h.testLocal ? " test_local=\"1\"" : "", cutAttr.c_str(), ex( text ).c_str() ); continue; } rw::emitTo( out, "", ex( h.ref ).c_str(), h.tip.c_str(), ex( h.date ).c_str(), ex( h.path ).c_str(), h.line, whereisKindOf( h ), h.testLocal ? " test_local=\"1\"" : "", whereisSharedAttr( listed, h ).c_str(), cutAttr.c_str(), ex( text ).c_str() ); } ASSUME( shownCount == hitPage.end - hitPage.begin ); // = the LISTED rows AFTER this page, so shown + more == the listed rows from this page's offset // on. Un-paginated that is the historic "listed rows minus the 60 printed"; paged it is what the NEXT page holds // (and next_offset= on the root says where to ask for it). if( hitPage.end < listedCount ) { rw::emitTo( out, "", listedCount - hitPage.end ); } // The elided ref rows, COUNTED and one pasteable call away: count= is exact (every kind="ref" row of the hit list, // whatever page this is), and next= lists exactly them. Not a element: is this listing's own // page remainder, and a reader (or a gate) that reads " 0 ) { rw::emitTo( out, "", listed.refsElided, nextAttrXml( whereisRefsNext( res ) ) ); } } // The window one --whereis page prints over its listed rows. An explicit --limit overrides maxHits (the caller's 60-hit // display default, or SIZE_MAX under --detail); --offset skips whole rows and clamps at the end, so offset-past-the-end is // an empty page rather than an out-of-range read. maxHits can be SIZE_MAX, which pageWindow's int limit cannot carry — // clamp the "no explicit --limit" arm to the row count instead of overflowing it. inline PageWindow whereisHitWindow( std::size_t listedCount, std::size_t maxHits, int pageLimit, int pageOffset ) { const int rowCap = pageLimit > 0 ? pageLimit : ( maxHits >= listedCount ? int( listedCount ) : int( maxHits ) ); return pageWindow( listedCount, rowCap, pageOffset ); } // Contract-level defect: this verb said hits="2560" and printed 60, and // --limit/--offset were accepted and ignored, so a paging loop over it never advanced and never ended. // `pageLimit`/`pageOffset` (0 = un-paginated) window the hit list, which is already deterministically // ordered (HEAD first, then refs). The root gains shown= + capped= UNCONDITIONALLY — disclosing the silent // cap is the point, so that one attribute pair is the deliberate break in the un-paginated byte shape. // stays: it is the same fact from the other end (what this page did NOT print), and its // arithmetic contract (shown + dropped == the rows still ahead) is kept exact. // // Paging lives in its own entry point rather than as two defaulted parameters on writeWhereis() so the // un-paginated contract — the one the MCP `whereis` verb calls — keeps its exact three-argument shape. inline void writeWhereisListedPage( std::FILE* out, const WhereResult& res, std::size_t maxHits, int pageLimit, int pageOffset, const ListedHits& listed ) { std::vector esc; const XmlEscaper ex = [ & ]( std::string_view s ) { return std::string( escapeXml( s, esc ) ); }; const PageWindow hitPage = whereisHitWindow( listed.rows.size(), maxHits, pageLimit, pageOffset ); rw::emitRaw( out, "", out ); const std::string headDate = whereisHeadDate( res ); writeWhereisRoot( out, res, WhereisPageView{ listed, hitPage, pageLimit, pageOffset }, headDate, ex ); writeWhereisSelectorNotes( out, res, ex ); // The history lane, when it was asked for: what the probe did, then this symbol's own verdict. // §L10: the oracle answers "did any line carrying this name ever leave the tree", and a doc that merely // QUOTED the symbol (a stale capture file, a plan) counts as a line carrying the name — so a symbol very // much alive on HEAD could still get v="removed", citing the doc's deletion rather than the symbol's. // The fix is narrower than "suppress whenever on-head=1": a doc-only mention (a design doc quoting a name // whose code WAS deleted) also sets on-head=1 lexically, and that IS real rot the fate row must still // name — suppressing on plain on-head= would silence exactly that true positive. What distinguishes the // two is head_labels=: "index" means the PARSED index confirmed a real definition on HEAD (not just a // text hit), which is a claim strong enough to override the line-removal oracle outright. So the fate // row is printed unless the index itself already proved the symbol is defined on HEAD. writeWhereisFate( out, res, ex ); writeWhereisUnparsed( out, res, listed.asked ); writeWhereisTextCut( out, res, WhereisPageView{ listed, hitPage, pageLimit, pageOffset }, cutRows, ex ); writeWhereisRows( out, res, listed, hitPage, headDate, ex ); rw::emitRaw( out, "" ); } // THE DEFAULT, MEASURED (review B1, then D1). First the definitions: the Defs page is served whenever it SHOWS more // definitions than the All page under the row cap (a capped All page can list fewer), whatever its bytes. Only when both // pages show the same definitions do the bytes decide. The Defs page pays a fixed overhead — listing=, the element // and, under the compact legend, their readings — that eliding one or two short ref rows does not repay: on 24 of 58 // sampled symbols the Defs page was LONGER than the All page while listing fewer rows. So both pages are rendered and // Defs is served only when it is STRICTLY shorter both as written (the full legend, whose prose is the same text on // both pages, so this compares the payload) and as the compact dialect delivers it (its per-answer readings ride only // the page that carries their attribute, so a reading the All page's ref rows trigger can tip either way). A tie serves All: equal // bytes, more rows. Both pages are rendered UNPAGED over the same row cap, so the choice is a fact of the answer, not // of --limit/--offset: every page of one answer lists the same rows. A render that fails serves All (the whole answer). inline ListedHits whereisServedListing( const WhereResult& res, std::size_t maxHits, WhereisListing listing ) { if( listing != WhereisListing::ShorterOfDefsAll ) { ListedHits named = listedHits( res, listing ); named.asked = listing; // an explicit listing rides the (writeWhereisUnparsed) return named; } ListedHits defs = listedHits( res, WhereisListing::Defs ); if( defs.attr.empty() ) { return defs; // Defs already degraded to every hit (no ref row, or no def row): nothing to compare } ListedHits all = listedHits( res, WhereisListing::All ); // COMPLETE ANSWERS FIRST (orchestrator ruling 2026-10-08, review D1). Under the shared row cap the All page can list // FEWER definitions than the Defs page: HEAD's ref rows fill the cap before the branch definitions arrive. Both // lists are in hit-list order and Defs is the def subsequence, so the definitions the All page shows are a PREFIX // of the Defs page's: comparing the counts compares the sets. More definitions wins whatever its bytes; only equal // sets fall to the bytes rule below. // Counted as DEFINITION ROWS OF THE WHOLE LIST: on the Defs page a representative row stands for defRefs rows // (foldSharedBranchDefs), on the All page every row is its own. The prefix argument still holds: each def row // the All page shows is either a representative (on the Defs page no later than its own def-subsequence // position, so within the first n) or folds into one that precedes it, and every (representative, ref) pair // is counted once in defRefs — so the Defs page's weighted count is at least the All page's. const auto defsShown = [ & ]( const ListedHits& l ) { const std::size_t n = std::min( l.rows.size(), maxHits ); const bool weighted = l.attr == "defs"; std::size_t shown = 0; for( std::size_t k = 0; k < n; ++k ) { const WhereHit& h = res.hits[ l.rows[ k ] ]; shown += !h.isDef ? 0 : ( weighted ? std::max( h.defRefs, 1 ) : 1 ); } return shown; }; const std::size_t defsOnDefsPage = defsShown( defs ); const std::size_t defsOnAllPage = defsShown( all ); ENSURES( defsOnDefsPage >= defsOnAllPage, "the All page's shown definitions are a prefix of the Defs page's" ); if( defsOnDefsPage > defsOnAllPage ) { return defs; } const Rendered defsPage = renderToString( [ & ]( std::FILE* f ) { writeWhereisListedPage( f, res, maxHits, 0, 0, defs ); } ); const Rendered allPage = renderToString( [ & ]( std::FILE* f ) { writeWhereisListedPage( f, res, maxHits, 0, 0, all ); } ); const bool defsShorter = defsPage.ok && allPage.ok && defsPage.text.size() < allPage.text.size() && compactDeliveredBytesOrWritten( defsPage.text, "whereis" ) < compactDeliveredBytesOrWritten( allPage.text, "whereis" ); return defsShorter ? std::move( defs ) : std::move( all ); } // One --whereis answer page: the listing it serves (whereisServedListing), then that page, its long lines cut to previews // only when that makes THIS page shorter (review C2). The listing is chosen on the WHOLE-line pages (ListedHits::textPreview // is false there), so a preview never flips the default listing. A page holding a line over kWhereisTextWholeMaxBytes is // then rendered both ways, and the cut one is served only when it is STRICTLY shorter both as written and as the compact // dialect delivers it, the rule whereisServedListing applies to its two pages: its , its rows' t_bytes= and the // PREVIEW clause or the compact readings can cost more than cutting a line just past the threshold saves (a 395-byte line // saved 92 B and its disclosure cost 250). So a served page is byte-identical to the same page whole, or strictly shorter // in both postures. A tie, or a render that fails, serves the page whole (the whole answer). inline void writeWhereisPage( std::FILE* out, const WhereResult& res, std::size_t maxHits, int pageLimit, int pageOffset, WhereisListing listing ) { // The LISTED rows (WhereisListing): indices into res.hits, in the hit list's own order, so a listing is a // subsequence of the whole list and its rows print byte-identically to the same rows of listing=all. const ListedHits listed = whereisServedListing( res, maxHits, listing ); ListedHits previewed = listed; previewed.textPreview = !res.textWhole; if( whereisPageCutRows( res, previewed, whereisHitWindow( listed.rows.size(), maxHits, pageLimit, pageOffset ) ) == 0 ) { writeWhereisListedPage( out, res, maxHits, pageLimit, pageOffset, listed ); return; } const Rendered cutPage = renderToString( [ & ]( std::FILE* f ) { writeWhereisListedPage( f, res, maxHits, pageLimit, pageOffset, previewed ); } ); const Rendered wholePage = renderToString( [ & ]( std::FILE* f ) { writeWhereisListedPage( f, res, maxHits, pageLimit, pageOffset, listed ); } ); const bool cutShorter = cutPage.ok && wholePage.ok && cutPage.text.size() < wholePage.text.size() && compactDeliveredBytesOrWritten( cutPage.text, "whereis" ) < compactDeliveredBytesOrWritten( wholePage.text, "whereis" ); if( cutShorter || wholePage.ok ) { rw::emitRaw( out, ( cutShorter ? cutPage.text : wholePage.text ).c_str() ); return; } writeWhereisListedPage( out, res, maxHits, pageLimit, pageOffset, listed ); } // The un-paginated form — unchanged contract, for callers that want the whole (capped) listing. inline void writeWhereis( std::FILE* out, const WhereResult& res, std::size_t maxHits ) { writeWhereisPage( out, res, maxHits, 0, 0, WhereisListing::All ); } }} // namespace rw::crossref