#pragma once // serialize.h — minified, escaped XML serialization. Streamed through a 64 KB // buffer (no whole-document string), terse schema, every name/path XML-escaped. #include "infra/emit.h" // rw::formatTo — snprintf's shape kept (stack buffer, snprintf's return) #include "infra/os.h" // rw::os::open_memstream — the est_tokens charge buffers #include // std::format_to_n — the appendf lambdas append through it directly #include "model.h" #include "memguard.h" // #350: memguard::limitSpelling — memory_limit= is spelled as the --max-memory value that raises it #include "extentsuspect.h" // extent honesty: extent_suspect= reason spellings (extent::extentSuspectReasons) #include "nextverb.h" // P3 (L7): next= on the top-ranked row #include "countfloor.h" // count-floor: implementorFloors — the implementors_floor= evidence #include "arch.h" // P3: builtinLayer() — the file-node layer= tag #include "memberowner.h" // QUALNAME-1: MemberOwners — the row folds a member's sc= into n="Owner.name" #include "graph.h" // H6/F2: definitionCountOfName — the ONE resolver behind --lego's defs= single-pick disclosure #include "graphlegend.h" // R-E fix (2026-08-19): rw::rootRelPathsLegend — the ONE root= definition #include "lintrules.h" // §P9.4: langOfPath / dependencyCapable — packDeps' dep_files= denominator #include "resolve.h" // S6-C: canonicalId() — the `id=` canonical symbol string (shared with the resolver) #include "redact.h" // deterministic secret redaction of emitted body content (opt-out --no-redact) #include "docparse.h" // docparse::detail::readWholeFile — --pack-top-n reads each served file through the one whole-file reader #include "infra/sortutil.h" // numeric-key radix helpers for rank/file score order #include "infra/jsonesc.h" // F9: jsonesc::utf8SeqLen — the canonical UTF-8-sequence-length core (was duplicated here) #include "infra/strkern.h" // S5: appendCleanRun — the run-copy skip that replaces escapeXml's per-byte switch #include "notes.h" // L3: field-notes NoteIndex — the retrieval-time surfacing lookup (INERT when null) #include "pageview.h" // §P8: pageWindow / pageDisclosure — the shared --limit/--offset contract (packDeps) #include "sarif.h" // R-E (2026-08-17): rootRelativeUri/rootPrefixOf — the same root= single-root-only // strip --grep's emitGrepReport uses, reused here so the two verbs cannot diverge #include "prconverge.h" // W2-F: RankDisclosure + the pr_iters= / pr_converged= spellings (CLI and MCP share them) #include "historyread.h" // K51: history_unread= on every root that prints churn=/amp= #include "gitmine.h" // F3 (H2H-Graft): RecentFile — the map's rows are the churn-decay miner's own product #include "lexical.h" // MAP-STEM: rw::subtokens — the one subtoken rule (lexLowerByte rides with it) #include "mapinventory.h" // D7: the map scope's tier — computeMapInventory, kMapInventoryLegend #include #include // std::iota — codeFirstKeep #include #include #include // §H7 degrade seam: std::getenv for the non-release fault switch #include #include #include #include // back_inserter (reorderDocsAfterCode) #include #include namespace rw { // XML 1.0 forbids the C0 control set even ESCAPED (0x00-0x08, 0x0B, 0x0C, 0x0E-0x1F — only \t \n \r // are legal Chars): a stray form-feed in an otherwise-legal source file would break the G4 xmllint // gate. Substitute a space — deterministic, applied identically in escapeXml and the CDATA copies. inline char xmlSafeByte( char c ) noexcept { return ( static_cast( c ) < 0x20 && c != '\t' && c != '\n' && c != '\r' ) ? ' ' : c; } // ── §B12.7 + CA4 verifier F-MED-1 — THE C0 DIALECT DIVERGENCE, AND ITS TELL ───────────────────────────── // The XML and JSON dialects do not agree about C0 controls and invalid UTF-8, and they CANNOT: XML 1.0 // forbids the C0 set even escaped (only \t \n \r are legal Chars) so xmlSafeByte must substitute a space, // and an invalid sequence must become '?', while JSON has \u00XX for every control byte. jsonesc.h's own // header records the decision from the other side: the JSON escaper is the FAITHFUL one and stays faithful, // because normalizing there would make the two dialects agree by making BOTH lossy. // // What was wrong was not the divergence — it was that the lossy side said nothing. ``, whose // entire §B1.7 point is being the VERBATIM copy of the user's task, silently was not; and `EmittedBody::text` // is recorded at packBodies' push_back BEFORE appendCdataSafe, so a def holding ESC + Latin-1 reached XML at // 140 B scrubbed and JSON at 148 B raw with nothing anywhere saying so. // // So the lossy side now DISCLOSES: `scrubbed="1"` on the XML surface that lost bytes, and its JSON twin // beside the same field — the JSON is byte-faithful and does not need the warning for itself, but a consumer // diffing the two dialects is owed the fact from either one. THE MARKERS ARE A CLOSED SET, and each XML one // has exactly one JSON counterpart: `task_scrubbed=`/`route_scrubbed=` on the root (ctxRootOpen) twin // `"task_xml_scrubbed"`/`"route_xml_scrubbed"` (ctxRootJsonScrubKeys, right below it — one function, so a // call site cannot emit the field and skip the question), and `` twins `"xml_scrubbed"` on // the JSON body object. test/bodydialectcheck.sh arm (H) sweeps that table in BOTH directions rather than // the one marker that already worked. This predicate is the single decision behind all six: true iff // escapeXml/appendCdataSafe would CHANGE bytes (a C0 other than \t\n\r, or an invalid UTF-8 sequence). // Entity-escaping (& < > " ') is NOT lossy — it round-trips exactly — and must not set the flag, or every // document on earth would wear it. inline bool xmlScrubIsLossy( std::string_view s ) noexcept { const char* d = s.data(); const std::size_t n = s.size(); for( std::size_t i = 0; i < n; ) { const unsigned char c = static_cast( d[i] ); if( c < 0x20 && c != '\t' && c != '\n' && c != '\r' ) { return true; // xmlSafeByte -> ' ' } if( c < 0x80 ) { ++i; continue; } const int len = jsonesc::utf8SeqLen( d, i, n ); if( len == 0 ) { return true; // escapeXml -> '?', appendCdataSafe -> '?' } i += std::size_t( len ); } return false; } // W3FIX M2 — the three LEGAL control chars (\t \n \r) still cannot be written LITERALLY by escapeXml, for two // independent reasons that both bite the §B1.7 verbatim task attribute: // (a) G4 forbids a '\n' anywhere outside CDATA, and `--for=$'a\nb'` put one straight into `` // — a two-line document out of a serializer whose contract is one line; // (b) XML ATTRIBUTE-VALUE NORMALIZATION (XML 1.0 §3.3.3) replaces a literal \t/\n/\r in an attribute with a // SPACE at parse time, so even where the raw byte was tolerated the parser handed back a DIFFERENT // string than the user typed — §B1.7's whole point is that the attribute is the verbatim copy. // A numeric character reference survives normalization exactly (the parser un-escapes it AFTER normalizing), // so ` ` round-trips to '\n' while emitting no literal newline. Emitted for element text too: there a // char reference parses back to the identical character, so one rule serves both seams and no call site has // to know which kind of node it is writing into. Near-golden-neutral, not fully: a plain --whereis over a // .tsv file (bench/recalleval labels) legitimately carries real tabs into t= attributes, where the old code // emitted a literal tab an XML parser silently normalized to a space — the reference there is a FIX, not a // hostile-input path (seam-verifier NIT, 2026-07-29). Everywhere else on this repo the references appear // only on hostile input. inline const char* xmlControlCharRef( char c ) noexcept { switch( c ) { case '\t': return " "; case '\n': return " "; case '\r': return " "; default: return nullptr; } } // F9: utf8SeqLen used to be duplicated here (byte-identical logic to jsonesc.h's copy, just re-commented) — // now forwards to the one canonical core in jsonesc.h (zero project includes, so pulling it in here adds no // cycle risk). Kept as a `using` so every call site below (escapeXml, appendCdataSafe) is unchanged. using jsonesc::utf8SeqLen; // escape & < > " ' (ampersand FIRST) into `out`; returns a view into it. Reused per call. Byte < 0x20 is // scrubbed to a space (xmlSafeByte) or, for the three legal control chars, written as a numeric character // reference (xmlControlCharRef — see M2 above: G4 + attribute-value normalization); an invalid UTF-8 sequence // (A4-F20) is scrubbed to '?' so the emitted name/path/doc-comment/sig text is always well-formed XML AND // valid UTF-8 regardless of source bytes. // S5 — THE BYTE SET IS THE CONTRACT. Everything below that is NOT in this set is copied through // unchanged by the switch's `default:` arm, so the run loop may memcpy it in bulk without looking at it; // everything that IS in the set still goes through the SAME switch, one byte at a time, unchanged. The // set is therefore derivable from the switch and must be re-derived with it: the five entity bytes, the // whole C0 range (\t \n \r become character references, every other C0 is scrubbed to a space by // xmlSafeByte), and every byte >= 0x80 (utf8SeqLen decides whether the sequence is copied or scrubbed // to '?'). 0x7F is deliberately absent — xmlSafeByte passes DEL through, so it is a clean-run byte. // test/emitescapecheck.sh's MUT arm exists because a set one member short is otherwise silent. inline constexpr strkern::Byteset256 kXmlEscapeByteset = [] { strkern::Byteset256 set; set.addRange( 0x00, 0x1F ); set.add( '&' ); set.add( '<' ); set.add( '>' ); set.add( '"' ); set.add( '\'' ); set.addRange( 0x80, 0xFF ); return set; }(); inline std::string_view escapeXml( std::string_view s, std::vector& out ) { out.clear(); out.reserve( s.size() + 16 ); const auto put = [ & ]( const char* lit ) { while( *lit ) { out.push_back( *lit++ ); } }; const char* d = s.data(); const std::size_t n = s.size(); // Init and increment skip to the next byte the switch actually has an opinion about, copying // everything before it in one insert. On ordinary source text — names, paths, signatures, // doc-comments — that run is the whole string: one scan and one memcpy for the whole call. for( std::size_t i = strkern::appendCleanRun( d, 0, n, kXmlEscapeByteset, out ); i < n; i = strkern::appendCleanRun( d, i, n, kXmlEscapeByteset, out ) ) { const char c = d[i]; switch( c ) { case '&': put( "&" ); ++i; break; case '<': put( "<" ); ++i; break; case '>': put( ">" ); ++i; break; case '"': put( """ ); ++i; break; case '\'': put( "'" ); ++i; break; case '\t': case '\n': case '\r': put( xmlControlCharRef( c ) ); ++i; break; // M2: verbatim round-trip, no literal control byte default: if( static_cast( c ) < 0x80 ) { out.push_back( xmlSafeByte( c ) ); ++i; } else if( const int len = utf8SeqLen( d, i, n ); len == 0 ) { out.push_back( '?' ); ++i; } // scrub invalid UTF-8 else { for( int k = 0; k < len; ++k ) { out.push_back( d[i + k] ); } i += std::size_t( len ); } } } return std::string_view( out.data(), out.size() ); } // M12 (capture-audit-2026-09-04, lane L9): under multi-root, every one of the graph-count verbs // (--callers/--callees/--uses) drops root= (correctly — there IS no single root=) but discloses NOTHING in // its place, so a reader of `ripwire src test --callers=SYM` has no way to learn what the `