#pragma once #if !defined( RIPWIRE_INGEST_TU ) #error "ingest_astquery.h is a SECTION of src/ingest.cpp's translation unit - include it only from ingest.cpp (see the ingest-family split note there)" #endif #include "infra/emit.h" // rw::emitTo / emitRaw / formatTo — THE emitter and its siblings #include // %.*s (precision, pointer) collapses to one view // ingest_astquery.h — the narrow-parse query services, moved VERBATIM from ingest.cpp in the // 2026-08-29 split: the --match/--lint shared AST-query pass (capture text + predicate evaluation, // the per-file newline-offset index, grouped query compilation, the did-you-mean node-kind hints, // astQueryGrouped/astQuery and the grammar-applicability disclosure), the pattern surface's grammar // census (supportedPatternGrammars/eligiblePatternFiles), the R-H span tiers (spanTiersOfFiles), // the unreachable-code walk behind the built-in lint rule, and collectGatedLocalNames. These are // rw-level services other TUs reach through ingest.h — they sat AFTER ingest() outside the unnamed // namespace, so unlike the other sections this one reopens `namespace rw` alone and is included at // the very END of ingest.cpp, exactly where its content sat. Same RIPWIRE_INGEST_TU guard: any // includer other than ingest.cpp is a compile error. namespace rw { // ---- the compiled #match? / #not-match? regexes of ONE query, resolved when the query is ---------- // One compiled regex per ( query, string id ), built when the query is compiled instead of once per MATCH. // // WHY. `passesPredicates` ran `std::regex_search( lhs, std::regex( rhs ) )` per match, per file, and `rhs` // is a CONSTANT owned by the TSQuery — `ts_query_string_value_for_id` hands back the same bytes every time. // So the most expensive constructor in the standard library was answering a question whose answer never // changes. A 1 ms `sample` of `--lint` over the go corpus (44,376 busy leaf samples) put the // `std::basic_regex` subtree at 13.52% of busy CPU with 100% of it owned by passesPredicates, and regex // CONSTRUCTION — `__parse_ERE_dupl_symbol`, `__parse_atom`, the `__state` vector's growth — at ~46.7% of // every malloc leaf in the run. Whole-query regex counts are single digits; the table is tiny. // // KEYED BY STRING ID, NOT BY TEXT. `value_id` indexes the query's own string table, so // `ts_query_string_count` sizes an exact O(1) lookup and no hashing, no comparison and no allocation is // left on the per-match path. -1 = this string is not a precompilable regex argument; -2 = it IS one and // the guard REFUSED it (src/regexguard.h: malformed, non-portable, or catastrophic backtracking). // // THE REFUSAL IS PART OF THE CONTRACT. A malformed pattern used to throw out of the per-match constructor, // get caught, and leave `ok = true` — i.e. filter NOTHING. Precompiling moved that throw from the match to // the build, and -2 reproduces it for the BUILT-IN rule packs, whose patterns are constants of this binary. // A USER's pattern (--match, --lint-rules) is different: filtering nothing for it is a row the predicate never // decided, printed as if it had. So every refusal is also recorded in `refusals`, which the caller's // AstQueryGroup::regexRefusedOut collects and --match / --lint-rules refuse by name, like --regex. // test/astqueryregexcheck.sh arm C4, test/regexguardcheck.sh arm (a). // // A CAPTURE-TYPED ARGUMENT IS NEVER IN HERE. `(#match? @a @b)`'s pattern is the matched node's own text — // per match by construction — and stays dynamic (arm D). // // THREADING. Built single-threaded with the query, then SHARED by every worker that evaluates predicates. // Concurrent use of `const` standard-library operations is data-race-free ([res.on.data.races]), and // `std::regex_search`'s state lives in the algorithm, not in the pattern; arm F is the empirical half. struct PredicateRegexTable { std::vector slotOfStringId; // per query-string id: -1 not precompiled, -2 refused, >=0 index into res std::vector res; std::vector refusals; // "'PATTERN' refused: REASON", one per refused string id, build order }; // Walk every predicate of every pattern once and compile the constant #match?/#not-match? arguments. // It walks with forEachPredicate, the same walk passesPredicates uses below — the two must agree about which // argument is the pattern, and a second spelling of that rule is how a filter quietly changes meaning. inline PredicateRegexTable buildPredicateRegexTable( const TSQuery* q ) { PredicateRegexTable table; if( q == nullptr ) { return table; } table.slotOfStringId.assign( ts_query_string_count( q ), -1 ); const std::uint32_t patterns = ts_query_pattern_count( q ); for( std::uint32_t patternIndex = 0; patternIndex < patterns; ++patternIndex ) { forEachPredicate( q, patternIndex, [ & ]( std::span group ) { const std::string_view op = predicateOperator( q, group ); if( ( op != "match?" && op != "not-match?" ) || group.size() < 3 ) { return true; } const TSQueryPredicateStep& arg = group[2]; if( arg.type != TSQueryPredicateStepTypeString || arg.value_id >= table.slotOfStringId.size() || table.slotOfStringId[ arg.value_id ] != -1 ) { return true; // capture-typed, out of range, or already decided } const std::string pattern( queryString( q, arg.value_id ) ); RegexCompile compiled = compileGuardedRegex( pattern, kRegexEcmaScript ); // std::regex's default flags, as before if( compiled.refusal ) { table.slotOfStringId[ arg.value_id ] = -2; // refused — the built-in packs keep the old per-match catch table.refusals.push_back( "'" + pattern + "' refused: " + *compiled.refusal ); return true; } table.res.push_back( std::move( compiled.regex ) ); table.slotOfStringId[ arg.value_id ] = static_cast( table.res.size() - 1 ); return true; } ); } return table; } // ONE #match? / #not-match? predicate, decided. Its own function rather than an inline block because the // three states below cost passesPredicates +19 cognitive complexity inline, on a function already well over // the ccx bar — and because the states are the whole contract of the precompile and deserve to be read in // one place: // * slot >= 0 — a precompiled constant pattern. The common case, and the point of the table. // * slot == -2 — a constant pattern the guard REFUSED. Filter NOTHING, which is exactly what the old // per-match `catch( ... ) { ok = true; }` did when the same construction threw at the same pattern (a user // group never gets this far: its refusal was collected at compile time and the verb refused). // * slot == -1 — no constant to precompile (a Capture-typed argument, whose pattern is per-match text). // Compile it here, per match, as before — through the guard. // A predicate the guard cannot DECIDE filters nothing, as it always did, and says WHY: the per-match text was refused // by the screen, the per-match text does not compile, the engine abandoned the match, or the captured text was too // long to hand the engine at all (F-B4). The caller records that cause for a user's group // (noteUndecidedPredicate), so the verb's refusal names the right fix. `rhs` is the pattern text in every state (the // constant, or the per-match capture); it is only COMPILED in the last one. `lhs` (the captured node's own text) can // be arbitrarily large — kCallerStackBytesFloor bounds it the same way skillscan.h and --arch bound theirs. enum class MatchPredicateOutcome : std::uint8_t { Pass, Fail, TextScreened, TextUncompilable, Abandoned, Skipped }; inline MatchPredicateOutcome evalMatchPredicate( bool negated, const std::string& lhs, const std::string& rhs, const PredicateRegexTable& rx, const TSQueryPredicateStep& arg ) { const bool constant = ( arg.type == TSQueryPredicateStepTypeString && arg.value_id < rx.slotOfStringId.size() ); const std::int32_t slot = constant ? rx.slotOfStringId[ arg.value_id ] : -1; if( slot == -2 ) { return MatchPredicateOutcome::Pass; // the pattern did not compile ⇒ this predicate filters nothing } RegexVerdict verdict = RegexVerdict::Miss; if( slot >= 0 ) { verdict = rx.res[ std::size_t( slot ) ].search( lhs, kCallerStackBytesFloor ); } else { const RegexCompile perMatch = compileGuardedRegex( rhs, kRegexEcmaScript ); // a Capture-typed argument: this match's own text if( perMatch.refusal ) { return perMatch.isScreened ? MatchPredicateOutcome::TextScreened : MatchPredicateOutcome::TextUncompilable; } verdict = perMatch.regex.search( lhs, kCallerStackBytesFloor ); } if( verdict == RegexVerdict::Exhausted ) { return MatchPredicateOutcome::Abandoned; } if( verdict == RegexVerdict::Skipped ) { return MatchPredicateOutcome::Skipped; } return ( ( verdict == RegexVerdict::Hit ) != negated ) ? MatchPredicateOutcome::Pass : MatchPredicateOutcome::Fail; } inline std::uint32_t lineAtByte( const std::vector& nlOffsets, std::uint32_t bytePos ) noexcept; // defined below // Where a predicate is being evaluated, for the undecided path only: the user group's sink (null for the built-in // packs, which record nothing), the file, and its newline index for the reported line. struct PredicateSite { AstRegexUndecided* sink; std::uint32_t fileId; const std::vector& nlOffsets; }; // Record one undecided evaluation against a user's group. The site's position is the match's first capture; the // reason is re-derived here, on the rare path, rather than carried through every evaluation that decided. inline void noteUndecidedPredicate( const PredicateSite& site, const TSQueryMatch& m, MatchPredicateOutcome outcome, const std::string& pattern ) { if( site.sink == nullptr ) { return; } const std::uint32_t byte = ( m.capture_count != 0 ) ? ts_node_start_byte( m.captures[0].node ) : 0; if( outcome == MatchPredicateOutcome::Abandoned ) { site.sink->note( AstRegexUndecidedCause::Abandoned, site.fileId, byte, lineAtByte( site.nlOffsets, byte ), pattern, kRegexAbandonedReason ); return; } if( outcome == MatchPredicateOutcome::Skipped ) { site.sink->note( AstRegexUndecidedCause::Skipped, site.fileId, byte, lineAtByte( site.nlOffsets, byte ), pattern, kRegexOversizeReason ); return; } const RegexCompile refused = compileGuardedRegex( pattern, kRegexEcmaScript ); site.sink->note( outcome == MatchPredicateOutcome::TextScreened ? AstRegexUndecidedCause::TextScreened : AstRegexUndecidedCause::TextUncompilable, site.fileId, byte, lineAtByte( site.nlOffsets, byte ), pattern, refused.refusal.value_or( std::string() ) ); } // evaluate a pattern's query predicates against a match — the string predicates #eq? / #not-eq? / #any-of? / // #not-any-of? (src/infra/tspredicates.h, shared with the tags pass, #355) and #match? / #not-match? (ECMAScript // regex, --match/--lint only). ts_query never applies these itself; without this, #eq? is a no-op. // The step-group walk is forEachPredicate — the same one buildPredicateRegexTable walks, so the two agree about which // steps a predicate consumes by construction. The operator name is read in TWO statements (queryString, whose note // records the gcc-on-x86-64 evaluation-order bug that once made every predicate here a silent no-op). inline bool passesPredicates( const TSQuery* q, const PredicateRegexTable& rx, const TSQueryMatch& m, std::string_view src, const PredicateSite& site ) { bool ok = true; forEachPredicate( q, m.pattern_index, [ & ]( std::span pr ) { const StringPredicate decided = evalStringPredicate( q, pr, m, src ); if( decided != StringPredicate::NotString ) { ok = ( decided == StringPredicate::Pass ); return ok; } const std::string_view op = predicateOperator( q, pr ); if( ( op != "match?" && op != "not-match?" ) || pr.size() < 3 ) { return true; // unknown operator or short group ⇒ don't filter } std::string lhs, rhs; const auto argText = [ & ]( const TSQueryPredicateStep& s, std::string& out ) -> bool { if( s.type == TSQueryPredicateStepTypeCapture ) { std::string_view view; // the capture's text: captureView, the same reader the string predicates use if( !captureView( m, s.value_id, src, view ) ) { return false; } out = std::string( view ); return true; } if( s.type == TSQueryPredicateStepTypeString ) { out = std::string( queryString( q, s.value_id ) ); return true; } return false; }; if( !argText( pr[1], lhs ) || !argText( pr[2], rhs ) ) { return true; // can't evaluate → don't filter } const MatchPredicateOutcome outcome = evalMatchPredicate( op == "not-match?", lhs, rhs, rx, pr[2] ); ok = ( outcome != MatchPredicateOutcome::Fail ); // undecided ⇒ filters nothing, as it always did if( ok && outcome != MatchPredicateOutcome::Pass ) { noteUndecidedPredicate( site, m, outcome, rhs ); } return ok; } ); return ok; } // ---- per-file newline-offset index → O(log n) 1-based line lookup (A4 perf) ---- // Replaces the per-capture "scan [0,startByte) counting '\n'" (byte-0 rescan, O(startByte) EACH match) // with one O(fileBytes) pass + a binary search per capture. Byte-identical result: // line(b) = 1 + (# of '\n' at offset < b) = 1 + lower_bound(offsets, b) position. // The pass itself rides rw::findByte (src/infra/fixedStr.h) — a NEON/SSE2 find-'\n' kernel that is EXACT, so the // offsets are bit-identical to the byte-at-a-time loop this replaced and determinism is untouched. '\r' is // not a line break here and never was. bench/bench_newline_ab.cpp races the two against libc memchr and // asserts all three agree byte-for-byte before it reports a number; the kernel won at ~1.4x over memchr. inline std::vector buildNewlineOffsets( std::string_view src ) { PROFILE_SCOPE_DESCRIBE( "strings: buildNewlineOffsets (byte scan for newline)" ); std::vector off; const char* const begin = src.data(); const char* first = begin; const char* const last = begin + src.size(); while( first < last ) { first = rw::findByte( first, last, '\n' ); // NEON/SSE2 kernel, exact — same answer as the byte loop it replaced if( first == last ) { break; } off.push_back( std::uint32_t( first - begin ) ); ++first; } return off; } inline std::uint32_t lineAtByte( const std::vector& nlOffsets, std::uint32_t bytePos ) noexcept { return std::uint32_t( 1 + ( std::lower_bound( nlOffsets.begin(), nlOffsets.end(), bytePos ) - nlOffsets.begin() ) ); } // One AstMatch row from one [startByte,endByte) span of one file. THE single place a match's snippet is // cut: the 120-byte cap, the UTF-8 continuation-byte back-off that stops the cut splitting a codepoint, // and the whitespace scrub that keeps the row on one line. Shared by the query walk's capture emitter and // by the pattern walk — two callers producing byte-different snippets for the same span would be a // difference no reader could explain, and a second copy of this is exactly the new-clone-of-reused-helper // shape --quality-delta flags. inline AstMatch makeAstMatch( std::uint32_t fileId, std::string_view bytes, const std::vector& nlOffsets, std::uint32_t a, std::uint32_t b, std::string tag ) { PROFILE_SCOPE_DESCRIBE( "strings: capture text substr + whitespace scrub" ); std::size_t cutLen = std::min( b - a, 120u ); if( cutLen < b - a ) { while( cutLen > 0 && ( static_cast( bytes[a + cutLen] ) & 0xC0 ) == 0x80 ) { --cutLen; } } std::string text( bytes.substr( a, cutLen ) ); for( char& ch : text ) { if( ch == '\n' || ch == '\r' || ch == '\t' ) { ch = ' '; } } return AstMatch{ fileId, a, b, lineAtByte( nlOffsets, a ), std::move( tag ), std::move( text ) }; } // ---- shared AST-query pass (--match / --lint) ---- // One compiled query, plus the group it answers for. Grouping is a property of the QUERY, not of the walk: // a worker executes every query a file's grammar has and files the captures into that query's own bucket. struct GroupedQuery { TSQuery* query = nullptr; std::string tag; std::uint32_t groupIndex = 0; PredicateRegexTable rx; // this query's compiled #match? patterns — built with the query, read per match }; // Every query one grammar has to answer, in BOTH shapes. `perSpec` is one compiled query per spec, the // literal thing each spec asked for. `combined` is all of those specs' patterns compiled into ONE query. // // The difference is the number of TREE WALKS. ts_query_cursor_exec walks the subtree once per query, so // forty-odd single-pattern queries walked every C++ file forty-odd times to ask forty-odd independent // questions of the same nodes -- and the profile put 87% of --lint's whole cost inside that loop. A query // holding forty patterns walks once and runs them off one shared automaton; each match's `pattern_index` // says which spec answered, so nothing about the RESULT changes. // // Both shapes are kept: `combined` is what the workers run, `perSpec` is the fallback if the concatenated // source does not compile (or does not report the pattern count its parts add up to), and it also owns the // tag and group each pattern belongs to. struct GrammarQueries { std::vector perSpec; TSQuery* combined = nullptr; // nullptr = degraded to one tree walk per spec PredicateRegexTable combinedRx; // the combined query's own compiled #match? patterns std::vector patternOwner; // combined pattern index -> index into perSpec }; // Compile every group's specs for ONE grammar, in both shapes GrammarQueries holds. Called once per // grammar the corpus can reach, from its own thread: the language tables it reads are immutable and each // call touches nothing but its own result. GrammarQueries compileGrammarQueries( const TSLanguage* g, const std::vector& groups ) { GrammarQueries gqs; std::string combinedSrc; for( std::size_t groupIndex = 0; groupIndex < groups.size(); ++groupIndex ) { if( groups[groupIndex].specs == nullptr ) { continue; } for( const AstQuerySpec& spec : *groups[groupIndex].specs ) { if( astQueryNestsTooDeep( spec.query ) ) { continue; // never handed to the compiler — reported once, by name, where uncompiled specs are } std::uint32_t off = 0; TSQueryError err = TSQueryErrorNone; TSQuery* q = ts_query_new( g, spec.query.data(), static_cast( spec.query.size() ), &off, &err ); if( q == nullptr ) { continue; // this spec is not valid for this grammar -- a C++ query simply does not fire on Python } const std::uint32_t patterns = ts_query_pattern_count( q ); for( std::uint32_t patternIndex = 0; patternIndex < patterns; ++patternIndex ) { gqs.patternOwner.push_back( static_cast( gqs.perSpec.size() ) ); } gqs.perSpec.push_back( { q, spec.tag, static_cast( groupIndex ), buildPredicateRegexTable( q ) } ); combinedSrc.append( spec.query ); combinedSrc.push_back( '\n' ); // a spec may end in a `;` line comment; never let it swallow the next } } if( !gqs.perSpec.empty() ) { std::uint32_t off = 0; TSQueryError err = TSQueryErrorNone; TSQuery* comb = ts_query_new( g, combinedSrc.data(), static_cast( combinedSrc.size() ), &off, &err ); if( comb != nullptr && ts_query_pattern_count( comb ) == static_cast( gqs.patternOwner.size() ) ) { gqs.combined = comb; gqs.combinedRx = buildPredicateRegexTable( comb ); } else { // Patterns that each compile alone are not GUARANTEED to compile together, and a pattern count that // does not add up would misattribute every tag. Either way the per-spec walks are still correct -- // only slower -- so this degrades rather than fails. if( comb != nullptr ) { ts_query_delete( comb ); } DISCLOSE( Diagnostics::answerUnchanged, "the per-spec walks emit the same captures, total-key sorted: only slower", "astQuery: combined per-grammar query did not compile - falling back to one tree walk per spec" ); } } return gqs; } // Run one grammar's compiled specs over one file's tree and file every capture under its spec's group. The query half of // astQueryGrouped's per-file work, shared with the parse worker's construct pass (collectConstructHits below) so the // two cannot capture differently from the same tree. inline void execGrammarQueries( TSQueryCursor* cur, const GrammarQueries& qs, TSNode root, std::uint32_t fileId, std::string_view bytes, const std::vector& nlOffsets, const std::vector& groups, std::vector>& perGroupHits ) { // Every capture of one match, filed under the spec that owns the pattern. Shared by both // execution shapes below so the ONE walk and the fallback walks cannot drift apart. const auto emitCaptures = [ & ]( const TSQueryMatch& m, const GroupedQuery& owner ) { for( std::uint16_t c = 0; c < m.capture_count; ++c ) { const TSNode n = m.captures[c].node; const std::uint32_t a = ts_node_start_byte( n ), b = ts_node_end_byte( n ); if( a >= b || b > bytes.size() ) { continue; } // The snippet cut (120 bytes, UTF-8-safe, whitespace-scrubbed) lives in // makeAstMatch — the pattern walk emits rows through the same helper. perGroupHits[owner.groupIndex].push_back( makeAstMatch( fileId, bytes, nlOffsets, a, b, owner.tag ) ); } }; if( qs.combined != nullptr ) { TSQuery* const q = qs.combined; // ONE walk for every spec this grammar has ts_query_cursor_exec( cur, q, root ); TSQueryMatch m; while( ts_query_cursor_next_match( cur, &m ) ) { if( m.pattern_index >= qs.patternOwner.size() ) { continue; // unreachable: patternOwner was verified against ts_query_pattern_count } const GroupedQuery& owner = qs.perSpec[ qs.patternOwner[ m.pattern_index ] ]; if( !passesPredicates( q, qs.combinedRx, m, bytes, { groups[ owner.groupIndex ].regexUndecidedOut, fileId, nlOffsets } ) ) { continue; // honour #eq? / #match? etc. — predicates are per PATTERN, so this reads the right ones } emitCaptures( m, owner ); } return; } for( const GroupedQuery& gq : qs.perSpec ) { ts_query_cursor_exec( cur, gq.query, root ); TSQueryMatch m; while( ts_query_cursor_next_match( cur, &m ) ) { if( !passesPredicates( gq.query, gq.rx, m, bytes, { groups[ gq.groupIndex ].regexUndecidedOut, fileId, nlOffsets } ) ) { continue; // honour #eq? / #match? etc. } emitCaptures( m, gq ); } } } // octocode F3: the "compiled for no grammar" refusal (below) used to hand back the query verbatim and // nothing else, so `(call_expresion)` — one deleted 's' away from the real `call_expression` — got no // nearer a fix than staring at the S-expression. This is the hint: pull every token that LOOKS like a // node-kind reference out of the failed query text, edit-distance each against the UNION of every linked // grammar's own node-kind vocabulary (ts_language_symbol_count/name — the grammar's own runtime-exposed // truth, never a hand-maintained list this tree would have to keep in sync), and report the closest. // // Candidate extraction is a plain text scan, not a second ts_query_new attempt: the query already failed to // compile against EVERY linked grammar, so there is no successful parse to introspect. A node-kind token is // an identifier immediately after `(` — `(call_expression ...)`, `(binary_expression left: (identifier))` — // outside a quoted anonymous-token literal (`"+"`) and outside a `;` line comment; the bare wildcard `_` // ("any node") is excluded, and predicates (`#eq?`) / field negation (`!decorator`) never start with an // identifier char so they are excluded by construction, not by a special case. A FIELD name (`left:`) is // never captured either — it precedes a `:`, never a `(`. // // Vocabulary: TSSymbolTypeRegular and TSSymbolTypeSupertype only — the two symbol kinds a query ever names // bare. TSSymbolTypeAnonymous is a literal token (written as a quoted string, never a bare identifier) and // TSSymbolTypeAuxiliary is grammar-internal machinery; suggesting either as "the kind you meant" would be // a hint the reader could not type back into a query. Several extensions share one grammar object // (.cpp/.cc/.cxx -> tree_sitter_cpp); each distinct TSLanguage* is walked once. When more than one grammar // defines the same kind name, kLangTable's fixed row order decides which grammar the hint names — a pure // function of the table, independent of HashMap iteration order. // // Deterministic across candidate tokens too: smaller edit distance wins, a tie breaks on the lexicographically // smaller resulting KIND NAME — never on which candidate token or which grammar was tried first. Bandwidth // cutoff matches didyoumean.h's own kMaxEditDistance (3): beyond that a "hint" is noise, not help, and the // hint stays empty (the same honest "no plausible near-miss" contract as didYouMean()). struct NodeKindHint { std::string kind; std::string grammar; }; // both empty ⇒ no candidate was close enough static std::vector extractCandidateNodeKinds( std::string_view query ) { std::vector out; bool inString = false; for( std::size_t i = 0; i < query.size(); ++i ) { const char c = query[i]; if( inString ) { if( c == '\\' ) { ++i; continue; } // escape: skip the escaped byte, same rule astQueryShape uses if( c == '"' ) { inString = false; } continue; } if( c == '"' ) { inString = true; continue; } if( c == ';' ) // line comment: everything to end-of-line is inert { while( i + 1 < query.size() && query[i + 1] != '\n' ) { ++i; } continue; } if( c != '(' ) { continue; } std::size_t j = i + 1; while( j < query.size() && std::isspace( static_cast( query[j] ) ) ) { ++j; } if( j >= query.size() || !( std::isalpha( static_cast( query[j] ) ) || query[j] == '_' ) ) { continue; // "(#eq?" / "(!decorator" / "(\"literal\"" — none of these is a node-kind token } const std::size_t start = j; while( j < query.size() && ( std::isalnum( static_cast( query[j] ) ) || query[j] == '_' ) ) { ++j; } std::string tok( query.substr( start, j - start ) ); if( tok != "_" && std::find( out.begin(), out.end(), tok ) == out.end() ) { out.push_back( std::move( tok ) ); } } return out; } static NodeKindHint nearestNodeKindHint( std::string_view query ) { NodeKindHint hint; const std::vector candidates = extractCandidateNodeKinds( query ); if( candidates.empty() ) { return hint; // nothing that even looks like a node-kind token (e.g. a bare syntax error) } // The union of every linked grammar's own vocabulary, first-grammar-in-table-order wins per name. HashMap kindGrammar; // node-kind name -> owning grammar's display name kindGrammar.reserve( 8192 ); std::vector tried; for( const LangEntry& le : kLangTable ) { if( le.grammar == nullptr || le.querySub.empty() ) { continue; // no grammar (markdown) or nothing to attribute a --match hit against } const TSLanguage* g = le.grammar(); if( std::find( tried.begin(), tried.end(), g ) != tried.end() ) { continue; // several extensions share one grammar object } tried.push_back( g ); const std::uint32_t symCount = ts_language_symbol_count( g ); for( std::uint32_t s = 0; s < symCount; ++s ) { const TSSymbolType ty = ts_language_symbol_type( g, static_cast( s ) ); if( ty != TSSymbolTypeRegular && ty != TSSymbolTypeSupertype ) { continue; } const char* nm = ts_language_symbol_name( g, static_cast( s ) ); if( nm == nullptr || *nm == '\0' ) { continue; } kindGrammar.try_emplace( std::string_view( nm ), le.querySub ); // first grammar in table order wins } } if( kindGrammar.empty() ) { return hint; } constexpr int kMaxEditDistance = 3; // same bandwidth as didYouMean()'s symbol-name cutoff int bestDist = kMaxEditDistance + 1; for( const std::string& cand : candidates ) { const std::string_view nearest = rw::nearestNameByEditDistance( kindGrammar.begin(), kindGrammar.end(), cand, kMaxEditDistance, []( const auto& kv ) -> std::string_view { return kv.first; } ); if( nearest.empty() ) { continue; } const int dist = rw::boundedEditDistance( nearest, cand, kMaxEditDistance ); const bool better = hint.kind.empty() || dist < bestDist || ( dist == bestDist && std::string( nearest ) < hint.kind ); if( better ) { bestDist = dist; hint.kind = std::string( nearest ); hint.grammar = std::string( kindGrammar.at( nearest ) ); } } return hint; } // octocode F3: the refusal loop's own trailer, extracted so that loop's own branch count doesn't grow — a // caller that asked for neither field pays one pointer-compare and returns, same as before this existed. static void recordNodeKindHint( const AstQueryGroup& group, const std::string& query ) { if( group.nearestKindOut == nullptr && group.nearestGrammarOut == nullptr ) { return; } const NodeKindHint hint = nearestNodeKindHint( query ); if( group.nearestKindOut != nullptr ) { group.nearestKindOut->push_back( hint.kind ); } if( group.nearestGrammarOut != nullptr ) { group.nearestGrammarOut->push_back( hint.grammar ); } } // Defined further down this file, next to the rest of the unreachable-code check's helpers, and // forward-declared here so the shared file walk can drive it — the same split as // ingest()/astQueryGrouped() already use above. inline void ur_walkTree( TSNode root, std::uint32_t fileId, std::string_view src, const std::vector& nlOffsets, std::vector& hits ); // Drive every BUILT-IN WALK group over one already-parsed file, each into its own bucket. Called from the // shared worker loop with the tree and newline index the query groups are about to use, which is the whole // point: a walk group exists so a non-query check can stop re-reading and re-parsing the corpus for itself. // R2: one file's worth of pattern matching, kept out of runWalkGroups' dispatch body for the same reason // ur_walkTree is — the dispatcher stays a two-line switch over walk kinds, and each walk's own logic (and // its own reasons to grow) lives next to itself. inline void pat_walkTree( const pattern::PatternProgramSet* set, TSNode root, std::uint32_t fileId, std::string_view bytes, const std::vector& nlOffsets, const TSLanguage* grammar, std::vector& hits, std::atomic* ellipsisCappedOut, std::atomic* qualifiedUnmatchedOut, MemberNearMissTally* memberOut ) { if( set == nullptr ) { return; // a Pattern group with no programs is a caller bug, but never a crash } const pattern::PatternProgram* prog = set->forGrammar( grammar ); if( prog == nullptr ) { return; // this file's grammar is one the pattern did not resolve for — unresolved_in= says so } std::vector> spans; pattern::MatchStats stats; pattern::findMatches( *prog, root, bytes, spans, pattern::kMaxHits, stats ); if( ellipsisCappedOut != nullptr && stats.ellipsisCappedCount != 0 ) { // Relaxed is right: nothing else is published alongside it and the only reader runs after the pool // has joined. Addition is associative, so the total does not depend on which worker got here first. ellipsisCappedOut->fetch_add( stats.ellipsisCappedCount, std::memory_order_relaxed ); } if( qualifiedUnmatchedOut != nullptr && stats.qualifiedUnmatchedCount != 0 ) { qualifiedUnmatchedOut->fetch_add( stats.qualifiedUnmatchedCount, std::memory_order_relaxed ); // same reduction, same reason } if( memberOut != nullptr && ( stats.memberUnmatchedCount | stats.memberCappedCount ) != 0 ) { memberOut->counted.fetch_add( stats.memberUnmatchedCount, std::memory_order_relaxed ); // same reduction, same reason memberOut->capped.fetch_add( stats.memberCappedCount, std::memory_order_relaxed ); memberOut->arrow.fetch_add( stats.memberArrowCount, std::memory_order_relaxed ); memberOut->optchain.fetch_add( stats.memberOptChainCount, std::memory_order_relaxed ); } for( const auto& [a, b] : spans ) { if( a < b && b <= bytes.size() ) { hits.push_back( makeAstMatch( fileId, bytes, nlOffsets, a, b, std::string() ) ); } } } // `grammar` is the language THIS file was parsed with: the pattern walk needs it because one --pattern // string compiles to a different node shape per grammar, and the wrong program against the right tree // would silently match nothing. The unreachable-code walk ignores it (its rule is kind-name based). // --quality-delta's handler/placeholder shapes (src/handlershape.h): the shape logic is a pure function of // (tree, bytes, language) and lives there; this adapter only turns its spans into rows, through the same // makeAstMatch cut every other walk uses. The tag is the shape name. inline void hs_walkTree( TSNode root, std::uint32_t fileId, std::string_view bytes, const std::vector& nlOffsets, Lang lang, std::vector& hits ) { std::vector spans; hshape::walkHandlerShapes( root, bytes, lang, spans ); for( const hshape::ShapeSpan& span : spans ) { if( span.startByte < span.endByte && span.endByte <= bytes.size() ) { hits.push_back( makeAstMatch( fileId, bytes, nlOffsets, span.startByte, span.endByte, std::string( span.tag ) ) ); } } } // `lang` is the file's language: the handler-shape walk reads a handler differently per grammar, and // several grammars share node-type names (catch_clause is JS, Java, C# and C++). inline void runWalkGroups( const std::vector& groups, TSNode root, std::uint32_t fileId, std::string_view bytes, const std::vector& nlOffsets, const TSLanguage* grammar, Lang lang, std::vector>& perGroupHits ) { for( std::size_t groupIndex = 0; groupIndex < groups.size(); ++groupIndex ) { if( groups[groupIndex].walk == AstWalk::UnreachableCode ) { ur_walkTree( root, fileId, bytes, nlOffsets, perGroupHits[groupIndex] ); } else if( groups[groupIndex].walk == AstWalk::HandlerShapes ) { hs_walkTree( root, fileId, bytes, nlOffsets, lang, perGroupHits[groupIndex] ); } else if( groups[groupIndex].walk == AstWalk::Pattern ) { pat_walkTree( groups[groupIndex].patternPrograms, root, fileId, bytes, nlOffsets, grammar, perGroupHits[groupIndex], groups[groupIndex].ellipsisCappedOut, groups[groupIndex].qualifiedUnmatchedOut, groups[groupIndex].memberOut ); } } } // §L3: grammar-applicability disclosure for AstQueryGroup::grammarsOut / eligibleFilesOut (see ingest.h for // the field docs). A SEPARATE probe over the full kLangTable rather than a read of astQueryGrouped's own // byGrammar table, because byGrammar only ever holds grammars the CORPUS is present for — exactly the // information this exists to supply when the honest answer is "none of them" (a query that compiles fine // for java/csharp/typescript but the corpus is Python-only). Cost is bounded by kLangTable's size (~37 // rows) per requesting group's specs, paid only when a caller asks — every existing --lint / --lint-rules // call site leaves both fields null, so this is a no-op for them. static void computeGrammarDisclosure( const IngestResult& ing, const std::vector& groups ) { for( const AstQueryGroup& grp : groups ) { if( grp.grammarsOut == nullptr && grp.eligibleFilesOut == nullptr ) { continue; } if( grp.specs == nullptr ) { continue; // a walk-only group has no query to probe grammars for } std::vector triedGrammars; // dedup tracker: several extensions share one grammar std::vector okGrammars; // grammars at least one spec in this group compiled against std::vector okNames; // grammarsOut dedup: TWO distinct grammar OBJECTS can // share one display NAME (tree_sitter_typescript and // tree_sitter_tsx are both querySub "typescript"; the // CUDA grammar reuses "cpp") — okGrammars stays one // entry per compiling OBJECT (eligible_files needs // every one of them), grammarsOut stays one per NAME. for( const LangEntry& le : kLangTable ) { if( le.grammar == nullptr || le.querySub.empty() ) { continue; // no grammar (markdown) or no tags.scm surface to compile a --match query against } const TSLanguage* g = le.grammar(); if( std::find( triedGrammars.begin(), triedGrammars.end(), g ) != triedGrammars.end() ) { continue; } triedGrammars.push_back( g ); bool compiledAny = false; for( const AstQuerySpec& spec : *grp.specs ) { if( astQueryNestsTooDeep( spec.query ) ) { continue; } std::uint32_t off = 0; TSQueryError err = TSQueryErrorNone; if( TSQuery* probe = ts_query_new( g, spec.query.data(), static_cast( spec.query.size() ), &off, &err ) ) { ts_query_delete( probe ); compiledAny = true; break; } } if( compiledAny ) { okGrammars.push_back( g ); if( grp.grammarsOut != nullptr ) { const std::string name( le.querySub ); if( std::find( okNames.begin(), okNames.end(), name ) == okNames.end() ) { okNames.push_back( name ); grp.grammarsOut->push_back( name ); } } } } if( grp.eligibleFilesOut != nullptr ) { std::size_t eligible = 0; for( std::size_t fileId = 0; fileId < ing.files.size(); ++fileId ) { // #157: a file the nesting guard refused is never scanned (see the walk's own skip above), // so it must not count as "eligible" here — eligible_files= would otherwise claim a file was // scanned that the same run's own refusal excluded from the walk. if( fileId < ing.nestRefusedFile.size() && ing.nestRefusedFile[ fileId ] != 0 ) { continue; } const std::string ext = lowerExtensionOf( diskPath( ing, std::uint32_t( fileId ) ) ); const LangEntry* fle = lookupLangOfCrawledFile( ext ); if( fle == nullptr || fle->grammar == nullptr ) { continue; } if( std::find( okGrammars.begin(), okGrammars.end(), fle->grammar() ) != okGrammars.end() ) { ++eligible; } } *grp.eligibleFilesOut = eligible; } } } // Every constant #match?/#not-match? pattern the guard refused, per group that asked (regexRefusedOut): read off // the per-SPEC tables, whose group is known (the combined query's table holds the same patterns and no group). // One spec compiles for several grammars, so the same refusal arrives more than once, and byGrammar is a hash // map — sorted and de-duplicated, the list is a pure function of the patterns. static void collectPredicateRegexRefusals( const HashMap& byGrammar, const std::vector& groups ) { for( const auto& [ grammar, queries ] : byGrammar ) { for( const GroupedQuery& gq : queries.perSpec ) { std::vector* const out = groups[ gq.groupIndex ].regexRefusedOut; if( out != nullptr ) { out->insert( out->end(), gq.rx.refusals.begin(), gq.rx.refusals.end() ); } } } for( const AstQueryGroup& group : groups ) { if( group.regexRefusedOut != nullptr ) { rw::sortutil::sortUnique( *group.regexRefusedOut ); } } } std::vector> astQueryGrouped( const IngestResult& ing, const std::vector& groups, std::vector* keptBytesOut ) { std::vector> out( groups.size() ); if( keptBytesOut != nullptr ) { keptBytesOut->assign( ing.files.size(), std::string() ); // sized BEFORE the pool starts: workers only ever write distinct slots } bool anySpecs = false; bool anyWalk = false; bool allGroupsFiltered = !groups.empty(); // L7: every group names a mustContain literal for( const AstQueryGroup& group : groups ) { anySpecs = anySpecs || ( group.specs != nullptr && !group.specs->empty() ); anyWalk = anyWalk || ( group.walk != AstWalk::None ); allGroupsFiltered = allGroupsFiltered && !group.mustContain.empty(); } // A walk group is work even with no spec anywhere: it needs the parse, not a compiled query. if( ( !anySpecs && !anyWalk ) || ing.files.empty() ) { return out; } // The grammars this CORPUS can actually reach. ts_query_new is not cheap -- compiling every spec // against every one of the sixteen linked grammars was the single largest serial cost of a `--lint` // run, and on a corpus holding one language fifteen sixteenths of it answered a question no file // could ask. A grammar with no file to run on contributes no match, so not compiling for it changes // nothing a caller can see -- except the "did not compile for ANY grammar" disclosure below, which is // a statement about the QUERY and not about the corpus, and is therefore still decided over the full // table. // The `.h` remap is deliberately conservative: whether a header is ObjC is a fact about its BYTES, // read per file inside the walk, so any `.h` at all admits the ObjC grammar here. std::vector presentGrammars; { bool anyCHeader = false; for( std::size_t fileId = 0; fileId < ing.files.size(); ++fileId ) { const std::string ext = lowerExtensionOf( diskPath( ing, std::uint32_t( fileId ) ) ); anyCHeader = anyCHeader || ext == ".h"; const LangEntry* le = lookupLangOfCrawledFile( ext ); if( le == nullptr || le->grammar == nullptr ) { continue; } const TSLanguage* g = le->grammar(); if( std::find( presentGrammars.begin(), presentGrammars.end(), g ) == presentGrammars.end() ) { presentGrammars.push_back( g ); } } const LangEntry* objcLe = anyCHeader ? lookupLang( ".m" ) : nullptr; if( objcLe != nullptr && objcLe->grammar != nullptr && std::find( presentGrammars.begin(), presentGrammars.end(), objcLe->grammar() ) == presentGrammars.end() ) { presentGrammars.push_back( objcLe->grammar() ); } } // Compile each spec against every DISTINCT grammar it is valid for (up front, single-threaded). Queries // are immutable after creation → shared read-only across workers; only the cursor is per-thread. // ONE GRAMMAR PER THREAD. Compiling is per-grammar independent work over read-only language tables -- // the same assumption the ingest prewarm already makes when it launches ts_query_new off-thread -- and // it was the longest SERIAL stretch of a --lint run: the file walk that follows is fully parallel, so // a single-threaded compile set the floor on the whole verb. Results land in a vector indexed by the // deterministic presentGrammars order and are installed in that order afterwards, so no thread's // arrival time reaches the map, let alone the output. HashMap byGrammar; { PROFILE_SCOPE_DESCRIBE( "astQuery: compile queries per grammar" ); std::vector compiledPerGrammar( presentGrammars.size() ); { unsigned compileHw = std::thread::hardware_concurrency(); if( compileHw == 0 ) { compileHw = 1; } const unsigned compileThreads = static_cast( std::min( compileHw, std::max( presentGrammars.size(), std::size_t( 1 ) ) ) ); std::atomic nextGrammar{ 0 }; std::vector compilers; compilers.reserve( compileThreads ); for( unsigned worker = 0; worker < compileThreads; ++worker ) { compilers.emplace_back( [ & ]() noexcept { for( ;; ) { const std::size_t grammarIndex = nextGrammar.fetch_add( 1, std::memory_order_relaxed ); if( grammarIndex >= presentGrammars.size() ) { break; } compiledPerGrammar[grammarIndex] = compileGrammarQueries( presentGrammars[grammarIndex], groups ); } } ); } for( std::thread& th : compilers ) { th.join(); } } for( std::size_t grammarIndex = 0; grammarIndex < presentGrammars.size(); ++grammarIndex ) { if( !compiledPerGrammar[grammarIndex].perSpec.empty() ) { byGrammar.emplace( presentGrammars[grammarIndex], std::move( compiledPerGrammar[grammarIndex] ) ); } } } for( std::size_t groupIndex = 0; groupIndex < groups.size(); ++groupIndex ) // warn once if a spec compiled for NO grammar (malformed query) { if( groups[groupIndex].specs == nullptr ) { continue; } for( const AstQuerySpec& spec : *groups[groupIndex].specs ) { if( astQueryNestsTooDeep( spec.query ) ) { rw::emitTo( stderr, "ripwire: AST query refused: it nests deeper than {} levels, and the query compiler recurses per level\n", kMaxAstQueryNesting ); if( groups[groupIndex].uncompiledOut ) { groups[groupIndex].uncompiledOut->push_back( spec.query ); } continue; } bool any = false; for( const auto& [g, qs] : byGrammar ) { for( const GroupedQuery& gq : qs.perSpec ) { if( gq.groupIndex == groupIndex && gq.tag == spec.tag ) { any = true; break; } } if( any ) { break; } } // Nothing in the corpus could run it — but "did not compile for ANY grammar" is a claim about // the QUERY (§P0.1: a malformed query's zero must never be presented as a measurement), so it // is settled against the grammars this corpus does NOT hold before it is made. First success // wins; a C++ query on a Python-only tree is valid and stays silent, exactly as when every // grammar was compiled up front. for( const LangEntry& absent : kLangTable ) { if( any ) { break; } if( absent.grammar == nullptr ) { continue; } const TSLanguage* ag = absent.grammar(); if( byGrammar.find( ag ) != byGrammar.end() ) { continue; // already tried above, and it did not compile } std::uint32_t off = 0; TSQueryError err = TSQueryErrorNone; if( TSQuery* probe = ts_query_new( ag, spec.query.data(), static_cast( spec.query.size() ), &off, &err ) ) { ts_query_delete( probe ); any = true; } } if( !any ) { rw::emitTo( stderr, "ripwire: AST query did not compile for any grammar: {}\n", std::string_view( spec.query.data(), spec.query.size() ) ); if( groups[groupIndex].uncompiledOut ) { groups[groupIndex].uncompiledOut->push_back( spec.query ); } recordNodeKindHint( groups[groupIndex], spec.query ); // octocode F3: opt-in, see above } } } // §L3: grammar-applicability disclosure, opt-in per group (grammarsOut / eligibleFilesOut) — a standalone // pass so the existing groups[] loops above stay exactly as complex as they were for every caller that // doesn't ask for this (--lint, --lint-rules leave both null; zero cost, zero shape change for them). computeGrammarDisclosure( ing, groups ); collectPredicateRegexRefusals( byGrammar, groups ); // opt-in per group, the same way (regexRefusedOut) const std::size_t nfiles = ing.files.size(); unsigned hw = std::thread::hardware_concurrency(); if( hw == 0 ) { hw = 1; } const unsigned nthreads = static_cast( std::min( hw, nfiles ) ); // NO mid-flight global cap: a shared match counter raced by workers makes WHICH matches survive the // cap scheduling-dependent (nondeterministic --lint/--match on repos past maxMatches). Collect every // file's matches fully (per-file counts are naturally bounded and short-lived), sort deterministically, // THEN truncate to maxMatches — cap membership becomes a pure function of the input. std::vector>> tHits( nthreads, std::vector>( groups.size() ) ); std::atomic nextSlot{ 0 }; std::vector pool; pool.reserve( nthreads ); // BIGGEST FILE FIRST (longest-processing-time-first) -- the same work order, for the same reason, that // the ingest parse pool builds before it fans out. Handing files out in crawl order leaves the corpus's // largest translation unit (ripwire's own src/main.cpp: 653 KB, 40x the median) to be picked up near the // END of the queue, where it runs alone against an otherwise idle pool and sets the wall time by itself. // Sorting the queue by descending on-disk size puts the stragglers first and lets the small files fill in // behind them. WHICH thread parses WHICH file was never part of the output -- captures are bucketed per // group and sorted on a total key after the join -- so this changes scheduling and nothing else. std::vector walkOrder( nfiles ); std::iota( walkOrder.begin(), walkOrder.end(), std::uint32_t( 0 ) ); { std::vector fileByteSize( nfiles, 0 ); std::error_code ec; for( std::size_t fileId = 0; fileId < nfiles; ++fileId ) { ec.clear(); fileByteSize[fileId] = fs::file_size( diskPath( ing, std::uint32_t( fileId ) ), ec ); if( ec ) { fileByteSize[fileId] = 0; } } std::stable_sort( walkOrder.begin(), walkOrder.end(), [ & ]( std::uint32_t a, std::uint32_t b ) noexcept { if( fileByteSize[a] != fileByteSize[b] ) { return fileByteSize[a] > fileByteSize[b]; } return a < b; } ); } for( unsigned t = 0; t < nthreads; ++t ) { pool.emplace_back( [ &, t ]() noexcept { ParserGuard pg; if( pg.p == nullptr ) { return; } TSQueryCursor* cur = ts_query_cursor_new(); std::string readBuf; // worker-local scratch, reused across files unless the read is retained below for( ;; ) { const std::size_t slot = nextSlot.fetch_add( 1, std::memory_order_relaxed ); if( slot >= nfiles ) { break; } const std::size_t fileId = walkOrder[slot]; try { // #157: stay consistent with ingest's own refusal — a file the nesting guard refused // before ingest ever extracted a fact from it must not be handed to THIS parse either, // or the structural-query walk (--match/--pattern/--lint) would return hits from content // the map itself declined to index. Ask ing.nestRefusedFile (model.h) rather than // re-running the prescan: same source of truth, no second guard to drift out of sync. if( fileId < ing.nestRefusedFile.size() && ing.nestRefusedFile[ fileId ] != 0 ) { continue; } const std::string& path = diskPath( ing, std::uint32_t( fileId ) ); // multi-root: labeled ing.files → on-disk path const std::string ext = lowerExtensionOf( path ); const LangEntry* le = lookupLangOfCrawledFile( ext ); if( le == nullptr ) { continue; } if( !readFile( path, readBuf ) ) { continue; } if( looksBinary( readBuf ) ) { continue; } if( allGroupsFiltered && std::none_of( groups.begin(), groups.end(), [ & ]( const AstQueryGroup& gq ) { return readBuf.find( gq.mustContain ) != std::string::npos; } ) ) { continue; // L7: no group's literal occurs in this file, so no group can match in it } if( ext == ".h" && looksObjC( readBuf ) ) { if( const LangEntry* objcLe = lookupLang( ".m" ) ) { le = objcLe; } } if( le->grammar == nullptr ) { continue; // markdown — no grammar (would deref a null fn ptr) } const TSLanguage* g = le->grammar(); const auto it = byGrammar.find( g ); const bool hasQueries = ( it != byGrammar.end() && !it->second.perSpec.empty() ); if( !hasQueries && !anyWalk ) { continue; // no spec applies to this grammar, and no built-in walk wants the tree either } // THE retention point, and the reason it is here rather than at any of the exits below: // handing the read over BEFORE the tree is built means every path that follows works from // the retained slot, so no exit can forget to keep it and no branch can keep it twice. When // nothing is retaining, `bytes` binds the worker's own scratch and the loop reuses one // buffer exactly as it always did. Markdown is already gone by here — a file with no // grammar has no symbol a later pass could ask about. std::string& bytes = ( keptBytesOut != nullptr ) ? ( ( *keptBytesOut )[fileId] = std::move( readBuf ) ) : readBuf; if( !ts_parser_set_language( pg.p, g ) || !grammarAbiOk( g ) ) { continue; } // .astro: the SAME frontmatter restriction the ingest parse applies. Without it this pass // would read the template the symbol index cannot see, and --lint/--match would report // positions no other verb can corroborate. IncludedRangeGuard rangeGuard; if( restrictAstroToFrontmatter( pg.p, *le, bytes, rangeGuard ) != AstroFrontmatter::Ok ) { continue; } TSTree* tree = nullptr; { PROFILE_SCOPE_DESCRIBE( "astQuery/worker: tree-sitter parse" ); tree = ts_parser_parse_string( pg.p, nullptr, bytes.data(), static_cast( bytes.size() ) ); } if( !tree ) { continue; } const TSNode root = ts_tree_root_node( tree ); const std::vector nlOffsets = buildNewlineOffsets( bytes ); // one pass, then binary-search per capture // Built-in walk groups first: they read the SAME tree and the SAME newline index the // query groups below use, into their own per-group bucket, so nothing crosses over. if( anyWalk ) { PROFILE_SCOPE_DESCRIBE( "astQuery/worker: built-in tree walk" ); runWalkGroups( groups, root, std::uint32_t( fileId ), bytes, nlOffsets, g, le->lang, tHits[t] ); } if( !hasQueries ) { ts_tree_delete( tree ); continue; // walk-only file — no compiled spec for this grammar } PROFILE_SCOPE_DESCRIBE( "astQuery/worker: cursor exec + captures" ); execGrammarQueries( cur, it->second, root, std::uint32_t( fileId ), bytes, nlOffsets, groups, tHits[t] ); ts_tree_delete( tree ); } catch( ... ) { /* per-file degrade — never abort the pass */ } } ts_query_cursor_delete( cur ); } ); } for( std::thread& th : pool ) { th.join(); } for( auto& [g, qs] : byGrammar ) { for( GroupedQuery& gq : qs.perSpec ) { ts_query_delete( gq.query ); } if( qs.combined != nullptr ) { ts_query_delete( qs.combined ); } } // Per group, exactly what a standalone pass produced: merge the per-thread buckets in thread order, // sort on the total key, then spend each tag's own budget. Nothing crosses a group boundary. for( std::size_t groupIndex = 0; groupIndex < groups.size(); ++groupIndex ) { const bool hasSpecs = ( groups[groupIndex].specs != nullptr && !groups[groupIndex].specs->empty() ); if( !hasSpecs && groups[groupIndex].walk == AstWalk::None ) { continue; // a caller may pass an inert slot to keep its group indices stable } std::vector merged; std::size_t tot = 0; for( const auto& perGroup : tHits ) { tot += perGroup[groupIndex].size(); } merged.reserve( tot ); for( auto& perGroup : tHits ) { for( auto& m : perGroup[groupIndex] ) { merged.push_back( std::move( m ) ); } } std::sort( merged.begin(), merged.end(), [ & ]( const AstMatch& x, const AstMatch& y ) // deterministic order { if( ing.files[x.fileId] != ing.files[y.fileId] ) { return ing.files[x.fileId] < ing.files[y.fileId]; } if( x.startByte != y.startByte ) { return x.startByte < y.startByte; } if( x.endByte != y.endByte ) { return x.endByte < y.endByte; // nested same-start captures (outer+inner call at one byte) need this or std::sort leaks thread arrival order } return x.tag < y.tag; // equal keys ⇒ identical records (text derives from [start,end)) — order among them can't affect output } ); // A built-in walk group has no spec table to budget against, and emits ONE tag — so the per-tag // cap below degenerates to a single truncation of the sorted list, which is byte-for-byte the tail // the standalone spelling ran (collect all, sort on the same total key, resize to maxMatches). if( !hasSpecs ) { if( merged.size() > groups[groupIndex].maxMatches ) { merged.resize( groups[groupIndex].maxMatches ); } out[groupIndex] = std::move( merged ); continue; } // ── deterministic PER-SPEC cap (§P0.2), applied AFTER the sort so the survivors are a pure function of // the input. One POOLED budget let the noisiest query eat it: `(number_literal)` alone filled 5000, the // pool was path-sorted then cut, and every other rule was starved of the tail of the tree — `--lint` // reported goto=1 on a tree with two and do-while=0 on a tree with one. Each tag now spends its OWN // budget, which is exactly what a separate pass per spec would have produced (same collected set, same // (file, startByte) order within a tag) at the cost of ONE tree walk instead of N. const std::vector& specs = *groups[groupIndex].specs; HashMap tagSlot; tagSlot.reserve( specs.size() * 2 ); for( const AstQuerySpec& spec : specs ) { const std::uint32_t nextSlot = static_cast( tagSlot.size() ); tagSlot.emplace( spec.tag, nextSlot ); // duplicate tags share one budget, by design } std::vector keptPerTag( tagSlot.size(), 0 ); std::vector keep; keep.reserve( merged.size() ); for( AstMatch& m : merged ) { const auto slotIt = tagSlot.find( m.tag ); ASSUME( slotIt != tagSlot.end() ); // every emitted tag came from a spec std::size_t& keptCount = keptPerTag[ slotIt->second ]; if( keptCount >= groups[groupIndex].maxMatches ) { continue; // this spec's own budget is spent — never another's } ++keptCount; keep.push_back( std::move( m ) ); } out[groupIndex] = std::move( keep ); } return out; } // R2: the grammars the pattern surface serves, derived from kLangTable so it can never disagree with the // crawler about which extension is which language. One row per distinct grammar OBJECT — .cu's CUDA // grammar shares querySub "cpp" with .cpp, and both need their own compiled program even though the // disclosure used to print one name for both. (.ts and .tsx were the other example of this until #285: // tree_sitter_tsx used to borrow querySub "typescript" the same way CUDA borrows "cpp" — but #285 needed // JSX-only node types in .tsx's query, which do not exist in the plain typescript grammar, so .tsx now has // its own querySub "tsx" — see kLangTable's .tsx row. The two are no longer a shared-querySub pair; cpp/cuda // is the mechanism's one surviving real example.) Membership is decided by pattern.h's template table: a // family with no wrap templates is a family this verb does not serve, stated in exactly one place. // kLangTable order makes the result deterministic without a sort. // The DISCLOSURE label for one grammar object, given the labels already handed out. querySub is the // TEMPLATE key and is deliberately shared by dialects that genuinely have nothing pattern-relevant to // diverge on — the C++ tags.scm and the C++ pattern templates are what compile against tree_sitter_cuda — // but a shared disclosure NAME is how V-3 happened: `grammars="cpp"` asserted the C++ grammar resolved on // a run where only the CUDA object had, while eligible_files=, keyed on the object, counted the .cpp file // as unscanned. The first object to claim a querySub keeps it verbatim (so every single-dialect language's // output is unchanged); a later object under the same key is qualified by the extension that introduced // it — "cpp/cu". DERIVED, not enumerated, so a dialect grammar added tomorrow cannot silently re-collide by // being forgotten in a table. static std::string patternGrammarLabel( std::string_view querySub, std::string_view ext, const std::vector& taken ) { bool claimed = false; for( const pattern::GrammarRow& r : taken ) { claimed = claimed || ( r.label == querySub ); } if( !claimed ) { return std::string( querySub ); } const std::string_view bare = ( !ext.empty() && ext.front() == '.' ) ? ext.substr( 1 ) : ext; return std::string( querySub ) + "/" + std::string( bare ); } // --slice (lane/paper-slice): path -> grammar object, through the ONE crawl rule (lowerExtensionOf + // kLangTable) -- see the ingest.h declaration for why this lives here and not in slice.h. const ::TSLanguage* sliceGrammarForFile( std::string_view path ) { const LangEntry* le = lookupLang( lowerExtensionOf( path ) ); if( le == nullptr || le->grammar == nullptr ) { return nullptr; } return le->grammar(); } std::vector supportedPatternGrammars() { std::vector rows; rows.reserve( kLangTable.size() ); for( const LangEntry& le : kLangTable ) { if( le.grammar == nullptr || le.querySub.empty() || pattern::templatesFor( le.querySub ) == nullptr ) { continue; } const TSLanguage* g = le.grammar(); bool seen = false; for( const pattern::GrammarRow& r : rows ) { seen = seen || ( r.grammar == g ); } if( !seen ) { rows.push_back( { g, le.querySub, patternGrammarLabel( le.querySub, le.ext, rows ) } ); } } return rows; } PatternFileCensus eligiblePatternFiles( const IngestResult& ing, const pattern::PatternProgramSet& set ) { const std::vector served = supportedPatternGrammars(); PatternFileCensus census; for( std::size_t fileId = 0; fileId < ing.files.size(); ++fileId ) { // #157: excluded from both eligible= and skipped= — the walk never reaches it (see the walk's own // skip), so counting it as either would claim a fact about a file this run never scanned. if( fileId < ing.nestRefusedFile.size() && ing.nestRefusedFile[ fileId ] != 0 ) { continue; } const std::string ext = lowerExtensionOf( diskPath( ing, std::uint32_t( fileId ) ) ); const LangEntry* le = lookupLangOfCrawledFile( ext ); if( le == nullptr || le->grammar == nullptr ) { continue; } const TSLanguage* g = le->grammar(); if( set.forGrammar( g ) != nullptr ) { ++census.eligibleCount; continue; } // Not resolved for. It only counts as SKIPPED if this verb serves the grammar at all — a .rb or a // .json file is not "skipped", it is out of scope, and unsupported= already says so. for( const pattern::GrammarRow& r : served ) { if( r.grammar == g ) { ++census.skippedCount; break; } } } return census; } // The single-group spelling every standalone caller uses. One walk, one group — byte-identical to the // hand-written pass this replaced, and there is exactly ONE file-walk implementation to keep correct. std::vector astQuery( const IngestResult& ing, const std::vector& specs, std::size_t maxMatches, std::vector* uncompiledOut ) { const std::vector one{ { &specs, maxMatches, uncompiledOut } }; std::vector> got = astQueryGrouped( ing, one ); return std::move( got[0] ); } // ---- R-H span tiers: the narrow single-file parse entry (declared in ingest.h — read its header first) -- // // Node-type classification is a rule over the type NAME, not a per-grammar table, and that is deliberate: // twelve grammars spell the same two concepts a dozen ways (comment / line_comment / block_comment / // multiline_comment / documentation_comment / html_comment; string / string_literal / raw_string_literal / // interpreted_string_literal / verbatim_string_literal / template_string / string_content), and a hand-kept // per-grammar table is exactly the surface that goes stale the next time a grammar is vendored in. The two // substring rules below cover every one of those spellings by construction; the exact-match table carries // only the spellings that DON'T contain either word. // // Substring, not prefix/suffix: `raw_string_literal` and `documentation_comment` both need it. The exact // table must stay exact — a substring rule for "str" would classify `struct_specifier` as a string. inline constexpr std::string_view kSpanTierExactStringTypes[] = { "char_literal", // C/C++/Rust "character_literal", // Java/C# "line_str_text", // Swift — the TEXT inside a "…" literal "raw_str_part", // Swift raw strings "heredoc_body", // Bash/Ruby "heredoc_content", // Bash }; // Code (the default) unless the node's own type says otherwise. inline SpanTier spanTierOfNodeType( const char* type ) noexcept { if( type == nullptr ) { return SpanTier::Code; } if( std::strstr( type, "comment" ) != nullptr ) { return SpanTier::Comment; } if( std::strstr( type, "string" ) != nullptr ) { return SpanTier::String; } for( const std::string_view exact : kSpanTierExactStringTypes ) { if( exact.compare( type ) == 0 ) { return SpanTier::String; } } return SpanTier::Code; } // Collect one tree's OUTERMOST comment/string spans. Explicit stack, not recursion: a generated file can // nest thousands of nodes deep (the YAML grammar's own 254-level indent bug is the standing reminder), and // a query-time pass may not be the thing that overflows the stack. A classified node is recorded and NOT // descended into, so `string_content` inside `string_literal` cannot produce a second, overlapping span. static void collectSpanTiers( TSNode root, std::uint32_t byteCount, SpanTierMap& out ) { std::vector stack; std::vector kids; // reused across nodes — a warm walk allocates nothing per node ChildCursor cursor( root ); stack.push_back( root ); while( !stack.empty() ) { const TSNode n = stack.back(); stack.pop_back(); const SpanTier tier = spanTierOfNodeType( ts_node_type( n ) ); const std::uint32_t a = ts_node_start_byte( n ), b = ts_node_end_byte( n ); if( tier != SpanTier::Code ) { if( a < b && b <= byteCount ) { out.startByte.push_back( a ); out.endByte.push_back( b ); out.tier.push_back( std::uint8_t( tier ) ); } continue; // do not descend: the span is already claimed, whole } // ALL children, not just the named ones — a comment is an `extra` in most grammars and several // spell it as an anonymous node, so a named-only walk silently misses exactly the tier this // function exists to find. // Collected once, then pushed in REVERSE so the stack pops left to right — the same visit order // the indexed loop had, at O(children) instead of O(children²). The width here is the FILE's: this // walk starts at the root, and a comment is an extra spliced straight into the child array, so a // 16 000-comment file made --grep's tier pass 56× the plain map of the same file before this became // a cursor (test/childwalkscalecheck.sh, arm B3; the rule is on src/infra/tschildren.h). collectChildren( n, cursor.cur, kids ); for( std::size_t c = kids.size(); c > 0; --c ) { stack.push_back( kids[ c - 1 ] ); } } // The stack walk emits in DFS pop order, which is not byte order once a subtree is skipped; the // classify path binary-searches, so sort here once rather than making every lookup linear. std::vector order( out.startByte.size() ); std::iota( order.begin(), order.end(), std::uint32_t( 0 ) ); std::stable_sort( order.begin(), order.end(), [ & ]( std::uint32_t x, std::uint32_t y ) noexcept { return out.startByte[x] < out.startByte[y]; } ); SpanTierMap sorted; sorted.startByte.reserve( order.size() ); sorted.endByte.reserve( order.size() ); sorted.tier.reserve( order.size() ); for( const std::uint32_t index : order ) { sorted.startByte.push_back( out.startByte[index] ); sorted.endByte.push_back( out.endByte[index] ); sorted.tier.push_back( out.tier[index] ); } out.startByte = std::move( sorted.startByte ); out.endByte = std::move( sorted.endByte ); out.tier = std::move( sorted.tier ); } // ─── P4.1 — the span-tier MEMO (the --grep fast path) ───────────────────────────────────────────── // // WHY. R-H's tier pass parses each HIT file with tree-sitter for one purpose: to learn where that file's // comment and string spans are. Measured on a 4,175-file corpus, that parse was the single largest cost // of a warm --grep answer — 48.6 ms of 227 ms, larger than buildGraph — and it was repeated verbatim on // every later grep of the same unchanged file, because nothing carried the result between runs. // SpanTierMap is a PURE FUNCTION of (file bytes, grammar), so it memoizes: one small blob per // (path, parserVer) under the SAME cache-dir ladder every other ripwire cache uses, written by the pass // that first paid for it and read by every later one. See bench/PROFILE.md for the ledger row. // // WHY IT IS A CORRECTNESS SURFACE, NOT A PERFORMANCE ONE. The tier ladder SUPPRESSES comment/string rows // whenever the code tier is non-empty, so a stale span map does not merely slow an answer down — it // silently removes rows from one. Freshness is therefore the ingest cache's own A4-P7 stat gate rather // than a weaker one, with all four conditions required: // - the blob's recorded PATH matches byte-for-byte (so a 64-bit filename hash collision cannot alias // two files — the hash only picks the filename, it never stands in for identity); // - the file's CURRENT (sizeBytes, mtimeNs, ctimeNs) still equal the triple recorded when it was parsed; // - that recorded mtime is STRICTLY OLDER than the blob's own on-disk mtime (the racy rule — a file // rewritten inside one mtime tick of the blob reads as changed, never as fresh); // - the parserVer in the filename matches, so any grammar/.scm change orphans every old blob at once. // A miss is silent and self-healing: parse, then write. Every failure mode this cache has degrades in the // SAFE direction — a blob that cannot be read, parsed or trusted is simply ignored and the file is // re-parsed, which is byte-for-byte the pre-memo behaviour. // // WHAT USED TO BE THE RESIDUAL, and where it went: a rewrite landing on the SAME byte size AND the same // mtime, at least one mtime tick before the blob was written, was indistinguishable from no rewrite and was // served a stale map. It shares its fix with the parse cache's own warm-run stat gate (ingest_cache.h, // A4-P7): st_ctime is recorded in the third slot and compared, so the utimes() that restores an mtime — and // any write at all — is visible for free out of the ::stat the caller already took. What remains outside it // is the same narrow set named there: a clock moved backward, raw block-device writes, and a filesystem // with no distinct ctime. --no-cache still exists and the gate still diffs cold against warm rather than // trusting either. // // DISCLOSED LIMIT, stated rather than implied: this memo does not make the tier pass cheaper the FIRST // time a file is classified, and a repo whose files change on every run never warms it. It is a warm-path // optimization and nothing else; the disclosed tier budgets (kGrepTierFileBudget / kGrepTierByteBudget) // are unchanged and still bound the cold cost. --no-cache turns it off at the caller, like every other // ripwire cache, so the reproducibility switch stays one switch. constexpr std::uint32_t kSpanTierMemoMagic = 0x53544d31u; // 'STM1' constexpr std::uint32_t kSpanTierMemoVersion = 2u; // 2: the record gains ctimeNs (card A3 follow-up) — a FORMAT change, so v1 blobs are simply never loaded and age out constexpr std::uint32_t kSpanTierMemoMaxSpans = 8u << 20; // refuse to trust (or write) an implausible span count // The SIZE FLOOR, and why there is one. Parse cost is linear in bytes (E5's 42 ns/B), so a small file's // tier map is cheaper to recompute than to look up — while its blob costs the same cache-dir slot as a // large one's. That matters because the dir-wide hygiene sweep (quality.h sweepStaleCacheBlobsOnce) caps // the whole "ripwire-" family by COUNT as well as by age and total bytes: a memo that wrote one blob per // small file would flood that budget and start evicting the main parse caches, trading a few ms of grep // for a full re-ingest. Measured on the 4,175-file benchmark corpus, source files at or above this floor // are 12.9% of the file count but 86.1% of the bytes — so the floor keeps essentially all of the saving // while writing roughly eight times fewer blobs. Files below it take the ordinary read-and-parse path, // which is exactly the pre-memo behaviour; nothing about their answer changes. constexpr long long kSpanTierMemoMinBytes = 32ll << 10; // Composed exactly the way every OTHER blob family is (quality.h): one fixed-width identity hex per key // field, then shaKeyedCachePath to assemble and shard the name. Two properties come free and are the reason // to reuse rather than hand-roll a fourth name builder — cacheRootKeyHex realpath-normalizes before hashing, // so two spellings of one file share a blob; and exclConfigHex folds extractionIdentityTag(), so a // kParserVer/kCacheVersion bump renames every memo blob at once, which is the same self-healing invalidation // the parse cache already has. (The hand-rolled fixed-buffer name builder this replaces was flagged as a // 60-token clone of those very builders by --quality-delta, and the detector was right.) inline std::string spanTierMemoPath( const std::string& diskPath ) { return quality::shaKeyedCachePath( "stier", quality::cacheRootKeyHex( diskPath ), quality::exclConfigHex( {}, "stier" ), std::to_string( kSpanTierMemoVersion ) ); } // Read the memo for `diskPath` at the stat the caller just took. Returns false — leaving `out` untouched — // on ANY doubt whatsoever; there is deliberately no partial-trust path. inline bool spanTierMemoLoad( const std::string& diskPath, const StatInfo& now, SpanTierMap& out ) { if( now.sizeBytes < 0 || now.mtimeNs < 0 || now.ctimeNs < 0 ) { return false; // unstatable ⇒ nothing to compare against ⇒ never trusted } const std::string blobPath = spanTierMemoPath( diskPath ); const StatInfo blobStat = statSizeTimes( blobPath ); if( blobStat.mtimeNs < 0 || now.mtimeNs >= blobStat.mtimeNs ) { return false; // absent, or the racy rule: the file is not provably older than the blob } std::ifstream blobIn( blobPath, std::ios::binary ); if( !blobIn ) { return false; } const auto u32 = [ & ]() -> std::uint32_t { std::uint32_t v = 0; blobIn.read( reinterpret_cast( &v ), sizeof( v ) ); return v; }; const auto i64 = [ & ]() -> long long { long long v = 0; blobIn.read( reinterpret_cast( &v ), sizeof( v ) ); return v; }; if( u32() != kSpanTierMemoMagic || u32() != kSpanTierMemoVersion ) { return false; } // Short-circuit order is irrelevant here and MUST be: every i64() has to be consumed so the next // field lands at its wire offset. Written as three reads into locals for exactly that reason. const long long recSize = i64(); const long long recMtime = i64(); const long long recCtime = i64(); if( recSize != now.sizeBytes || recMtime != now.mtimeNs || recCtime != now.ctimeNs ) { return false; } const std::uint32_t pathLen = u32(); if( !blobIn || pathLen != diskPath.size() ) { return false; } std::string recordedPath( pathLen, '\0' ); blobIn.read( recordedPath.data(), std::streamsize( pathLen ) ); if( !blobIn || recordedPath != diskPath ) { return false; // filename-hash collision (or a moved blob) — identity is the PATH, not its hash } const std::uint32_t spanCount = u32(); if( !blobIn || spanCount > kSpanTierMemoMaxSpans ) { return false; } SpanTierMap loaded; loaded.startByte.resize( spanCount ); loaded.endByte.resize( spanCount ); loaded.tier.resize( spanCount ); if( spanCount > 0 ) { blobIn.read( reinterpret_cast( loaded.startByte.data() ), std::streamsize( spanCount ) * std::streamsize( sizeof( std::uint32_t ) ) ); blobIn.read( reinterpret_cast( loaded.endByte.data() ), std::streamsize( spanCount ) * std::streamsize( sizeof( std::uint32_t ) ) ); blobIn.read( reinterpret_cast( loaded.tier.data() ), std::streamsize( spanCount ) ); } if( !blobIn ) { return false; // truncated / torn blob — re-parse rather than classify from half a map } // Every tier byte is external input: the memo carries no checksum, so a flipped or hand-written byte arrives // here intact. A value at or past kSpanTierCount is not a tier, and search.h's grepApplySpanTiers counts hits // into a per-tier array indexed by it — before this check, an out-of-bounds write on the stack. const bool tiersInRange = std::all_of( loaded.tier.begin(), loaded.tier.end(), []( const std::uint8_t tier ) noexcept { return tier < kSpanTierCount; } ); if( !VALIDATE( tiersInRange ) ) { DISCLOSE( Diagnostics::answerUnchanged, "the memo is refused and the file re-parsed: the tiers are recomputed, never guessed", "grep: span-tier memo carries a tier byte past SpanTier — memo refused, the file is re-parsed" ); return false; } loaded.isParsed = true; out = std::move( loaded ); return true; } // Write the memo. Best-effort in every direction: a failure loses only the speed-up, never an answer. // Temp-then-rename so a torn write can never be observed as a complete blob (the same discipline // atomicWriteQSnap uses); the temp name carries pid + a process-unique counter, so the parallel tier // workers — which each own a distinct file slot — cannot collide even across concurrent ripwire runs. inline void spanTierMemoStore( const std::string& diskPath, const StatInfo& now, const SpanTierMap& map ) { if( now.sizeBytes < 0 || now.mtimeNs < 0 || now.ctimeNs < 0 || !map.isParsed ) { return; } const std::size_t spanCount = map.startByte.size(); if( spanCount != map.endByte.size() || spanCount != map.tier.size() || spanCount > kSpanTierMemoMaxSpans ) { return; } static std::atomic tempSeq{ 0 }; const std::string blobPath = spanTierMemoPath( diskPath ); char suffix[ 64 ]; // 4 B literal + %d at 11 + 1 B + %llu at 20 = 36 B worst case into 64 — see test/fixedbufsweep.sh's census rw::formatTo( suffix, sizeof( suffix ), ".tmp{}-{}", int( os::getpid() ), static_cast( tempSeq.fetch_add( 1, std::memory_order_relaxed ) ) ); const std::string tempPath = blobPath + suffix; { std::ofstream out( tempPath, std::ios::binary | std::ios::trunc ); if( !out ) { return; } const auto putU32 = [ & ]( std::uint32_t v ) { out.write( reinterpret_cast( &v ), sizeof( v ) ); }; const auto putI64 = [ & ]( long long v ) { out.write( reinterpret_cast( &v ), sizeof( v ) ); }; putU32( kSpanTierMemoMagic ); putU32( kSpanTierMemoVersion ); putI64( now.sizeBytes ); putI64( now.mtimeNs ); putI64( now.ctimeNs ); putU32( std::uint32_t( diskPath.size() ) ); out.write( diskPath.data(), std::streamsize( diskPath.size() ) ); putU32( std::uint32_t( spanCount ) ); if( spanCount > 0 ) { out.write( reinterpret_cast( map.startByte.data() ), std::streamsize( spanCount ) * std::streamsize( sizeof( std::uint32_t ) ) ); out.write( reinterpret_cast( map.endByte.data() ), std::streamsize( spanCount ) * std::streamsize( sizeof( std::uint32_t ) ) ); out.write( reinterpret_cast( map.tier.data() ), std::streamsize( spanCount ) ); } out.flush(); if( !out ) { std::error_code rmEc; fs::remove( fs::path( tempPath ), rmEc ); return; } } std::error_code renameEc; fs::rename( fs::path( tempPath ), fs::path( blobPath ), renameEc ); if( renameEc ) { std::error_code rmEc; fs::remove( fs::path( tempPath ), rmEc ); } } // The two seams the tier worker actually calls: "is this file worth a memo at all" lives HERE, once, so the // worker's own body states the intent (try the memo, else parse; record what the parse cost) and carries // neither the enable flag nor the size floor as extra branches of its own. // // The LOAD is consulted before the READ, not merely before the parse: a hit pins the exact (path, size, // mtime) whose bytes produced the stored map, and every content-dependent decision the worker makes below it // — the binary sniff, the .h-to-ObjC grammar reroute, the grammar choice itself — is a pure function of those // same bytes, so re-reading them could only re-derive what the blob already proves. One thing deliberately // does NOT move on a hit: SpanTierBatch::bytesParsed counts bytes handed to tree-sitter, and on a memo hit // none were. A count that grew on a run which parsed nothing would be a small lie in exactly the place this // file's honesty rules care about. // // The STORE is keyed to the stat taken BEFORE the read, and written after isParsed is set — so the blob is // exactly the map this answer used, and a file rewritten between the stat and the read stores a pair that no // longer matches it and is therefore rejected on the next load rather than trusted. inline bool spanTierMemoTryLoad( bool useMemo, const std::string& diskPath, const StatInfo& now, SpanTierMap& out ) { return useMemo && now.sizeBytes >= kSpanTierMemoMinBytes && spanTierMemoLoad( diskPath, now, out ); } inline void spanTierMemoTryStore( bool useMemo, const std::string& diskPath, const StatInfo& now, const SpanTierMap& map ) { if( useMemo && now.sizeBytes >= kSpanTierMemoMinBytes ) { spanTierMemoStore( diskPath, now, map ); } } SpanTierBatch spanTiersOfFiles( std::span diskPaths, bool useMemo ) { SpanTierBatch batch; batch.perFile.resize( diskPaths.size() ); if( diskPaths.empty() ) { return batch; } // BIGGEST FILE FIRST, the same longest-processing-time-first order (and the same reason) as // astQueryGrouped's pool: hand the corpus's largest translation unit out first so it cannot strand an // otherwise-idle pool at the tail. E5 design condition 2 — this is the pattern it named, scoped to the // caller's file list instead of ing.files. const std::size_t fileCount = diskPaths.size(); std::vector walkOrder( fileCount ); std::iota( walkOrder.begin(), walkOrder.end(), std::uint32_t( 0 ) ); { std::vector fileByteSize( fileCount, 0 ); std::error_code ec; for( std::size_t fileIndex = 0; fileIndex < fileCount; ++fileIndex ) { ec.clear(); fileByteSize[fileIndex] = fs::file_size( diskPaths[fileIndex], ec ); if( ec ) { fileByteSize[fileIndex] = 0; } } std::stable_sort( walkOrder.begin(), walkOrder.end(), [ & ]( std::uint32_t a, std::uint32_t b ) noexcept { if( fileByteSize[a] != fileByteSize[b] ) { return fileByteSize[a] > fileByteSize[b]; } return a < b; } ); } unsigned hw = std::thread::hardware_concurrency(); if( hw == 0 ) { hw = 1; } const unsigned threadCount = static_cast( std::min( hw, fileCount ) ); std::atomic nextSlot{ 0 }; std::atomic bytesParsed{ 0 }; const auto worker = [ & ]() noexcept { ParserGuard pg; if( pg.p == nullptr ) { DISCLOSE( "span tiers: no tree-sitter parser — hits stay unclassified (never suppressed)" ); return; } std::string bytes; try { for( ;; ) { const std::size_t slot = nextSlot.fetch_add( 1, std::memory_order_relaxed ); if( slot >= fileCount ) { break; } const std::size_t fileIndex = walkOrder[slot]; const std::string& path = diskPaths[fileIndex]; const std::string ext = lowerExtensionOf( path ); const LangEntry* le = lookupLang( ext ); if( le == nullptr || le->grammar == nullptr ) { continue; // markdown and every unsupported extension: unclassifiable, and it stays that way } const StatInfo fileStat = statSizeTimes( path ); if( spanTierMemoTryLoad( useMemo, path, fileStat, batch.perFile[fileIndex] ) ) { continue; // isParsed was set by the loader; the slot is owned by this worker alone } bytes.clear(); if( !readFile( path, bytes ) || looksBinary( bytes ) ) { continue; } if( ext == ".h" && looksObjC( bytes ) ) { if( const LangEntry* objcLe = lookupLang( ".m" ) ) { le = objcLe; // the SAME reroute the crawl and the AST walk both apply } } const TSLanguage* g = le->grammar(); if( g == nullptr || !ts_parser_set_language( pg.p, g ) || !grammarAbiOk( g ) ) { continue; } // .astro: the SAME frontmatter restriction, so a span tier cannot disagree with the index. IncludedRangeGuard rangeGuard; if( restrictAstroToFrontmatter( pg.p, *le, bytes, rangeGuard ) != AstroFrontmatter::Ok ) { continue; } TSTree* tree = nullptr; { PROFILE_SCOPE_DESCRIBE( "spanTiers/worker: tree-sitter parse" ); tree = ts_parser_parse_string( pg.p, nullptr, bytes.data(), static_cast( bytes.size() ) ); } if( tree == nullptr ) { continue; } collectSpanTiers( ts_tree_root_node( tree ), std::uint32_t( bytes.size() ), batch.perFile[fileIndex] ); batch.perFile[fileIndex].isParsed = true; // slot owned by this worker alone ts_tree_delete( tree ); bytesParsed.fetch_add( bytes.size(), std::memory_order_relaxed ); spanTierMemoTryStore( useMemo, path, fileStat, batch.perFile[fileIndex] ); } } catch( ... ) // a throw escaping a worker thread is std::terminate — degrade to unclassified instead { DISCLOSE( "span tiers: parse worker degraded (exception swallowed) — files left unclassified" ); } }; if( threadCount <= 1 ) { worker(); } else { // symmetric bare scope: the workers live exactly as long as the parse batch std::vector pool; pool.reserve( threadCount ); for( unsigned t = 0; t < threadCount; ++t ) { pool.emplace_back( worker ); } for( std::thread& w : pool ) { w.join(); } } batch.bytesParsed = bytesParsed.load( std::memory_order_relaxed ); for( const SpanTierMap& m : batch.perFile ) { if( m.isParsed ) { ++batch.parsedFileCount; } else { ++batch.unparsedFileCount; } } return batch; } // ---- unreachable-code detection (built-in --lint rule "unreachable-code") ---- // // A GENUINE block node — a brace-delimited statement list whose direct children are the block's // statements (NOT a switch body's case list, NOT a case body). Only these are scanned: within one, // a statement's straight-line successor is a plain sibling, so "code after an unconditional exit is // dead" holds syntactically. Case bodies (children of case_statement) are deliberately NOT blocks // here, so `break; x();` inside a case is never flagged (conservative — no false positives). inline bool ur_isBlockNode( const char* t ) noexcept { return std::strcmp( t, "compound_statement" ) == 0 // C / C++ / ObjC || std::strcmp( t, "block" ) == 0 // Python / Java (and other {…} blocks) || std::strcmp( t, "statement_block" ) == 0; // JS / TS } // An UNCONDITIONAL terminator statement: once seen at a block level, control cannot fall through to // the next sibling. `goto` is intentionally EXCLUDED — a following statement can be a label target, // so flagging it would be a false positive (the #1 trap this check must avoid). `return_statement` // covers C-family + Python; `raise_statement` is Python's throw. inline bool ur_isTerminator( const char* t ) noexcept { return std::strcmp( t, "return_statement" ) == 0 || std::strcmp( t, "break_statement" ) == 0 || std::strcmp( t, "continue_statement" ) == 0 || std::strcmp( t, "throw_statement" ) == 0 // C++ / ObjC / Java / JS || std::strcmp( t, "raise_statement" ) == 0; // Python } // A node that is NOT a real statement for reachability purposes — skip it when looking for the next // sibling after a terminator (comments, and the block's own braces/colon punctuation). tree-sitter // exposes comments as named siblings inside a block; a comment after `return` is not dead CODE. inline bool ur_isSkippableSibling( TSNode n ) noexcept { if( !ts_node_is_named( n ) ) { return true; // '{', '}', ';', ':' punctuation tokens } const char* t = ts_node_type( n ); return std::strcmp( t, "comment" ) == 0; } // A jump TARGET sibling: a label or a case makes the following statements reachable out-of-line, so // the moment one appears after a terminator we STOP scanning this block (never flag past it). This // is the second false-positive guard (belt-and-braces with excluding goto and not scanning case // bodies): even a stray label inside a plain block halts the dead-code claim. inline bool ur_isJumpTarget( const char* t ) noexcept { return std::strcmp( t, "labeled_statement" ) == 0 // C-family `lbl:` (goto/switch fallthrough target) || std::strcmp( t, "case_statement" ) == 0 // C-family switch case/default || std::strcmp( t, "case" ) == 0 // grammar variants || std::strcmp( t, "default" ) == 0; } // Walk one file's AST (iterative frame-stack DFS — the cc_walk shape, NO recursion). For every block // node, scan its direct children left-to-right; the FIRST non-skippable statement after a terminator // (with no intervening jump target) is unreachable → one finding at that statement's start byte. inline void ur_walkTree( TSNode root, std::uint32_t fileId, std::string_view src, const std::vector& nlOffsets, std::vector& hits ) // A4-F25: NOT noexcept — allocates (see cc_walk) { struct UrFrame { TSNode node; std::uint16_t depth; }; std::vector stack; stack.reserve( 64 ); stack.push_back( { root, 0 } ); ChildCursor cursor( root ); std::vector kids; kids.reserve( 64 ); while( !stack.empty() ) { const UrFrame frame = stack.back(); stack.pop_back(); if( frame.depth > 512 ) { continue; // pathological-AST guard (file capped at 1 MB) } const TSNode n = frame.node; const std::uint16_t childDepth = static_cast( frame.depth + 1 ); const char* t = ts_node_type( n ); collectChildren( n, cursor.cur, kids ); // one collection serves the block scan AND the descent // If this is a genuine block, scan its statement siblings for a post-terminator statement. if( ur_isBlockNode( t ) ) { bool sawTerminator = false; for( const TSNode c : kids ) { const char* ct = ts_node_type( c ); if( sawTerminator ) { if( ur_isSkippableSibling( c ) ) { continue; // comment / punctuation → not code, keep looking } if( ur_isJumpTarget( ct ) ) { break; // label/case → reachable out-of-line → stop, no flag } // First real statement after an unconditional exit in this block → UNREACHABLE. const std::uint32_t a = ts_node_start_byte( c ), b = ts_node_end_byte( c ); if( a < b && b <= src.size() ) { const std::uint32_t line = lineAtByte( nlOffsets, a ); std::size_t cutLen = std::min( b - a, 120u ); if( cutLen < b - a ) { // UTF-8-safe truncation (serialize.h/astQuery pattern) while( cutLen > 0 && ( static_cast( src[a + cutLen] ) & 0xC0 ) == 0x80 ) { --cutLen; } } std::string text( src.substr( a, cutLen ) ); for( char& ch : text ) { if( ch == '\n' || ch == '\r' || ch == '\t' ) { ch = ' '; } } hits.push_back( { fileId, a, b, line, std::string( "unreachable-code" ), std::move( text ) } ); } break; // one finding per block — the first dead statement; the rest are consequential noise } else if( ur_isSkippableSibling( c ) ) { continue; // comments/punctuation don't set the terminator flag } else if( ur_isTerminator( ct ) ) { sawTerminator = true; // arm: the NEXT real sibling is unreachable } // else: an ordinary statement — reachable; a terminator inside it (nested block/branch) // is handled when we descend into that child's own block, never at THIS level. } } // Descend into every child (blocks nest — a function body holds inner blocks, and non-block // statements like if/for CONTAIN blocks we must still reach). Push in reverse for L-to-R order. for( std::size_t i = kids.size(); i > 0; --i ) { stack.push_back( { kids[ i - 1 ], childDepth } ); } } } // local-variable-indexing plan, Phase 2 (docs/LOCALS_INDEXING.md) — see ingest.h's own comment for the // full contract. Definition lives HERE (outside the anonymous namespace above) purely for LINKAGE — it // must be externally callable to satisfy ingest.h's declaration — while every helper it calls // (ln_extractDeclaratorIdentifiers / ln_declaratorIdentifiers / ln_declDepth / ln_collectLocalDecls) stays // anonymous-namespace-scoped next to cc_walk/complexityOf, which they mirror. std::vector collectGatedLocalNames( std::string_view defBytes, std::uint32_t defStartLine, Lang lang ) { std::vector out; if( !localsCountedLang( lang ) || defBytes.empty() ) { return out; // MVP scope (model.h::localsCountedLang) — degrade to empty, never assert on a caller mistake } const TSLanguage* grammar = ( lang == Lang::C ) ? tree_sitter_c() : tree_sitter_cpp(); TSParser* parser = ts_parser_new(); if( parser == nullptr ) { return out; } ts_parser_set_language( parser, grammar ); TSTree* tree = ts_parser_parse_string( parser, nullptr, defBytes.data(), std::uint32_t( defBytes.size() ) ); if( tree == nullptr ) { ts_parser_delete( parser ); return out; } const TSNode root = ts_tree_root_node( tree ); // the def parses as a single top-level function_definition inside a translation_unit — descend into // the translation_unit's children (bounded: one file-worth of def text, already size-capped upstream). ChildCursor cursor( root ); forEachChild( root, cursor.cur, [ & ]( TSNode child ) { ln_collectLocalDecls( child, child, 512, out, defStartLine, defBytes ); return true; } ); ts_tree_delete( tree ); ts_parser_delete( parser ); return out; } // QD-TANGLE detail (ingest.h tangleDetailOf): ONE function's steering locals and deep regions, by re-parsing its file and re-running // the same fused walk with a detail sink. The node is the smallest one that starts at the definition's first byte and ends at its last // — the def node complexityOf measured at ingest (a body-sibling or literal-bound def measures a different node, and gets no detail). TangleDetailFacts tangleDetailOf( std::string_view fileBytes, std::string_view path, std::uint32_t startByte, std::uint32_t endByte ) { TangleDetailFacts out; const LangEntry* le = lookupLang( lowerExtensionOf( path ) ); if( le != nullptr && le->grammar != nullptr && lowerExtensionOf( path ) == ".h" && looksObjC( fileBytes ) ) { le = lookupLang( ".m" ); // the SAME reroute the crawl applies } if( le == nullptr || le->grammar == nullptr || startByte >= endByte || endByte > fileBytes.size() ) { return out; } ParserGuard pg; const TSLanguage* g = le->grammar(); if( pg.p == nullptr || g == nullptr || !ts_parser_set_language( pg.p, g ) || !grammarAbiOk( g ) ) { return out; } const TreeGuard tree( ts_parser_parse_string( pg.p, nullptr, fileBytes.data(), std::uint32_t( fileBytes.size() ) ) ); // freed on every exit if( tree.tree == nullptr ) { return out; } TSNode n = ts_node_descendant_for_byte_range( ts_tree_root_node( tree.tree ), startByte, endByte - 1u ); while( !ts_node_is_null( n ) && ts_node_start_byte( n ) > startByte ) { n = parentOf( n ); } if( !ts_node_is_null( n ) && ts_node_start_byte( n ) == startByte && ts_node_end_byte( n ) == endByte ) { complexityOf( n, fileBytes, le->lang, &out ); out.ok = true; } return out; } // ---- definitionLinesInBlobs (ingest.h): the tags path over blobs from other git refs ---- // // The DISCLOSE sink of one blob: a degrade sets its slot's status, which is exactly what the caller reads (a blob // that is not Parsed never confirms a definition, and --whereis says how many such blobs it met). struct BlobDefsSlot { enum class DisclosureWhy : std::uint8_t { NoParser, // ts_parser_new / ts_query_cursor_new gave nothing: the worker parses no blob GrammarRejected, // the grammar could not be set on the parser (ABI): this blob is not parsed ParseNull, // tree-sitter returned no tree ExtractPartial, // the tags pass disclosed a shortfall (ExtractShortfall): its definitions may be incomplete WorkerThrew, // an exception inside the worker: the slot keeps Failed }; BlobDefinitions& slot; void disclose( DisclosureWhy ) noexcept // every reason records the same fact: no complete answer for this blob { slot.lines.clear(); slot.status = BlobDefsStatus::Failed; } }; // The worker's non-markdown extraction over one blob, in its order: the .astro frontmatter restriction, the parse, the // member-macro re-parse when the first parse holds errors, the `#if 0` ranges, the definitions-only tags pass. False when // the slot is already final (a template-only .astro, or a degrade it discloses). inline bool tagsDefinitionsOfBlob( std::string_view bytes, const LangEntry& le, TSParser* parser, TSQueryCursor* cursor, MemberMacroReparse& macroWork, std::vector& defs, BlobDefinitions& slot, std::string_view fileStem, TagsCapture mode ) { IncludedRangeGuard rangeGuard; const AstroFrontmatter astro = restrictAstroToFrontmatter( parser, le, bytes, rangeGuard ); if( astro == AstroFrontmatter::Unterminated ) { DISCLOSE( ( BlobDefsSlot{ slot } ), BlobDefsSlot::DisclosureWhy::ExtractPartial, "whereis: an .astro blob opens a frontmatter fence it never closes — not extracted" ); return false; } if( astro == AstroFrontmatter::None ) { slot.status = BlobDefsStatus::Parsed; // a template-only .astro: the index extracts nothing from it either return false; } TreeGuard tree( parseTree( parser, bytes ) ); if( tree.get() == nullptr ) { DISCLOSE( ( BlobDefsSlot{ slot } ), BlobDefsSlot::DisclosureWhy::ParseNull, "whereis: a blob did not parse — its rows confirm no definition" ); return false; } if( ts_node_has_error( ts_tree_root_node( tree.get() ) ) ) // a clean parse has no error bytes: the re-parse never adopts { (void)measureHealthAdoptingMemberMacroReparse( parser, le.lang, bytes, tree, macroWork ); // may adopt it, as the worker does } const TSNode root = ts_tree_root_node( tree.get() ); const std::vector ppDead = preprocDeadRangesFor( le, root, bytes ); std::vector refs; std::vector binds; std::vector incs; ExtractShortfall shortfall; captureTagsFacts( cursor, le, 0, bytes, root, defs, refs, binds, incs, ppDead, shortfall, mode, fileStem ); if( shortfall.isShort ) { DISCLOSE( ( BlobDefsSlot{ slot } ), BlobDefsSlot::DisclosureWhy::ExtractPartial, "whereis: the tags pass over a blob was partial — its rows confirm no definition" ); return false; } return true; } // One blob through the crawl's language rule and the parse-pool worker's extraction steps, in the worker's order, into // the definitions HEAD's index would make SYMBOLS of. `scan` is a one-slot scan the nesting refusal records into (its // --skipped bookkeeping has no reader here: the slot's Unread status is this blob's disclosure). Null when the slot is // already final (no grammar, unread, a degrade it discloses, a template-only .astro); else the language the blob read as. inline const LangEntry* rawDefsOfOneBlob( const BlobText& blob, TSParser* parser, TSQueryCursor* cursor, MemberMacroReparse& macroWork, IngestFileScan& scan, BlobDefinitions& slot, TagsCapture mode, std::vector& defs ) { const std::string ext = lowerExtensionOf( blob.path ); const LangEntry* le = lookupLang( ext ); if( le == nullptr ) { slot.status = BlobDefsStatus::NoGrammar; return nullptr; } if( blob.bytes.size() > kDefaultMaxFileBytes || looksBinary( blob.bytes ) ) { slot.status = BlobDefsStatus::Unread; // the crawl's own ceiling and binary sniff: the index skips these too return nullptr; } if( le->ext == ".h" && looksObjC( blob.bytes ) ) { if( const LangEntry* objcLe = lookupLang( ".m" ) ) { le = objcLe; } } const std::string pathText( blob.path ); scan.nestRefusedBytes[ 0 ] = 0; if( refuseNesting( *le, blob.bytes, pathText.c_str(), 0, scan ) ) { slot.status = BlobDefsStatus::Unread; return nullptr; } if( !prepareParserFor( parser, *le ) ) { DISCLOSE( ( BlobDefsSlot{ slot } ), BlobDefsSlot::DisclosureWhy::GrammarRejected, "whereis: a blob's grammar could not be set — its rows confirm no definition" ); return nullptr; } if( le->lang == Lang::Markdown ) { TreeGuard mdTree( parseTree( parser, blob.bytes ) ); if( mdTree.get() == nullptr ) { DISCLOSE( ( BlobDefsSlot{ slot } ), BlobDefsSlot::DisclosureWhy::ParseNull, "whereis: a markdown blob did not parse — its rows confirm no definition" ); return nullptr; } std::vector refs; extractMarkdown( 0, blob.bytes, fs::path( pathText ).stem().string(), ts_tree_root_node( mdTree.get() ), defs, refs ); } else if( !tagsDefinitionsOfBlob( blob.bytes, *le, parser, cursor, macroWork, defs, slot, cjsExportStem( le->lang, pathText ), mode ) ) { return nullptr; // the slot says why (template-only .astro: Parsed with nothing; a degrade: Failed) } // The model build's steps that decide which captured definitions become SYMBOLS (ingest.cpp 3a..3a-ter), so a // blob answers what HEAD's index answers for the same bytes: overlapping patterns collapse to one definition, an // ObjC @interface declaration shadowed by its same-file @implementation drops, and a field leaves the symbol // universe (fields are not Symbols, so on HEAD a field line is never a def site either). dedupRawDefs( defs ); collapseObjCDeclDefs( defs ); (void)partitionFieldDefs( defs ); return le; } inline void definitionLinesOfOneBlob( const BlobText& blob, std::string_view name, TSParser* parser, TSQueryCursor* cursor, MemberMacroReparse& macroWork, IngestFileScan& scan, BlobDefinitions& slot ) { std::vector defs; if( rawDefsOfOneBlob( blob, parser, cursor, macroWork, scan, slot, TagsCapture::DefinitionsOnly, defs ) == nullptr ) { return; } for( const RawDef& d : defs ) { if( d.name == name ) { slot.lines.push_back( d.line ); } } std::sort( slot.lines.begin(), slot.lines.end() ); slot.status = BlobDefsStatus::Parsed; } // The query set, single-threaded and BEFORE any worker: every grammar a blob can reach — its extension's, plus ObjC's for a // .h (the content reroute is decided inside the worker). compiledQueryFor only ever READS the cache, so a grammar is // compiled here once and the cache owns it from then on (CompiledQueryCache frees it at teardown). inline void installBlobQueries( std::span blobs ) { std::vector want; const auto need = [ & ]( const LangEntry* le ) { if( le == nullptr || le->grammar == nullptr || le->querySub.empty() || compiledQueryCache().find( le->grammar() ) != compiledQueryCache().end() ) { return; } if( std::none_of( want.begin(), want.end(), [ & ]( const LangEntry* w ) { return w->grammar() == le->grammar(); } ) ) { want.push_back( le ); } }; for( const BlobText& b : blobs ) { const std::string ext = lowerExtensionOf( b.path ); need( lookupLang( ext ) ); if( ext == ".h" ) { need( lookupLang( ".m" ) ); } } for( const LangEntry* le : want ) { if( TSQuery* q = compileQueryStandalone( *le ); q != nullptr ) // a compile error stays absent: captureTagsFacts discloses it { compiledQueryCache()[ le->grammar() ] = q; } } } // One parse worker: its own parser, query cursor and scratch, claiming blobs in `walkOrder` until none is left. Each blob // writes only its own slot of `out` (and whatever `oneBlob` writes at that blob's index). template inline void blobPoolWorker( std::span blobs, std::span walkOrder, std::atomic& nextSlot, std::span out, const OneBlob& oneBlob ) noexcept { ParserGuard pg; TSQueryCursor* cursor = ts_query_cursor_new(); if( pg.p == nullptr || cursor == nullptr ) { for( std::size_t s = nextSlot.fetch_add( 1, std::memory_order_relaxed ); s < blobs.size(); s = nextSlot.fetch_add( 1, std::memory_order_relaxed ) ) { DISCLOSE( ( BlobDefsSlot{ out[ walkOrder[ s ] ] } ), BlobDefsSlot::DisclosureWhy::NoParser, "whereis: no tree-sitter parser for a blob worker" ); } if( cursor != nullptr ) { ts_query_cursor_delete( cursor ); } return; } MemberMacroReparse macroWork; IngestFileScan scan = makeFileScan( std::vector{ std::string() } ); for( std::size_t s = nextSlot.fetch_add( 1, std::memory_order_relaxed ); s < blobs.size(); s = nextSlot.fetch_add( 1, std::memory_order_relaxed ) ) { BlobDefinitions& slot = out[ walkOrder[ s ] ]; try { oneBlob( walkOrder[ s ], pg.p, cursor, macroWork, scan, slot ); } catch( ... ) // a throw escaping a worker thread is std::terminate: the blob degrades instead { DISCLOSE( ( BlobDefsSlot{ slot } ), BlobDefsSlot::DisclosureWhy::WorkerThrew, "whereis: a blob worker threw — that blob confirms no definition" ); } } ts_query_cursor_delete( cursor ); } // The blob pool: queries installed single-threaded first, then hardware_concurrency() workers, biggest blob first. template inline std::vector runBlobPool( std::span blobs, const OneBlob& oneBlob ) { std::vector out( blobs.size() ); if( blobs.empty() ) { return out; } warmFieldIdTable(); // idempotent; the AST walks behind captureTagsFacts read it lock-free installBlobQueries( blobs ); std::vector walkOrder( blobs.size() ); std::iota( walkOrder.begin(), walkOrder.end(), std::uint32_t( 0 ) ); std::stable_sort( walkOrder.begin(), walkOrder.end(), [ & ]( std::uint32_t a, std::uint32_t b ) noexcept { return blobs[ a ].bytes.size() > blobs[ b ].bytes.size(); } ); // biggest first: no straggler at the tail const unsigned threadCount = static_cast( std::min( std::max( 1u, std::thread::hardware_concurrency() ), blobs.size() ) ); std::atomic nextSlot{ 0 }; std::vector pool; pool.reserve( threadCount ); for( unsigned t = 0; t < threadCount; ++t ) { pool.emplace_back( [ & ]() noexcept { blobPoolWorker( blobs, walkOrder, nextSlot, out, oneBlob ); } ); } for( std::thread& w : pool ) { w.join(); } return out; } std::vector definitionLinesInBlobs( std::span blobs, std::string_view name ) { return runBlobPool( blobs, [ & ]( std::uint32_t at, TSParser* parser, TSQueryCursor* cursor, MemberMacroReparse& macroWork, IngestFileScan& scan, BlobDefinitions& slot ) { definitionLinesOfOneBlob( blobs[ at ], name, parser, cursor, macroWork, scan, slot ); } ); } // QD-TANGLE D4 (ingest.h): every callable's tangle inputs in each blob, by the index's own extraction and cc_walk. std::vector tangleInputsInBlobs( std::span blobs ) { std::vector out( blobs.size() ); const std::vector status = runBlobPool( blobs, [ & ]( std::uint32_t at, TSParser* parser, TSQueryCursor* cursor, MemberMacroReparse& macroWork, IngestFileScan& scan, BlobDefinitions& slot ) { std::vector defs; const LangEntry* le = rawDefsOfOneBlob( blobs[ at ], parser, cursor, macroWork, scan, slot, TagsCapture::DefinitionsAndShape, defs ); if( le == nullptr ) { return; } BlobTangleDefs& mine = out[ at ]; mine.lang = le->lang; for( const RawDef& d : defs ) { if( d.kind == SymKind::Function || d.kind == SymKind::Method ) { mine.defs.push_back( { d.scope, d.name, tangleInputsFrom( d.loc, d.maxNest, d.deepLoc, d.steer, tangleDispatchShare( d.deepLoc, d.deepDispatch, d.nestArmCut ) ), rawDefBodyStart( d ), d.endByte } ); } } slot.status = BlobDefsStatus::Parsed; } ); for( std::size_t i = 0; i < out.size(); ++i ) { out[ i ].status = status[ i ].status; if( status[ i ].status != BlobDefsStatus::Parsed ) { out[ i ].defs.clear(); } } return out; } // ---- QD-CONSTRUCTS-AT-INGEST: the --quality-delta construct hits, found in the parse worker's own tree ---------- // findQualityConstructs (lintrules.h) used to re-read and re-parse every file on every --quality-delta call — the // largest item of a warm working-tree call (43 / 22 / 68 % of envoy / cockroach / vscode instructions, ML Stage 0 // 11-13). The parse worker already holds the tree, so it runs the SAME two groups over it here (the kErrorMaskRules // specs through execGrammarQueries, the HandlerShapes walk through runWalkGroups), the hits ride the file's cache // record, and a warm call reads them back with no file I/O. // // IDENTITY with the re-parse: the worker parses with the same grammar row (lookupLangOfCrawledFile plus the same .h // ObjC sniff), after the same nest refusal, binary sniff and .astro frontmatter restriction astQueryGrouped applies, // and calls this BEFORE the member-macro re-parse can swap the tree. Two classes of file the re-parse reads are not // handed here, and both can hold no hit: Markdown and reST (the markdown grammar has no comment, handler or stub node // kind and no kErrorMaskRules query compiles for it). test/qdconstructscheck.sh compares the two lists on real trees. // // The compiled specs are per kLangTable row, compiled once per process on first use (call_once) and owned by a // function-local static, the house shape of compiledQueryCache(). Lines are resolved only when a file has a row. namespace { struct ConstructQueryTable { std::vector specs = errorMaskQuerySpecs(); std::vector groups = constructQueryGroups( specs ); // borrows `specs`: the table never moves std::array compiled; std::array byRow; ConstructQueryTable() = default; ConstructQueryTable( const ConstructQueryTable& ) = delete; ConstructQueryTable& operator=( const ConstructQueryTable& ) = delete; ~ConstructQueryTable() { for( GrammarQueries& qs : byRow ) { for( GroupedQuery& gq : qs.perSpec ) { ts_query_delete( gq.query ); } if( qs.combined != nullptr ) { ts_query_delete( qs.combined ); } } } }; ConstructQueryTable& constructQueryTable() { static ConstructQueryTable table; return table; } } // namespace void collectConstructHits( const LangEntry& le, std::uint32_t fileId, std::string_view bytes, TSNode root, TSQueryCursor* cur, std::vector& out ) { PROFILE_SCOPE_DESCRIBE( "ingest/extractFile: quality construct hits" ); ConstructQueryTable& table = constructQueryTable(); const std::size_t row = std::size_t( &le - kLangTable.data() ); ASSUME( row < kLangTable.size(), "a parse worker's LangEntry is a kLangTable row (lookupLang answers into the table)" ); const TSLanguage* grammar = le.grammar(); std::call_once( table.compiled[row], [ & ] { table.byRow[row] = compileGrammarQueries( grammar, table.groups ); } ); // No newline index up front: almost every file has no row, so lines are filled in only when one does. const std::vector noLines; std::vector> perGroup( table.groups.size() ); runWalkGroups( table.groups, root, fileId, bytes, noLines, grammar, le.lang, perGroup ); if( !table.byRow[row].perSpec.empty() ) { execGrammarQueries( cur, table.byRow[row], root, fileId, bytes, noLines, table.groups, perGroup ); } if( perGroup[0].empty() && perGroup[1].empty() ) { return; } const std::vector nlOffsets = buildNewlineOffsets( bytes ); for( std::vector& rows : perGroup ) { for( AstMatch& m : rows ) { m.line = lineAtByte( nlOffsets, m.startByte ); } } appendFileConstructHits( perGroup[0], perGroup[1], bytes, out ); } // QD-PROJECTION-FIXED (TRAIN-1111g fix 1): the construct tables of the rows this ingest will parse, compiled before its // parse workers start — one thread per row, as the workers' call_once would have, joined here — so the footprint they add // is read with no other work of this ingest running. A row compiled by an earlier ingest adds nothing. The total is what // the --quality-delta=A..B projection counts once (memguard.h FIXED BYTES); a reading that went down records nothing. namespace { std::atomic constructTablesHeld{ 0 }; // constant-initialized: no first-use order to get wrong } // namespace std::uint64_t constructTablesFootprint() noexcept { return constructTablesHeld.load( std::memory_order_relaxed ); } void warmConstructTables( const std::array& rows ) { PROFILE_SCOPE_DESCRIBE( "ingest/compile-queries: construct tables" ); ConstructQueryTable& table = constructQueryTable(); std::vector todo; for( std::size_t row = 0; row < rows.size(); ++row ) { if( rows[row] && kLangTable[row].grammar != nullptr ) { todo.push_back( row ); } } if( todo.empty() ) { return; } const std::uint64_t before = os::mem_footprint(); std::vector compilers; compilers.reserve( todo.size() ); for( const std::size_t row : todo ) { compilers.emplace_back( [ &table, row ]() noexcept { std::call_once( table.compiled[row], [ & ] { table.byRow[row] = compileGrammarQueries( kLangTable[row].grammar(), table.groups ); } ); } ); } for( std::thread& th : compilers ) { th.join(); } const std::uint64_t after = os::mem_footprint(); if( after > before ) { constructTablesHeld.fetch_add( after - before, std::memory_order_relaxed ); } } } // namespace rw — ingest_astquery.h section of ingest.cpp