#pragma once #include "infra/emit.h" // rw::emitTo / emitRaw / formatTo — THE emitter and its siblings #include "gitcmd.h" // rw::gitCmd — every git child starts with --no-optional-locks -c core.fsmonitor=false #if !defined( RIPWIRE_INGEST_TU ) #error "ingest_crawl.h is a SECTION of src/ingest.cpp's translation unit - include it only from ingest.cpp (see the ingest-family split note there)" #endif #include "infra/tablelookup.h" // findByField — the same lookup wrap's agentTarget uses #include "infra/sortutil.h" // svLess — string_view order without libstdc++'s length subtraction (#343) #include "memguard.h" // #350: memguard::Watch — the crawl stops when the memory guard says so // ingest_crawl.h — crawl + parse setup, moved VERBATIM from ingest.cpp in the 2026-08-29 split: the // limits/skip config, the extension -> {lang, grammar, query} table (lookupLang), capture-role and // DEF-name policy (roleOf/finalSegment/defNameFromCapture), skip rules, §L1 parse health, the // JSON/YAML/MD nest-depth guards, the deterministic crawl with its drop taxonomy (collectSources), // file IO (readFile/readFilePrefix), the A4-P7 stat-gate helpers, and the embedded tags-query text + // compiled-query cache/prewarm. Everything the ingest pipeline needs BEFORE a file is parsed. Same // contract as every ingest_*.h: reopens `namespace rw` and the unnamed namespace inside it — one TU, // one unnamed namespace, internal linkage unchanged, zero new API surface — under the // RIPWIRE_INGEST_TU guard. namespace rw { namespace { // ---- limits / skip config (all in one place) ---- // The per-file byte ceiling is now a RUNTIME value (default kDefaultMaxFileBytes = 4 MB, ingest.h), // threaded through collectSources so --max-file-size can override it. Kept here as the last-resort // fallback for any caller that somehow crawls with a zero ceiling. // (kBinarySniffCap moved to ingest.h — see rw::looksBinary there, now shared with grep's aux-file scan.) // saveCache's balanced lexical-index merge carries odd runs with memcpy. Keep the // payload's byte-copy contract explicit while preserving the former pair ordering. struct LexPair { std::uint64_t hash; std::uint32_t slot; friend constexpr bool operator<( const LexPair& lhs, const LexPair& rhs ) { return lhs.hash < rhs.hash || ( lhs.hash == rhs.hash && lhs.slot < rhs.slot ); } }; static_assert( std::is_trivially_copyable_v ); // ---- the extension -> {lang, grammar fn, query file} table (DOD, no per-file switch) ---- using LangFn = const TSLanguage* (*)( void ); struct LangEntry { std::string_view ext; // file extension incl. leading dot Lang lang; LangFn grammar; std::string_view querySub; // key into the configure-generated embedded tags.scm table }; // The one extension whose parse is RESTRICTED to a sub-range of the file (its `---` frontmatter). // Named once so the table row and ingest_sidecap.h's restrictAstroToFrontmatter cannot drift apart. inline constexpr std::string_view kAstroExt = ".astro"; // The Ruby extension whose top level is a view (a Jbuilder template), named once for the row and ingest_sidecap.h's // captureRubyBareCalls call, which binds the template's `json` there. inline constexpr std::string_view kJbuilderExt = ".jbuilder"; // Order does not matter (linear scan); kept grouped by language for readability. // The extent is EXACT, not headroom: it was 32 with 32 rows, .toml made it 33, .pyi made it 34 and the // .yml/.yaml pair made it 36, the .php/.phtml/.lua trio made it 40, the .ex/.exs pair made it 42, the // .rst/.adoc/.org/.mdx prose quartet made it 46, .dart made it 47, .kt made it 48, .hxx made it 49, // .gd made it 50, .astro made it 51, two later extensions made it 53, Clojure's four made it 57, and .proto makes it 58. Sizing it to the row count makes // `std::array present` (the grammar-prewarm set, // below) exact too, and it turns "added a row and forgot the extent" into a compile error rather than a // silent drop. // // THE PLAIN-TEXT PROSE FORMATS (.rst/.adoc/.org/.mdx), on the SAME grammar and the same walk as markdown. // docparse.h's kMarkdownGrammarExts is the NAME list every reader-facing prose lens consults; the // static_assert under this table ties the two together so neither can grow alone. Four SINGLE-PURPOSE // prose extensions — an extension that exists for nothing but documents — that the crawl indexed nowhere // before 2026-09-09, so a repository whose decision history lives in `docs/adr/*.rst` got "0 relevant of 0 // document files" out of --recall. // // WHY THE MARKDOWN GRAMMAR AND NOT AN EXTRACTOR PER FORMAT. Measured, not assumed. reStructuredText's // title underlines (`=====`, `-----`) ARE setext headings, so the existing section tier tiles a real .rst // document with no new code at all and --recall serves the DECISION rather than the whole file. On a real // 292-file, 2.28 MB astropy `.rst` documentation tree, five pre-registered questions at three budgets: // heading-tiled answered 15/15 while the same prose with its setext underlines removed (the one-unit // control) answered 9/15, and tiled was CHEAPER at every budget (mean est_tokens 1299/2659/5379 against // 1436/2781/6041 at max-tokens 2000/4000/8000). An extractor per format would have bought that same // benefit for a docText copy of every byte, a second body-resolution rule and a fifth parser to keep // deterministic. `=` and `-` cover 1321 of that corpus's 1948 underlines (67.8%); `*`/`^`/`"`/`~`/`+`/`#` // titles are read as prose, which costs recall precision inside a file and nothing else. // // SUPERSEDED FOR .rst (#409): the setext reading above got `=` and `-` titles and read every directive as a paragraph, so // .rst now goes through ingest_rest.h's parser; the grammar row stays as its Lang::Markdown identity. The rest of this note // stands for .adoc/.org/.mdx. // // WHAT THIS HONESTLY DOES NOT DO, disclosed because --recall says `section-granular` only when it is true: // AsciiDoc's `== Section` and Org-mode's `* Heading` are NOT markdown headings (the former is a paragraph, // the latter a list item), so those files carry the file-level node alone and serve as ONE whole-file // unit. A heading detector per format is a later lane with its own measurement. `.mdx` is markdown with // JSX, which the block grammar already reads as html blocks (opaque). Gate: test/textdocscheck.sh. constexpr std::array kLangTable = {{ { ".cpp", Lang::Cpp, &tree_sitter_cpp, "cpp" }, { ".cc", Lang::Cpp, &tree_sitter_cpp, "cpp" }, { ".cxx", Lang::Cpp, &tree_sitter_cpp, "cpp" }, // `.metal` = Metal Shading Language, a C++14 dialect — so it rides the C++ grammar and the C++ // tags.scm, no grammar or query of its own (fixes a real bug: an entire shader half was invisible, // `--callers=ml_styleFor` returned 0). MEASURED on a real 45-shader Metal application tree (864 KB) // before this entry was added, not assumed: 0.81% of bytes land inside an ERROR subtree under the C++ grammar // (the C grammar: 12.3% — 15x worse; real C++ in the same repo: 0.00%), and every one of the 249 // distinct `kernel`/`vertex`/`fragment` entry points is still captured, because tree-sitter's error // recovery localises the MSL-only qualifier to a single token and keeps the enclosing // function_definition intact. A control experiment that blanked every MSL-only keyword // (kernel/vertex/fragment/constant/device/threadgroup) before parsing cut ERROR nodes 1068->122 but // produced a BYTE-IDENTICAL def-name and ref-name set — so no pre-parse scrub is carried here; it // would be pure risk for zero graph gain. Residual known noise: an anonymous `enum : uint { ... }` // recovers as a NAMED enum, minting 18 junk t="type" symbols named `uint` across those 45 files // (0.04% of that repo's index) — accepted, not special-cased. // Lang::Cpp (not a new Lang::Metal) is deliberate: MSL and the C++/ObjC++ host share ONE call // namespace through dual-compile headers (`#if __METAL_VERSION__`), which is the whole point of // indexing shaders — a separate Lang would need a Metal<->Cpp bridge in langCompatible AND would // silently drop out of every `lang == Lang::Cpp` C-family behaviour (scope qualification, clone // detection, C-family lint) for no benefit. { ".metal", Lang::Cpp, &tree_sitter_cpp, "cpp" }, // Metal Shading Language (MSL) — see above // `.cu`/`.cuh` = CUDA C++, on the VENDORED tree-sitter-cuda grammar — NOT the Metal-style ride on // the C++ grammar, and that difference is MEASURED, not assumed (2026-08-04 probe, the fixture now // at test/cudafix/): under tree_sitter_cpp every definition survived error recovery (__global__/ // __device__/__launch_bounds__/template kernels — all 12 defs extracted) and device-side call edges // resolved, but every `kernel<<>>( args )` LAUNCH site produced no call reference at // all — `--callers=rk_reduceSum` returned count=0 — and a `__constant__` module table failed to // extract. Losing every host→kernel edge is the exact failure the Metal entry exists to prevent // (`--callers=ml_styleFor` = 0), so CUDA earns the real grammar Metal measurably did not need. // The former §7b limit (a `__constant__ float T[ 64 ];` module table yielded no symbol) is CLOSED as // of kParserVer 60: the real gap was the missing initializer — the r3 q10 patterns required an // init_declarator, and the cudaMemcpyToSymbol idiom never has one. tags.scm now carries structural // uninitialized-declaration patterns and cudaMemorySpaceQualifierOf gates them at capture time // (`__constant__` case-blind; `__device__`/`__managed__` behind the SCREAMING gate) — pinned positive // in test/cudacheck.sh §7b, which also pins the by-design non-goals: a lower-case mutable `__device__` // global stays out, and an uninitialized declaration with NO memory-space qualifier (the plain-C++ // extern/static/alignas shapes) is dropped, so non-CUDA C++ maps are unchanged. // tree-sitter-cuda is a GENERATED superset of tree-sitter-cpp (grammar.js requires cpp's): its // `kernel_call_expression` is aliased to `call_expression` with a `function:` field, so the C++ // tags.scm ("cpp" below) compiles against it unchanged and launches extract as ordinary calls. // Lang::Cpp (not a new Lang::Cuda) for the same deliberate reason as Metal: host and device share // ONE call namespace through dual-compile headers (`#ifdef __CUDACC__`), and a separate Lang would // need a bridge in langCompatible while dropping out of every C-family behaviour for no benefit. { ".cu", Lang::Cpp, &tree_sitter_cuda, "cpp" }, // CUDA C++ — see above { ".cuh", Lang::Cpp, &tree_sitter_cuda, "cpp" }, // CUDA header (dual-compile lives here) // `.h` stays C++-owned (deliberate, L3): a C header parses acceptably under the C++ grammar and // `.h` ownership between C/C++/ObjC is inherently ambiguous without a project-config signal this // tool doesn't have (an .m/.mm sibling reroutes via looksObjC() below; there is no analogous C // content-sniff — `#include ` alone isn't a reliable C-vs-C++ discriminant). graph.h's // langCompatible bridges Cpp<->C (like the existing Cpp<->ObjC bridge) so a `.c` DEFINITION still // resolves against a call/declaration living in its (C++-parsed) `.h`. { ".h", Lang::Cpp, &tree_sitter_cpp, "cpp" }, { ".hpp", Lang::Cpp, &tree_sitter_cpp, "cpp" }, { ".hh", Lang::Cpp, &tree_sitter_cpp, "cpp" }, // A4 (found-items 2026-09-17): `.hxx` (a C++ header spelling, same status as `.hpp`/`.hh`) had no row // here — every OTHER per-extension table in the tree (flipimpact.h's dead-code header set, layout.h's // --layout scan, lintrules.h, quality.h's isHeaderPath/isTestScriptPath twin, resolve.h's include // resolver, verbs_lint.h) already lists `.hxx` alongside `.h`/`.hpp`/`.hh`, so a repository that spells // its headers `.hxx` was invisible to the crawl even though every downstream table was ready for it. { ".hxx", Lang::Cpp, &tree_sitter_cpp, "cpp" }, { ".c", Lang::C, &tree_sitter_c, "c" }, // plain C (L3) — was entirely invisible before this table gained its own row { ".py", Lang::Python, &tree_sitter_python, "python" }, { ".pyi", Lang::Python, &tree_sitter_python, "python" }, // typing stub — often a library's ONLY Python-visible API (a Rust/C core's whole Python surface lives in one .pyi) { ".go", Lang::Go, &tree_sitter_go, "go" }, { ".rs", Lang::Rust, &tree_sitter_rust, "rust" }, { ".ts", Lang::TypeScript, &tree_sitter_typescript, "typescript" }, // #285: .tsx gets its OWN querySub ("tsx", queries/tsx/tags.scm), not "typescript" — the tsx // grammar is a superset of the plain one, but a query naming a JSX-only node type (added for // JSX element call edges) fails to compile against the plain grammar WHOLESALE, which would have // taken every .ts symbol/reference down with it. See queries/typescript/tags.scm's header. { ".tsx", Lang::TypeScript, &tree_sitter_tsx, "tsx" }, { ".mts", Lang::TypeScript, &tree_sitter_typescript, "typescript" }, { ".cts", Lang::TypeScript, &tree_sitter_typescript, "typescript" }, // Astro: the FRONTMATTER ONLY, via the one ts_parser_set_included_ranges call this build makes // (ingest_sidecap.h). Rides Lang::TypeScript deliberately — langCompatible() admits only same-Lang // pairs, so a Lang of its own would not resolve a frontmatter call into the .ts service it names, // which is the entire point of issue #67. Same shape as .tsx/.mts above and .metal/.cu below. { kAstroExt, Lang::TypeScript, &tree_sitter_typescript, "typescript" }, { ".swift", Lang::Swift, &tree_sitter_swift, "swift" }, { ".m", Lang::ObjC, &tree_sitter_objc, "objc" }, // Objective-C { ".mm", Lang::ObjC, &tree_sitter_objc, "objc" }, // Objective-C++ (ObjC layer + C-style; C++ partial) { ".js", Lang::JavaScript, &tree_sitter_javascript, "javascript" }, // JavaScript — parent grammar of TypeScript { ".jsx", Lang::JavaScript, &tree_sitter_javascript, "javascript" }, // JSX (the JS grammar parses jsx natively) { ".mjs", Lang::JavaScript, &tree_sitter_javascript, "javascript" }, // ES module { ".cjs", Lang::JavaScript, &tree_sitter_javascript, "javascript" }, // CommonJS module { ".sh", Lang::Bash, &tree_sitter_bash, "bash" }, // shell script { ".bash", Lang::Bash, &tree_sitter_bash, "bash" }, { ".zsh", Lang::Bash, &tree_sitter_bash, "bash" }, // zsh — parsed with the bash grammar (superset-ish; partial) { ".java", Lang::Java, &tree_sitter_java, "java" }, // Java — classes/interfaces/enums/methods/ctors + calls { ".rb", Lang::Ruby, &tree_sitter_ruby, "ruby" }, // Ruby — class/module/def + method calls + require // A Rake task file and a Jbuilder view are plain Ruby (test/rubyrakejbuildercheck.sh, parser version 135): what a // task or a JSON view calls is a caller like any other. Outside any class their self is known — Rake's `main`, the // view — and graph.h RubyTopSelf reads it; a `.jbuilder` file's `json` is the template's (ingest_ruby.h). { ".rake", Lang::Ruby, &tree_sitter_ruby, "ruby" }, { kJbuilderExt, Lang::Ruby, &tree_sitter_ruby, "ruby" }, { ".json", Lang::Json, &tree_sitter_json, "json" }, // JSON — top-level + 2nd-level object keys as t="sec"; DATA, no call edges { ".toml", Lang::Toml, &tree_sitter_toml, "toml" }, // TOML — [table] headers + their keys as t="sec"; DATA, no call edges { ".yml", Lang::Yaml, &tree_sitter_yaml, "yaml" }, // YAML — mapping keys (mdepth<=2, seqs transparent) as t="sec"; DATA, no call edges { ".yaml", Lang::Yaml, &tree_sitter_yaml, "yaml" }, // YAML sibling extension (k8s manifests favour it) { ".cs", Lang::CSharp, &tree_sitter_c_sharp, "csharp" }, // C# — classes/structs/interfaces/records/enums/methods/props + calls // PHP rides tree-sitter-php's `php/` sub-grammar (NOT `php_only/`) — see CMakeLists. That grammar // treats everything outside `` as `text`, which is why a `.phtml` template and a Blade // view (`*.blade.php` — still a `.php` extension) index without an ERROR subtree instead of parsing // as one. Both extensions therefore share one row shape; no separate template tier exists or is needed. { ".php", Lang::Php, &tree_sitter_php, "php" }, // PHP — classes/interfaces/traits/enums/functions/methods + calls { ".phtml", Lang::Php, &tree_sitter_php, "php" }, // PHP template (markup + islands) — same grammar, same query { ".ex", Lang::Elixir, &tree_sitter_elixir, "elixir" }, { ".exs", Lang::Elixir, &tree_sitter_elixir, "elixir" }, { ".dart", Lang::Dart, &tree_sitter_dart, "dart" }, // Lua: no classes, no imports. The five function-definition spellings and the one call node are the // whole extractable structure (queries/lua/tags.scm states the metatable/dynamic-dispatch floor). { ".lua", Lang::Lua, &tree_sitter_lua, "lua" }, // Lua — function/method defs (5 shapes) + calls // GDScript (.gd): Godot's language. A .gd FILE IS A CLASS BODY — `class_name` names it, top-level // `func`/`var` are its members — which is why queries/gdscript/tags.scm captures file-scope defs as // function/var rather than needing an enclosing class node. `.tscn`/`.tres`/`.gdshader` are NOT // indexed: they are scene/resource/shader formats with their own grammars, and none is vendored here. { ".gd", Lang::GDScript, &tree_sitter_gdscript, "gdscript" }, // GDScript — class/func/var/const/enum/signal defs + calls // Kotlin: `.kts` (Gradle script DSL) is deliberately NOT a row here yet — its trailing-lambda // density needs its own parse-quality probe before riding this grammar; `.kt` only for now. { ".kt", Lang::Kotlin, &tree_sitter_kotlin, "kotlin" }, // Kotlin — classes/objects/interfaces/functions + calls; JVM-bridged to Java (graph.h langCompatible) { ".clj", Lang::Clojure, &tree_sitter_clojure, "clojure" }, { ".cljs", Lang::Clojure, &tree_sitter_clojure, "clojure" }, { ".cljc", Lang::Clojure, &tree_sitter_clojure, "clojure" }, { ".bb", Lang::Clojure, &tree_sitter_clojure, "clojure" }, // Protocol Buffers: message / enum / service / rpc / enum value definitions, no call edges (queries/proto/tags.scm). { ".proto", Lang::Proto, &tree_sitter_proto, "proto" }, { ".md", Lang::Markdown, &tree_sitter_markdown, "" }, // Markdown DOC tier — headings/sections via extractMarkdown()'s custom tree walk; NO tags.scm (query stays "") { ".markdown", Lang::Markdown, &tree_sitter_markdown, "" }, // sibling extension, same walk { ".rst", Lang::Markdown, &tree_sitter_markdown, "" }, // reStructuredText — read by ingest_rest.h, no parse with this grammar { ".adoc", Lang::Markdown, &tree_sitter_markdown, "" }, // AsciiDoc — whole-file unit (see above) { ".org", Lang::Markdown, &tree_sitter_markdown, "" }, // Org-mode — whole-file unit (see above) { ".mdx", Lang::Markdown, &tree_sitter_markdown, "" }, // MDX — markdown with JSX islands }}; // SIBLING-COMPLETENESS GUARD (METHODOLOGY §3), at COMPILE time. docparse.h owns the NAME list every // reader-facing prose lens consults; this table owns the GRAMMAR rows. Adding a format to one and not the // other is exactly the defect this lane fixed — `.rst` counted as prose for ordering while the crawl // indexed it nowhere — so it is made impossible rather than merely documented. constexpr bool everyMarkdownGrammarExtHasARow() noexcept { for( const std::string_view ext : docparse::kMarkdownGrammarExts ) { // findIndexByField, not findByField: GCC rejects the pointer-vs-null compare of an array-derived // address in constant evaluation under the sanitizer flags (#347); the index form is clean on // every front end. const std::size_t rowIdx = findIndexByField( kLangTable, &LangEntry::ext, ext ); if( rowIdx == std::size( kLangTable ) || kLangTable[rowIdx].lang != Lang::Markdown ) { return false; } } return true; } static_assert( everyMarkdownGrammarExtHasARow(), "every docparse::kMarkdownGrammarExts entry needs a kLangTable row on Lang::Markdown — " "a prose format admitted by one and not the other is indexed nowhere while every lens calls it prose" ); // SIBLING-COMPLETENESS GUARD #2, the same idea for the verb-time language classifier. lintrules.h's langOfPath buckets // an indexed file by language for --lint-rules, --deps/--arch, co-change's dep_capable=, --nonlocal-state and the lint // catalog, from its own kLintExtRows table. That table used to be "kept in sync by hand", and hand sync is how PR #233 // lost its `.gd` row. This is the one translation unit that sees both tables, so the sync is asserted here: // - every kLangTable row carries an extension and a grammar (a zero-filled tail row has neither); // - a CODE row (model.h isCodeLang) is in kLintExtRows with the SAME Lang, and a data/doc row is not; // - every kLintExtRows row names a kLangTable row with the same Lang (a lint row the crawl never admits is dead). // Each check returns the first offending ROW INDEX, and the table's size when clean. It never returns an extension // string: the first draft did, with "" for clean, and a zero-filled row's extension is also "", so the check passed // the very defect it exists for. The index is what the compiler's note prints ("'41 == 48'"). constexpr std::size_t firstCrawlRowLangOfPathMisbuckets() noexcept { for( std::size_t index = 0; index < kLangTable.size(); ++index ) { const LangEntry& row = kLangTable[index]; const std::size_t lintIdx = findIndexByField( kLintExtRows, &LintExtRow::ext, row.ext ); // index form: #347 const bool lintFound = lintIdx != std::size( kLintExtRows ); const bool isMirrored = lintFound && kLintExtRows[lintIdx].lang == row.lang; // The grammar-pointer-vs-null clause is the ONE part of this guard GCC cannot constant-evaluate // when the grammar TUs are ASan-instrumented ("'(tree_sitter_cpp == 0)' is not a constant // expression", #347); clang accepts it under the same flags. The degrade is scoped to exactly // that front end + flag pair — the "REAL but reduced" posture CMakeLists announces for the GCC // sanitizer stack — and every other build asserts the grammar non-null in full. #if defined( __SANITIZE_ADDRESS__ ) && !defined( __clang__ ) const bool grammarNull = false; #else const bool grammarNull = row.grammar == nullptr; #endif if( row.ext.empty() || grammarNull || ( isCodeLang( row.lang ) ? !isMirrored : lintFound ) ) { return index; } } return kLangTable.size(); } constexpr std::size_t firstLangOfPathRowTheCrawlNeverAdmits() noexcept { for( std::size_t index = 0; index < std::size( kLintExtRows ); ++index ) { const LintExtRow& row = kLintExtRows[index]; const std::size_t crawlIdx = findIndexByField( kLangTable, &LangEntry::ext, row.ext ); // index form: #347 if( crawlIdx == std::size( kLangTable ) || kLangTable[crawlIdx].lang != row.lang ) { return index; } } return std::size( kLintExtRows ); } static_assert( firstCrawlRowLangOfPathMisbuckets() == kLangTable.size(), "a kLangTable row is empty, or a CODE row is missing from lintrules.h kLintExtRows (or names another Lang), " "or a data/doc row is in it — langOfPath would call an indexed file the wrong language" ); static_assert( firstLangOfPathRowTheCrawlNeverAdmits() == std::size( kLintExtRows ), "a lintrules.h kLintExtRows row names an extension kLangTable does not admit under the same Lang" ); // The language of a file the crawl ALREADY admitted. A `.txt` reaches files[] only when SphinxTxtRoots declared it, so here // it reads as reST; lookupLang itself keeps answering no, since it is the admission test for every other caller. const LangEntry* lookupLang( std::string_view ext ) noexcept; inline const LangEntry* lookupLangOfCrawledFile( std::string_view ext ) noexcept { return lookupLang( ext == ".txt" ? std::string_view( ".rst" ) : ext ); } const LangEntry* lookupLang( std::string_view ext ) noexcept { return findByField( kLangTable, &LangEntry::ext, ext ); } std::string lowerExtensionOf( std::string_view path ) { const std::size_t slash = path.find_last_of( '/' ); const std::size_t base = ( slash == std::string_view::npos ) ? 0 : slash + 1; const std::size_t dot = path.find_last_of( '.' ); if( dot == std::string_view::npos || dot <= base ) { return {}; } std::string ext; ext.reserve( path.size() - dot ); for( std::size_t i = dot; i < path.size(); ++i ) { ext.push_back( static_cast( std::tolower( static_cast( path[i] ) ) ) ); } return ext; } // ---- capture-name prefix -> role. @definition.* -> DEF, @reference.* -> REF, @import.* -> IMP. ---- // `Import` is the shared import-capture vocabulary of #358: one `@import.path` per WRITTEN specifier, // declared in the grammar's own tags.scm and normalised by ONE specifier normaliser per DepDialect // (src/ingest_importcap.h), in place of the per-language extractors this replaces. enum class CapRole : std::uint8_t { Ignore, NameOnly, Def, Ref, Import }; // Map the part AFTER "definition."/"reference." to a SymKind. Falls back to Other. SymKind defKind( std::string_view tail ) noexcept { if( tail == "function" ) { return SymKind::Function; } if( tail == "method" ) { return SymKind::Method; } if( tail == "class" ) { return SymKind::Class; } if( tail == "struct" ) { return SymKind::Struct; } if( tail == "interface" ) { return SymKind::Interface; } if( tail == "var" ) { return SymKind::Var; } if( tail == "constant" ) { return SymKind::Var; } if( tail == "cjsexport" ) // JS `module.exports.NAME = fn` / `exports.NAME = fn` — gated (isCjsExportTarget) { return SymKind::Function; } if( tail == "classfactory" ) // JS/TS `const Sub = Base.extend('Sub', …)` — gated (isJsClassFactory) { return SymKind::Class; } if( tail == "protomethod" ) // JS `Foo.prototype.NAME = fn` — gated (isPrototypeMemberTarget) { return SymKind::Method; } if( tail == "enummember" ) // Python `NAME = value` in an enum-family class — gated (isPyEnumMemberTarget) { return SymKind::Var; } if( tail == "testmacroblock" ) // LB-E: doctest/Catch2 `TEST_CASE( "title" ) { … }` — gated (testMacroBlockPartsOf) { return SymKind::Function; } if( tail == "module" ) { return SymKind::Other; } if( tail == "macro" ) { // macro-edges round: honest kind (was Function). C preproc_def/preproc_function_def, C++ // preproc_function_def, Rust macro_definition — t="macro", a disclosed-degraded callable. return SymKind::Macro; } if( tail == "field" ) // member variable (card A3) — gated (dropGatedCapture: static members / non-self targets drop) { return SymKind::Field; } if( tail == "type" ) { return SymKind::Struct; // typedef/alias/enum bucket (every grammar but Go's; model.h NamedType note) } // honesty-small-068: the Go type_spec split — the grammar's `type:` field PROVES the form, so the kind says it. if( tail == "definedtype" ) // `type N string`, `type L[T any] []T`, `type W Other` — a defined type, not a struct { return SymKind::NamedType; } if( tail == "functype" ) // `type F func(...)` { return SymKind::FuncType; } if( tail == "alias" ) // `type A = B` { return SymKind::Alias; } if( tail == "section" ) { return SymKind::Section; // JSON object keys (t="sec"), same kind as markdown headings } if( tail == "yamlkey" ) // YAML mapping keys — gated (yamlKeyCaptureDropped: depth cut + merge key) { return SymKind::Section; } return SymKind::Other; } CapRole roleOf( std::string_view cap, SymKind& kindOut ) noexcept { constexpr std::string_view kDef = "definition."; constexpr std::string_view kRef = "reference."; constexpr std::string_view kImp = "import."; if( cap == "name" ) { return CapRole::NameOnly; } if( cap.size() > kDef.size() && cap.substr( 0, kDef.size() ) == kDef ) { kindOut = defKind( cap.substr( kDef.size() ) ); return CapRole::Def; } if( cap.size() > kRef.size() && cap.substr( 0, kRef.size() ) == kRef ) { return CapRole::Ref; } // @import.* is tested AFTER @reference.* on purpose: the two families must stay disjoint, and an // earlier arm silently swallowing an import capture would look like a working query. if( cap.size() > kImp.size() && cap.substr( 0, kImp.size() ) == kImp ) { return CapRole::Import; } return CapRole::Ignore; // @doc, @local.scope, etc. } // ---- final identifier segment (ns::f / pkg.F / mod::f -> last segment) ---- std::string finalSegment( std::string_view raw ) // allocates a std::string → not noexcept { // A C++ OPERATOR name (`operator<`, `operator<<`, `operator<=>`, `operator bool`) is the one place a // name legitimately carries `<`/`::`-looking punctuation that is NOT a scope/type-arg separator — the // `<`-strip and "::"/"." splits below would mangle it to a bare `operator`. Detect the `operator` // keyword segment (bare `operator...` or the last `Scope::operator...`) and return it whole. Guarded on // a following non-identifier char so a normal identifier that merely starts with "operator" (e.g. // `operatorId`) still falls through to the generic path unchanged. if( const std::size_t op = raw.rfind( "operator" ); op != std::string_view::npos ) { const bool atSegStart = ( op == 0 ) || ( raw[ op - 1 ] == ':' ) || ( raw[ op - 1 ] == '.' ); const std::size_t after = op + 8; // one-past `operator` const bool isOpToken = after >= raw.size() || !( std::isalnum( static_cast( raw[ after ] ) ) || raw[ after ] == '_' ); if( atSegStart && isOpToken ) { return std::string( raw.substr( op ) ); // `operator<<`, `operator bool`, etc. — verbatim } } // FIX #1 (generic base/impl): drop a type-argument list before ANYTHING else. A base/impl name // handed back as `generic_type` carries `<...>` (Java `Base`, Rust `Wrapper`) whose // bare form (`Base`/`Wrapper`) is what byName keys on. The strip MUST precede the "::"/"." split, // or a `::` INSIDE the args (`Foo`) would be mistaken for the segment separator. A name never // legitimately contains a bare '<' (only a type-argument list opens one), so truncating at the FIRST // '<' is safe; the C++/TS bare-identifier path has no '<' → no-op (byte-identical). One carve-out: // a type-argument list always FOLLOWS an identifier, so a name that STARTS with '<' is not a generic // — it is a Swift operator function (`<`, `<=`, `<+>`), which the strip would erase to "". if( const std::size_t lt = raw.find( '<' ); lt != std::string_view::npos && lt > 0 ) { raw = raw.substr( 0, lt ); } // already a bare identifier in nearly all cases; defensive split on '.' and "::". std::size_t pos = raw.rfind( "::" ); if( pos != std::string_view::npos ) { raw = raw.substr( pos + 2 ); } pos = raw.rfind( '.' ); if( pos != std::string_view::npos ) { raw = raw.substr( pos + 1 ); } // trim trailing whitespace a `Base ` / `Wrapper ` spacing left behind (grammars usually // hand back tight text, but a space before the '<' would otherwise leave `Base ` after the strip). while( !raw.empty() && ( raw.back() == ' ' || raw.back() == '\t' ) ) { raw.remove_suffix( 1 ); } return std::string( raw ); } // ---- the DEF-name policy: which captured names get finalSegment's scope split, and which are whole ---- // Most code languages want the split — `ns::f` / `pkg.F` must key on the bare `f` that byName resolves. // The DATA-CONFIG languages want the opposite, because there a `.` is part of the NAME and not a scope // separator: a TOML table header IS its dotted spelling, so `[tool.ruff.lint]` must be findable as // `tool.ruff.lint` rather than as `lint` — a name that collides with every other `lint` in a repo and makes // `--grep=tool.ruff` miss the very table it names. // // JSON belongs here for the SAME reason and was ALREADY wrong before TOML existed: a package.json // dependency `"lodash.merge"` was indexed as `merge` (measured; the TOML round's sibling sweep is what // surfaced it). YAML is the third tenant: a `dotted.plain.key:` is one key whose name contains dots. // Covering all three is the sibling-completeness rule docs/METHODOLOGY.md §3 calls the dominant // defect class here — fixing the instance and leaving its sibling broken is the failure it names. // // For data-config languages, preserving dots cannot widen the call graph: they emit no @reference // captures, and graph.h's langCompatible isolates them from code languages. // Elixir DOES emit call references. Its module/protocol names retain their dotted spelling here; // function-definition captures are already bare names. captureTagsFacts still applies finalSegment // to reference names, so preserving Elixir definition captures does not create a byName mismatch. // // This lives beside finalSegment rather than inside captureTagsFacts on purpose — the caller is a very // large function already over the complexity bar, and a policy branch buried in it is both invisible and // a measured regression (--quality-delta scored the inline ternary at +3 ccx). /// Return an owned definition lookup name, preserving config keys and Elixir module names verbatim. std::string defNameFromCapture( Lang lang, std::string_view raw ) { if( lang == Lang::Json || lang == Lang::Toml || lang == Lang::Yaml || lang == Lang::Elixir || lang == Lang::Clojure ) { return std::string( raw ); } return finalSegment( raw ); } // ---- skip rules ---- bool isDenylistedName( std::string_view name ) noexcept { auto endsWith = [ name ]( std::string_view suf ) noexcept { return name.size() >= suf.size() && name.substr( name.size() - suf.size() ) == suf; }; if( name == "package-lock.json" ) { return true; // npm lockfile — huge, machine-generated, zero config value } if( name == "npm-shrinkwrap.json" ) { return true; // npm lockfile variant (same shape/size as package-lock) } if( endsWith( ".min.js" ) ) { return true; } if( endsWith( "_pb2.py" ) ) { return true; } if( endsWith( ".pb.go" ) ) { return true; } return false; } // (looksBinary moved to ingest.h as rw::looksBinary — unqualified lookup below still finds it, same namespace.) // §L1 — PARSE HEALTH, measured on the tree the ingest ALREADY built (no second parse, no second read). // // What it answers: of the files that ARE in the index, which ones did the parser only partly understand, // and which ones look machine-written? Before this, a corpus of 252 deliberately-invalid Python files // reported oversize="0" — "index complete" — while every symbol drawn from it was garbage. // // COST. `ts_node_has_error` is a flag on the subtree, so a clean file pays one O(1) test and nothing // else; the walk below descends only into children that carry the flag, so it is proportional to the // damage rather than to the file. The whitespace sample is a bounded 4 KB scan of bytes already in cache. // // WHY TOP-MOST ERROR SPANS. An ERROR node's subtree is itself full of error-flagged nodes; summing all of // them would count the same bytes many times and produce a ratio above 1. Descent stops at the outermost // ERROR, so errBytes is a true byte measure of "what the parser could not interpret". MISSING nodes are // zero-width by construction (the parser inserted a token that was not there), so they contribute to // errNodes and nothing to errBytes — which is exactly why BOTH numbers are disclosed, not just a ratio. // // errNodes/errBytes ALSO count invalid UTF-8 byte sequences found in the leading whitespace-sample window // (one per bad sequence, since tree-sitter's error recovery does not reliably flag them as ERROR/MISSING — // a garbage byte run can parse as an unrecognized leaf with no error node at all). FileHealth measureFileHealth( TSNode root, std::string_view bytes ) { FileHealth h; h.fileBytes = std::uint32_t( bytes.size() > 0xFFFFFFFFull ? 0xFFFFFFFFull : bytes.size() ); // Walks the sample codepoint-by-codepoint (jsonesc::utf8SeqLen, already the shared UTF-8 validator // for mcp.h/ccjson.h) rather than byte-by-byte: a whitespace byte is only meaningful outside a // multi-byte sequence, and this lets the same pass also catch invalid UTF-8 — see below — for free. // A bad sequence resyncs one byte at a time, same as any decoder recovering from garbage. Bad // sequences are recorded as ascending START POSITIONS (the scan runs left to right), which is what // lets the ERROR walk below dedup against them with std::lower_bound instead of a per-span scan. const std::size_t sample = bytes.size() < kHealthWsSampleBytes ? bytes.size() : kHealthWsSampleBytes; std::uint32_t ws = 0; std::vector badUtf8Positions; for( std::size_t i = 0; i < sample; ) { const int seqLen = jsonesc::utf8SeqLen( bytes.data(), i, bytes.size() ); if( seqLen == 0 ) { badUtf8Positions.push_back( std::uint32_t( i ) ); ++i; continue; } if( seqLen == 1 ) { const unsigned char c = ( unsigned char ) bytes[ i ]; if( c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v' ) { ++ws; } } i += std::size_t( seqLen ); } h.wsBytes = ws; // Invalid UTF-8 in the leading sample is unparseable content by construction, but tree-sitter's own // error recovery does not reliably surface it as an ERROR/MISSING node (a garbage byte run can be // swallowed as an unrecognized leaf with no error flag at all — confirmed empirically on a random-byte // .kt file: ts_node_has_error(root) came back false). Fold it into errNodes/errBytes rather than adding // a parallel disclosure field: it is exactly what those two attributes already mean to a reader — // "bytes this build could not interpret" — just found by a byte-level scan instead of a tree walk. // A position that falls inside a top-most ERROR span below is the SAME problem tree-sitter already // flagged there, and counting it twice would push err_ratio (errBytes/fileBytes) past its documented // <=1.0 ceiling — so coverage is tallied WHILE walking and folded in ONCE, after, rather than adding // every position up front and backing out duplicates mid-walk: `h` holds a correct value at every // point in this function, never a transiently over-counted one a reader mid-function could observe. if( !ts_node_has_error( root ) ) { // no ERROR span for a bad sequence to overlap — every one found is a genuinely new finding h.errNodes += std::uint32_t( badUtf8Positions.size() ); h.errBytes += std::uint32_t( badUtf8Positions.size() ); return h; } std::vector stack; ChildCursor cursor( root ); // reused across nodes — this walk never recurses stack.push_back( root ); std::uint32_t coveredBadUtf8 = 0; // badUtf8Positions entries already inside a counted top-most ERROR span while( !stack.empty() ) { const TSNode n = stack.back(); stack.pop_back(); if( ts_node_is_error( n ) ) { ++h.errNodes; const std::uint32_t lo = ts_node_start_byte( n ); const std::uint32_t hi = ts_node_end_byte( n ); h.errBytes += hi > lo ? hi - lo : 0u; if( hi > lo && !badUtf8Positions.empty() ) { const auto lo_it = std::lower_bound( badUtf8Positions.begin(), badUtf8Positions.end(), lo ); const auto hi_it = std::lower_bound( lo_it, badUtf8Positions.end(), hi ); coveredBadUtf8 += std::uint32_t( hi_it - lo_it ); } continue; // top-most only — see the note above } if( ts_node_is_missing( n ) ) { ++h.errNodes; continue; } // O(children), not O(children²). The root of a RECOVERED file is exactly where the width is // largest and least controlled — one comment flood plus one unparseable token measured 56× the // identical flood with no error in it before this became a cursor (test/childwalkscalecheck.sh, // arm B4; the rule is on src/infra/tschildren.h). Filtered in place: `stack` is the work list, // and only the children that carry an error belong on it. forEachChild( n, cursor.cur, [ &stack ]( TSNode c ) { if( ts_node_has_error( c ) || ts_node_is_missing( c ) ) { stack.push_back( c ); } return true; } ); } // Fold in only the bad-UTF-8 positions NOT already covered by a top-most ERROR span above. const std::uint32_t uncoveredBadUtf8 = std::uint32_t( badUtf8Positions.size() ) - coveredBadUtf8; h.errNodes += uncoveredBadUtf8; h.errBytes += uncoveredBadUtf8; return h; } // True when raw bracket/brace nesting exceeds kMaxJsonNestDepth — degenerate or hostile DATA, never config // (found live by bench/multiswe: tree-sitter-json's error recovery is superlinear on unclosed // nesting; a 100 KB file of "[[[[…" from nlohmann/json's own parser-torture suite measured 43 s). One O(n) // byte scan, quote-aware (a bracket inside a JSON string does not open a level), fully deterministic — // never a wall-clock parse timeout, which would break the byte-identical-output contract. bool jsonNestsTooDeep( std::string_view bytes ) noexcept { std::uint32_t depth = 0; bool inString = false, escaped = false; for( char c : bytes ) { if( inString ) { if( escaped ) { escaped = false; } else if( c == '\\' ) { escaped = true; } else if( c == '"' ) { inString = false; } continue; } if( c == '"' ) { inString = true; } else if( c == '[' || c == '{' ) { if( ++depth > kMaxJsonNestDepth ) { return true; } } else if( c == ']' || c == '}' ) { if( depth > 0 ) { --depth; } } } return false; } // True when block indentation implies a scanner indent stack anywhere near tree-sitter-yaml's // serialize() cliff — see kMaxYamlNestDepth in ingest.h for the defect arithmetic. Like jsonNestsTooDeep, // one deterministic O(n) byte scan BEFORE any parse — never a wall-clock timeout, which would break the // byte-identical-output contract. It OVER-approximates the scanner's indent stack, and the direction is // the whole design: an over-count skips a degenerate file with a disclosed stderr note; an under-count // would hand the parser a file whose state serialization corrupts memory. Three over-approximations: // - a stack of open indent COLUMNS, popped when a line dedents to or past them; every content line // charges its own column (a wrapped plain-scalar continuation counts like a key line); // - each leading `- ` / `? ` / `: ` block marker after the indent opens one more level on its line // (`- - - x` builds three scanner levels on one unindented line); // - the verdict charges 2 stack slots per open column, because a block mapping and a block sequence // can open at the SAME column (`key:` over `- item` at indent 0 is two scanner pushes, one column). // Lines inside block scalars and multi-line quoted values are indistinguishable without a parse and // count like any other line — over-approximation again (a block scalar would need 30+ distinct // increasing indents to trigger). Flow nesting (`{`/`[`) is deliberately NOT counted: the scanner's // serialize stack grows only on BLOCK begins (verified against every push_ind site in the vendored // scanner.c), so a deep one-line flow document is bounded by the generic parse path, not this one. bool yamlNestsTooDeep( std::string_view bytes ) noexcept { constexpr std::uint32_t kColCap = kMaxYamlNestDepth; // more slots than can survive the 2x verdict below std::uint32_t openCols[ kColCap ]; std::uint32_t openCount = 0; std::size_t i = 0; const std::size_t byteCount = bytes.size(); while( i < byteCount ) { // leading indent (tabs are invalid YAML block indentation; charging them anyway only over-counts) std::uint32_t col = 0; while( i < byteCount && ( bytes[ i ] == ' ' || bytes[ i ] == '\t' ) ) { ++col; ++i; } // block markers: `- ` (sequence entry), `? ` (explicit key), `: ` (explicit value) std::uint32_t markerLevels = 0; while( i + 1 < byteCount && ( bytes[ i ] == '-' || bytes[ i ] == '?' || bytes[ i ] == ':' ) && ( bytes[ i + 1 ] == ' ' || bytes[ i + 1 ] == '\t' ) ) { ++markerLevels; i += 2; col += 2; } const bool blankOrComment = ( i >= byteCount || bytes[ i ] == '\n' || bytes[ i ] == '\r' || bytes[ i ] == '#' ); if( !blankOrComment || markerLevels > 0 ) { const std::uint32_t base = col - 2u * markerLevels; while( openCount > 0 && openCols[ openCount - 1 ] >= base ) { --openCount; } // dedent pops for( std::uint32_t m = 0; m <= markerLevels; ++m ) // this line's level + one per marker { if( openCount >= kColCap ) { return true; // proxy-stack overflow IS the too-deep verdict } openCols[ openCount ] = base + 2u * m; ++openCount; } if( 2u * openCount > kMaxYamlNestDepth ) { return true; } } while( i < byteCount && bytes[ i ] != '\n' ) { ++i; } // rest of line is content, not structure if( i < byteCount ) { ++i; } } return false; } // True when blockquote/list nesting implies an open-blocks stack anywhere near tree-sitter-markdown's // serialize() cliff — see kMaxMdBlockDepth in ingest.h for the defect arithmetic. Like the json/yaml // prescans: one deterministic O(n) byte scan BEFORE any parse, over-approximating in the safe // direction (an over-count skips a degenerate file with a disclosed stderr note; an under-count would // hand the parser a file whose state serialization corrupts memory). Per line, the estimate is: // - one slot per leading '>' blockquote marker (spaces/tabs may interleave: `> > >` is depth 3); // - one slot per list marker in the leading run (`-`/`*`/`+` or `N.`/`N)` followed by space — // CommonMark nests a NEW list per marker, so `- - - x` opens three blocks on one line); // - plus remaining leading indent width / 2 (a nested list level costs ~2 columns, so deep // indentation built across earlier lines shows up as this line's indent). // The scanner's stack does accumulate ACROSS lines, but only by way of markers/indent that some line // exhibits — the deepest line's estimate bounds the stack the scanner can hold there. Stateless per // line, so a thematic break (`---`, 3 markers) or ASCII art never comes near 200. bool mdNestsTooDeep( std::string_view bytes ) noexcept { std::size_t i = 0; const std::size_t byteCount = bytes.size(); while( i < byteCount ) { std::uint32_t slots = 0; std::uint32_t indent = 0; bool leading = true; while( i < byteCount && bytes[ i ] != '\n' && leading ) { const char c = bytes[ i ]; if( c == ' ' || c == '\t' ) { ++indent; ++i; } else if( c == '>' ) { ++slots; ++i; } else if( ( c == '-' || c == '*' || c == '+' ) && ( i + 1 >= byteCount || bytes[ i + 1 ] == ' ' || bytes[ i + 1 ] == '\t' || bytes[ i + 1 ] == '\n' ) ) { ++slots; ++i; // the marker only; its following space rides the indent branch (over-counts, safely) } else if( c >= '0' && c <= '9' ) { std::size_t d = i; while( d < byteCount && bytes[ d ] >= '0' && bytes[ d ] <= '9' ) { ++d; } if( d < byteCount && ( bytes[ d ] == '.' || bytes[ d ] == ')' ) && ( d + 1 >= byteCount || bytes[ d + 1 ] == ' ' || bytes[ d + 1 ] == '\t' || bytes[ d + 1 ] == '\n' ) ) { ++slots; i = d + 1; // digits + the ./) delimiter; the space rides the indent branch } else { leading = false; } } else { leading = false; } } if( slots + indent / 2u > kMaxMdBlockDepth ) { return true; } while( i < byteCount && bytes[ i ] != '\n' ) { ++i; } if( i < byteCount ) { ++i; } } return false; } // ── the Kotlin string-template nesting prescan (kotlinStringsNestTooDeep) and its three lexical helpers ────────────── // The byte length of the character literal that starts at bytes[ i ] == '\'', or 0 when none does. The grammar's shape // is `'` (an escape, or ONE codepoint that is not a quote or a newline) `'`, and the parser's internal lexer takes it // whole, so a quote inside one (`'"'`) is never offered to the external scanner as a string start. inline std::size_t kotlinCharLiteralLength( std::string_view bytes, std::size_t i ) noexcept { const auto byteAt = [ & ]( std::size_t k ) noexcept -> unsigned char { return k < bytes.size() ? static_cast( bytes[ k ] ) : 0u; }; const unsigned char first = byteAt( i + 1 ); std::size_t length = 0; if( first == '\\' ) { length = ( byteAt( i + 2 ) == 'u' ) ? 8u : 4u; // '\uXXXX' or a one-character escape } else if( first != 0u && first != '\'' && first != '\n' && first != '\r' ) { std::size_t codepointBytes = 1; if( ( first >> 5 ) == 0x6u ) { codepointBytes = 2; } else if( ( first >> 4 ) == 0xEu ) { codepointBytes = 3; } else if( ( first >> 3 ) == 0x1Eu ) { codepointBytes = 4; } length = codepointBytes + 2u; } return ( length > 0 && byteAt( i + length - 1 ) == '\'' ) ? length : 0u; } // The index just past the `/* … */` comment whose `/*` starts at bytes[ i ], nesting exactly as the vendored scanner's // scan_multiline_comment does — including its reading of an unterminated comment, which runs to end of input. inline std::size_t kotlinBlockCommentEnd( std::string_view bytes, std::size_t i ) noexcept { std::size_t k = i + 2; std::size_t depth = 1; bool afterStar = false; while( k < bytes.size() ) { const char c = bytes[ k ]; ++k; if( c == '*' ) { afterStar = true; } else if( c == '/' && afterStar ) { afterStar = false; if( --depth == 0 ) { return k; } } else { afterStar = false; if( c == '/' && k < bytes.size() && bytes[ k ] == '*' ) { ++depth; ++k; } } } return k; } // In Kotlin CODE: the index just past a token that is consumed WHOLE before the external scanner can see a quote inside // it — a `//` comment, a nested `/* */` comment, a character literal, a backtick identifier — or `i` itself when none // starts at bytes[ i ]. inline std::size_t kotlinCodeTriviaEnd( std::string_view bytes, std::size_t i ) noexcept { const std::size_t byteCount = bytes.size(); const char c = bytes[ i ]; const char next = ( i + 1 < byteCount ) ? bytes[ i + 1 ] : '\0'; if( c == '/' && next == '/' ) { const std::size_t newline = bytes.find( '\n', i ); return ( newline == std::string_view::npos ) ? byteCount : newline; } if( c == '/' && next == '*' ) { return kotlinBlockCommentEnd( bytes, i ); } if( c == '\'' ) { return i + kotlinCharLiteralLength( bytes, i ); } if( c == '`' ) { const std::size_t close = bytes.find_first_of( "`\r\n", i + 1 ); const bool named = close != std::string_view::npos && bytes[ close ] == '`' && close > i + 1; return named ? close + 1 : i; } return i; } enum class KotlinStringEvent : std::uint8_t { None, OpenInterpolation, CloseString }; // One step of the vendored scanner's scan_string_content at bytes[ i ], inside an open string of the given shape: the // index after the bytes it consumes, and whether those bytes opened an interpolation or closed the string. // `$` a run of at least the string's `$` prefix followed by `{` opens an interpolation; any other run is content. // `\` skips the byte after it, and `\$` the byte after the `$` too: the scanner's loop falls through to its bottom // advance, so `\$${` is content, never an interpolation. Before a quote the string's shape decides. In a // single-quoted string `\$"` CLOSES it (upstream's own reading, mirrored because the stack follows it) and `\"` is // content. In a triple-quoted string `\` is no escape before a quote, bare or as `\$` (vendored patch 002), so the // quote is read again by the triple-quote close test. // `"` closes a single-quoted string; a run of three or more closes a triple-quoted one, and shorter runs are content. inline std::pair kotlinStringStep( std::string_view bytes, std::size_t i, bool tripleQuoted, std::size_t dollars ) noexcept { const std::size_t byteCount = bytes.size(); const auto byteAt = [ & ]( std::size_t k ) noexcept -> char { return k < byteCount ? bytes[ k ] : '\0'; }; const auto runOf = [ & ]( char c ) noexcept { std::size_t run = 0; while( i + run < byteCount && bytes[ i + run ] == c ) { ++run; } return run; }; switch( bytes[ i ] ) { case '$': { const std::size_t run = runOf( '$' ); const bool opens = run >= dollars && byteAt( i + run ) == '{'; return { i + run + ( opens ? 1u : 0u ), opens ? KotlinStringEvent::OpenInterpolation : KotlinStringEvent::None }; } case '\\': { const bool escapesDollar = byteAt( i + 1 ) == '$'; const std::size_t quoteAt = i + ( escapesDollar ? 2u : 1u ); if( tripleQuoted && byteAt( quoteAt ) == '"' ) { return { quoteAt, KotlinStringEvent::None }; // the triple-quote close test reads this quote again } if( escapesDollar ) { return { i + 3, byteAt( i + 2 ) == '"' ? KotlinStringEvent::CloseString : KotlinStringEvent::None }; } return { i + 2, KotlinStringEvent::None }; } case '"': { if( !tripleQuoted ) { return { i + 1, KotlinStringEvent::CloseString }; } const std::size_t run = runOf( '"' ); return { i + run, run >= 3 ? KotlinStringEvent::CloseString : KotlinStringEvent::None }; } default: { return { i + 1, KotlinStringEvent::None }; } } } // True when string-template nesting would take tree-sitter-kotlin's scanner string stack past kMaxKotlinStringNestDepth — // see that constant in ingest.h for the defect. Like the json/yaml/markdown prescans: one deterministic O(n) byte scan // BEFORE any parse, never a wall-clock timeout. Unlike them it is not a shape ESTIMATE, because the stack it bounds // changes in exactly two places — a string START pushes one entry and a string END pops one — so this scan MIRRORS the // vendored scanner's state machine (scan_string_start / scan_string_content; kotlinStringStep above carries the string // half). In code — top level, or inside an interpolation, which closes at the `}` balancing its `${` — a quote after an // optional `$` run is a string START, and kotlinCodeTriviaEnd skips what the parser's internal lexer takes whole first. // What a byte mirror cannot see is the parser's ERROR RECOVERY, which is why the ceiling sits 4x under the cliff rather // than at it, and why the vendored patch that turns the scanner's own abort() into a refused push is a second, // independent layer and not a formality. bool kotlinStringsNestTooDeep( std::string_view bytes ) noexcept { struct NestFrame { bool isString = false; bool tripleQuoted = false; std::size_t dollars = 1; // a string: how long a `$` run must be to open an interpolation inside it std::size_t openBraces = 0; // an interpolation: `{` opened inside this `${ … }` and not yet closed }; // String and interpolation frames strictly alternate above top-level code, so twice the ceiling bounds the stack — // and the string push that would pass the ceiling IS the verdict, so it is reached before the array could fill. std::array frames {}; std::size_t frameCount = 0; std::uint32_t stringDepth = 0; const std::size_t byteCount = bytes.size(); std::size_t i = 0; while( i < byteCount ) { NestFrame* const top = ( frameCount > 0 ) ? &frames[ frameCount - 1 ] : nullptr; if( top != nullptr && top->isString ) { const auto [ next, event ] = kotlinStringStep( bytes, i, top->tripleQuoted, top->dollars ); i = next; if( event == KotlinStringEvent::CloseString ) { --frameCount; --stringDepth; } else if( event == KotlinStringEvent::OpenInterpolation && frameCount < frames.size() ) { frames[ frameCount ] = NestFrame{}; ++frameCount; } continue; } const std::size_t afterTrivia = kotlinCodeTriviaEnd( bytes, i ); if( afterTrivia != i ) { i = afterTrivia; continue; } std::size_t dollarRun = 0; while( i + dollarRun < byteCount && bytes[ i + dollarRun ] == '$' ) { ++dollarRun; } if( i + dollarRun < byteCount && bytes[ i + dollarRun ] == '"' ) { if( stringDepth >= kMaxKotlinStringNestDepth || frameCount >= frames.size() ) { return true; } const std::size_t quote = i + dollarRun; const bool triple = quote + 2 < byteCount && bytes[ quote + 1 ] == '"' && bytes[ quote + 2 ] == '"'; frames[ frameCount ] = NestFrame{ true, triple, std::clamp( dollarRun, 1u, 255u ), 0u }; // the scanner caps its prefix at 255 ++frameCount; ++stringDepth; i = quote + ( triple ? 3u : 1u ); continue; } if( top != nullptr && bytes[ i ] == '{' ) { ++top->openBraces; } else if( top != nullptr && bytes[ i ] == '}' ) { if( top->openBraces == 0 ) { --frameCount; // the brace that balances `${` closes the interpolation: back inside its string } else { --top->openBraces; } } i += ( dollarRun > 0 ) ? dollarRun : 1u; } return false; } // A .h defaults to C++, but an Objective-C header (@interface/@protocol) must use the objc grammar or its // class/protocol structure is lost to the C++ parser. Cheap content peek (first 8 KB) for the distinctive // '@' declarations. @ is not valid C++ outside a string/comment — and "outside a string/comment" is load- // bearing, not negligible (kParserVer 74): a C++ header ABOUT Objective-C handling says "@interface" in a // doc comment (src/ingest_model.h's collapseObjCDeclDefs contract did), and the raw substring search this // replaces rerouted the whole header to the objc grammar, which cannot parse namespaces/lambdas/noexcept — // every C++ symbol in the file was shredded at EXTRACTION with no --skipped row and no floor. So the scan // masks // and /* */ comments plus string/char literals (raw strings included) and only counts an '@' that // sits in live code. Truncation honesty: a comment/string/raw-delimiter left OPEN at the 8 KB window edge // masks the rest of the window — a real @interface past an 8 KB leading comment is missed, exactly as it // was under the substring search's window. // The two literal-skip helpers, shared shape: given the opening byte's index, return the index of the // LAST byte the literal consumed (the caller's ++i steps past it), or npos when the window ends inside // the literal (the caller stops scanning — the truncation-honesty case above). // // Ordinary "…" / '…' with backslash escapes. An unterminated-on-line literal returns the newline's own // index, so the scan RESUMES on the next line — a lone quote in broken code must not mask the rest of // the window. std::size_t sniffSkipQuoted( std::string_view head, std::size_t openIndex, char quote ) noexcept { for( std::size_t i = openIndex + 1; i < head.size(); ++i ) { if( head[ i ] == '\\' ) { ++i; continue; } // skip the escaped char (incl. \") if( head[ i ] == quote || head[ i ] == '\n' ) { return i; } } return std::string_view::npos; } // R"delim( … )delim" — no escapes, so skip to its exact closing sequence, allocation-free (rIndex names // the R; the prefix letters u8/u/U/L before it are plain identifier chars to this scan and need no // special casing — the R adjacent to the quote is the discriminant). std::size_t sniffSkipRawString( std::string_view head, std::size_t rIndex ) noexcept { const std::size_t open = head.find( '(', rIndex + 2 ); if( open == std::string_view::npos ) { return std::string_view::npos; } const std::string_view delim = head.substr( rIndex + 2, open - ( rIndex + 2 ) ); for( std::size_t close = open + 1; ; ++close ) { close = head.find( ')', close ); if( close == std::string_view::npos ) { return std::string_view::npos; } if( head.compare( close + 1, delim.size(), delim ) == 0 && close + 1 + delim.size() < head.size() && head[ close + 1 + delim.size() ] == '"' ) { return close + 1 + delim.size(); // the closing '"' } } } // One masked-region dispatch: does a comment or literal START at index i? Returns the region's last // byte (the caller's ++i steps past it), npos when the region is open at the window edge, or i itself // when no region starts here (i.e. head[ i ] is live code). Comment notes: line comments end at EOL (a // trailing backslash-continuation only matters to a real compiler; the next line re-enters the scan // harmlessly). The char-literal guard: a ' whose LEFT neighbor is an identifier/digit char is a C++14 // digit separator (1'000'000) or a literal-suffix boundary, not a literal opener — treating it as one // would swallow real code up to the next stray quote. std::size_t sniffSkipMaskedRegion( std::string_view head, std::size_t i ) noexcept { const std::size_t npos = std::string_view::npos; const char c = head[ i ]; const char next = ( i + 1 < head.size() ) ? head[ i + 1 ] : '\0'; if( c == '/' && next == '/' ) { return head.find( '\n', i + 2 ); } if( c == '/' && next == '*' ) { const std::size_t close = head.find( "*/", i + 2 ); return ( close == npos ) ? npos : close + 1; } if( c == 'R' && next == '"' ) { return sniffSkipRawString( head, i ); } if( c == '"' ) { return sniffSkipQuoted( head, i, '"' ); } const bool identBefore = i > 0 && ( std::isalnum( static_cast( head[ i - 1 ] ) ) || head[ i - 1 ] == '_' ); if( c == '\'' && !identBefore ) { return sniffSkipQuoted( head, i, '\'' ); } return i; } bool looksObjC( std::string_view bytes ) noexcept { const std::string_view head = bytes.substr( 0, bytes.size() < 8192 ? bytes.size() : 8192 ); for( std::size_t i = 0; i < head.size(); ++i ) { const std::size_t last = sniffSkipMaskedRegion( head, i ); if( last != i ) { if( last == std::string_view::npos ) { return false; } // region open at the window edge i = last; continue; } // live code: the three distinctive ObjC declaration keywords, same tokens as the substring era. if( head[ i ] == '@' && ( head.compare( i + 1, 9, "interface" ) == 0 || head.compare( i + 1, 8, "protocol" ) == 0 || head.compare( i + 1, 14, "implementation" ) == 0 ) ) { return true; } } return false; } // ---- deterministic crawl: collect candidate source paths, then SORT ---- // excludeLabel (multi-root A12): non-empty ⇒ --exclude substrings match `