/** * HTML page analyzer using htmlparser2's streaming tokenizer. * * Extracts SEO-relevant data from a page's HTML: title, meta description, * headings, images, links, canonical, OG tags, structured data, robots meta, * word count, hreflang. * * Deliberately NOT a DOM parser: the previous cheerio implementation built a * full DOM (~5-10x the HTML's size) per page, and with 25 concurrent parses * on a 128MB isolate that was the audit engine's dominant OOM cause. The * tokenizer keeps only the accumulated text and extracted fields in memory. */ import { Parser } from "htmlparser2"; import { normalizeUrl, isSameOrigin } from "./url-utils"; import type { PageAnalysis, PageLink } from "./types"; const SKIPPED_LINK_PROTOCOLS = /^(javascript:|mailto:|tel:|#)/; /** Subtrees whose text is not visible content. */ const NON_CONTENT_TAGS = new Set(["script", "style", "noscript", "svg"]); const HEADING_LEVELS: Record = { h1: 1, h2: 2, h3: 3, h4: 4, h5: 5, h6: 6, }; const MAX_ANCHOR_CHARS = 200; /** * Per-page caps on the extracted collections. Crawler-trap and mega-menu * pages can carry thousands of links/images per page, and crawled pages sit * in memory in 25-page persist batches — uncapped collections were part of * the audit engine's exceededMemory profile. Counts derived from these * arrays saturate at the cap on such pathological pages. */ const MAX_EXTRACTED_LINKS = 1_000; const MAX_EXTRACTED_IMAGES = 1_000; interface OpenAnchor { href: string; rel: string; text: string[]; } /** * Analyze an HTML string and extract all SEO-relevant data. */ export function analyzeHtml( html: string, pageUrl: string, statusCode: number, responseTimeMs: number, redirectUrl: string | null = null, ): PageAnalysis { let title: string | null = null; let titleDepth = 0; let titleDone = false; // parse5 (the old DOM path) treats

(); let openAnchor: OpenAnchor | null = null; // Visible text: prefer text inside an explicit ; when the document // never opens one (fragments), fall back to all non-head text. Both // exclude NON_CONTENT_TAGS subtrees. let suppressDepth = 0; let bodyDepth = 0; let headDepth = 0; let sawBody = false; const bodyParts: string[] = []; const fallbackParts: string[] = []; const handleMetaTag = (attribs: Record) => { const content = attribs["content"]; if (attribs["name"] === "description") { metaDescription ??= content?.trim() ?? ""; } else if (attribs["name"] === "robots") { robotsMeta ??= content ?? null; } else if (attribs["property"] === "og:title") { ogTitle ??= content ?? null; } else if (attribs["property"] === "og:description") { ogDescription ??= content ?? null; } else if (attribs["property"] === "og:image") { ogImage ??= content ?? null; } }; const handleLinkTag = (attribs: Record) => { if (attribs["rel"] === "canonical") { canonical ??= attribs["href"] ?? null; } else if (attribs["rel"] === "alternate" && attribs["hreflang"]) { hreflangTags.push(attribs["hreflang"]); } }; const closeAnchor = () => { if (!openAnchor) return; const { href, rel, text } = openAnchor; openAnchor = null; if (linksByTarget.size >= MAX_EXTRACTED_LINKS) return; const resolved = normalizeUrl(href, pageUrl); if (!resolved || linksByTarget.has(resolved)) return; const anchor = text .join("") .replace(/\s+/g, " ") .trim() .slice(0, MAX_ANCHOR_CHARS); linksByTarget.set(resolved, { targetUrl: resolved, anchor: anchor || null, isInternal: isSameOrigin(resolved, pageUrl), isNofollow: rel.split(/\s+/).includes("nofollow"), }); }; const parser = new Parser( { onopentag(name, attribs) { if (NON_CONTENT_TAGS.has(name)) { suppressDepth += 1; } if (name === "noscript") noscriptDepth += 1; if (noscriptDepth > 0) return; switch (name) { case "title": // Ignore inside <svg> — only the document title counts. if (!titleDone && suppressDepth === 0) { titleDepth += 1; if (title === null) title = ""; } break; case "head": headDepth += 1; break; case "body": bodyDepth += 1; sawBody = true; break; case "meta": handleMetaTag(attribs); break; case "link": handleLinkTag(attribs); break; case "img": if (images.length < MAX_EXTRACTED_IMAGES) { images.push({ src: attribs["src"] ?? null, alt: "alt" in attribs ? attribs["alt"] : null, }); } break; case "script": if (attribs["type"] === "application/ld+json") { hasStructuredData = true; } break; case "a": { // HTML forbids nested <a>; browsers implicitly close the open // one, and the tokenizer has no tree correction, so mirror that. closeAnchor(); const href = attribs["href"]; if (href && !SKIPPED_LINK_PROTOCOLS.test(href)) { openAnchor = { href, rel: attribs["rel"]?.toLowerCase() ?? "", text: [], }; } break; } } const headingLevel = HEADING_LEVELS[name]; if (headingLevel !== undefined) { headingOrder.push(headingLevel); if (headingLevel === 1 && openH1 === null) openH1 = []; } }, ontext(text) { if (suppressDepth > 0) return; if (titleDepth > 0) { if (title !== null) title += text; return; } if (openH1) openH1.push(text); if (openAnchor) openAnchor.text.push(text); if (bodyDepth > 0) { bodyParts.push(text); } else if (headDepth === 0) { fallbackParts.push(text); } }, onclosetag(name) { if (NON_CONTENT_TAGS.has(name) && suppressDepth > 0) { suppressDepth -= 1; } if (name === "noscript" && noscriptDepth > 0) { noscriptDepth -= 1; return; } if (noscriptDepth > 0) return; if (name === "title" && titleDepth > 0) { titleDepth -= 1; if (titleDepth === 0) titleDone = true; } if (name === "head" && headDepth > 0) headDepth -= 1; if (name === "body" && bodyDepth > 0) bodyDepth -= 1; if (name === "a") closeAnchor(); if (name === "h1" && openH1) { h1s.push(openH1.join("").trim()); openH1 = null; } }, }, // Defaults (non-XML mode): lowercased tag/attribute names, decoded // entities — matching what the DOM-based implementation saw. ); parser.write(html); parser.end(); const rawText = (sawBody ? bodyParts : fallbackParts).join(""); const bodyText = rawText.replace(/\s+/g, " ").trim(); const wordCount = bodyText ? bodyText.split(/\s+/).length : 0; return { url: pageUrl, statusCode, redirectUrl, responseTimeMs, title: (title ?? "").trim(), metaDescription: metaDescription ?? "", canonical, robotsMeta, ogTitle, ogDescription, ogImage, h1s, headingOrder, wordCount, bodyText, images, links: Array.from(linksByTarget.values()), hasStructuredData, hreflangTags, }; }