import { followSafely, UnsafeUrlError } from "./safe-url.js"; import { isDomainProxyNeeded, markDomainProxyNeeded, } from "./session.js"; export interface ScrapeOptions { url: string; formats?: Array<"markdown" | "html" | "links">; retainImages?: "all" | "none"; removeSelectors?: string[]; render?: boolean; timeout?: number; bypassCache?: boolean; cache?: boolean; } export interface ScrapeResult { success: boolean; url: string; title: string; markdown: string; html?: string; links?: string[]; statusCode: number; contentHash: string; tier: "cache" | "static" | "sandbox" | "cloudflare" | "browser" | "impersonate" | "proxy-impersonate"; latencyMs: number; error?: string; } const domainCookies = new Map(); const domainUserAgents = new Map(); const BROWSER_USER_AGENTS = [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", ]; export function getBrowserHeaders( targetUrl: string, extraHeaders?: Record ): Record { const parsed = new URL(targetUrl); const ua = domainUserAgents.get(parsed.hostname) ?? BROWSER_USER_AGENTS[ Math.abs(hashCode(targetUrl)) % BROWSER_USER_AGENTS.length ]; let hostCookie: string | undefined; try { hostCookie = domainCookies.get(parsed.hostname); } catch { // ignore invalid URL } return { "User-Agent": ua, Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.9", "Accept-Encoding": "gzip, deflate, br", "Cache-Control": "no-cache", Pragma: "no-cache", "Sec-Ch-Ua": '"Chromium";v="136", "Google Chrome";v="136", "Not.A/Brand";v="99"', "Sec-Ch-Ua-Mobile": "?0", "Sec-Ch-Ua-Platform": '"macOS"', "Sec-Fetch-Dest": "document", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-Site": "none", "Sec-Fetch-User": "?1", "Upgrade-Insecure-Requests": "1", ...(hostCookie ? { Cookie: hostCookie } : {}), ...extraHeaders, }; } export function setDomainCookie(hostname: string, cookieHeader: string): void { if (hostname && cookieHeader) { domainCookies.set(hostname, cookieHeader); } } /** * User-Agent paired with a domain's replayed session cookies. Cloudflare's * `cf_clearance` (and similar bound tokens) only validate when the exact UA * that solved the challenge is reused, so replay sets both together. */ export function setDomainUserAgent(hostname: string, userAgent: string): void { if (hostname && userAgent) { domainUserAgents.set(hostname, userAgent); } } function hashCode(str: string): number { let hash = 0; for (let i = 0; i < str.length; i++) { hash = (hash << 5) - hash + str.charCodeAt(i); hash |= 0; } return hash; } /** Markers that only appear on an AWS WAF challenge interstitial. */ const AWS_WAF_CHALLENGE_MARKERS = [ "awswafcookiedomainlist", "awswafintegration", "aws-waf-token", 'id="challenge-container"', "/__challenge_", ]; /** * True when an HTML response (with optional status) is an AWS WAF challenge * interstitial: a marker match proves it regardless of status; otherwise a * documented HTTP 202 whose payload is a `