import { HttpCrawler, ProxyConfiguration, SessionPool } from 'crawlee'; const proxyConfiguration = new ProxyConfiguration({/* opts */}); const crawler = new HttpCrawler({ // To use the proxy IP session rotation logic, you must turn the proxy usage on. proxyConfiguration, // Overrides default Session pool configuration. sessionPool: new SessionPool({ maxPoolSize: 100 }), // Set to true if you want the crawler to save cookies per session, // and set the cookie header to request automatically (default is true). saveResponseCookies: true, async requestHandler({ session, body }) { const title = /(.*?)<\/title>/.exec(body as string)?.[1]; if (title === 'Blocked') { session?.retire(); } else if (title === 'Not sure if blocked, might also be a connection error') { session?.markBad(); } else { // session.markGood() - this step is done automatically in BasicCrawler. } }, });