'use strict' const { isBrowserlessError, ensureError, browserTimeout } = require('@browserless/errors') const createIsolatedFunction = require('isolated-function') const requireOneOf = require('require-one-of') const pTimeout = require('p-timeout') const pReflect = require('p-reflect') const pRetry = require('p-retry') const { AbortError } = pRetry const { SLOT } = createIsolatedFunction const createRunFunction = require('./function') const path = require('path') // `isolated-function` reads these to know which dependencies a snippet requires // are already on disk, so it can skip installing them. Walking up from a // dependency only lands on the consumer's `node_modules` under a flat install: // with pnpm it lands inside the isolated store, which holds that dependency's // own dependencies and nothing else, so every snippet paid a fresh install. // `module.paths` is the resolution chain Node itself would use from here, so it // reaches the consumer under either layout. const nodeModulesOf = file => { const marker = `${path.sep}node_modules${path.sep}` return file.slice(0, file.lastIndexOf(marker) + marker.length - 1) } const nodePaths = [ ...new Set([nodeModulesOf(require.resolve('@cloudflare/puppeteer')), ...module.paths]) ] const stringify = fn => fn.toString().trim().replace(/;$/, '') const getTargetId = async page => { let session try { session = await page.createCDPSession() const { targetInfo } = await session.send('Target.getTargetInfo') return targetInfo.targetId } catch { return undefined } finally { // The lookup failing is retryable, and a supplied page outlives the call, // so a session left attached here accumulates on someone else's page. if (session) await pReflect(session.detach()) } } // `goto` hands the normal path a `{ userAgent, viewport }` descriptor, which a // snippet can read. A supplied page has no `goto`, so read the same two facts off // the page itself rather than letting the snippet see `device` on one path only. const readDevice = async page => { if (typeof page?.evaluate !== 'function' || typeof page.viewport !== 'function') return undefined const { value: userAgent } = await pReflect(page.evaluate(() => navigator.userAgent)) const viewport = page.viewport() // All of it or none: a descriptor missing half its fields still reads as a // descriptor, so a snippet gets `undefined` from a field instead of a value. return userAgent && viewport ? { userAgent, viewport } : undefined } const isHttpResponse = response => response != null && typeof response.status === 'function' const isPageNotFound = error => error?.message === createRunFunction.PAGE_NOT_FOUND // A supplied page need not be a full Puppeteer page, so neither accessor is // assumed: the isolate connects over this endpoint and cannot run without it. const wsEndpointOf = page => { const browser = typeof page.browser === 'function' ? page.browser() : undefined return typeof browser?.wsEndpoint === 'function' ? browser.wsEndpoint() : undefined } const isPlainObject = value => { if (value === null || typeof value !== 'object') return false const prototype = Object.getPrototypeOf(value) return prototype === null || prototype === Object.prototype } const assertArgs = args => { if (args !== undefined && !isPlainObject(args)) { throw new TypeError('Expected `args` to be a plain object') } } const serializeResponse = response => ({ status: response.status(), statusText: response.statusText(), url: response.url(), ok: response.ok(), headers: response.headers(), remoteAddress: response.remoteAddress(), timing: response.timing(), fromCache: response.fromCache(), fromServiceWorker: response.fromServiceWorker() }) module.exports = ({ tmpdir } = {}) => { const isolatedFunction = createIsolatedFunction({ tmpdir, nodePaths }) const runFunction = createRunFunction(isolatedFunction) const toPageValues = extendPage => { if (!extendPage) return const values = Object.fromEntries( Object.entries(extendPage).filter(([, value]) => typeof value !== 'function') ) return Object.keys(values).length ? values : undefined } const createFunction = ( fn, { getBrowserless = requireOneOf(['browserless']), retry = 2, timeout = 30000, gotoOpts, extendPage, hostPage, needsBrowser: needsBrowserOverride, getPage, args: baseArgs, ...opts } = {} ) => { assertArgs(baseArgs) const code = stringify(fn) const usesPage = createRunFunction.isUsingPage(code) const needsNetwork = needsBrowserOverride === true || createRunFunction.needsBrowser(code, { ...extendPage, ...hostPage }, usesPage) const source = createRunFunction.buildTemplate(SLOT, { usesPage, needsBrowser: needsNetwork, extendPage, hostPage }) const pageValues = toPageValues(extendPage) let browserPromise const getBrowser = async () => { if (!browserPromise) { browserPromise = Promise.resolve(getBrowserless()).catch(error => { browserPromise = undefined throw error }) } return browserPromise } // A per-call `code` override is bundled on its own, so this reads the merged // options rather than `code`: the template built above only matches the // snippet this function was created with. const withPrebuiltSource = (runFunctionOpts, network) => runFunctionOpts.code === code ? { ...runFunctionOpts, needsNetwork: network, source } : runFunctionOpts const splitArgs = ({ args: callArgs, ...callOpts }) => { assertArgs(callArgs) if (baseArgs === undefined && callArgs === undefined) return { callOpts } return { callOpts, args: { ...baseArgs, ...callArgs } } } const buildRunOpts = async ({ page, device, response, url, call, strictTarget = false }) => { if (!page) throw new Error(createRunFunction.PAGE_NOT_FOUND) const targetId = await getTargetId(page) if (strictTarget && !targetId) throw new Error(createRunFunction.PAGE_NOT_FOUND) const resolvedDevice = device ?? (await readDevice(page)) const browserWSEndpoint = wsEndpointOf(page) if (!browserWSEndpoint) throw new Error('Browser WebSocket endpoint not found') const { callOpts, args } = call return withPrebuiltSource( { url, code, device: resolvedDevice, extendPage, hostPage, ...opts, ...callOpts, ...(pageValues && { pageValues }), ...(isHttpResponse(response) && { _response: serializeResponse(response) }), browserWSEndpoint, targetId, ...(strictTarget && { strictTarget: true }), ...(args && { args }) }, needsNetwork ) } const settle = result => { if (result.isFulfilled) return result const error = ensureError(result.value) if (isBrowserlessError(error)) throw error return result } // The page was navigated by whoever handed it over, so there is no `goto` // and no page to close: its owner decides when it dies. `timeout` is not // forwarded to `runFunction`, so this path has to apply it itself. A // timeout rejects the call and leaves the page open. The snippet and its // isolate subprocess keep running until the snippet returns or the page // is closed. The retry loop stops, so no later attempt starts. const runWithGivenPage = async (url, call) => { let isRejected = false // The caller holds its rejection, so nothing further should start. Called // from `run`, which is what enforces it: p-retry waits its backoff before // calling `run` again and the timeout can land inside that gap. Called // again from `onFailedAttempt` only to end the loop there and then, rather // than leaving a backoff timer to expire first — no attempt can get past // the check in `run` either way. const abortIfTimedOut = () => { if (isRejected) throw new AbortError() } const run = async () => { abortIfTimedOut() // Asked again per attempt: the supplied page may have died with the // fault being retried, so the caller gets to hand over a live one. const { page, device, response } = await getPage() const result = await runFunction( await buildRunOpts({ page, device, response, url, call, strictTarget: true }) ) // A miss inside the isolate comes back as a rejected result. It is not // a user-code failure: the supplied page was never found. if (!result.isFulfilled && isPageNotFound(ensureError(result.value))) { throw new Error(createRunFunction.PAGE_NOT_FOUND) } return settle(result) } // There is no context to replace here, so this is the only retry the // supplied-page path gets. Attaching to the page is what fails // transiently: `buildRunOpts` opens a CDP session and the isolate // connects over the websocket endpoint, and both can lose a race the // next attempt wins. const task = () => pRetry(run, { retries: retry, onFailedAttempt: error => { if (error.name === 'AbortError') throw error abortIfTimedOut() const isRetryable = error.code === 'EBRWSRCONTEXTCONNRESET' || error.code === 'EPROTOCOL' || isPageNotFound(error) if (!isRetryable) throw error } }) return pTimeout(task(), timeout, () => { isRejected = true throw browserTimeout({ timeout }) }) } const runWithBrowser = async (url, call) => { const browser = await getBrowser() const browserless = await browser.createContext() return browserless .withPage((page, goto) => async () => { const { device, response } = await goto(page, { url, timeout, ...gotoOpts }) return settle( await runFunction(await buildRunOpts({ page, device, response, url, call })) ) })() .finally(() => browserless.destroyContext()) } const runWithoutBrowser = async (url, { callOpts, args }) => { const result = await runFunction( withPrebuiltSource( { url, code, extendPage, hostPage, ...opts, ...callOpts, ...(pageValues && { pageValues }), ...(args && { args }) }, false ) ) return settle(result) } return async (url, fnOpts = {}) => { // The default only replaces `undefined`. A `null` options object used to // be spread into the run options, which ignores it, so it stays empty. const call = splitArgs(fnOpts ?? {}) if (!needsNetwork) return runWithoutBrowser(url, call) return getPage ? runWithGivenPage(url, call) : runWithBrowser(url, call) } } createFunction.teardown = () => isolatedFunction.teardown() createFunction.shells = isolatedFunction.shells return createFunction } module.exports.isHttpResponse = isHttpResponse module.exports.serializeResponse = serializeResponse module.exports.inspect = createRunFunction.inspect