import type { LaunchOptions, Page } from 'puppeteer-core' import { EventEmitter } from 'node:events' export interface ResourceData { page: Page [key: string]: any } export interface JobInstance { resources: ResourceData /** * Called to close the related resources */ close: () => Promise } export interface WorkerInstance { jobInstance: () => Promise /** * Closes the worker (called when the cluster is about to shut down) */ close: () => Promise /** * Repair is called when there is a problem with the worker (like call or close throwing * an error) */ repair: () => Promise } export type ConcurrencyImplementationClassType = new (options: LaunchOptions, puppeteer: any) => ConcurrencyImplementation export default abstract class ConcurrencyImplementation { protected options: LaunchOptions protected puppeteer: any /** * @param options Options that should be provided to puppeteer.launch * @param puppeteer puppeteer object (like puppeteer or puppeteer-core) */ constructor(options: LaunchOptions, puppeteer: any) /** * Initializes the manager */ abstract init(): Promise /** * Closes the manager (called when cluster is about to shut down) */ abstract close(): Promise /** * Creates a worker and returns it */ abstract workerInstance(perBrowserOptions: LaunchOptions | undefined): Promise } interface ClusterOptions { concurrency: number | ConcurrencyImplementationClassType maxConcurrency: number workerCreationDelay: number puppeteerOptions: LaunchOptions perBrowserOptions: LaunchOptions[] | undefined monitor: boolean timeout: number retryLimit: number retryDelay: number skipDuplicateUrls: boolean sameDomainDelay: number puppeteer: any } declare type Partial = { [P in keyof T]?: T[P]; } export type ClusterOptionsArgument = Partial interface TaskFunctionArguments { page: Page data: JobData worker: { id: number } } export declare type TaskFunction = (arg: TaskFunctionArguments) => Promise // hacky solution to get around Cluster.d.ts private fields export class Cluster extends EventEmitter { static CONCURRENCY_PAGE: number static CONCURRENCY_CONTEXT: number static CONCURRENCY_BROWSER: number public options: ClusterOptions public perBrowserOptions: LaunchOptions[] | null public workers: unknown[] public workersAvail: unknown[] public workersBusy: unknown[] public workersStarting: number public allTargetCount: number public jobQueue: { size: () => number } public errorCount: number public taskFunction: TaskFunction | null public idleResolvers: (() => void)[] public waitForOneResolvers: ((data: JobData) => void)[] public browser: ConcurrencyImplementation | null public isClosed: boolean public startTime: number public nextWorkerId: number public monitoringInterval: unknown public display: { log: (str: string) => void, resetCursor: () => void, close: () => void } | null public duplicateCheckUrls: Set public lastDomainAccesses: Map public systemMonitor: { getCpuUsage: () => number, getMemoryUsage: () => number } public checkForWorkInterval: unknown static launch(options: ClusterOptionsArgument): Promise> public constructor() public init: unknown public launchWorker: unknown task(taskFunction: TaskFunction): Promise public nextWorkCall: number public workCallTimeout: unknown public work: unknown public doWork: unknown public lastLaunchedWorkerTime: number public allowedToStartWorker: unknown public isTaskFunction: unknown public queueJob: unknown queue(data: JobData, taskFunction?: TaskFunction): Promise queue(taskFunction: TaskFunction): Promise execute(data: JobData, taskFunction?: TaskFunction): Promise execute(taskFunction: TaskFunction): Promise idle(): Promise waitForOne(): Promise close(): Promise public monitor: unknown }