import * as errors from './exceptions.js' import { DEFAULT_PRIORITY, UNKNOWN_JOB_RETRY_DELAY } from './constants.js' import { executeChannel } from './tracing_channels.js' import type { Job } from './job.js' import type { AcquiredJob } from './contracts/adapter.js' import type { QueueConfigResolver, ResolvedJobOptions } from './queue_config_resolver.js' import type { JobClass, JobContext, JobFactory, JobOptions, JobRetention, QueueManagerConfig, RetryConfig, } from './types/main.js' import type { JobExecuteMessage } from './types/tracing_channels.js' import { parseTimeout } from './utils.js' type PermanentFailureReason = 'timeout' | 'no-retries' | 'max-attempts' | 'stalled' /** * A timed out handler that had not settled when the timeout fired. The * attempt is over, but the handler keeps running until it returns: callers * wait for this promise before they run anything in its place. */ type TimedOutExecution = { timedOutExecution?: Promise } export type JobExecutionOutcome = | { type: 'completed'; removeOnComplete?: JobRetention } | ({ type: 'retry' retryAt?: Date /** Set when the job goes back to the queue because no class has its name. */ jobNotFound?: true } & TimedOutExecution) | ({ type: 'failed' reason: PermanentFailureReason error: Error removeOnFail?: JobRetention failedHookError?: Error } & TimedOutExecution) | { type: 'initialization-failed' error: Error removeOnFail?: JobRetention } export interface JobExecuteOptions { /** * A job whose name has no registered class goes back to the queue, to run * again 30 seconds later, while it has fewer attempts than this. During a * rolling deploy, an old worker can take a job dispatched by new code: a * worker that knows it runs it later. Defaults to 0: the job fails. */ unknownJobRetries?: number } type JobExecutionRuntimeDependencies = { /** Returns undefined when no class is registered under the name. */ resolveJob: (jobName: string) => JobClass | undefined | Promise configResolver: QueueConfigResolver jobFactory?: JobFactory executionWrapper?: NonNullable } type FailureDecision = | { type: 'retry'; retryAt?: Date } | { type: 'failed' reason: PermanentFailureReason hookError: Error } /** * No class is registered under the job name. Only the runtime throws it, so * it cannot be confused with an E_JOB_NOT_FOUND from the job's own code. */ class JobNotRegistered extends errors.E_JOB_NOT_FOUND {} const noopExecutionWrapper: NonNullable = async (run) => run() /** * Execute one in-process Job attempt through a single lifecycle interface. * * Resolution, instantiation, context construction, hydration, timeout, * tracing, failure hooks, and retry classification live behind this seam. * Callers retain only their adapter-specific scheduling and persistence work. */ export class JobExecutionRuntime { readonly #resolveJob: JobExecutionRuntimeDependencies['resolveJob'] readonly #configResolver: QueueConfigResolver readonly #jobFactory?: JobFactory readonly #executionWrapper: NonNullable constructor({ resolveJob, configResolver, jobFactory, executionWrapper, }: JobExecutionRuntimeDependencies) { this.#resolveJob = resolveJob this.#configResolver = configResolver this.#jobFactory = jobFactory this.#executionWrapper = executionWrapper ?? noopExecutionWrapper } /** * Execute a single acquired Job attempt and return the adapter-facing outcome. */ execute( job: AcquiredJob, queue: string, options: JobExecuteOptions = {} ): Promise { const executeMessage: JobExecuteMessage = { job, queue } const startTime = performance.now() const run = () => executeChannel.tracePromise(async () => { try { return await this.#executeAttempt(job, queue, executeMessage, options) } finally { executeMessage.duration = Number((performance.now() - startTime).toFixed(2)) } }, executeMessage) return this.#executionWrapper(run, job, queue) } /** * Fail a Job that stalled more than `maxStalledCount` times, without running it. * * The `failed()` hook receives `E_JOB_STALLED`, and the outcome carries the * Job's own retention so the caller finalizes it like any permanent failure. * The attempt is not traced, since the Job does not execute. */ async failStalled( job: AcquiredJob, queue: string, maxStalledCount: number ): Promise { let instance: Job let options: JobOptions try { ;({ instance, options } = await this.#instantiate(job)) } catch (error) { return { type: 'initialization-failed', error: error as Error, removeOnFail: this.#configResolver.resolveJobOptions(queue).removeOnFail, } } const error = new errors.E_JOB_STALLED([job.name, maxStalledCount]) const retention = this.#configResolver.resolveJobOptions(queue, options) instance.$hydrate(job.payload, this.#createContext(job, queue)) let failedHookError: Error | undefined try { await instance.failed?.(error) } catch (hookError) { failedHookError = hookError as Error } return { type: 'failed', reason: 'stalled', error, removeOnFail: retention.removeOnFail, failedHookError, } } async #instantiate(job: AcquiredJob): Promise<{ instance: Job; options: JobOptions }> { const JobClass = await this.#resolveJob(job.name) if (!JobClass) throw new JobNotRegistered([job.name]) const instance = this.#jobFactory ? await this.#jobFactory(JobClass) : new JobClass() return { instance, options: JobClass.options || {} } } async #executeAttempt( job: AcquiredJob, queue: string, executeMessage: JobExecuteMessage, executeOptions: JobExecuteOptions ): Promise { let instance: Job let options: JobOptions | undefined let timeout: number | undefined try { ;({ instance, options } = await this.#instantiate(job)) // An invalid Job.options.timeout fails the Job without retrying it. // Registration rejects one, but a job registered with hotReload is // imported again for every execution, after that check. timeout = parseTimeout(this.#configResolver.resolveJobOptions(queue, options).timeout) } catch (error) { const initializationError = error as Error executeMessage.error = initializationError // Only a name with no class goes back to the queue: an E_JOB_NOT_FOUND // thrown while loading or constructing a registered job, for another // name, fails it like any other initialization error. if ( error instanceof JobNotRegistered && job.attempts < (executeOptions.unknownJobRetries ?? 0) ) { const retryAt = new Date(Date.now() + UNKNOWN_JOB_RETRY_DELAY) executeMessage.status = 'retrying' executeMessage.nextRetryAt = retryAt return { type: 'retry', retryAt, jobNotFound: true } } executeMessage.status = 'failed' return { type: 'initialization-failed', error: initializationError, removeOnFail: this.#configResolver.resolveJobOptions(queue, options).removeOnFail, } } const context = this.#createContext(job, queue) const resolvedOptions = this.#configResolver.resolveJobOptions(queue, options) const retryConfig = this.#configResolver.resolveRetryConfig(queue, options) const timedOut: TimedOutExecution = {} try { await this.#executeJob(instance, job.payload, context, timeout, timedOut) executeMessage.status = 'completed' return { type: 'completed', removeOnComplete: resolvedOptions.removeOnComplete } } catch (error) { const executionError = error as Error const decision = this.#resolveFailure( job.name, resolvedOptions, retryConfig, executionError, job.attempts ) executeMessage.error = executionError if (decision.type === 'retry') { executeMessage.status = 'retrying' executeMessage.nextRetryAt = decision.retryAt return { ...decision, ...timedOut } } executeMessage.status = 'failed' let failedHookError: Error | undefined try { await instance.failed?.(decision.hookError) } catch (error) { failedHookError = error as Error } return { type: 'failed', reason: decision.reason, error: executionError, removeOnFail: resolvedOptions.removeOnFail, failedHookError, ...timedOut, } } } #createContext(job: AcquiredJob, queue: string): JobContext { return { jobId: job.id, name: job.name, attempt: job.attempts + 1, queue, priority: job.priority ?? DEFAULT_PRIORITY, acquiredAt: new Date(job.acquiredAt), stalledCount: job.stalledCount ?? 0, ...(job.scheduleId ? { scheduleId: job.scheduleId } : {}), } } async #executeJob( instance: Job, payload: unknown, context: JobContext, timeout: number | undefined, timedOut: TimedOutExecution ): Promise { if (timeout === undefined) { instance.$hydrate(payload, context) return instance.execute() } const signal = AbortSignal.timeout(timeout) instance.$hydrate(payload, context, signal) const { abortPromise, cleanupAbortListener } = this.#createTimeoutAbortRace( signal, instance.constructor.name, timeout ) // Unset when execute() throws synchronously: nothing keeps running then. let execution: Promise | undefined try { execution = instance.execute() await Promise.race([execution, abortPromise]) } catch (error) { // Cancellation is cooperative: a handler that has not settled yet keeps // running, so the caller must keep its slot until it returns. if (execution && error instanceof errors.E_JOB_TIMEOUT) { timedOut.timedOutExecution = Promise.resolve(execution).then( () => {}, () => {} ) } throw error } finally { cleanupAbortListener() } } #resolveFailure( jobName: string, options: ResolvedJobOptions, retryConfig: RetryConfig, error: Error, attempts: number ): FailureDecision { if (error instanceof errors.E_JOB_TIMEOUT && options.failOnTimeout) { return { type: 'failed', reason: 'timeout', hookError: error } } if (typeof retryConfig.maxRetries === 'undefined' || retryConfig.maxRetries <= 0) { return { type: 'failed', reason: 'no-retries', hookError: error } } if (attempts >= retryConfig.maxRetries) { return { type: 'failed', reason: 'max-attempts', hookError: new errors.E_JOB_MAX_ATTEMPTS_REACHED([jobName], { cause: error }), } } if (retryConfig.backoff) { return { type: 'retry', retryAt: retryConfig.backoff().getNextRetryAt(attempts + 1), } } return { type: 'retry' } } #createTimeoutAbortRace(signal: AbortSignal, runtimeJobName: string, timeout: number) { let abortHandler: (() => void) | undefined const abortPromise = new Promise((_, reject) => { abortHandler = () => { reject(new errors.E_JOB_TIMEOUT([runtimeJobName, timeout])) } if (signal.aborted) { abortHandler() return } signal.addEventListener('abort', abortHandler, { once: true }) }) return { abortPromise, cleanupAbortListener: () => { if (abortHandler) { signal.removeEventListener('abort', abortHandler) } }, } } }