import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { createServer, type AddressInfo } from 'node:net' import Knex from 'knex' import type { Knex as KnexTypes } from 'knex' import { test } from '@japa/runner' import type { Assert } from '@japa/assert' import { Redis, type RedisOptions } from 'ioredis' import { CronExpressionParser } from 'cron-parser' import { MemoryAdapter } from './_mocks/memory_adapter.js' import { redis, RedisAdapter } from '../src/drivers/redis_adapter.js' import { knex, KnexAdapter } from '../src/drivers/knex_adapter.js' import { KnexQueueSchemaService } from '../src/services/knex_queue_schema.js' import { registerDriverTestSuite } from './_utils/register_driver_test_suite.js' import { registerWorkWatcherSuite } from './_utils/register_work_watcher_suite.js' import { withRedisWriteSpy } from './_utils/with_redis_write_spy.js' import { withKnexQuerySpy } from './_utils/with_knex_query_spy.js' import { pauseKnexDeletes } from './_utils/pause_deletes.js' import { registerRetentionRaceSuite } from './_utils/register_retention_race_suite.js' import { MemoryLogger } from './_mocks/memory_logger.js' import { startSilentProxy } from './_utils/silent_proxy.js' import { subscriptionHeartbeat } from '../src/drivers/wake_messages.js' import { pgDeleteFailureSql, registerPruneFailureSuite, } from './_utils/register_prune_failure_suite.js' import { startSqliteWriteContender } from './_utils/sqlite_write_contender.js' const KEY_PREFIX = 'boringnode::queue::test::' test('redis factory should reuse a connection from another ioredis package copy', async ({ assert, }) => { class ForeignRedis { readonly lazyConnect = true readonly enableOfflineQueue = false readonly maxRetriesPerRequest = 0 lastKey?: string defineCommand() {} async zcard(key: string) { this.lastKey = key return 42 } } const connection = new ForeignRedis() const adapter = redis(connection as unknown as Redis)() assert.equal(await adapter.size(), 42) assert.equal(connection.lastKey, 'jobs::default::pending') }) test.group('Adapter | Memory', (group) => { let adapter: MemoryAdapter group.each.teardown(async () => { await adapter?.destroy() }) registerDriverTestSuite({ test, createAdapter: () => { adapter = new MemoryAdapter() return adapter }, supportsConcurrency: false, }) }) test('redis factory logs the errors of the connection it creates', async ({ assert }) => { const logger = new MemoryLogger() // Nothing listens on port 1: the connection fails once and gives up. const adapter = redis({ host: '127.0.0.1', port: 1, retryStrategy: () => null })({ logger }) try { for (let wait = 0; wait < 50 && logger.logs.length === 0; wait++) { await new Promise((resolve) => setTimeout(resolve, 20)) } const errorLogs = logger.logs.filter((entry) => entry.level === 'error') assert.lengthOf(errorLogs, 1) assert.match(String(errorLogs[0].obj?.err), /ECONNREFUSED/) } finally { await adapter.destroy().catch(() => {}) } }) test('redis factory leaves the errors of an app connection to the app', ({ assert }) => { const connection = new Redis({ lazyConnect: true }) redis(connection)({ logger: new MemoryLogger() }) assert.equal(connection.listenerCount('error'), 0) }) test.group('Adapter | Redis', (group) => { let connection: Redis group.each.setup(async () => { connection = new Redis({ host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), keyPrefix: KEY_PREFIX, db: 15, // Use db 15 for tests so we can safely flush it }) // Flush before test await connection.flushdb() return async () => { await connection.quit() } }) registerDriverTestSuite({ test, createAdapter: () => new RedisAdapter(connection), }) registerWorkWatcherSuite({ test, createAdapter: () => new RedisAdapter(connection), }) test('dispatches and watches for an ACL user without channel permissions', async ({ assert, cleanup, }) => { // New ACL users get no channel permission by default since Redis 7. await connection.call( 'ACL', 'SETUSER', 'queue-no-channels', 'on', '>pw', '~*', '+@all', 'resetchannels' ) const restricted = new Redis({ host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), keyPrefix: KEY_PREFIX, db: 15, username: 'queue-no-channels', password: 'pw', }) cleanup(async () => { restricted.disconnect() await connection.call('ACL', 'DELUSER', 'queue-no-channels') }) const adapter = new RedisAdapter(restricted) const watcher = await adapter.watch(['default']) cleanup(() => watcher.close()) const waiting = watcher.wait(2000, new AbortController().signal, 0) await adapter.pushOn('default', { id: 'acl-job', name: 'TestJob', payload: {}, attempts: 0 }) assert.isTrue(await waiting) }) test('claims a schedule written elsewhere once its marker wakes the worker', async ({ assert, cleanup, }) => { const worker = new RedisAdapter(connection) const watcher = await worker.watch(['default']) cleanup(() => watcher.close()) const writer = new RedisAdapter(connection) await writer.upsertSchedule({ id: 'later', name: 'TestJob', payload: {}, everyMs: 60_000, timezone: 'UTC', nextRunAt: new Date(Date.now() + 60_000), }) watcher.reset() assert.isNull(await worker.claimDueSchedule()) // Due before the next due time this worker read. await writer.upsertSchedule({ id: 'sooner', name: 'TestJob', payload: {}, everyMs: 60_000, timezone: 'UTC', nextRunAt: new Date(Date.now() + 100), }) assert.isTrue(await watcher.wait(1000, new AbortController().signal, 0)) watcher.reset() assert.equal((await worker.claimDueSchedule())?.id, 'sooner') }) test('claims a schedule written elsewhere within a second while busy', async ({ assert, cleanup, }) => { const worker = new RedisAdapter(connection) const watcher = await worker.watch(['default']) cleanup(() => watcher.close()) assert.isNull(await worker.claimDueSchedule()) // A busy worker does not wait, so no marker reaches it. await new RedisAdapter(connection).upsertSchedule({ id: 'while-busy', name: 'TestJob', payload: {}, everyMs: 60_000, timezone: 'UTC', nextRunAt: new Date(Date.now()), }) assert.isNull(await worker.claimDueSchedule()) await new Promise((resolve) => setTimeout(resolve, 1100)) assert.equal((await worker.claimDueSchedule())?.id, 'while-busy') }) test('does not cache a claim that a schedule marker overtook', async ({ assert, cleanup }) => { const direct = new Redis(connection.options) cleanup(() => direct.disconnect()) // Holds the reply of the next schedule claim, which finds nothing. const held = Promise.withResolvers() let holding = true const evaluate = direct.eval.bind(direct) as (...args: unknown[]) => Promise direct.eval = (async (...args: unknown[]) => { const reply = await evaluate(...args) if (holding && args[2] === 'schedules::due') { holding = false await held.promise } return reply }) as typeof direct.eval const worker = new RedisAdapter(direct) const watcher = await worker.watch(['default']) cleanup(() => watcher.close()) const claiming = worker.claimDueSchedule() await new Promise((resolve) => setTimeout(resolve, 50)) // Written after the claim read the due index: its marker wakes the worker. const waiting = watcher.wait(1000, new AbortController().signal, 0) await new RedisAdapter(connection).upsertSchedule({ id: 'after-claim', name: 'TestJob', payload: {}, everyMs: 60_000, timezone: 'UTC', nextRunAt: new Date(Date.now()), }) assert.isTrue(await waiting) held.resolve() assert.isNull(await claiming) assert.equal((await worker.claimDueSchedule())?.id, 'after-claim') }) test('claims every time once its watcher closes', async ({ assert }) => { const worker = new RedisAdapter(connection) const watcher = await worker.watch(['default']) assert.isNull(await worker.claimDueSchedule()) await watcher.close() // Polling now: no marker announces this schedule. await new RedisAdapter(connection).upsertSchedule({ id: 'after-close', name: 'TestJob', payload: {}, everyMs: 60_000, timezone: 'UTC', nextRunAt: new Date(Date.now()), }) assert.equal((await worker.claimDueSchedule())?.id, 'after-close') }) test('claims a schedule whose marker was lost with its connection', async ({ assert, cleanup, }) => { shortenHeartbeat(cleanup) const target = { host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), } const proxy = await startSilentProxy(target) cleanup(() => proxy.close()) // Only the blocking connection goes through the proxy. const direct = new Redis({ ...target, keyPrefix: KEY_PREFIX, db: 15 }) direct.duplicate = (override?: Partial) => new Redis({ host: '127.0.0.1', port: proxy.port, keyPrefix: KEY_PREFIX, db: 15, ...override }) cleanup(() => direct.disconnect()) const worker = new RedisAdapter(direct, false, new MemoryLogger()) const watcher = await worker.watch(['default']) cleanup(() => watcher.close()) assert.isNull(await worker.claimDueSchedule()) const waiting = watcher.wait(2000, new AbortController().signal, 0) await new Promise((resolve) => setTimeout(resolve, 50)) proxy.silence() // Redis pops its marker for the silent connection: the reply is lost. await new RedisAdapter(connection).upsertSchedule({ id: 'lost-marker', name: 'TestJob', payload: {}, everyMs: 60_000, timezone: 'UTC', nextRunAt: new Date(Date.now()), }) assert.isTrue(await waiting) assert.equal((await worker.claimDueSchedule())?.id, 'lost-marker') }).timeout(5000) test('refuses to watch for an ACL user that cannot block', async ({ assert, cleanup }) => { await connection.call( 'ACL', 'SETUSER', 'queue-no-blocking', 'on', '>pw', '~*', '&*', '+@all', '-@blocking' ) const restricted = new Redis({ host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), keyPrefix: KEY_PREFIX, db: 15, username: 'queue-no-blocking', password: 'pw', }) cleanup(async () => { restricted.disconnect() await connection.call('ACL', 'DELUSER', 'queue-no-blocking') }) // The worker polls instead of logging a failed wait every second. await assert.rejects(() => new RedisAdapter(restricted).watch(['default']), /NOPERM/) }) test('wakes one waiting watcher per marker', async ({ assert, cleanup }) => { const writer = new RedisAdapter(connection) const job = (id: string) => ({ id, name: 'TestJob', payload: {}, attempts: 0 }) // Pushed while every worker is busy: two jobs leave one marker. await writer.pushOn('default', job('first')) await writer.pushOn('default', job('second')) const watchers = await Promise.all([ new RedisAdapter(connection).watch(['default']), new RedisAdapter(connection).watch(['default']), ]) const controller = new AbortController() cleanup(async () => { controller.abort() await Promise.all(watchers.map((watcher) => watcher.close())) }) const woken: number[] = [] for (const [index, watcher] of watchers.entries()) { void watcher.wait(5000, controller.signal, 0).then((wake) => wake && woken.push(index)) } await new Promise((resolve) => setTimeout(resolve, 200)) assert.lengthOf(woken, 1) // The woken worker takes a job while another remains: the next one wakes. await writer.popFrom('default') await new Promise((resolve) => setTimeout(resolve, 200)) assert.sameMembers(woken, [0, 1]) }) test('hands a due time to another watcher when its watcher stops waiting', async ({ assert, cleanup, }) => { const first = await new RedisAdapter(connection).watch(['default']) const second = await new RedisAdapter(connection).watch(['default']) cleanup(() => Promise.all([first.close(), second.close()]).then(() => {})) const busy = new AbortController() const firstWait = first.wait(5000, busy.signal, 0) await new Promise((resolve) => setTimeout(resolve, 50)) const writer = new RedisAdapter(connection) await writer.pushLaterOn( 'default', { id: 'later', name: 'TestJob', payload: {}, attempts: 0 }, 300 ) await new Promise((resolve) => setTimeout(resolve, 100)) // The first watcher holds the due time, then its worker gets busy. busy.abort() assert.isFalse(await firstWait) const start = Date.now() assert.isTrue(await second.wait(2000, new AbortController().signal, 0)) assert.closeTo(Date.now() - start, 200, 100) }) test('aborts a blocking connection stuck in its handshake', async ({ assert, cleanup }) => { // Accepts connections and never answers. const blackhole = createServer(() => {}) await new Promise((resolve) => blackhole.listen(0, '127.0.0.1', resolve)) cleanup(() => void blackhole.close()) const port = (blackhole.address() as AddressInfo).port const direct = new Redis(connection.options) direct.duplicate = (override?: Partial) => new Redis({ ...connection.options, host: '127.0.0.1', port, ...override }) cleanup(() => direct.disconnect()) const controller = new AbortController() const adapter = new RedisAdapter(direct, false, new MemoryLogger()) const watching = adapter.watch(['default'], controller.signal) setTimeout(() => controller.abort(), 100) await assert.rejects(() => watching) }).timeout(2000) test('gives up on a blocking connection that does not start', async ({ assert, cleanup }) => { shortenHeartbeat(cleanup) // Accepts connections and never answers, as a connection whose command was lost. const blackhole = createServer(() => {}) await new Promise((resolve) => blackhole.listen(0, '127.0.0.1', resolve)) cleanup(() => void blackhole.close()) const port = (blackhole.address() as AddressInfo).port const direct = new Redis(connection.options) direct.duplicate = (override?: Partial) => new Redis({ ...connection.options, host: '127.0.0.1', port, ...override }) cleanup(() => direct.disconnect()) // Without an abort, the worker falls back to polling. const adapter = new RedisAdapter(direct, false, new MemoryLogger()) await assert.rejects(() => adapter.watch(['default']), /did not start in time/) }).timeout(2000) test('reconnects a blocking connection that stops answering', async ({ assert, cleanup }) => { shortenHeartbeat(cleanup) const target = { host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), } const proxy = await startSilentProxy(target) cleanup(() => proxy.close()) // Only the blocking connection goes through the proxy. const direct = new Redis({ ...target, keyPrefix: KEY_PREFIX, db: 15 }) direct.duplicate = (override?: Partial) => new Redis({ host: '127.0.0.1', port: proxy.port, keyPrefix: KEY_PREFIX, db: 15, ...override }) cleanup(() => direct.disconnect()) const logger = new MemoryLogger() const watcher = await new RedisAdapter(direct, false, logger).watch(['default']) cleanup(() => watcher.close()) const silences = () => logger.logs.filter((entry) => entry.message.includes('connection is silent')).length const waiting = watcher.wait(2000, new AbortController().signal, 0) await new Promise((resolve) => setTimeout(resolve, 50)) proxy.silence() const writer = new RedisAdapter(connection) const job = (id: string) => ({ id, name: 'TestJob', payload: {}, attempts: 0 }) // The marker of this push may be lost with the silent connection: the // watcher wakes to look once it gives up on it. await writer.pushOn('default', job('during-silence')) assert.isTrue(await waiting) await writer.popFrom('default') assert.equal(silences(), 1) // The new connection receives the next marker. const next = watcher.wait(2000, new AbortController().signal, 0) await writer.pushOn('default', job('after-reconnection')) assert.isTrue(await next) assert.equal(silences(), 1) }).timeout(5000) test('reconnects a blocking connection the server closes', async ({ assert, cleanup }) => { const logger = new MemoryLogger() const watcher = await new RedisAdapter(connection, false, logger).watch(['default']) cleanup(() => watcher.close()) const waiting = watcher.wait(3000, new AbortController().signal, 0) await new Promise((resolve) => setTimeout(resolve, 50)) // Its reply may be lost: the watcher looks for work at once, not after // the timeout of BZPOPMIN. const start = Date.now() const clients = (await connection.client('LIST')) as string const blocked = clients.split('\n').find((line) => line.includes('cmd=bzpopmin'))! await connection.client('KILL', 'ID', blocked.match(/id=(\d+)/)![1]) assert.isTrue(await waiting) assert.isBelow(Date.now() - start, 500) // The new connection receives the next marker. const next = watcher.wait(3000, new AbortController().signal, 0) await new RedisAdapter(connection).pushOn('default', { id: 'after-kill', name: 'TestJob', payload: {}, attempts: 0, }) assert.isTrue(await next) assert.isBelow(Date.now() - start, 1000) }).timeout(5000) test('listSchedules should use bounded network round-trips as schedule count grows', async ({ assert, }) => { const adapter = new RedisAdapter(connection) for (let i = 0; i < 50; i++) { await adapter.upsertSchedule({ id: `perf-schedule-${i}`, name: 'PerfJob', payload: { i }, everyMs: 60_000, timezone: 'UTC', }) } const { result: schedules, writes } = await withRedisWriteSpy({ connection, run: () => adapter.listSchedules(), }) assert.lengthOf(schedules, 50) assert.isAtMost( writes, 4, `Expected bounded write count with pipelining, got ${writes} writes for 50 schedules` ) }) test('claimDueSchedule should use bounded network round-trips when many schedules are not due', async ({ assert, }) => { const adapter = new RedisAdapter(connection) const futureRunAt = new Date(Date.now() + 60_000) for (let i = 0; i < 50; i++) { const id = `future-schedule-${i}` await adapter.upsertSchedule({ id, name: 'FutureJob', payload: { i }, everyMs: 60_000, timezone: 'UTC', }) await adapter.updateSchedule(id, { nextRunAt: futureRunAt }) } const { result: claimed, writes } = await withRedisWriteSpy({ connection, run: () => adapter.claimDueSchedule(), }) assert.isNull(claimed) assert.isAtMost( writes, 2, `Expected bounded claim writes, got ${writes} writes for 50 future schedules` ) }) test('deleteSchedule should not leave ghost index under write-failure chaos', async ({ assert, }) => { const adapter = new RedisAdapter(connection) const id = 'chaos-delete-schedule' await adapter.upsertSchedule({ id, name: 'ChaosJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }) const { writes } = await withRedisWriteSpy({ connection, run: () => adapter.deleteSchedule(id), onWrite: (writeCount) => { if (writeCount === 2) { throw new Error('chaos: second network write blocked') } }, }) const scheduleExists = await connection.exists(`schedules::data::${id}`) const indexContains = await connection.sismember('schedules::index', id) assert.equal(scheduleExists, 0) assert.equal(indexContains, 0) assert.equal( writes, 1, 'deleteSchedule should be emitted in a single write window to avoid partial state' ) }) test('completeJob should not delete a newer TTL dedup lock when Redis keyPrefix is disabled', async ({ assert, }) => { const redisOptions = { host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), db: 15, keyPrefix: '', } const inspectorConnection = new Redis(redisOptions) const adapter = redis(redisOptions)() const queue = 'raw-ttl-clean-queue' const dedupId = 'TestJob::raw-ttl-clean-1' const dedupKey = `jobs::${queue}::dedup::${dedupId}` await connection.flushdb() try { await adapter.pushOn(queue, { id: 'raw-ttl-clean-uuid-1', name: 'TestJob', payload: { n: 1 }, attempts: 0, dedup: { id: dedupId, ttl: 80 }, }) const first = await adapter.popFrom(queue) assert.equal(first!.id, 'raw-ttl-clean-uuid-1') await new Promise((r) => setTimeout(r, 150)) const second = await adapter.pushOn(queue, { id: 'raw-ttl-clean-uuid-2', name: 'TestJob', payload: { n: 2 }, attempts: 0, dedup: { id: dedupId, ttl: 10_000 }, }) assert.equal(second && typeof second === 'object' && second.outcome, 'added') assert.equal(await inspectorConnection.get(dedupKey), 'raw-ttl-clean-uuid-2') await adapter.completeJob(first!, queue, true) assert.equal(await inspectorConnection.get(dedupKey), 'raw-ttl-clean-uuid-2') const third = await adapter.pushOn(queue, { id: 'raw-ttl-clean-uuid-3', name: 'TestJob', payload: { n: 3 }, attempts: 0, dedup: { id: dedupId, ttl: 10_000 }, }) assert.equal(third && typeof third === 'object' && third.outcome, 'skipped') assert.equal(third && typeof third === 'object' && third.jobId, 'raw-ttl-clean-uuid-2') } finally { await connection.flushdb() await adapter.destroy() await inspectorConnection.quit() } }) test('history pruning should not delete a newer dedup lock when Redis keyPrefix is disabled', async ({ assert, }) => { const redisOptions = { host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), db: 15, keyPrefix: '', } const inspectorConnection = new Redis(redisOptions) const adapter = redis(redisOptions)() const queue = 'raw-finalize-prune-queue' const dedupId = 'TestJob::raw-finalize-prune-1' const dedupKey = `jobs::${queue}::dedup::${dedupId}` await connection.flushdb() try { await adapter.pushOn(queue, { id: 'raw-finalize-prune-uuid-1', name: 'TestJob', payload: { n: 1 }, attempts: 0, dedup: { id: dedupId, ttl: 80 }, }) const first = await adapter.popFrom(queue) assert.equal(first!.id, 'raw-finalize-prune-uuid-1') await adapter.completeJob(first!, queue, { count: 1 }) await new Promise((r) => setTimeout(r, 150)) const second = await adapter.pushOn(queue, { id: 'raw-finalize-prune-uuid-2', name: 'TestJob', payload: { n: 2 }, attempts: 0, dedup: { id: dedupId }, }) assert.equal(second && typeof second === 'object' && second.outcome, 'added') assert.equal(await inspectorConnection.get(dedupKey), 'raw-finalize-prune-uuid-2') const popped = await adapter.popFrom(queue) assert.equal(popped!.id, 'raw-finalize-prune-uuid-2') await adapter.completeJob(popped!, queue, { count: 1 }) assert.equal(await inspectorConnection.get(dedupKey), 'raw-finalize-prune-uuid-2') const third = await adapter.pushOn(queue, { id: 'raw-finalize-prune-uuid-3', name: 'TestJob', payload: { n: 3 }, attempts: 0, dedup: { id: dedupId }, }) assert.equal(third && typeof third === 'object' && third.outcome, 'skipped') assert.equal(third && typeof third === 'object' && third.jobId, 'raw-finalize-prune-uuid-2') } finally { await connection.flushdb() await adapter.destroy() await inspectorConnection.quit() } }) test('history pruning keeps the data of a job id pushed again', async ({ assert }) => { const adapter = new RedisAdapter(connection) const queue = 'reused-id-prune-queue' const retention = { count: 2 } async function failNext(expectedId: string) { const job = await adapter.popFrom(queue) assert.equal(job!.id, expectedId) await adapter.failJob(job!, queue, new Error('boom'), retention) } await adapter.pushOn(queue, { id: 'op-1', name: 'TestJob', payload: { copy: 1 }, attempts: 0 }) await failNext('op-1') await adapter.pushOn(queue, { id: 'op-1', name: 'TestJob', payload: { copy: 2 }, attempts: 0 }) const live = await adapter.popFrom(queue) assert.equal(live!.id, 'op-1') // The failed history now holds op-1, a, b: the old op-1 entry is pruned. for (const id of ['a', 'b']) { await adapter.pushOn(queue, { id, name: 'TestJob', payload: {}, attempts: 0 }) await failNext(id) } const record = await adapter.getJob('op-1', queue) assert.equal(record?.status, 'active') assert.deepEqual(record?.data.payload, { copy: 2 }) // Once the live copy completes, a stale failed entry cannot delete its data. await adapter.pushOn(queue, { id: 'op-2', name: 'TestJob', payload: { copy: 1 }, attempts: 0 }) await failNext('op-2') await adapter.pushOn(queue, { id: 'op-2', name: 'TestJob', payload: { copy: 2 }, attempts: 0 }) const second = await adapter.popFrom(queue) await adapter.completeJob(second!, queue, retention) for (const id of ['c', 'd']) { await adapter.pushOn(queue, { id, name: 'TestJob', payload: {}, attempts: 0 }) await failNext(id) } const completed = await adapter.getJob('op-2', queue) assert.equal(completed?.status, 'completed') assert.deepEqual(completed?.data.payload, { copy: 2 }) }) test('recoverStalledJobs should not delete a newer dedup lock when Redis keyPrefix is disabled', async ({ assert, }) => { const redisOptions = { host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), db: 15, keyPrefix: '', } const inspectorConnection = new Redis(redisOptions) const adapter = redis(redisOptions)() const queue = 'raw-stall-dedup-queue' const dedupId = 'TestJob::raw-stall-dedup-1' const dedupKey = `jobs::${queue}::dedup::${dedupId}` await connection.flushdb() try { await adapter.pushOn(queue, { id: 'raw-stall-dedup-uuid-1', name: 'TestJob', payload: { n: 1 }, attempts: 0, stalledCount: 0, dedup: { id: dedupId, ttl: 80 }, }) const first = await adapter.popFrom(queue) assert.equal(first!.id, 'raw-stall-dedup-uuid-1') await new Promise((r) => setTimeout(r, 150)) const second = await adapter.pushOn(queue, { id: 'raw-stall-dedup-uuid-2', name: 'TestJob', payload: { n: 2 }, attempts: 0, dedup: { id: dedupId }, }) assert.equal(second && typeof second === 'object' && second.outcome, 'added') assert.equal(await inspectorConnection.get(dedupKey), 'raw-stall-dedup-uuid-2') // First job still active + stalled. With maxStalledCount=0 it is handed back to be failed. const { recovered, exceeded } = await adapter.recoverStalledJobs(queue, 10, 0, 100) assert.equal(recovered, 0) assert.lengthOf(exceeded, 1) await adapter.failJob(exceeded[0], queue, new Error('stalled')) assert.equal(await inspectorConnection.get(dedupKey), 'raw-stall-dedup-uuid-2') const third = await adapter.pushOn(queue, { id: 'raw-stall-dedup-uuid-3', name: 'TestJob', payload: { n: 3 }, attempts: 0, dedup: { id: dedupId }, }) assert.equal(third && typeof third === 'object' && third.outcome, 'skipped') assert.equal(third && typeof third === 'object' && third.jobId, 'raw-stall-dedup-uuid-2') } finally { await connection.flushdb() await adapter.destroy() await inspectorConnection.quit() } }) test('dedup keys are removed when the connection uses a key prefix', async ({ assert }) => { const adapter = new RedisAdapter(connection) const push = (queue: string, id: string) => adapter.pushOn(queue, { id, name: 'TestJob', payload: {}, attempts: 0, dedup: { id }, }) const dedupExists = (queue: string, id: string) => connection.exists(`jobs::${queue}::dedup::${id}`) await push('prefix-complete', 'completed-job') const completed = await adapter.popFrom('prefix-complete') await adapter.completeJob(completed!, 'prefix-complete') assert.equal(await dedupExists('prefix-complete', 'completed-job'), 0) await push('prefix-fail', 'failed-job') const failed = await adapter.popFrom('prefix-fail') await adapter.failJob(failed!, 'prefix-fail', new Error('boom')) assert.equal(await dedupExists('prefix-fail', 'failed-job'), 0) await push('prefix-prune', 'pruned-job') const pruned = await adapter.popFrom('prefix-prune') await adapter.completeJob(pruned!, 'prefix-prune', { count: 1 }) // History is ordered by completion time; keep both completions in distinct milliseconds. await new Promise((resolve) => setTimeout(resolve, 2)) await push('prefix-prune', 'kept-job') const kept = await adapter.popFrom('prefix-prune') await adapter.completeJob(kept!, 'prefix-prune', { count: 1 }) assert.equal(await dedupExists('prefix-prune', 'pruned-job'), 0) assert.equal(await dedupExists('prefix-prune', 'kept-job'), 1) await push('prefix-stalled', 'stalled-job') await adapter.popFrom('prefix-stalled') await new Promise((resolve) => setTimeout(resolve, 5)) const { exceeded } = await adapter.recoverStalledJobs('prefix-stalled', 0, 0, 100) await adapter.failJob(exceeded[0], 'prefix-stalled', new Error('stalled')) assert.equal(await dedupExists('prefix-stalled', 'stalled-job'), 0) }) test('completeJob does not delete a newer dedup lock when the connection uses a key prefix', async ({ assert, }) => { const adapter = new RedisAdapter(connection) const queue = 'prefixed-ttl-clean-queue' const dedupId = 'TestJob::prefixed-ttl-clean-1' const dedupKey = `jobs::${queue}::dedup::${dedupId}` await adapter.pushOn(queue, { id: 'prefixed-ttl-clean-uuid-1', name: 'TestJob', payload: { n: 1 }, attempts: 0, dedup: { id: dedupId, ttl: 80 }, }) const first = await adapter.popFrom(queue) // The first lock expires while its job is still running, so a second job takes the id. await new Promise((resolve) => setTimeout(resolve, 150)) await adapter.pushOn(queue, { id: 'prefixed-ttl-clean-uuid-2', name: 'TestJob', payload: { n: 2 }, attempts: 0, dedup: { id: dedupId, ttl: 10_000 }, }) await adapter.completeJob(first!, queue) assert.equal(await connection.get(dedupKey), 'prefixed-ttl-clean-uuid-2') }) test('dedup replace should return skipped when stored job_data is malformed JSON', async ({ assert, }) => { const adapter = new RedisAdapter(connection) const queue = 'malformed-dedup-queue' const dataKey = `jobs::${queue}::data` await adapter.pushOn(queue, { id: 'malformed-uuid-1', name: 'TestJob', payload: { version: 1 }, attempts: 0, dedup: { id: 'TestJob::malformed-1', ttl: 10_000, replace: true }, }) await connection.hset(dataKey, 'malformed-uuid-1', '{not valid json') const second = await adapter.pushOn(queue, { id: 'malformed-uuid-2', name: 'TestJob', payload: { version: 2 }, attempts: 0, dedup: { id: 'TestJob::malformed-1', ttl: 10_000, replace: true }, }) assert.equal(second && typeof second === 'object' && second.outcome, 'skipped') assert.equal(second && typeof second === 'object' && second.jobId, 'malformed-uuid-1') const stored = await connection.hget(dataKey, 'malformed-uuid-1') assert.equal(stored, '{not valid json') }) test('dedup: orphan dedup pointer is reclaimed when job data is missing', async ({ assert }) => { const adapter = new RedisAdapter(connection) const queue = 'orphan-dedup-queue' const dataKey = `jobs::${queue}::data` const dedupKey = `jobs::${queue}::dedup::TestJob::orphan-1` await adapter.pushOn(queue, { id: 'orphan-uuid-1', name: 'TestJob', payload: { version: 1 }, attempts: 0, dedup: { id: 'TestJob::orphan-1' }, }) // Simulate the pointer outliving the job data (e.g. an external pruner // removes the hash entry and pending ZSET entry while the dedup key has // not expired yet). The dedup pointer remains pointing at a vanished id. const pendingKey = `jobs::${queue}::pending` await connection.hdel(dataKey, 'orphan-uuid-1') await connection.zrem(pendingKey, 'orphan-uuid-1') const before = await connection.get(dedupKey) assert.equal(before, 'orphan-uuid-1', 'dedup pointer should still reference the orphaned id') // A fresh dispatch should treat the orphan pointer as reclaimable and add // a new job, repointing the dedup key to the new winner. const second = await adapter.pushOn(queue, { id: 'orphan-uuid-2', name: 'TestJob', payload: { version: 2 }, attempts: 0, dedup: { id: 'TestJob::orphan-1' }, }) assert.equal(second && typeof second === 'object' && second.outcome, 'added') assert.equal(second && typeof second === 'object' && second.jobId, 'orphan-uuid-2') const after = await connection.get(dedupKey) assert.equal(after, 'orphan-uuid-2', 'dedup pointer should be reclaimed for the new job') const size = await adapter.sizeOf(queue) assert.equal(size, 1) }) test('retryJob should keep retrying legacy jobs without Redis metadata', async ({ assert }) => { const adapter = new RedisAdapter(connection) adapter.setWorkerId('worker-1') const queue = 'legacy-retry-empty-array-payload-queue' await adapter.pushOn(queue, { id: 'legacy-retry-empty-array-uuid-1', name: 'TestJob', payload: { empty: [], }, attempts: 2, }) const first = await adapter.popFrom(queue) await adapter.retryJob(first!, queue) const retried = await adapter.popFrom(queue) assert.deepEqual(retried!.payload, { empty: [], }) assert.equal(retried!.attempts, 3) }) test('retained Redis jobs keep metadata-backed payload overrides', async ({ assert }) => { const adapter = new RedisAdapter(connection) adapter.setWorkerId('worker-1') const queue = 'metadata-retained-replace-queue' const metadataKey = `jobs::${queue}::metadata` await adapter.pushOn(queue, { id: 'metadata-retained-uuid-1', name: 'TestJob', payload: { version: 1 }, attempts: 0, dedup: { id: 'TestJob::metadata-retained-1', ttl: 10_000, replace: true }, }) await adapter.pushOn(queue, { id: 'metadata-retained-uuid-2', name: 'TestJob', payload: { version: 2, empty: [], }, attempts: 0, dedup: { id: 'TestJob::metadata-retained-1', ttl: 10_000, replace: true }, }) assert.exists(await connection.hget(metadataKey, 'metadata-retained-uuid-1')) const job = await adapter.popFrom(queue) await adapter.completeJob(job!, queue, false) const record = await adapter.getJob('metadata-retained-uuid-1', queue) assert.exists(await connection.hget(metadataKey, 'metadata-retained-uuid-1')) assert.deepEqual(record!.data.payload, { version: 2, empty: [], }) }) test('Redis metadata is removed when a job is completed without retention', async ({ assert, }) => { const adapter = new RedisAdapter(connection) adapter.setWorkerId('worker-1') const queue = 'metadata-clean-complete-queue' const metadataKey = `jobs::${queue}::metadata` await adapter.pushOn(queue, { id: 'metadata-clean-uuid-1', name: 'TestJob', payload: { version: 1 }, attempts: 0, dedup: { id: 'TestJob::metadata-clean-1', ttl: 10_000, replace: true }, }) await adapter.pushOn(queue, { id: 'metadata-clean-uuid-2', name: 'TestJob', payload: { version: 2 }, attempts: 0, dedup: { id: 'TestJob::metadata-clean-1', ttl: 10_000, replace: true }, }) assert.exists(await connection.hget(metadataKey, 'metadata-clean-uuid-1')) const job = await adapter.popFrom(queue) await adapter.completeJob(job!, queue, true) assert.isNull(await connection.hget(metadataKey, 'metadata-clean-uuid-1')) }) test('Redis metadata is removed when retained history pruning removes a job', async ({ assert, }) => { const adapter = new RedisAdapter(connection) adapter.setWorkerId('worker-1') const queue = 'metadata-prune-history-queue' const metadataKey = `jobs::${queue}::metadata` await adapter.pushOn(queue, { id: 'metadata-prune-uuid-1', name: 'TestJob', payload: { version: 1 }, attempts: 0, dedup: { id: 'TestJob::metadata-prune-1', ttl: 10_000, replace: true }, }) await adapter.pushOn(queue, { id: 'metadata-prune-uuid-2', name: 'TestJob', payload: { version: 2 }, attempts: 0, dedup: { id: 'TestJob::metadata-prune-1', ttl: 10_000, replace: true }, }) assert.exists(await connection.hget(metadataKey, 'metadata-prune-uuid-1')) const first = await adapter.popFrom(queue) await adapter.completeJob(first!, queue, { count: 1 }) await new Promise((resolve) => setTimeout(resolve, 5)) await adapter.pushOn(queue, { id: 'metadata-prune-uuid-3', name: 'TestJob', payload: { version: 3 }, attempts: 0, dedup: { id: 'TestJob::metadata-prune-2', ttl: 10_000, replace: true }, }) await adapter.pushOn(queue, { id: 'metadata-prune-uuid-4', name: 'TestJob', payload: { version: 4 }, attempts: 0, dedup: { id: 'TestJob::metadata-prune-2', ttl: 10_000, replace: true }, }) assert.exists(await connection.hget(metadataKey, 'metadata-prune-uuid-3')) const second = await adapter.popFrom(queue) await adapter.completeJob(second!, queue, { count: 1 }) assert.isNull(await connection.hget(metadataKey, 'metadata-prune-uuid-1')) assert.exists(await connection.hget(metadataKey, 'metadata-prune-uuid-3')) }) test('Redis metadata is removed when a permanently stalled job is failed', async ({ assert }) => { const adapter = new RedisAdapter(connection) adapter.setWorkerId('worker-1') const queue = 'metadata-stalled-clean-queue' const metadataKey = `jobs::${queue}::metadata` await adapter.pushOn(queue, { id: 'metadata-stalled-uuid-1', name: 'TestJob', payload: { version: 1 }, attempts: 0, dedup: { id: 'TestJob::metadata-stalled-1', ttl: 10_000, replace: true }, }) await adapter.pushOn(queue, { id: 'metadata-stalled-uuid-2', name: 'TestJob', payload: { version: 2 }, attempts: 0, dedup: { id: 'TestJob::metadata-stalled-1', ttl: 10_000, replace: true }, }) assert.exists(await connection.hget(metadataKey, 'metadata-stalled-uuid-1')) await adapter.popFrom(queue) await new Promise((resolve) => setTimeout(resolve, 20)) const { recovered, exceeded } = await adapter.recoverStalledJobs(queue, 10, 0, 100) assert.equal(recovered, 0) assert.lengthOf(exceeded, 1) await adapter.failJob(exceeded[0], queue, new Error('stalled')) assert.isNull(await connection.hget(metadataKey, 'metadata-stalled-uuid-1')) assert.isNull(await adapter.getJob('metadata-stalled-uuid-1', queue)) }) test('migrate makes pre-existing schedules claimable from the due index', async ({ assert }) => { const adapter = new RedisAdapter(connection) // Simulate pre-upgrade schedule data: write hash + index directly, skip ZSET const id = 'pre-existing-schedule' const pastRunAt = (Date.now() - 5_000).toString() await connection .multi() .hset(`schedules::${id}`, { id, name: 'LegacyJob', payload: '{}', status: 'active', every_ms: '60000', timezone: 'UTC', next_run_at: pastRunAt, last_run_at: '', run_count: '0', created_at: Date.now().toString(), }) .sadd('schedules::index', id) .exec() // Without backfill, ZSET has no entry so claim returns null const beforeBackfill = await adapter.claimDueSchedule() assert.isNull(beforeBackfill) await adapter.migrate() const score = await connection.zscore('schedules::due', id) assert.equal(Number(score), Number(pastRunAt)) const afterBackfill = await adapter.claimDueSchedule() assert.isNotNull(afterBackfill) assert.equal(afterBackfill!.id, id) }) test('migrate moves 0.7 schedule hashes, including one whose id is due', async ({ assert }) => { const adapter = new RedisAdapter(connection) const pastRunAt = (Date.now() - 5_000).toString() const legacySchedule = (id: string) => ({ id, name: 'LegacyJob', payload: JSON.stringify({ id }), status: 'active', every_ms: '60000', timezone: 'UTC', next_run_at: pastRunAt, run_count: '0', created_at: Date.now().toString(), }) // In 0.7, the hash of a schedule with the id `due` lives at the due index key. await connection .multi() .hset('schedules::due', legacySchedule('due')) .hset('schedules::legacy', legacySchedule('legacy')) .sadd('schedules::index', 'due', 'legacy') .exec() await adapter.migrate() assert.equal(await connection.exists('schedules::legacy'), 0) assert.equal(await connection.type('schedules::due'), 'zset') assert.deepEqual((await adapter.getSchedule('due'))!.payload, { id: 'due' }) assert.deepEqual((await adapter.getSchedule('legacy'))!.payload, { id: 'legacy' }) assert.sameMembers( [(await adapter.claimDueSchedule())!.id, (await adapter.claimDueSchedule())!.id], ['due', 'legacy'] ) }) test('migrate keeps the current hash when a legacy copy also exists', async ({ assert }) => { const adapter = new RedisAdapter(connection) const id = 'migrated-twice' await adapter.upsertSchedule({ id, name: 'CurrentJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }) await connection.hset(`schedules::${id}`, { id, name: 'LegacyJob', status: 'active' }) await adapter.migrate() assert.equal(await connection.exists(`schedules::${id}`), 0) assert.equal((await adapter.getSchedule(id))!.name, 'CurrentJob') }) test('migrate handles a 0.7 id whose legacy key is the new key of another id', async ({ assert, }) => { const adapter = new RedisAdapter(connection) // The 0.7 hash of `data::x` is stored at `schedules::data::x`, the new key of `x`. await connection .multi() .hset('schedules::x', { id: 'x', name: 'XJob', status: 'active' }) .hset('schedules::data::x', { id: 'data::x', name: 'DataXJob', status: 'active' }) .sadd('schedules::index', 'x', 'data::x') .exec() await adapter.migrate() assert.equal((await adapter.getSchedule('x'))!.name, 'XJob') assert.equal((await adapter.getSchedule('data::x'))!.name, 'DataXJob') }) test('migrate aborts without changes when a destination belongs to another key', async ({ assert, }) => { const adapter = new RedisAdapter(connection) // 0.7 cron finalization recreates `schedules::data::foo` with only `next_run_at` when // the schedule `data::foo` is deleted while it is being finalized. That key is not // indexed and is the new key of the schedule `foo`. await connection .multi() .hset('schedules::foo', { id: 'foo', name: 'FooJob', status: 'active' }) .hset('schedules::data::foo', { next_run_at: Date.now().toString() }) .sadd('schedules::index', 'foo') .exec() await assert.rejects(() => adapter.migrate(), /Cannot migrate schedule "foo"/) assert.equal(await connection.hget('schedules::foo', 'name'), 'FooJob') await connection.del('schedules::data::foo') await adapter.migrate() assert.equal((await adapter.getSchedule('foo'))!.name, 'FooJob') }) test('schedules named after index keys do not collide with them', async ({ assert }) => { const adapter = new RedisAdapter(connection) const ids = ['due', 'index', 'regular'] const dueAt = new Date(Date.now() - 1_000) for (const id of ids) { await adapter.upsertSchedule({ id, name: 'TestJob', payload: { id }, everyMs: 60_000, timezone: 'UTC', }) await adapter.updateSchedule(id, { nextRunAt: dueAt }) } assert.sameMembers( (await adapter.listSchedules()).map((schedule) => schedule.id), ids ) const claimed: string[] = [] for (const _ of ids) { claimed.push((await adapter.claimDueSchedule())!.id) } assert.sameMembers(claimed, ids) assert.deepEqual((await adapter.getSchedule('index'))!.payload, { id: 'index' }) }) test('defining a schedule still stored at its 0.7 key fails until migrate runs', async ({ assert, }) => { const adapter = new RedisAdapter(connection) const id = 'paused-cron' const nextRunAt = (Date.now() + 3_600_000).toString() const config = { id, name: 'CronJob', payload: {}, cronExpression: '0 3 * * *', timezone: 'UTC', nextRunAt: new Date(Date.now() + 60_000), } // A schedule paused by 0.7, which the application defines at boot before migrate(). await connection .multi() .hset(`schedules::${id}`, { id, name: 'CronJob', payload: '{}', status: 'paused', cron_expression: '0 3 * * *', timezone: 'UTC', next_run_at: nextRunAt, run_count: '42', created_at: Date.now().toString(), }) .sadd('schedules::index', id) .exec() await assert.rejects(() => adapter.upsertSchedule(config), /run migrate\(\)/) assert.equal(await connection.exists(`schedules::data::${id}`), 0) await adapter.migrate() await adapter.upsertSchedule(config) const schedule = await adapter.getSchedule(id) assert.equal(schedule!.status, 'paused') assert.equal(schedule!.runCount, 42) assert.equal(schedule!.nextRunAt!.getTime(), Number(nextRunAt)) }) test('defining a schedule again succeeds when its 0.7 key holds something else', async ({ assert, }) => { const adapter = new RedisAdapter(connection) const config = { name: 'TestJob', payload: {}, everyMs: 60_000, timezone: 'UTC' } // `schedules::data::x`, the 0.7 key of `data::x`, is the new key of `x`, and // the 0.7 keys of `due` and `index` are the indexes. const ids = ['x', 'data::x', 'due', 'index'] for (const id of [...ids, ...ids]) { await adapter.upsertSchedule({ ...config, id }) } assert.sameMembers( (await adapter.listSchedules()).map((schedule) => schedule.id), ids ) }) test('defining a schedule succeeds when its 0.7 key is not indexed', async ({ assert }) => { const adapter = new RedisAdapter(connection) // migrate() only moves indexed schedules, so this hash would block the upsert forever. await connection.hset('schedules::orphan', { id: 'orphan', name: 'OrphanJob' }) await adapter.upsertSchedule({ id: 'orphan', name: 'TestJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }) assert.equal((await adapter.getSchedule('orphan'))!.name, 'TestJob') }) test('migrate is idempotent', async ({ assert }) => { const adapter = new RedisAdapter(connection) const nextRunAt = Date.now() + 30_000 await adapter.upsertSchedule({ id: 'idempotent-schedule', name: 'TestJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }) await adapter.updateSchedule('idempotent-schedule', { nextRunAt: new Date(nextRunAt), }) await connection .multi() .del('schedules::due') .zadd('schedules::due', Date.now() - 10_000, 'orphaned-schedule') .exec() await adapter.migrate() const firstMembers = await connection.zrange('schedules::due', 0, '-1', 'WITHSCORES') await adapter.migrate() const secondMembers = await connection.zrange('schedules::due', 0, '-1', 'WITHSCORES') assert.deepEqual(firstMembers, ['idempotent-schedule', nextRunAt.toString()]) assert.deepEqual(secondMembers, firstMembers) }) test('migrate runs in one atomic Redis command', async ({ assert }) => { const adapter = new RedisAdapter(connection) const id = 'atomic-backfill-schedule' await connection .multi() .hset(`schedules::data::${id}`, { id, status: 'active', next_run_at: (Date.now() + 30_000).toString(), }) .sadd('schedules::index', id) .exec() const { writes } = await withRedisWriteSpy({ connection, run: () => adapter.migrate(), }) assert.equal(writes, 1) }) test('concurrent migration and schedule writes leave the due index canonical', async ({ assert, cleanup, }) => { const secondConnection = new Redis({ host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), keyPrefix: KEY_PREFIX, db: 15, }) cleanup(async () => { await secondConnection.quit() }) const adapter = new RedisAdapter(connection) const secondAdapter = new RedisAdapter(secondConnection) for (let i = 0; i < 20; i++) { const id = `migration-write-schedule-${i}` const nextRunAt = Date.now() + 30_000 + i await Promise.all([ adapter.migrate(), secondAdapter.upsertSchedule({ id, name: 'MigrationWriteJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }), ]) await Promise.all([ adapter.migrate(), secondAdapter.updateSchedule(id, { nextRunAt: new Date(nextRunAt) }), ]) assert.equal(Number(await connection.zscore('schedules::due', id)), nextRunAt) } }) test('stale ZSET score is self-healed during claim', async ({ assert }) => { const adapter = new RedisAdapter(connection) const id = 'stale-score-schedule' const futureRunAt = Date.now() + 60_000 await adapter.upsertSchedule({ id, name: 'StaleJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }) await adapter.updateSchedule(id, { nextRunAt: new Date(futureRunAt) }) // Corrupt the ZSET score to a past value while hash still says future await connection.zadd('schedules::due', Date.now() - 10_000, id) const claimed = await adapter.claimDueSchedule() assert.isNull(claimed, 'should not claim when hash says schedule is not due yet') // ZSET score should have been repaired to match the hash const repairedScore = await connection.zscore('schedules::due', id) assert.equal(Number(repairedScore), futureRunAt) }) test('schedule lifecycle keeps the due index aligned with canonical state', async ({ assert, }) => { const adapter = new RedisAdapter(connection) const id = 'lifecycle-schedule' const firstRunAt = Date.now() + 30_000 const pausedRunAt = firstRunAt + 30_000 await adapter.upsertSchedule({ id, name: 'LifecycleJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }) await adapter.updateSchedule(id, { nextRunAt: new Date(firstRunAt) }) assert.equal(Number(await connection.zscore('schedules::due', id)), firstRunAt) await adapter.updateSchedule(id, { status: 'paused', nextRunAt: new Date(pausedRunAt), }) assert.isNull(await connection.zscore('schedules::due', id)) await adapter.updateSchedule(id, { nextRunAt: new Date(pausedRunAt + 30_000) }) assert.isNull(await connection.zscore('schedules::due', id)) await adapter.updateSchedule(id, { status: 'active' }) assert.equal(Number(await connection.zscore('schedules::due', id)), pausedRunAt + 30_000) await adapter.updateSchedule(id, { nextRunAt: null }) assert.isNull(await connection.zscore('schedules::due', id)) await adapter.updateSchedule(id, { nextRunAt: new Date(firstRunAt) }) await adapter.deleteSchedule(id) assert.isNull(await connection.zscore('schedules::due', id)) }) test('schedule mutations update canonical state and its index in one command', async ({ assert, }) => { const adapter = new RedisAdapter(connection) const id = 'atomic-schedule-mutation' const { writes: upsertWrites } = await withRedisWriteSpy({ connection, run: () => adapter.upsertSchedule({ id, name: 'AtomicJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }), }) assert.equal(upsertWrites, 1) const { writes: updateWrites } = await withRedisWriteSpy({ connection, run: () => adapter.updateSchedule(id, { nextRunAt: new Date(Date.now() + 30_000) }), }) assert.equal(updateWrites, 1) }) test('concurrent resume and next-run updates leave the due index canonical', async ({ assert, cleanup, }) => { const secondConnection = new Redis({ host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), keyPrefix: KEY_PREFIX, db: 15, }) cleanup(async () => { await secondConnection.quit() }) const adapter = new RedisAdapter(connection) const secondAdapter = new RedisAdapter(secondConnection) const id = 'concurrent-resume-schedule' await adapter.upsertSchedule({ id, name: 'ConcurrentJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }) for (let i = 0; i < 20; i++) { const nextRunAt = Date.now() + 30_000 + i await adapter.updateSchedule(id, { status: 'paused' }) await Promise.all([ adapter.updateSchedule(id, { nextRunAt: new Date(nextRunAt) }), secondAdapter.updateSchedule(id, { status: 'active' }), ]) const schedule = await adapter.getSchedule(id) assert.equal(schedule!.status, 'active') assert.equal(Number(await connection.zscore('schedules::due', id)), nextRunAt) } }) test('concurrent upsert and next-run updates leave the due index canonical', async ({ assert, cleanup, }) => { const secondConnection = new Redis({ host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), keyPrefix: KEY_PREFIX, db: 15, }) cleanup(async () => { await secondConnection.quit() }) const adapter = new RedisAdapter(connection) const secondAdapter = new RedisAdapter(secondConnection) const id = 'concurrent-upsert-schedule' await adapter.upsertSchedule({ id, name: 'ConcurrentJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }) for (let i = 0; i < 20; i++) { const nextRunAt = Date.now() + 60_000 + i await adapter.updateSchedule(id, { status: 'paused' }) // The upsert keeps the status and next run the concurrent update sets, // in either order, since the timing does not change. await Promise.all([ adapter.updateSchedule(id, { status: 'active', nextRunAt: new Date(nextRunAt) }), secondAdapter.upsertSchedule({ id, name: 'ConcurrentJob', payload: { iteration: i }, everyMs: 60_000, timezone: 'UTC', }), ]) const schedule = await adapter.getSchedule(id) assert.equal(schedule!.status, 'active') assert.equal(schedule!.nextRunAt!.getTime(), nextRunAt) assert.equal(Number(await connection.zscore('schedules::due', id)), nextRunAt) } }) test('interval claims update the due index from the canonical hash', async ({ assert }) => { const adapter = new RedisAdapter(connection) const id = 'interval-index-schedule' await adapter.upsertSchedule({ id, name: 'IntervalJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }) await adapter.updateSchedule(id, { nextRunAt: new Date(Date.now() - 1_000) }) assert.equal((await adapter.claimDueSchedule())?.id, id) const schedule = await adapter.getSchedule(id) const score = await connection.zscore('schedules::due', id) assert.isNotNull(schedule!.nextRunAt) assert.equal(Number(score), schedule!.nextRunAt!.getTime()) }) test('cron claims update the due index from the canonical hash', async ({ assert }) => { const adapter = new RedisAdapter(connection) const id = 'cron-index-schedule' await adapter.upsertSchedule({ id, name: 'CronJob', payload: {}, cronExpression: '* * * * *', timezone: 'UTC', }) await adapter.updateSchedule(id, { nextRunAt: new Date(Date.now() - 1_000) }) assert.equal((await adapter.claimDueSchedule())?.id, id) const schedule = await adapter.getSchedule(id) const score = await connection.zscore('schedules::due', id) assert.isNotNull(schedule!.nextRunAt) assert.equal(Number(score), schedule!.nextRunAt!.getTime()) }) test('cron finalization does not undo a concurrent pause or delete', async ({ assert, cleanup, }) => { const secondConnection = new Redis({ host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), keyPrefix: KEY_PREFIX, db: 15, }) cleanup(async () => { await secondConnection.quit() }) const adapter = new RedisAdapter(connection) const secondAdapter = new RedisAdapter(secondConnection) for (const mutation of ['pause', 'delete'] as const) { const id = `cron-finalize-${mutation}` await adapter.upsertSchedule({ id, name: 'CronFinalizeJob', payload: {}, cronExpression: '* * * * *', timezone: 'UTC', }) await adapter.updateSchedule(id, { nextRunAt: new Date(Date.now() - 1_000) }) const originalEval = connection.eval.bind(connection) let releaseClaim!: () => void let claimReturned!: () => void const claimReleased = new Promise((resolve) => { releaseClaim = resolve }) const claimHasReturned = new Promise((resolve) => { claimReturned = resolve }) let gateNextEval = true connection.eval = (async (...args: Parameters) => { const result = await originalEval(...args) if (gateNextEval) { gateNextEval = false claimReturned() await claimReleased } return result }) as typeof connection.eval const claim = adapter.claimDueSchedule() await claimHasReturned if (mutation === 'pause') { await secondAdapter.updateSchedule(id, { status: 'paused' }) } else { await secondAdapter.deleteSchedule(id) } releaseClaim() await claim connection.eval = originalEval if (mutation === 'pause') { assert.equal((await adapter.getSchedule(id))!.status, 'paused') } else { assert.isNull(await adapter.getSchedule(id)) } assert.isNull(await connection.zscore('schedules::due', id)) if (mutation === 'pause') { const schedule = await adapter.getSchedule(id) assert.isNotNull(schedule!.nextRunAt) await secondAdapter.updateSchedule(id, { status: 'active' }) assert.equal( Number(await connection.zscore('schedules::due', id)), schedule!.nextRunAt!.getTime() ) const originalNow = Date.now Date.now = () => schedule!.nextRunAt!.getTime() try { assert.equal((await adapter.claimDueSchedule())?.id, id) } finally { Date.now = originalNow } } } }) test('cron finalization does not apply a calculation from superseded configuration', async ({ assert, cleanup, }) => { const secondConnection = new Redis({ host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), keyPrefix: KEY_PREFIX, db: 15, }) cleanup(async () => { await secondConnection.quit() }) const adapter = new RedisAdapter(connection) const secondAdapter = new RedisAdapter(secondConnection) const id = 'cron-finalize-superseded-config' await adapter.upsertSchedule({ id, name: 'OriginalCronJob', payload: {}, cronExpression: '0 9 * * *', timezone: 'UTC', }) await adapter.updateSchedule(id, { nextRunAt: new Date(Date.now() - 1_000) }) const originalEval = connection.eval.bind(connection) let releaseClaim!: () => void let claimReturned!: () => void const claimReleased = new Promise((resolve) => { releaseClaim = resolve }) const claimHasReturned = new Promise((resolve) => { claimReturned = resolve }) let gateNextEval = true connection.eval = (async (...args: Parameters) => { const result = await originalEval(...args) if (gateNextEval) { gateNextEval = false claimReturned() await claimReleased } return result }) as typeof connection.eval const claim = adapter.claimDueSchedule() await claimHasReturned await secondAdapter.upsertSchedule({ id, name: 'UpdatedCronJob', payload: {}, cronExpression: '0 9 * * *', timezone: 'America/New_York', }) releaseClaim() await claim connection.eval = originalEval const schedule = await adapter.getSchedule(id) assert.equal(schedule!.name, 'UpdatedCronJob') assert.equal(schedule!.timezone, 'America/New_York') assert.isNull(schedule!.nextRunAt) assert.isNull(await connection.zscore('schedules::due', id)) }) test('cron finalization survives a concurrent upsert with the same timing', async ({ assert, cleanup, }) => { const secondConnection = new Redis({ host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), keyPrefix: KEY_PREFIX, db: 15, }) cleanup(async () => { await secondConnection.quit() }) const adapter = new RedisAdapter(connection) const secondAdapter = new RedisAdapter(secondConnection) const id = 'cron-finalize-same-timing-upsert' const config = { id, name: 'CronJob', cronExpression: '0 9 * * *', timezone: 'UTC', } const claimedOccurrence = new Date(Date.now() - 1_000) await adapter.upsertSchedule({ ...config, payload: { version: 1 }, nextRunAt: claimedOccurrence, }) const originalEval = connection.eval.bind(connection) let releaseClaim!: () => void let claimReturned!: () => void const claimReleased = new Promise((resolve) => { releaseClaim = resolve }) const claimHasReturned = new Promise((resolve) => { claimReturned = resolve }) let gateNextEval = true connection.eval = (async (...args: Parameters) => { const result = await originalEval(...args) if (gateNextEval) { gateNextEval = false claimReturned() await claimReleased } return result }) as typeof connection.eval // A redeploy runs the same schedule definition while the claim is in flight. Its next run // was computed before the claim: the claimed occurrence. const claim = adapter.claimDueSchedule() await claimHasReturned await secondAdapter.upsertSchedule({ ...config, payload: { version: 2 }, nextRunAt: claimedOccurrence, }) releaseClaim() await claim connection.eval = originalEval // The claim computed the next 9:00 run, which the same timing keeps valid. const schedule = await adapter.getSchedule(id) assert.deepEqual(schedule!.payload, { version: 2 }) assert.equal(schedule!.nextRunAt!.getUTCHours(), 9) assert.isAbove(schedule!.nextRunAt!.getTime(), Date.now()) assert.equal( Number(await connection.zscore('schedules::due', id)), schedule!.nextRunAt!.getTime() ) assert.isNull(await adapter.claimDueSchedule()) assert.equal((await adapter.getSchedule(id))!.runCount, 1) }) /** * Claims a due cron schedule and fails its finalization, as when the * process dies between the two scripts. */ async function claimWithoutFinalization(adapter: RedisAdapter) { const originalEval = connection.eval.bind(connection) let evalCount = 0 connection.eval = (async (...args: Parameters) => { if (++evalCount === 2) throw new Error('connection lost') return originalEval(...args) }) as typeof connection.eval const error = await adapter.claimDueSchedule().then( () => null, (claimError: Error) => claimError ) connection.eval = originalEval if (error?.message !== 'connection lost') { throw new Error('The claim was expected to fail its finalization') } } test('defining a schedule again finalizes a cron claim never finalized', async ({ assert }) => { const adapter = new RedisAdapter(connection) const id = 'cron-claim-never-finalized' const config = { id, name: 'CronJob', payload: {}, cronExpression: '0 9 * * *', timezone: 'UTC', } const claimedOccurrence = new Date(Date.now() - 1_000) await adapter.upsertSchedule({ ...config, nextRunAt: claimedOccurrence }) await claimWithoutFinalization(adapter) const { lastRunAt } = (await adapter.getSchedule(id))! // The next deploy computed its next run before the claim: the claimed occurrence. await adapter.upsertSchedule({ ...config, nextRunAt: claimedOccurrence }) const expectedNextRunAt = CronExpressionParser.parse('0 9 * * *', { currentDate: lastRunAt!, tz: 'UTC', }) .next() .toDate() const schedule = await adapter.getSchedule(id) assert.equal(schedule!.nextRunAt!.getTime(), expectedNextRunAt.getTime()) assert.equal(Number(await connection.zscore('schedules::due', id)), expectedNextRunAt.getTime()) assert.isNull(await connection.hget(`schedules::data::${id}`, 'claim_token')) // The claimed occurrence does not run again. assert.isNull(await adapter.claimDueSchedule()) assert.equal((await adapter.getSchedule(id))!.runCount, 1) }) test('a later claim finalizes a cron claim never finalized', async ({ assert, cleanup }) => { const originalNow = Date.now cleanup(() => { Date.now = originalNow }) const adapter = new RedisAdapter(connection) const id = 'cron-claim-never-finalized-then-claimed' const claimedAt = Date.parse('2030-01-01T09:00:01Z') await adapter.upsertSchedule({ id, name: 'CronJob', payload: {}, cronExpression: '0 9 * * *', timezone: 'UTC', nextRunAt: new Date('2030-01-01T09:00:00Z'), }) Date.now = () => claimedAt await claimWithoutFinalization(adapter) // The claim may still be finalized by its worker. Date.now = () => claimedAt + 1_000 assert.isNull(await new RedisAdapter(connection).claimDueSchedule()) assert.isNull((await adapter.getSchedule(id))!.nextRunAt) // Its worker died: the next claim finalizes it without running the claimed occurrence again. Date.now = () => claimedAt + 60_000 assert.isNull(await new RedisAdapter(connection).claimDueSchedule()) const nextRunAt = Date.parse('2030-01-02T09:00:00Z') const schedule = await adapter.getSchedule(id) assert.equal(schedule!.nextRunAt?.getTime(), nextRunAt) assert.equal(schedule!.runCount, 1) assert.equal(Number(await connection.zscore('schedules::due', id)), nextRunAt) Date.now = () => nextRunAt assert.equal((await new RedisAdapter(connection).claimDueSchedule())?.id, id) }) test('a cron claim finalized late by its worker and by another claimer runs once', async ({ assert, cleanup, }) => { const secondConnection = new Redis({ host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), keyPrefix: KEY_PREFIX, db: 15, }) const originalNow = Date.now const originalEval = connection.eval.bind(connection) cleanup(async () => { Date.now = originalNow connection.eval = originalEval await secondConnection.quit() }) const adapter = new RedisAdapter(connection) const secondAdapter = new RedisAdapter(secondConnection) const id = 'cron-claim-finalized-twice' const claimedAt = Date.parse('2030-01-01T09:00:01Z') // Every minute, so the occurrence after the claim is due when the second claimer runs. await adapter.upsertSchedule({ id, name: 'CronJob', payload: {}, cronExpression: '* * * * *', timezone: 'UTC', nextRunAt: new Date('2030-01-01T09:00:00Z'), }) let releaseClaim!: () => void let claimReturned!: () => void const claimReleased = new Promise((resolve) => { releaseClaim = resolve }) const claimHasReturned = new Promise((resolve) => { claimReturned = resolve }) let gateNextEval = true connection.eval = (async (...args: Parameters) => { const result = await originalEval(...args) if (gateNextEval) { gateNextEval = false claimReturned() await claimReleased } return result }) as typeof connection.eval // The first worker claims 09:00 and stalls before finalizing. Date.now = () => claimedAt const firstClaim = adapter.claimDueSchedule() await claimHasReturned // The second one finalizes the stalled claim, then claims 09:01. Date.now = () => Date.parse('2030-01-01T09:01:30Z') const secondClaimed = await secondAdapter.claimDueSchedule() releaseClaim() const firstClaimed = await firstClaim assert.equal(firstClaimed?.nextRunAt?.toISOString(), '2030-01-01T09:00:00.000Z') assert.equal(secondClaimed?.nextRunAt?.toISOString(), '2030-01-01T09:01:00.000Z') // The late finalization of the first worker does not move the next run back to 09:01. const schedule = await adapter.getSchedule(id) assert.equal(schedule!.runCount, 2) assert.equal(schedule!.nextRunAt?.toISOString(), '2030-01-01T09:02:00.000Z') assert.isNull(await secondAdapter.claimDueSchedule()) }) test('a manual trigger does not move the time a cron claim is finalized from', async ({ assert, }) => { const adapter = new RedisAdapter(connection) const id = 'cron-claim-never-finalized-then-triggered' const config = { id, name: 'CronJob', payload: {}, cronExpression: '0 9 * * *', timezone: 'UTC', } await adapter.upsertSchedule({ ...config, nextRunAt: new Date(Date.now() - 1_000) }) await claimWithoutFinalization(adapter) const { lastRunAt: claimedAt } = (await adapter.getSchedule(id))! // Schedule.trigger() records its run, here past the next 9:00. await adapter.updateSchedule(id, { runCount: 2, lastRunAt: new Date(claimedAt!.getTime() + 2 * 86_400_000), }) await adapter.upsertSchedule({ ...config, nextRunAt: new Date(Date.now() + 3_600_000) }) const expectedNextRunAt = CronExpressionParser.parse('0 9 * * *', { currentDate: claimedAt!, tz: 'UTC', }) .next() .toDate() const schedule = await adapter.getSchedule(id) assert.equal(schedule!.nextRunAt!.getTime(), expectedNextRunAt.getTime()) assert.isNull(await connection.hget(`schedules::data::${id}`, 'claimed_at')) }) test('defining a schedule again keeps a cron claim at its run limit finished', async ({ assert, }) => { const adapter = new RedisAdapter(connection) const id = 'cron-claim-never-finalized-at-limit' const config = { id, name: 'CronJob', payload: {}, cronExpression: '0 9 * * *', timezone: 'UTC', limit: 1, } await adapter.upsertSchedule({ ...config, nextRunAt: new Date(Date.now() - 1_000) }) await claimWithoutFinalization(adapter) await adapter.upsertSchedule({ ...config, nextRunAt: new Date(Date.now() + 3_600_000) }) const schedule = await adapter.getSchedule(id) assert.isNull(schedule!.nextRunAt) assert.equal(schedule!.runCount, 1) assert.isNull(await connection.zscore('schedules::due', id)) assert.isNull(await connection.hget(`schedules::data::${id}`, 'claim_token')) }) test('defining a schedule again keeps a cron claim past its end date finished', async ({ assert, }) => { const adapter = new RedisAdapter(connection) const id = 'cron-claim-never-finalized-past-end' // Yearly, so the next occurrence falls after the end date. const config = { id, name: 'CronJob', payload: {}, cronExpression: '0 0 1 1 *', timezone: 'UTC', to: new Date(Date.now() + 60_000), } await adapter.upsertSchedule({ ...config, nextRunAt: new Date(Date.now() - 1_000) }) await claimWithoutFinalization(adapter) await adapter.upsertSchedule({ ...config, nextRunAt: new Date(Date.now() + 30_000) }) const schedule = await adapter.getSchedule(id) assert.isNull(schedule!.nextRunAt) assert.isNull(await connection.zscore('schedules::due', id)) assert.isNull(await connection.hget(`schedules::data::${id}`, 'claim_token')) }) test('cron finalization survives a concurrent runtime metadata update', async ({ assert, cleanup, }) => { const secondConnection = new Redis({ host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), keyPrefix: KEY_PREFIX, db: 15, }) cleanup(async () => { await secondConnection.quit() }) const adapter = new RedisAdapter(connection) const secondAdapter = new RedisAdapter(secondConnection) const id = 'cron-finalize-concurrent-trigger' await adapter.upsertSchedule({ id, name: 'CronTriggerJob', payload: {}, cronExpression: '* * * * *', timezone: 'UTC', }) await adapter.updateSchedule(id, { nextRunAt: new Date(Date.now() - 1_000) }) const originalEval = connection.eval.bind(connection) let releaseClaim!: () => void let claimReturned!: () => void const claimReleased = new Promise((resolve) => { releaseClaim = resolve }) const claimHasReturned = new Promise((resolve) => { claimReturned = resolve }) let gateNextEval = true connection.eval = (async (...args: Parameters) => { const result = await originalEval(...args) if (gateNextEval) { gateNextEval = false claimReturned() await claimReleased } return result }) as typeof connection.eval const claim = adapter.claimDueSchedule() await claimHasReturned await secondAdapter.updateSchedule(id, { runCount: 2, lastRunAt: new Date(), }) releaseClaim() await claim connection.eval = originalEval const schedule = await adapter.getSchedule(id) assert.equal(schedule!.runCount, 2) assert.isNotNull(schedule!.nextRunAt) assert.equal( Number(await connection.zscore('schedules::due', id)), schedule!.nextRunAt!.getTime() ) }) test('cron finalization does not undo a cleared next run', async ({ assert, cleanup }) => { const secondConnection = new Redis({ host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), keyPrefix: KEY_PREFIX, db: 15, }) cleanup(async () => { await secondConnection.quit() }) const adapter = new RedisAdapter(connection) const secondAdapter = new RedisAdapter(secondConnection) const id = 'cron-finalize-cleared-next-run' await adapter.upsertSchedule({ id, name: 'CronClearedJob', payload: {}, cronExpression: '* * * * *', timezone: 'UTC', }) await adapter.updateSchedule(id, { nextRunAt: new Date(Date.now() - 1_000) }) const originalEval = connection.eval.bind(connection) let releaseClaim!: () => void let claimReturned!: () => void const claimReleased = new Promise((resolve) => { releaseClaim = resolve }) const claimHasReturned = new Promise((resolve) => { claimReturned = resolve }) let gateNextEval = true connection.eval = (async (...args: Parameters) => { const result = await originalEval(...args) if (gateNextEval) { gateNextEval = false claimReturned() await claimReleased } return result }) as typeof connection.eval const claim = adapter.claimDueSchedule() await claimHasReturned await secondAdapter.updateSchedule(id, { nextRunAt: null }) releaseClaim() await claim connection.eval = originalEval const schedule = await adapter.getSchedule(id) assert.isNull(schedule!.nextRunAt) assert.isNull(await connection.zscore('schedules::due', id)) }) test('stale cron finalization cannot modify a recreated schedule claim', async ({ assert, cleanup, }) => { const secondConnection = new Redis({ host: process.env.REDIS_HOST || 'localhost', port: Number.parseInt(process.env.REDIS_PORT || '6379', 10), keyPrefix: KEY_PREFIX, db: 15, }) cleanup(async () => { await secondConnection.quit() }) const adapter = new RedisAdapter(connection) const secondAdapter = new RedisAdapter(secondConnection) const id = 'cron-finalize-recreated-claim' const cronExpression = '0 9 * * *' await adapter.upsertSchedule({ id, name: 'OriginalCronJob', payload: {}, cronExpression, timezone: 'UTC', }) await adapter.updateSchedule(id, { nextRunAt: new Date(Date.now() - 1_000) }) const originalEval = connection.eval.bind(connection) let releaseFirstClaim!: () => void let firstClaimReturned!: () => void const firstClaimReleased = new Promise((resolve) => { releaseFirstClaim = resolve }) const firstClaimHasReturned = new Promise((resolve) => { firstClaimReturned = resolve }) let gateFirstEval = true connection.eval = (async (...args: Parameters) => { const result = await originalEval(...args) if (gateFirstEval) { gateFirstEval = false firstClaimReturned() await firstClaimReleased } return result }) as typeof connection.eval const firstClaim = adapter.claimDueSchedule() await firstClaimHasReturned await secondAdapter.deleteSchedule(id) await secondAdapter.upsertSchedule({ id, name: 'RecreatedCronJob', payload: {}, cronExpression, timezone: 'America/New_York', }) await secondAdapter.updateSchedule(id, { nextRunAt: new Date(Date.now() - 1_000) }) const originalSecondEval = secondConnection.eval.bind(secondConnection) let releaseSecondClaim!: () => void let secondClaimReturned!: () => void const secondClaimReleased = new Promise((resolve) => { releaseSecondClaim = resolve }) const secondClaimHasReturned = new Promise((resolve) => { secondClaimReturned = resolve }) let gateSecondEval = true secondConnection.eval = (async (...args: Parameters) => { const result = await originalSecondEval(...args) if (gateSecondEval) { gateSecondEval = false secondClaimReturned() await secondClaimReleased } return result }) as typeof secondConnection.eval const secondClaim = secondAdapter.claimDueSchedule() await secondClaimHasReturned releaseFirstClaim() await firstClaim const awaitingSecondFinalization = await adapter.getSchedule(id) assert.equal(awaitingSecondFinalization!.name, 'RecreatedCronJob') assert.equal(awaitingSecondFinalization!.timezone, 'America/New_York') assert.isNull(awaitingSecondFinalization!.nextRunAt) assert.isNull(await adapter.claimDueSchedule()) releaseSecondClaim() await secondClaim connection.eval = originalEval secondConnection.eval = originalSecondEval const finalized = await adapter.getSchedule(id) assert.isNotNull(finalized!.nextRunAt) assert.equal( Number(await connection.zscore('schedules::due', id)), finalized!.nextRunAt!.getTime() ) }) test('updating a deleted schedule does not recreate or index it', async ({ assert }) => { const adapter = new RedisAdapter(connection) const id = 'update-after-delete' await adapter.upsertSchedule({ id, name: 'DeletedScheduleJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }) await adapter.deleteSchedule(id) await adapter.updateSchedule(id, { status: 'active', nextRunAt: new Date(Date.now() - 1_000), runCount: 0, }) assert.isNull(await adapter.getSchedule(id)) assert.isNull(await connection.zscore('schedules::due', id)) assert.isNull(await adapter.claimDueSchedule()) }) test('claim removes a malformed due score and continues to a valid schedule', async ({ assert, }) => { const adapter = new RedisAdapter(connection) const malformedId = 'malformed-next-run-at' const validId = 'valid-after-malformed' await connection .multi() .hset(`schedules::data::${malformedId}`, { id: malformedId, status: 'active', next_run_at: 'not-a-number', }) .sadd('schedules::index', malformedId) .zadd('schedules::due', Date.now() - 2_000, malformedId) .exec() await adapter.upsertSchedule({ id: validId, name: 'ValidJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }) await adapter.updateSchedule(validId, { nextRunAt: new Date(Date.now() - 1_000) }) assert.equal((await adapter.claimDueSchedule())!.id, validId) assert.isNull(await connection.zscore('schedules::due', malformedId)) }) test('exhausted schedules are removed from the due index', async ({ assert }) => { const adapter = new RedisAdapter(connection) for (const config of [ { id: 'limited-interval', everyMs: 60_000, limit: 1 }, { id: 'limited-cron', cronExpression: '* * * * *', limit: 1 }, { id: 'ended-interval', everyMs: 60_000, to: new Date(Date.now() + 1_000) }, ]) { await adapter.upsertSchedule({ ...config, name: 'ExhaustedJob', payload: {}, timezone: 'UTC', }) await adapter.updateSchedule(config.id, { nextRunAt: new Date(Date.now() - 1_000) }) assert.equal((await adapter.claimDueSchedule())?.id, config.id) assert.isNull((await adapter.getSchedule(config.id))!.nextRunAt) assert.isNull(await connection.zscore('schedules::due', config.id)) } }) test('claim removes a due index member whose canonical hash is missing', async ({ assert }) => { const adapter = new RedisAdapter(connection) await connection.zadd('schedules::due', Date.now() - 1_000, 'missing-schedule') assert.isNull(await adapter.claimDueSchedule()) assert.isNull(await connection.zscore('schedules::due', 'missing-schedule')) }) }) /** * A jobs table as the previous versions left it: no unique index, and expired * owners that still hold their dedup id. */ /** Check watcher connections every 50 ms, with 300 ms to answer. */ function shortenHeartbeat(cleanup: (callback: () => void) => void) { const defaults = { ...subscriptionHeartbeat } Object.assign(subscriptionHeartbeat, { interval: 50, timeout: 300 }) cleanup(() => Object.assign(subscriptionHeartbeat, defaults)) } async function assertDedupMigration( assert: Assert, connection: ReturnType, tableName: string ) { const schemaService = new KnexQueueSchemaService(connection) const job = (id: string, dedupId: string, dedupAt: number) => ({ id, queue: 'default', status: 'completed', data: '{}', dedup_id: dedupId, dedup_at: dedupAt, dedup_ttl: 10, }) await connection.raw('DROP INDEX ??', [`${tableName}_dedup_uidx`]) await connection.schema.alterTable(tableName, (table) => table.index(['queue', 'dedup_id'])) await connection(tableName).insert([ job('older', 'shared', 1_000), job('newer', 'shared', 2_000), job('alone', 'alone', 1_000), ]) await schemaService.addDedupColumns(tableName) await schemaService.addDedupColumns(tableName) assert.deepEqual(await connection(tableName).select('id', 'dedup_id').orderBy('id'), [ { id: 'alone', dedup_id: 'alone' }, { id: 'newer', dedup_id: 'shared' }, { id: 'older', dedup_id: null }, ]) await assert.rejects(() => connection(tableName).insert(job('duplicate', 'shared', 3_000))) } test.group('Adapter | Knex (SQLite)', (group) => { let connection: ReturnType let adapter: KnexAdapter group.each.setup(async () => { // Each test gets a fresh in-memory database, so no cleanup needed connection = Knex({ client: 'better-sqlite3', connection: { filename: ':memory:', }, useNullAsDefault: true, }) // Create tables via KnexQueueSchemaService const schemaService = new KnexQueueSchemaService(connection) await schemaService.createJobsTable() await schemaService.createSchedulesTable() return async () => { await adapter?.destroy() await connection.destroy() } }) registerDriverTestSuite({ test, rejectsDuplicateJobIds: true, inTransaction: (operation) => connection.transaction((trx) => operation(new KnexAdapter({ connection: trx }))), createAdapter: () => { adapter = new KnexAdapter({ connection }) return adapter }, }) test('addDedupColumns should leave a dedup slot to its latest job', async ({ assert }) => { await assertDedupMigration(assert, connection, 'queue_jobs') }) test('dedup should reject a job id that is already stored', async ({ assert }) => { adapter = new KnexAdapter({ connection }) const job = { id: 'same-id', name: 'TestJob', payload: {}, attempts: 0 } await adapter.pushOn('default', { ...job, dedup: { id: 'first' } }) // The primary key violation itself, rather than a dispatch that runs again const error = await adapter.pushOn('default', { ...job, dedup: { id: 'second' } }).then( () => null, (rejection: { code?: string }) => rejection ) assert.equal(error?.code, 'SQLITE_CONSTRAINT_PRIMARYKEY') assert.equal(await adapter.sizeOf('default'), 1) }) test('dedup should respect a case-insensitive column collation', async ({ assert }) => { await connection.raw('DROP INDEX queue_jobs_dedup_uidx') await connection.raw('ALTER TABLE queue_jobs DROP COLUMN dedup_id') await connection.raw('ALTER TABLE queue_jobs ADD COLUMN dedup_id TEXT COLLATE NOCASE') await connection.raw( 'CREATE UNIQUE INDEX queue_jobs_dedup_uidx ON queue_jobs (queue, dedup_id)' ) adapter = new KnexAdapter({ connection }) const owner = await adapter.pushOn('default', { id: 'owner', name: 'TestJob', payload: {}, attempts: 0, dedup: { id: 'Order-A' }, }) const duplicate = await adapter.pushOn('default', { id: 'duplicate', name: 'TestJob', payload: {}, attempts: 0, dedup: { id: 'order-a' }, }) assert.deepEqual(owner, { outcome: 'added', jobId: 'owner' }) assert.deepEqual(duplicate, { outcome: 'skipped', jobId: 'owner' }) assert.equal(await adapter.sizeOf('default'), 1) }) test('listSchedules should execute a single SQL query in Knex adapter', async ({ assert }) => { const knexAdapter = new KnexAdapter({ connection }) for (let i = 0; i < 20; i++) { await knexAdapter.upsertSchedule({ id: `knex-list-${i}`, name: 'KnexPerfJob', payload: { i }, everyMs: 60_000, timezone: 'UTC', }) } const { result: schedules, queries } = await withKnexQuerySpy({ connection, run: () => knexAdapter.listSchedules(), }) assert.lengthOf(schedules, 20) const scheduleSelectQueries = queries.filter( (sql) => sql.includes('select') && sql.includes('queue_schedules') ) assert.lengthOf( scheduleSelectQueries, 1, `Expected a single schedule SELECT query, got ${scheduleSelectQueries.length}` ) }) test('deleteSchedule should execute a single SQL query in Knex adapter', async ({ assert }) => { const knexAdapter = new KnexAdapter({ connection }) const id = 'knex-delete-atomicity' await knexAdapter.upsertSchedule({ id, name: 'KnexDeleteJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }) const { queries } = await withKnexQuerySpy({ connection, run: () => knexAdapter.deleteSchedule(id), }) const schedule = await knexAdapter.getSchedule(id) assert.isNull(schedule) const scheduleDeleteQueries = queries.filter( (sql) => sql.includes('delete') && sql.includes('queue_schedules') ) assert.lengthOf( scheduleDeleteQueries, 1, `Expected a single schedule DELETE query, got ${scheduleDeleteQueries.length}` ) }) }) test.group('Adapter | Knex (PostgreSQL)', (group) => { let connection: ReturnType let adapter: KnexAdapter let schemaService: KnexQueueSchemaService const tableName = 'queue_jobs_test' const schedulesTableName = 'queue_schedules_test' group.each.setup(async () => { connection = Knex({ client: 'pg', connection: { host: process.env.PG_HOST || 'localhost', port: Number.parseInt(process.env.PG_PORT || '5432', 10), user: process.env.PG_USER || 'postgres', password: process.env.PG_PASSWORD || 'postgres', database: process.env.PG_DATABASE || 'queue_test', }, }) schemaService = new KnexQueueSchemaService(connection) // Clean up tables before each test await schemaService.dropJobsTable(tableName) await schemaService.dropSchedulesTable(schedulesTableName) // Create tables await schemaService.createJobsTable(tableName) await schemaService.createSchedulesTable(schedulesTableName) return async () => { await adapter?.destroy() await schemaService.dropJobsTable(tableName) await schemaService.dropSchedulesTable(schedulesTableName) await connection.destroy() } }) registerDriverTestSuite({ test, rejectsDuplicateJobIds: true, readsBesideTransaction: true, inTransaction: (operation) => connection.transaction((trx) => operation(new KnexAdapter({ connection: trx, tableName, schedulesTableName })) ), createAdapter: () => { adapter = new KnexAdapter({ connection, tableName, schedulesTableName }) return adapter }, }) registerWorkWatcherSuite({ test, inTransaction: (operation) => connection.transaction((trx) => operation( new KnexAdapter({ connection: trx, tableName, schedulesTableName, notifications: true }) ) ), createAdapter: () => new KnexAdapter({ connection, tableName, schedulesTableName, notifications: true }), }) test('announces a future delayed job past rows another transaction promotes', async ({ assert, cleanup, }) => { const row = (id: string, executeAt: number) => ({ id, queue: 'default', status: 'delayed', data: JSON.stringify({ id, name: 'TestJob', payload: {}, attempts: 0 }), execute_at: executeAt, }) const now = Date.now() const nextAt = now + 400 await connection(tableName).insert([ row('locked', now - 2000), row('own', now - 1000), row('next', nextAt), ]) const watcher = await new KnexAdapter({ connection, tableName, schedulesTableName, notifications: true, }).watch!(['default']) cleanup(() => watcher.close()) // Another worker holds the oldest due row while it promotes it. const trx = await connection.transaction() cleanup(async () => { await trx.rollback() }) await trx(tableName).where('id', 'locked').forUpdate().first() await new KnexAdapter({ connection, tableName, schedulesTableName, notifications: true, }).popFrom('default') watcher.reset() assert.isTrue(await watcher.wait(2000, new AbortController().signal, 0)) assert.isAtLeast(Date.now(), nextAt - 20, 'not woken for the locked row') }) /** * Replace the node-postgres client the adapter creates for LISTEN. Each * client gets the behavior `behave` returns for its 1-based creation * number: 'real' for a working connection, or a fake one. */ type FakeBehavior = { connect?: () => Promise query?: () => Promise end?: () => Promise } function fakeListenClients( cleanup: (callback: () => void) => void, behave: (index: number) => 'real' | FakeBehavior ) { type PgClient = new (settings: unknown) => { end(): Promise on(event: string, listener: (error?: Error) => void): unknown emit(event: string, error?: Error): unknown } const client = connection.client as KnexTypes.Client & { driver: { Client: PgClient } } const driver = client.driver const RealClient = driver.Client client.driver = { ...driver } cleanup(() => { client.driver = driver }) const created: Array<{ ended: boolean destroyed?: boolean emit: (event: string, error?: Error) => void }> = [] const FakeClient = class { #listeners = new Map void>() #behavior: FakeBehavior state = { ended: false, destroyed: false, emit: (event: string, error?: Error) => this.#listeners.get(event)?.(error), } // As node-postgres: the socket, destroyed when end() does not settle. connection = { stream: { destroy: () => (this.state.destroyed = true) } } constructor(behavior: FakeBehavior) { this.#behavior = behavior created.push(this.state) } on(event: string, listener: (error?: Error) => void) { this.#listeners.set(event, listener) } removeAllListeners() { this.#listeners.clear() } // As node-postgres: connect() never settles once ended in its handshake. async connect() { await this.#behavior.connect?.() } async query() { await this.#behavior.query?.() } async end() { this.state.ended = true await this.#behavior.end?.() } } client.driver.Client = function (settings: unknown) { const behavior = behave(created.length + 1) if (behavior !== 'real') return new FakeClient(behavior) const real = new RealClient(settings) const state = { ended: false, emit: (event: string, error?: Error) => real.emit(event, error), } created.push(state) const end = real.end.bind(real) real.end = () => { state.ended = true return end() } return real } as unknown as PgClient return created } test('rejects a LISTEN connection that receives no notification', async ({ assert, cleanup }) => { // As behind PgBouncer in transaction mode: LISTEN succeeds, nothing arrives. fakeListenClients(cleanup, () => ({})) const watching = new KnexAdapter({ connection, tableName, schedulesTableName, notifications: true, }) await assert.rejects(() => watching.watch!(['default']), /do not enable notifications/) }).timeout(5000) test('rejects a LISTEN probe whose connection fails', async ({ assert, cleanup }) => { // The probe sends its notification, then its connection fails. const created = fakeListenClients(cleanup, (index) => index === 1 ? 'real' : { async query() { setTimeout(() => created[1].emit('error', new Error('Connection terminated')), 10) }, } ) const watching = new KnexAdapter({ connection, tableName, schedulesTableName, notifications: true, }) await assert.rejects(() => watching.watch!(['default']), 'Connection terminated') }) test('destroys a LISTEN probe connection that does not end', async ({ assert, cleanup }) => { shortenHeartbeat(cleanup) const never = () => new Promise(() => {}) const created = fakeListenClients(cleanup, (index) => index === 1 ? 'real' : { connect: never, end: never } ) const watching = new KnexAdapter({ connection, tableName, schedulesTableName, notifications: true, }) await assert.rejects(() => watching.watch!(['default']), /do not enable notifications/) await new Promise((resolve) => setTimeout(resolve, 400)) assert.isTrue(created[1].destroyed) }).timeout(5000) test('aborts a watch whose connection settings lookup stalls', async ({ assert, cleanup }) => { const stalled = Knex({ client: 'pg', connection: () => new Promise(() => {}) }) cleanup(() => stalled.destroy()) const watching = new KnexAdapter({ connection: stalled, tableName, schedulesTableName, notifications: true, }) const controller = new AbortController() const watcher = watching.watch!(['default'], controller.signal) setTimeout(() => controller.abort(), 50) await assert.rejects(() => watcher) }).timeout(2000) test('aborts a first LISTEN connection still in its handshake', async ({ assert, cleanup }) => { const created = fakeListenClients(cleanup, () => ({ connect: () => new Promise(() => {}) })) const watching = new KnexAdapter({ connection, tableName, schedulesTableName, notifications: true, }) const controller = new AbortController() const watcher = watching.watch!(['default'], controller.signal) while (created.length < 1) await new Promise((resolve) => setTimeout(resolve, 5)) controller.abort() await assert.rejects(() => watcher) assert.isTrue(created[0].ended) }) test('rejects a LISTEN probe whose connection stalls', async ({ assert, cleanup }) => { // The LISTEN connection works, the probe never completes its handshake. fakeListenClients(cleanup, (index) => index === 1 ? 'real' : { connect: () => new Promise(() => {}) } ) const watching = new KnexAdapter({ connection, tableName, schedulesTableName, notifications: true, }) await assert.rejects(() => watching.watch!(['default']), /do not enable notifications/) }).timeout(5000) test('ignores a notification that is not a wake message', async ({ assert, cleanup }) => { const options = { connection, tableName, schedulesTableName, notifications: true } const watcher = await new KnexAdapter(options).watch!(['default']) cleanup(() => watcher.close()) watcher.reset() const waiting = watcher.wait(2000, new AbortController().signal, 0) await connection.raw(`select pg_notify(?, 'null')`, [tableName]) const job = { id: 'after-null', name: 'TestJob', payload: {}, attempts: 0 } await new KnexAdapter(options).pushOn('default', job) assert.isTrue(await waiting) }) test('watches tables found through the search path', async ({ assert, cleanup }) => { const schema = 'queue_search_path' await connection.raw('create schema if not exists ??', [schema]) cleanup(() => connection.raw('drop schema ?? cascade', [schema])) // Tables only the search path finds, unlike those of the other tests. const tables = { tableName: 'tenant_jobs', schedulesTableName: 'tenant_schedules' } for (const [table, like] of [ [tables.tableName, tableName], [tables.schedulesTableName, schedulesTableName], ]) { await connection.raw('create table ?? (like ?? including all)', [`${schema}.${table}`, like]) } const tenant = Knex({ ...connection.client.config, searchPath: [schema] }) cleanup(() => tenant.destroy()) const options = { connection: tenant, ...tables, notifications: true } const watcher = await new KnexAdapter(options).watch!(['default']) cleanup(() => watcher.close()) watcher.reset() const waiting = watcher.wait(2000, new AbortController().signal, 0) const job = { id: 'tenant-job', name: 'TestJob', payload: {}, attempts: 0 } await new KnexAdapter(options).pushOn('default', job) assert.isTrue(await waiting) }) test('does not wake again for a due schedule a concurrent claim holds', async ({ assert, cleanup, }) => { await connection(schedulesTableName).insert({ id: 'locked', name: 'TestJob', payload: '{}', every_ms: 60_000, timezone: 'UTC', next_run_at: Date.now() + 100, run_count: 0, status: 'active', created_at: Date.now(), }) const worker = new KnexAdapter({ connection, tableName, schedulesTableName, notifications: true, }) const watcher = await worker.watch!(['default']) cleanup(() => watcher.close()) // Another worker is claiming the schedule when it becomes due. const trx = await connection.transaction() cleanup(async () => { await trx.rollback() }) await trx(schedulesTableName).where('id', 'locked').forUpdate().first() await new Promise((resolve) => setTimeout(resolve, 150)) assert.isNull(await worker.claimDueSchedule()) watcher.reset() assert.isFalse( await watcher.wait(100, new AbortController().signal, 0), 'the commit of the other claim notifies' ) }) test('calls the connection provider again when its settings expire', async ({ assert, cleanup, }) => { let provided = 0 let expired = false const provider = Knex({ client: 'pg', connection: async () => { provided++ expired = false return { host: process.env.PG_HOST || 'localhost', port: Number.parseInt(process.env.PG_PORT || '5432', 10), user: process.env.PG_USER || 'postgres', password: process.env.PG_PASSWORD || 'postgres', database: process.env.PG_DATABASE || 'queue_test', application_name: `listen-${provided}`, expirationChecker: () => expired, } }, }) cleanup(() => provider.destroy()) const listenerName = async () => { const result = await connection.raw( // The LISTEN connection, not the probe that notifies through another one. `select application_name from pg_stat_activity where application_name like 'listen-%' and query not like '%pg_notify%'` ) return result.rows.map((row: { application_name: string }) => row.application_name) } const subscriber = new KnexAdapter({ connection: provider, notifications: true, tableName, schedulesTableName, logger: new MemoryLogger(), }) const watcher = await subscriber.watch!(['default']) cleanup(() => watcher.close()) const [first] = await listenerName() // Rotated credentials: the provider marks its settings expired. expired = true await connection.raw( `select pg_terminate_backend(pid) from pg_stat_activity where application_name = ?`, [first] ) await new Promise((resolve) => setTimeout(resolve, 1500)) const [second] = await listenerName() assert.isDefined(second) assert.notEqual(second, first) }) test('keeps the settings of the latest connection provider lookup', async ({ assert, cleanup, }) => { shortenHeartbeat(cleanup) // A lookup gated here answers once a later one restored LISTEN. let gate: PromiseWithResolvers | undefined let expired = false let provided = 0 const provider = Knex({ client: 'pg', connection: async () => { const name = gate ? 'listen-stale' : `listen-fresh-${++provided}` const answered = gate?.promise gate = undefined await answered return { host: process.env.PG_HOST || 'localhost', port: Number.parseInt(process.env.PG_PORT || '5432', 10), user: process.env.PG_USER || 'postgres', password: process.env.PG_PASSWORD || 'postgres', database: process.env.PG_DATABASE || 'queue_test', application_name: name, expirationChecker: () => expired, } }, }) cleanup(() => provider.destroy()) const listeners = async () => { const result = await connection.raw( `select application_name from pg_stat_activity where application_name like 'listen-%' and query not like '%pg_notify%'` ) return result.rows.map((row: { application_name: string }) => row.application_name) } const terminate = (name: string) => connection.raw( `select pg_terminate_backend(pid) from pg_stat_activity where application_name = ?`, [name] ) const waitFor = async (name: string, timeout: number) => { const deadline = Date.now() + timeout while (Date.now() < deadline) { const names = await listeners() if (names.includes(name)) return names await new Promise((resolve) => setTimeout(resolve, 50)) } return listeners() } const watcher = await new KnexAdapter({ connection: provider, notifications: true, tableName, schedulesTableName, logger: new MemoryLogger(), }).watch!(['default']) cleanup(() => watcher.close()) // The next lookup stalls past the startup deadline, the one after it works. const stale = Promise.withResolvers() gate = stale expired = true await terminate('listen-fresh-1') assert.include(await waitFor('listen-fresh-2', 8000), 'listen-fresh-2') expired = false stale.resolve() await new Promise((resolve) => setTimeout(resolve, 100)) // Reconnecting reuses the cached settings: those of a lookup after the stale one. await terminate('listen-fresh-2') await new Promise((resolve) => setTimeout(resolve, 1500)) const names = await listeners() assert.lengthOf(names, 1) assert.match(names[0], /^listen-fresh-/) }).timeout(15_000) test('does not reconnect a LISTEN connection that failed to start', async ({ assert, cleanup, }) => { const created = fakeListenClients(cleanup, () => ({ async query() { const error = new Error('Connection terminated') created[0].emit('error', error) throw error }, })) const subscriber = new KnexAdapter({ connection, tableName, schedulesTableName, logger: new MemoryLogger(), notifications: true, }) await assert.rejects(() => subscriber.watch!(['default']), 'Connection terminated') // Past the reconnection delay await new Promise((resolve) => setTimeout(resolve, 1200)) assert.lengthOf(created, 1) }) test('aborts a LISTEN reconnection still in its handshake when closed', async ({ assert, cleanup, }) => { // The first connection and its probe work, the next never completes its handshake. const created = fakeListenClients(cleanup, (index) => index <= 2 ? 'real' : { connect: () => new Promise(() => {}) } ) const subscriber = new KnexAdapter({ connection, tableName, schedulesTableName, logger: new MemoryLogger(), notifications: true, }) const watcher = await subscriber.watch!(['default']) created[0].emit('error', new Error('Connection terminated')) while (created.length < 3) await new Promise((resolve) => setTimeout(resolve, 20)) await watcher.close() assert.isTrue(created[2].ended) }) test('reconnects a LISTEN connection whose replacement stalls', async ({ assert, cleanup }) => { shortenHeartbeat(cleanup) // The first connection and its probe work, the next never completes its handshake. const created = fakeListenClients(cleanup, (index) => index === 3 ? { connect: () => new Promise(() => {}) } : 'real' ) const options = { connection, tableName, schedulesTableName, notifications: true } const watcher = await new KnexAdapter({ ...options, logger: new MemoryLogger() }).watch!([ 'default', ]) cleanup(() => watcher.close()) created[0].emit('error', new Error('Connection terminated')) // The stalled connection is given up, and the next one listens. const deadline = Date.now() + 8000 while (created.length < 5 && Date.now() < deadline) { await new Promise((resolve) => setTimeout(resolve, 50)) } assert.lengthOf(created, 5) assert.isTrue(created[2].ended) await new Promise((resolve) => setTimeout(resolve, 200)) watcher.reset() const waiting = watcher.wait(2000, new AbortController().signal, 0) const job = { id: 'after-stall', name: 'TestJob', payload: {}, attempts: 0 } await new KnexAdapter(options).pushOn('default', job) assert.isTrue(await waiting) }).timeout(10_000) test('signals no due time for a delayed job deduplicated away', async ({ assert, cleanup }) => { const options = { connection, tableName, schedulesTableName, notifications: true } const watcher = await new KnexAdapter(options).watch!(['default']) cleanup(() => watcher.close()) watcher.reset() const start = Date.now() const waiting = watcher.wait(2000, new AbortController().signal, 0) const writer = new KnexAdapter(options) const job = (id: string, dedup?: { id: string }) => ({ id, name: 'TestJob', payload: {}, attempts: 0, dedup, }) await writer.pushLaterOn('default', job('owner', { id: 'slot' }), 3000) // Skipped: no job is due at 200ms. Its time would hide the next one. await writer.pushLaterOn('default', job('duplicate', { id: 'slot' }), 200) await writer.pushLaterOn('default', job('next'), 500) assert.isTrue(await waiting) assert.isAtLeast(Date.now() - start, 450) }) test('signals jobs of a table whose name is too long for a channel', async ({ assert, cleanup, }) => { // A schema and a table name of 63 bytes each, the PostgreSQL maximum. const schema = `tenant_${'x'.repeat(56)}` const longTableName = `${schema}.queue_jobs_test` await connection.raw('create schema if not exists ??', [schema]) cleanup(() => connection.raw('drop schema ?? cascade', [schema])) await connection.raw('create table ?? (like ?? including all)', [longTableName, tableName]) const options = { tableName: longTableName, schedulesTableName, notifications: true } const subscriber = new KnexAdapter({ connection, ...options }) const watcher = await subscriber.watch!(['default']) cleanup(() => watcher.close()) watcher.reset() const waiting = watcher.wait(2000, new AbortController().signal, 0) const job = { id: 'long-table-job', name: 'TestJob', payload: {}, attempts: 0 } await connection.transaction((trx) => new KnexAdapter({ connection: trx, ...options }).pushOn('default', job) ) assert.isTrue(await waiting) assert.isNotNull(await subscriber.getJob(job.id, 'default')) }) test('reconnects a LISTEN connection that stops answering', async ({ assert, cleanup }) => { shortenHeartbeat(cleanup) const proxy = await startSilentProxy({ host: process.env.PG_HOST || 'localhost', port: Number.parseInt(process.env.PG_PORT || '5432', 10), }) cleanup(() => proxy.close()) // The LISTEN connection goes through the proxy. const knexClient = connection.client as KnexTypes.Client & { connectionConfigProvider?: () => Promise } knexClient.connectionConfigProvider = async () => ({ host: '127.0.0.1', port: proxy.port, user: process.env.PG_USER || 'postgres', password: process.env.PG_PASSWORD || 'postgres', database: process.env.PG_DATABASE || 'queue_test', }) cleanup(() => { knexClient.connectionConfigProvider = undefined }) const logger = new MemoryLogger() const watcher = await new KnexAdapter({ connection, tableName, schedulesTableName, logger, notifications: true, }).watch!(['default']) cleanup(() => watcher.close()) watcher.reset() const waiting = watcher.wait(4000, new AbortController().signal, 0) proxy.silence() // Its notification is lost: the new connection announces it. await new KnexAdapter({ connection, tableName, schedulesTableName, notifications: true, }).pushOn('default', { id: 'after-silence', name: 'TestJob', payload: {}, attempts: 0, }) assert.isTrue(await waiting) assert.include( logger.logs.map((entry) => String(entry.obj?.err)), 'Error: The LISTEN connection did not answer a ping' ) }).timeout(6000) test('a watcher woken for nothing sleeps through its cooldown', async ({ assert, cleanup }) => { const watcher = await new KnexAdapter({ connection, tableName, schedulesTableName, notifications: true, }).watch!(['default']) cleanup(() => watcher.close()) watcher.reset() const start = Date.now() const waiting = watcher.wait(2000, new AbortController().signal, 300) await new KnexAdapter({ connection, tableName, schedulesTableName, notifications: true, }).pushOn('default', { id: 'during-cooldown', name: 'TestJob', payload: {}, attempts: 0, }) assert.isTrue(await waiting) assert.closeTo(Date.now() - start, 300, 80) }) test('sends one notification per interval for a burst of pushes to several queues', async ({ assert, cleanup, }) => { const listener = (await connection.client.acquireRawConnection()) as { query(sql: string): Promise on(event: 'notification', listener: (message: { payload?: string }) => void): void } cleanup(() => connection.client.destroyRawConnection(listener)) const payloads: string[] = [] listener.on('notification', (message) => payloads.push(message.payload ?? '')) await listener.query(`LISTEN "${tableName}"`) const writer = new KnexAdapter({ connection, tableName, schedulesTableName, notifications: true, }) const start = Date.now() for (let index = 0; index < 50; index++) { // Spread over queues: the interval holds for the adapter. await writer.pushOn(`queue-${index % 5}`, { id: `burst-${index}`, name: 'TestJob', payload: {}, attempts: 0, }) } const elapsed = Date.now() - start await writer.destroy() await new Promise((resolve) => setTimeout(resolve, 50)) assert.isAtLeast(payloads.length, 1) assert.isAtMost(payloads.length, Math.ceil(elapsed / 50) + 1) }) test('watches without holding a connection of the pool', async ({ assert, cleanup }) => { // The worker holds a pool connection while its watcher starts. const trx = await connection.transaction() cleanup(async () => { await trx.rollback() }) await trx.raw('select 1') const pool = ( connection.client as KnexTypes.Client & { pool: { numUsed(): number; numFree(): number } } ).pool const before = pool.numUsed() + pool.numFree() const watcher = await new KnexAdapter({ connection, tableName, schedulesTableName, notifications: true, }).watch!(['default']) cleanup(() => watcher.close()) assert.equal(pool.numUsed() + pool.numFree(), before) }) test('knex() factory accepts a table name or options', async ({ assert }) => { const job = { id: 'factory-job', name: 'TestJob', payload: {}, attempts: 0 } await knex(connection, tableName)().pushOn('default', job) const polling = knex(connection, { tableName, schedulesTableName })() assert.isUndefined(polling.watch, 'polls by default') assert.isNotNull(await polling.getJob(job.id, 'default'), 'the string sets the jobs table') const listening = knex(connection, { tableName, schedulesTableName, notifications: true })() assert.isFunction(listening.watch) }) registerPruneFailureSuite({ test, createAdapter: (logger) => { adapter = new KnexAdapter({ connection, tableName, schedulesTableName, logger }) return adapter }, failDeletes: async (sqlstate, count) => { for (const statement of pgDeleteFailureSql(tableName, sqlstate, count)) { await connection.raw(statement) } }, inTransaction: (logger, operation) => connection.transaction((trx) => operation(new KnexAdapter({ connection: trx, tableName, schedulesTableName, logger })) ), }) test('addDedupColumns should leave a dedup slot to its latest job', async ({ assert }) => { await assertDedupMigration(assert, connection, tableName) }) test('listSchedules should execute a single SQL query in Knex PostgreSQL adapter', async ({ assert, }) => { const knexAdapter = new KnexAdapter({ connection, tableName, schedulesTableName }) for (let i = 0; i < 20; i++) { await knexAdapter.upsertSchedule({ id: `pg-list-${i}`, name: 'PgPerfJob', payload: { i }, everyMs: 60_000, timezone: 'UTC', }) } const { result: schedules, queries } = await withKnexQuerySpy({ connection, run: () => knexAdapter.listSchedules(), }) assert.lengthOf(schedules, 20) const scheduleSelectQueries = queries.filter( (sql) => sql.includes('select') && sql.includes(schedulesTableName) ) assert.lengthOf( scheduleSelectQueries, 1, `Expected a single schedule SELECT query, got ${scheduleSelectQueries.length}` ) }) test('deleteSchedule should execute a single SQL query in Knex PostgreSQL adapter', async ({ assert, }) => { const knexAdapter = new KnexAdapter({ connection, tableName, schedulesTableName }) const id = 'pg-delete-atomicity' await knexAdapter.upsertSchedule({ id, name: 'PgDeleteJob', payload: {}, everyMs: 60_000, timezone: 'UTC', }) const { queries } = await withKnexQuerySpy({ connection, run: () => knexAdapter.deleteSchedule(id), }) const schedule = await knexAdapter.getSchedule(id) assert.isNull(schedule) const scheduleDeleteQueries = queries.filter( (sql) => sql.includes('delete') && sql.includes(schedulesTableName) ) assert.lengthOf( scheduleDeleteQueries, 1, `Expected a single schedule DELETE query, got ${scheduleDeleteQueries.length}` ) }) test('concurrent dedup pushes should not both insert when no existing row is lockable', async ({ assert, }) => { const dedupId = 'TestJob::pg-concurrent-missing-row' const barrierFunction = 'queue_jobs_test_dedup_insert_barrier' const barrierTrigger = 'queue_jobs_test_dedup_insert_barrier_trigger' await connection.raw(` CREATE OR REPLACE FUNCTION ${barrierFunction}() RETURNS trigger AS $$ DECLARE attempts integer := 0; BEGIN IF NEW.dedup_id = '${dedupId}' THEN IF pg_try_advisory_lock(90312001) THEN LOOP EXIT WHEN NOT pg_try_advisory_lock(90312002); PERFORM pg_advisory_unlock(90312002); attempts := attempts + 1; IF attempts > 1000 THEN RAISE EXCEPTION 'timed out waiting for concurrent insert'; END IF; PERFORM pg_sleep(0.01); END LOOP; ELSE PERFORM pg_advisory_lock(90312002); END IF; END IF; RETURN NEW; END; $$ LANGUAGE plpgsql `) await connection.raw(` CREATE TRIGGER ${barrierTrigger} BEFORE INSERT ON ${tableName} FOR EACH ROW EXECUTE FUNCTION ${barrierFunction}() `) const createConnection = () => Knex({ client: 'pg', connection: { host: process.env.PG_HOST || 'localhost', port: Number.parseInt(process.env.PG_PORT || '5432', 10), user: process.env.PG_USER || 'postgres', password: process.env.PG_PASSWORD || 'postgres', database: process.env.PG_DATABASE || 'queue_test', }, pool: { min: 1, max: 1 }, }) const connectionA = createConnection() const connectionB = createConnection() const adapterA = new KnexAdapter({ connection: connectionA, tableName, schedulesTableName }) const adapterB = new KnexAdapter({ connection: connectionB, tableName, schedulesTableName }) try { const results = await Promise.all([ adapterA.pushOn('pg-dedup-race-queue', { id: 'pg-dedup-race-uuid-1', name: 'TestJob', payload: { n: 1 }, attempts: 0, dedup: { id: dedupId }, }), adapterB.pushOn('pg-dedup-race-queue', { id: 'pg-dedup-race-uuid-2', name: 'TestJob', payload: { n: 2 }, attempts: 0, dedup: { id: dedupId }, }), ]) const outcomes = results.map((result) => result && typeof result === 'object' ? result.outcome : undefined ) assert.equal(outcomes.filter((outcome) => outcome === 'added').length, 1) assert.equal(outcomes.filter((outcome) => outcome === 'skipped').length, 1) const count = await connection(tableName) .where('queue', 'pg-dedup-race-queue') .where('dedup_id', dedupId) .count<{ total: string }[]>('* as total') .first() assert.equal(Number(count?.total), 1) } finally { await adapterA.destroy() await adapterB.destroy() await connectionA.destroy() await connectionB.destroy() await connection.raw(`DROP TRIGGER IF EXISTS ${barrierTrigger} ON ${tableName}`) await connection.raw(`DROP FUNCTION IF EXISTS ${barrierFunction}()`) } }) test('retryJob should not violate dedup unique index after active TTL expires', async ({ assert, }) => { const knexAdapter = new KnexAdapter({ connection, tableName, schedulesTableName }) knexAdapter.setWorkerId('worker-1') const queue = 'pg-expired-active-retry-dedup-queue' const dedupId = 'TestJob::pg-expired-active-retry' await knexAdapter.pushOn(queue, { id: 'pg-expired-active-retry-uuid-1', name: 'TestJob', payload: { n: 1 }, attempts: 0, dedup: { id: dedupId, ttl: 30 }, }) const first = await knexAdapter.popFrom(queue) assert.equal(first!.id, 'pg-expired-active-retry-uuid-1') await new Promise((resolve) => setTimeout(resolve, 50)) const second = await knexAdapter.pushOn(queue, { id: 'pg-expired-active-retry-uuid-2', name: 'TestJob', payload: { n: 2 }, attempts: 0, dedup: { id: dedupId, ttl: 30 }, }) assert.equal(second && typeof second === 'object' && second.outcome, 'added') await knexAdapter.retryJob(first!, queue) const availableJobs = [await knexAdapter.popFrom(queue), await knexAdapter.popFrom(queue)] const availableIds = availableJobs.map((job) => job?.id).sort() assert.deepEqual(availableIds, [ 'pg-expired-active-retry-uuid-1', 'pg-expired-active-retry-uuid-2', ]) }) test('recoverStalledJobs should not violate dedup unique index after active TTL expires', async ({ assert, }) => { const knexAdapter = new KnexAdapter({ connection, tableName, schedulesTableName }) knexAdapter.setWorkerId('worker-1') const queue = 'pg-expired-active-stalled-dedup-queue' const dedupId = 'TestJob::pg-expired-active-stalled' await knexAdapter.pushOn(queue, { id: 'pg-expired-active-stalled-uuid-1', name: 'TestJob', payload: { n: 1 }, attempts: 0, dedup: { id: dedupId, ttl: 30 }, }) const first = await knexAdapter.popFrom(queue) assert.equal(first!.id, 'pg-expired-active-stalled-uuid-1') await new Promise((resolve) => setTimeout(resolve, 50)) const second = await knexAdapter.pushOn(queue, { id: 'pg-expired-active-stalled-uuid-2', name: 'TestJob', payload: { n: 2 }, attempts: 0, dedup: { id: dedupId, ttl: 30 }, }) assert.equal(second && typeof second === 'object' && second.outcome, 'added') const { recovered } = await knexAdapter.recoverStalledJobs(queue, 1, 1, 100) assert.equal(recovered, 1) const availableJobs = [await knexAdapter.popFrom(queue), await knexAdapter.popFrom(queue)] const availableIds = availableJobs.map((job) => job?.id).sort() assert.deepEqual(availableIds, [ 'pg-expired-active-stalled-uuid-1', 'pg-expired-active-stalled-uuid-2', ]) }) }) test.group('Adapter | Knex (MySQL)', (group) => { let connection: ReturnType let adapter: KnexAdapter let schemaService: KnexQueueSchemaService const tableName = 'knex_mysql_queue_jobs_test' const schedulesTableName = 'knex_mysql_queue_schedules_test' const config: KnexTypes.Config = { client: 'mysql2', connection: { host: process.env.MYSQL_HOST || 'localhost', port: Number.parseInt(process.env.MYSQL_PORT || '3307', 10), user: process.env.MYSQL_USER || 'root', password: process.env.MYSQL_PASSWORD || 'mysql', database: process.env.MYSQL_DATABASE || 'queue_test', }, } group.each.setup(async () => { connection = Knex(config) schemaService = new KnexQueueSchemaService(connection) await schemaService.dropJobsTable(tableName) await schemaService.dropSchedulesTable(schedulesTableName) await schemaService.createJobsTable(tableName) await schemaService.createSchedulesTable(schedulesTableName) return async () => { await adapter?.destroy() await schemaService.dropJobsTable(tableName) await schemaService.dropSchedulesTable(schedulesTableName) await connection.destroy() } }) registerDriverTestSuite({ test, rejectsDuplicateJobIds: true, readsBesideTransaction: true, inTransaction: (operation) => connection.transaction((trx) => operation(new KnexAdapter({ connection: trx, tableName, schedulesTableName })) ), supportsAtomicDedup: false, createAdapter: () => { adapter = new KnexAdapter({ connection, tableName, schedulesTableName }) return adapter }, }) registerRetentionRaceSuite({ test, manyConsumers: true, createAdapters: async (beforeDelete, cleanup) => { const first = pauseKnexDeletes(Knex(config), beforeDelete) const second = Knex(config) cleanup(async () => { await first.destroy() await second.destroy() }) return [ new KnexAdapter({ connection: first, tableName }), new KnexAdapter({ connection: second, tableName }), ] }, }) }) test.group('Adapter | Knex (SQLite file, two connections)', () => { const connectTwice = async (cleanup: (handler: () => Promise) => void) => { const directory = await mkdtemp(join(tmpdir(), 'queue-sqlite-')) const filename = join(directory, 'queue.sqlite') const connect = () => Knex({ client: 'better-sqlite3', connection: { filename }, useNullAsDefault: true, pool: { afterCreate: (db: any, done: any) => done(null, db.pragma('journal_mode = WAL')) }, }) const first = connect() const second = connect() cleanup(async () => { await first.destroy() await second.destroy() await rm(directory, { recursive: true, force: true }) }) return Object.assign([first, second] as const, { filename }) } test('popFrom and dedup pushes wait for a write from another process', async ({ assert, cleanup, }) => { const connections = await connectTwice(cleanup) await new KnexQueueSchemaService(connections[0]).createJobsTable() const adapter = new KnexAdapter({ connection: connections[0] }) adapter.setWorkerId('worker-1') const stopContender = await startSqliteWriteContender(connections.filename) const outcomes: string[] = [] try { for (let i = 0; i < 20; i++) { const job = { id: `job-${i}`, name: 'TestJob', payload: {}, attempts: 0 } for (let dispatch = 0; dispatch < 2; dispatch++) { const result = await adapter.pushOn('test-queue', { ...job, dedup: { id: `dedup-${i}` } }) outcomes.push(result?.outcome ?? 'none') } await adapter.pushLaterOn('test-queue', { ...job, id: `delayed-${i}` }, 0) assert.isNotNull(await adapter.popFrom('test-queue')) } } finally { await stopContender() } assert.deepEqual(outcomes, Array.from({ length: 20 }, () => ['added', 'skipped']).flat()) }) registerRetentionRaceSuite({ test, createAdapters: async (beforeDelete, cleanup) => { const [first, second] = await connectTwice(cleanup) await new KnexQueueSchemaService(first).createJobsTable() return [ new KnexAdapter({ connection: pauseKnexDeletes(first, beforeDelete) }), new KnexAdapter({ connection: second }), ] }, }) test('concurrent upserts from two connections all succeed', async ({ assert, cleanup }) => { const [first, second] = await connectTwice(cleanup) const schedulesTableName = 'queue_schedules' await new KnexQueueSchemaService(first).createSchedulesTable(schedulesTableName) const adapters = [first, second].map( (connection) => new KnexAdapter({ connection, schedulesTableName }) ) for (let round = 0; round < 5; round++) { const id = `two-connections-${round}` const base = Date.now() + 60_000 // Workers on two connections define the same schedule, then two timings. const results = await Promise.allSettled([ ...adapters.map((adapter) => adapter.upsertSchedule({ id, name: 'TestJob', payload: {}, everyMs: 60_000, timezone: 'UTC', nextRunAt: new Date(base + 60_000), }) ), ...adapters.map((adapter, index) => adapter.upsertSchedule({ id, name: 'TestJob', payload: {}, everyMs: (index + 2) * 60_000, timezone: 'UTC', nextRunAt: new Date(base + (index + 2) * 60_000), }) ), ]) assert.deepEqual( results.filter((result) => result.status === 'rejected'), [] ) const schedule = await adapters[0].getSchedule(id) assert.equal(schedule!.nextRunAt!.getTime() - base, schedule!.everyMs) } }) })