import { exists, hasFlag } from "@ioredis/commands"; import { EventEmitter } from "events"; import { AbortError, RedisError } from "redis-errors"; import asCallback from "standard-as-callback"; import Command from "../Command"; import ClusterAllFailedError from "../errors/ClusterAllFailedError"; import Pipeline from "../Pipeline"; import Redis from "../Redis"; import ScanStream from "../ScanStream"; import { addTransactionSupport, Transaction } from "../transaction"; import { Callback, ReplyMappingFromClusterOptions, ReplyMappingMode, ScanStreamOptions, WriteableStream, } from "../types"; import { CONNECTION_CLOSED_ERROR_MSG, Debug, defaults, noop, sample, shuffle, timeout, zipMap, } from "../utils"; import applyMixin from "../utils/applyMixin"; import Commander from "../utils/Commander"; import { ClusterOptions, ClusterOptionsWithReplyMapping, DEFAULT_CLUSTER_OPTIONS, } from "./ClusterOptions"; import ClusterSubscriber from "./ClusterSubscriber"; import ConnectionPool from "./ConnectionPool"; import DelayQueue from "./DelayQueue"; import { getConnectionName, getNodeKey, getUniqueHostnamesFromOptions, groupSrvRecords, NodeKey, nodeKeyToRedisOptions, NodeRole, normalizeNodeOptions, RedisOptions, waitForRedisReady, weightSrvRecords, } from "./util"; import Deque = require("denque"); import ClusterSubscriberGroup from "./ClusterSubscriberGroup"; import HimportCoordinator, { bindHimportCoordinator, cloneHimportFieldsets, getHimportBinding, interceptHimportControlCommand, isHimportControlCommand, setHimportRole, unbindHimportCoordinator, } from "../himport/HimportCoordinator"; const debug = Debug("cluster"); // Keyed by the command's promise rather than the command: a pipeline retried // as a whole gives its commands fresh promises and rejects, which must be // wrapped again when they are resent on their own. const REJECT_OVERWRITTEN_PROMISES = new WeakSet>(); type OfflineQueueItem = { command: Command; stream: WriteableStream; node: unknown; }; export type ClusterNode = | string | number | { host?: string | undefined; port?: number | undefined; }; export type ClusterStatus = | "end" | "close" | "wait" | "connecting" | "connect" | "ready" | "reconnecting" | "disconnecting"; /** * Client for the official Redis Cluster */ class Cluster< ReplyMapping extends ReplyMappingMode = "legacy" > extends Commander<{ type: "default"; mapping: ReplyMapping extends "resp3" ? "resp3" : "resp2"; }> { options: ClusterOptions; slots: NodeKey[][] = []; status: ClusterStatus; /** * @ignore */ _groupsIds: { [key: string]: number } = {}; /** * @ignore */ _groupsBySlot: number[] = Array(16384); /** * @ignore */ isCluster = true; private startupNodes: (string | number | object)[]; private connectionPool: ConnectionPool; private manuallyClosing: boolean; private retryAttempts = 0; private delayQueue: DelayQueue = new DelayQueue(); private offlineQueue = new Deque(); private subscriber: ClusterSubscriber; private shardedSubscribers: ClusterSubscriberGroup; private slotsTimer: ReturnType | null; private reconnectTimeout: ReturnType | null; private isRefreshing = false; private _refreshSlotsCacheCallbacks = []; private _autoPipelines: Map = new Map(); private _runningAutoPipelines: Set = new Set(); private _readyDelayedCallbacks: Callback[] = []; private subscriberGroupEmitter: EventEmitter | null; /** * Every time Cluster#connect() is called, this value will be * auto-incrementing. The purpose of this value is used for * discarding previous connect attampts when creating a new * connection. */ private connectionEpoch = 0; /** * Creates an instance of Cluster. */ //TODO: Add an option that enables or disables sharded PubSub // `redisOptions.replyMapping` carries the `ReplyMapping` class type parameter // so RESP3 return shapes are inferred from the options literal at construction, // mirroring the single-node `Redis` constructor. constructor( startupNodes: ClusterNode[], options: ClusterOptionsWithReplyMapping = {} ) { super(); EventEmitter.call(this); this.startupNodes = startupNodes; this.options = defaults({}, options, DEFAULT_CLUSTER_OPTIONS, this.options); this.options.himportFieldsets = cloneHimportFieldsets( this.options.himportFieldsets ); const himportCoordinator = this.options.himportFieldsets?.length ? new HimportCoordinator(this.options.himportFieldsets) : undefined; if (himportCoordinator) { bindHimportCoordinator(this, himportCoordinator, "cluster"); } if (this.options.shardedSubscribers) { this.createShardedSubscriberGroup(); } if ( this.options.redisOptions && this.options.redisOptions.keyPrefix && !this.options.keyPrefix ) { this.options.keyPrefix = this.options.redisOptions.keyPrefix; } // validate options if ( typeof this.options.scaleReads !== "function" && ["all", "master", "slave"].indexOf(this.options.scaleReads) === -1 ) { throw new Error( 'Invalid option scaleReads "' + this.options.scaleReads + '". Expected "all", "master", "slave" or a custom function' ); } if ( ["all", "master", "slave"].indexOf(this.options.subscriberNodeRole) === -1 ) { throw new Error( 'Invalid option subscriberNodeRole "' + this.options.subscriberNodeRole + '". Expected "all", "master" or "slave"' ); } // Below this, a PING cannot reliably be answered within one interval, and // event loop pauses would disconnect healthy subscribers. const MIN_HEALTH_CHECK_INTERVAL_MS = 1000; const healthCheckInterval = this.options.shardedSubscriberHealthCheckInterval; if ( healthCheckInterval !== null && !( typeof healthCheckInterval === "number" && Number.isFinite(healthCheckInterval) && healthCheckInterval >= MIN_HEALTH_CHECK_INTERVAL_MS ) ) { throw new Error( 'Invalid option shardedSubscriberHealthCheckInterval "' + healthCheckInterval + `". Expected null or a number of at least ${MIN_HEALTH_CHECK_INTERVAL_MS}` ); } const redisOptions = { ...(this.options.redisOptions ?? {}) }; delete (redisOptions as Partial).himportFieldsets; this.connectionPool = new ConnectionPool( redisOptions, this.options.clusterNodeRetryStrategy, himportCoordinator ? { onCreate: (redis, readOnly) => { bindHimportCoordinator( redis, himportCoordinator, readOnly ? "replica" : "master" ); }, onRoleChange: (redis, key, readOnly) => { setHimportRole(redis, readOnly ? "replica" : "master"); himportCoordinator.invalidate(redis); if (!readOnly) { waitForRedisReady(redis).then( () => { for (const definition of himportCoordinator.getDefinitions()) { const preparation = himportCoordinator.ensurePrepared( redis, definition ); preparation?.catch((error) => { this.emit("node error", error, key); }); } }, (error: Error) => { this.emit("node error", error, key); } ); } }, onRemove: (redis) => { unbindHimportCoordinator(redis); }, } : undefined ); this.connectionPool.on("-node", (redis, key) => { this.emit("-node", redis); }); this.connectionPool.on("+node", (redis) => { this.emit("+node", redis); }); this.connectionPool.on("drain", () => { if (this.status === "end") { return; } this.setStatus("close"); }); this.connectionPool.on("nodeError", (error, key) => { this.emit("node error", error, key); }); this.subscriber = new ClusterSubscriber( this.connectionPool, this, this.options.subscriberNodeRole ); if (this.options.scripts) { Object.entries(this.options.scripts).forEach(([name, definition]) => { this.defineCommand(name, definition); }); } if (this.options.lazyConnect) { this.setStatus("wait"); } else { this.connect().catch((err) => { debug("connecting failed: %s", err); }); } } /** * Connect to a cluster */ connect(): Promise { return new Promise((resolve, reject) => { if ( this.status === "connecting" || this.status === "connect" || this.status === "ready" ) { reject(new Error("Redis is already connecting/connected")); return; } if (this.status === "end") { this.retryAttempts = 0; if (this.options.shardedSubscribers) { this.subscriberGroupEmitter.removeAllListeners(); this.addSubscriberGroupListeners(); } } const epoch = ++this.connectionEpoch; this.setStatus("connecting"); this.resolveStartupNodeHostnames() .then((nodes) => { if (this.connectionEpoch !== epoch) { debug( "discard connecting after resolving startup nodes because epoch not match: %d != %d", epoch, this.connectionEpoch ); reject( new RedisError( "Connection is discarded because a new connection is made" ) ); return; } if (this.status !== "connecting") { debug( "discard connecting after resolving startup nodes because the status changed to %s", this.status ); reject(new RedisError("Connection is aborted")); this.handleAbortedConnect(); return; } this.connectionPool.reset(nodes); if (this.options.shardedSubscribers) { this.shardedSubscribers .reset(this.slots, this.connectionPool.getNodes("all")) .catch((err) => { // TODO should we emit an error event here? debug("Error while starting subscribers: %s", err); }); } const readyHandler = () => { this.setStatus("ready"); this.retryAttempts = 0; this.executeOfflineCommands(); this.resetNodesRefreshInterval(); resolve(); }; let closeListener: () => void = undefined; const refreshListener = () => { this.invokeReadyDelayedCallbacks(undefined); this.removeListener("close", closeListener); this.manuallyClosing = false; this.setStatus("connect"); // Startup nodes have no authoritative role metadata until the // first slots refresh completes. Automatic reconnects recover an // already-started subscriber while resetting the connection pool. if ( this.options.subscriberNodeRole !== "all" && !this.subscriber.isStarted() ) { this.subscriber.start(); } if (this.options.enableReadyCheck) { this.readyCheck((err, fail) => { if (err || fail) { debug( "Ready check failed (%s). Reconnecting...", err || fail ); if (this.status === "connect") { this.disconnect(true); } } else { readyHandler(); } }); } else { readyHandler(); } }; closeListener = () => { const error = new Error("None of startup nodes is available"); this.removeListener("refresh", refreshListener); this.invokeReadyDelayedCallbacks(error); reject(error); }; this.once("refresh", refreshListener); this.once("close", closeListener); this.once("close", this.handleCloseEvent.bind(this)); this.refreshSlotsCache((err) => { if (err && err.message === ClusterAllFailedError.defaultMessage) { Redis.prototype.silentEmit.call(this, "error", err); this.connectionPool.reset([]); } }); if (this.options.subscriberNodeRole === "all") { this.subscriber.start(); } if (this.options.shardedSubscribers) { this.shardedSubscribers.start().catch((err) => { // TODO should we emit an error event here? debug("Error while starting subscribers: %s", err); }); } }) .catch((err) => { this.setStatus("close"); this.handleCloseEvent(err); this.invokeReadyDelayedCallbacks(err); reject(err); }); }); } /** * Disconnect from every node in the cluster. */ disconnect(reconnect = false) { const status = this.status; this.setStatus("disconnecting"); if (!reconnect) { this.manuallyClosing = true; } if (this.reconnectTimeout && !reconnect) { clearTimeout(this.reconnectTimeout); this.reconnectTimeout = null; debug("Canceled reconnecting attempts"); } this.clearNodesRefreshInterval(); this.subscriber.stop(); if (this.options.shardedSubscribers) { this.shardedSubscribers.stop(); } // No node is left to emit "drain" while reconnecting. if (status === "wait" || (status === "reconnecting" && !reconnect)) { this.setStatus("close"); this.handleCloseEvent(); } else { this.connectionPool.reset([]); } } /** * Quit the cluster gracefully. */ quit(callback?: Callback<"OK">): Promise<"OK"> { const status = this.status; this.setStatus("disconnecting"); this.manuallyClosing = true; if (this.reconnectTimeout) { clearTimeout(this.reconnectTimeout); this.reconnectTimeout = null; } this.clearNodesRefreshInterval(); this.subscriber.stop(); if (this.options.shardedSubscribers) { this.shardedSubscribers.stop(); } if (status === "wait" || status === "reconnecting") { const ret = asCallback(Promise.resolve<"OK">("OK"), callback); // use setImmediate to make sure "close" event // being emitted after quit() is returned setImmediate( function () { this.setStatus("close"); this.handleCloseEvent(); }.bind(this) ); return ret; } return asCallback( Promise.all( this.nodes().map((node) => node.quit().catch((err) => { // Ignore the error caused by disconnecting since // we're disconnecting... if (err.message === CONNECTION_CLOSED_ERROR_MSG) { return "OK"; } throw err; }) ) ).then(() => "OK"), callback ); } /** * Create a new instance with the same startup nodes and options as the current one. * * @example * ```js * var cluster = new Redis.Cluster([{ host: "127.0.0.1", port: "30001" }]); * var anotherCluster = cluster.duplicate(); * ``` */ duplicate< OverrideOptions extends | ClusterOptionsWithReplyMapping | undefined = undefined >( overrideStartupNodes: ClusterNode[] = [], overrideOptions?: OverrideOptions ): Cluster> { const startupNodes = overrideStartupNodes.length > 0 ? overrideStartupNodes : this.startupNodes.slice(0); const options = Object.assign({}, this.options, overrideOptions); if (this.options.redisOptions && overrideOptions?.redisOptions) { options.redisOptions = Object.assign( {}, this.options.redisOptions, overrideOptions.redisOptions ); } return new Cluster(startupNodes, options) as Cluster< ReplyMappingFromClusterOptions >; } /** * Get nodes with the specified role */ nodes(role: NodeRole = "all"): Redis[] { if (role !== "all" && role !== "master" && role !== "slave") { throw new Error( 'Invalid role "' + role + '". Expected "all", "master" or "slave"' ); } return this.connectionPool.getNodes(role); } /** * This is needed in order not to install a listener for each auto pipeline * * @ignore */ delayUntilReady(callback: Callback) { this._readyDelayedCallbacks.push(callback); } /** * Get the number of commands queued in automatic pipelines. * * This is not available (and returns 0) until the cluster is connected and slots information have been received. */ get autoPipelineQueueSize(): number { let queued = 0; for (const pipeline of this._autoPipelines.values()) { queued += pipeline.length; } return queued; } /** * Refresh the slot cache * * @ignore */ refreshSlotsCache(callback?: Callback): void { if (callback) { this._refreshSlotsCacheCallbacks.push(callback); } if (this.isRefreshing) { return; } this.isRefreshing = true; const _this = this; const wrapper = (error?: Error) => { this.isRefreshing = false; for (const callback of this._refreshSlotsCacheCallbacks) { callback(error); } this._refreshSlotsCacheCallbacks = []; }; const nodes = shuffle(this.connectionPool.getNodes()); let lastNodeError = null; function tryNode(index: number) { if (index === nodes.length) { const error = new ClusterAllFailedError( ClusterAllFailedError.defaultMessage, lastNodeError ); return wrapper(error); } const node = nodes[index]; const key = `${node.options.host}:${node.options.port}`; debug("getting slot cache from %s", key); _this.getInfoFromNode(node, function (err) { switch (_this.status) { case "close": case "end": return wrapper(new Error("Cluster is disconnected.")); case "disconnecting": return wrapper(new Error("Cluster is disconnecting.")); } if (err) { _this.emit("node error", err, key); lastNodeError = err; tryNode(index + 1); } else { _this.emit("refresh"); wrapper(); } }); } tryNode(0); } /** * @ignore */ sendCommand(command: Command, stream?: WriteableStream, node?: any): unknown { if (this.status === "wait") { this.connect().catch(noop); } if (this.status === "end") { command.reject(new Error(CONNECTION_CLOSED_ERROR_MSG)); return command.promise; } if ( !stream && !node && this.status === "ready" && isHimportControlCommand(command) && interceptHimportControlCommand( this.connectionPool.getNodes("master"), command ) ) { return command.promise; } let to = this.options.scaleReads; if (to !== "master") { const isCommandReadOnly = command.isReadOnly || (exists(command.name) && hasFlag(command.name, "readonly")); if (!isCommandReadOnly) { to = "master"; } } // A pipeline goes to one node, so its first command picks the role. if (node) { node.to = node.to || to; to = node.to; } let targetSlot = node ? node.slot : command.getSlot(); const ttl = {}; const _this = this; // The connection the command was last sent on, used to detect // MOVED redirects that point back at the same node. let lastRedis: Redis | null = null; if (!node && !REJECT_OVERWRITTEN_PROMISES.has(command.promise)) { REJECT_OVERWRITTEN_PROMISES.add(command.promise); const reject = command.reject; command.reject = function (err) { const partialTry = tryConnection.bind(null, true); _this.handleError(err, ttl, { moved: function (slot, key) { debug("command %s is moved to %s", command.name, key); targetSlot = Number(slot); if (_this.slots[slot]) { _this.slots[slot][0] = key; } else { _this.slots[slot] = [key]; } _this._groupsBySlot[slot] = _this._groupsIds[_this.slots[slot].join(";")]; const mapped = _this.natMapper(key); const mappedKey = getNodeKey(mapped); if ( lastRedis && getNodeKey(lastRedis.options as RedisOptions) === mappedKey && _this.connectionPool.getInstanceByKey(mappedKey) === lastRedis ) { // The redirect points back at the connection that returned // the error, so it is stale (e.g. the endpoint is a proxy // whose backing server changed) and retrying on it would // loop forever. Replace it with a fresh one, unless another // command already did — then reuse the replacement. debug( "MOVED redirect points back at %s; recreating its connection", mappedKey ); _this.connectionPool.recreate(mapped); } else { _this.connectionPool.findOrCreate(mapped); } tryConnection(); debug("refreshing slot caches... (triggered by MOVED error)"); _this.refreshSlotsCache(); }, ask: function (slot, key) { debug("command %s is required to ask %s:%s", command.name, key); const mapped = _this.natMapper(key); _this.connectionPool.findOrCreate(mapped); tryConnection(false, `${mapped.host}:${mapped.port}`); }, tryagain: partialTry, clusterDown: partialTry, connectionClosed: partialTry, maxRedirections: function (redirectionError) { reject.call(command, redirectionError); }, defaults: function () { reject.call(command, err); }, }); }; const redirection = command.redirection; if (redirection) { // Follow the redirection the command received in a pipeline instead // of sending it to the node that redirected it again. Starting from // the batch's connection lets a MOVED that points back at it replace // the stale connection on the first try. command.redirection = undefined; lastRedis = redirection.from; command.reject(redirection.error); return command.promise; } } tryConnection(); function tryConnection(random?: boolean, asking?: string) { if (_this.status === "end") { command.reject(new AbortError("Cluster is ended.")); return; } let redis; if (_this.status === "ready" || command.name === "cluster") { if (node && node.redis) { redis = node.redis; } else if ( Command.checkFlag("ENTER_SUBSCRIBER_MODE", command.name) || Command.checkFlag("EXIT_SUBSCRIBER_MODE", command.name) ) { if ( _this.options.shardedSubscribers && (command.name == "ssubscribe" || command.name == "sunsubscribe") ) { // A zero-argument SUNSUBSCRIBE has no channel to compute a target // slot from, so it can't be routed to a single sharded subscriber. // Redis unsubscribes the connection from all of its shard channels // in that case; fan the command out across every sharded subscriber // that currently holds channels. if ( command.name == "sunsubscribe" && command.getKeys().length === 0 ) { _this.shardedSubscribers .sunsubscribeAll() .then((count) => command.resolve(count)) .catch((err) => command.reject(err)); return; } const sub = _this.shardedSubscribers.getResponsibleSubscriber(targetSlot); if (!sub) { command.reject( new AbortError(`No sharded subscriber for slot: ${targetSlot}`) ); return; } let status = -1; if (command.name == "ssubscribe") { status = _this.shardedSubscribers.addChannels(command.getKeys()); } if (command.name == "sunsubscribe") { status = _this.shardedSubscribers.removeChannels( command.getKeys() ); } if (status !== -1) { redis = sub.getInstance(); } else { command.reject( new AbortError( "Possible CROSSSLOT error: All channels must hash to the same slot" ) ); } } else { redis = _this.subscriber.getInstance(); if (!redis && _this.options.subscriberNodeRole !== "all") { command.reject( new AbortError( `No node matching subscriberNodeRole "${_this.options.subscriberNodeRole}" is available for the cluster subscriber` ) ); return; } } if (!redis) { command.reject(new AbortError("No subscriber for the cluster")); return; } } else { if (!random) { if (typeof targetSlot === "number" && _this.slots[targetSlot]) { const nodeKeys = _this.slots[targetSlot]; if (typeof to === "function") { const nodes = nodeKeys.map(function (key) { return _this.connectionPool.getInstanceByKey(key); }); redis = to(nodes, command); if (Array.isArray(redis)) { redis = sample(redis); } if (!redis) { redis = nodes[0]; } } else { let key: string; if (to === "all") { key = sample(nodeKeys); } else if (to === "slave" && nodeKeys.length > 1) { key = sample(nodeKeys, 1); } else { key = nodeKeys[0]; } redis = _this.connectionPool.getInstanceByKey(key); } } if (asking) { redis = _this.connectionPool.getInstanceByKey(asking); } } if (!redis && typeof to !== "function") { redis = _this.connectionPool.getSampleInstance(to); } if (!redis && to === "master" && _this.status === "ready") { // A replica would run a script instead of answering MOVED, so // handle this like a lost connection. handleError() only refreshes // the slots when it retries. if ( !(_this.options.retryDelayOnFailover > 0) || !(_this.options.maxRedirections > 0) ) { _this.refreshSlotsCache(); } command.reject(new Error(CONNECTION_CLOSED_ERROR_MSG)); return; } if (!redis) { redis = _this.connectionPool.getSampleInstance("all"); } // Commands of a pipeline are not rejected one by one: a batch that loses // a command here never flushes, and the commands that did reach a // connection stay in its queue and take the replies of later commands. if ( redis && !stream && !_this.options.enableOfflineQueue && redis.status !== "ready" && redis.status !== "wait" ) { command.reject(new Error(CONNECTION_CLOSED_ERROR_MSG)); return; } } if (node && !node.redis) { node.redis = redis; } } if (!redis && _this.options.enableOfflineQueue) { _this.offlineQueue.push({ command: command, stream: stream, node: node, }); return; } if (!redis) { command.reject( new Error( "Cluster isn't ready and enableOfflineQueue options is false" ) ); return; } lastRedis = redis; if (asking) { const himportBinding = getHimportBinding(redis); const managedSet = !stream && himportBinding?.role === "master" ? himportBinding.coordinator.classify(command) : undefined; if (managedSet) { waitForRedisReady(redis) .then(() => { if (command.isSettled) { return; } const preparation = himportBinding.coordinator.prepareCommand( redis, command ); return preparation ?? Promise.resolve(); }) .then( () => { if (command.isSettled) { return; } himportBinding.coordinator.allowNextSend(redis, command); redis.asking(); redis.sendCommand(command, stream); }, (error: Error) => { if (!command.isSettled) { command.reject(error); } } ); return; } redis.asking(); } redis.sendCommand(command, stream); } return command.promise; } sscanStream(key: string, options?: ScanStreamOptions) { return this.createScanStream("sscan", { key, options }); } sscanBufferStream(key: string, options?: ScanStreamOptions) { return this.createScanStream("sscanBuffer", { key, options }); } hscanStream(key: string, options?: ScanStreamOptions) { return this.createScanStream("hscan", { key, options }); } hscanBufferStream(key: string, options?: ScanStreamOptions) { return this.createScanStream("hscanBuffer", { key, options }); } zscanStream(key: string, options?: ScanStreamOptions) { return this.createScanStream("zscan", { key, options }); } zscanBufferStream(key: string, options?: ScanStreamOptions) { return this.createScanStream("zscanBuffer", { key, options }); } /** * @ignore */ handleError(error: Error, ttl: { value?: any }, handlers) { if (typeof ttl.value === "undefined") { ttl.value = this.options.maxRedirections; } else { ttl.value -= 1; } if (ttl.value <= 0) { handlers.maxRedirections( new Error("Too many Cluster redirections. Last error: " + error) ); return; } const errv = error.message.split(" "); if (errv[0] === "MOVED") { const slot = Number(errv[1]); // Reject redirects whose slot is not a valid integer in the cluster // range. The moved handlers use the slot to index the `slots` array; // a crafted token such as "__proto__" would otherwise resolve to // Array.prototype and pollute it globally. See // https://hackerone.com/reports/3761875 if (!Number.isInteger(slot) || slot < 0 || slot >= 16384) { handlers.defaults(); return; } const timeout = this.options.retryDelayOnMoved; if (timeout && typeof timeout === "number") { this.delayQueue.push( "moved", handlers.moved.bind(null, slot, errv[2]), { timeout } ); } else { handlers.moved(slot, errv[2]); } } else if (errv[0] === "ASK") { handlers.ask(errv[1], errv[2]); } else if (errv[0] === "TRYAGAIN") { this.delayQueue.push("tryagain", handlers.tryagain, { timeout: this.options.retryDelayOnTryAgain, }); } else if ( errv[0] === "CLUSTERDOWN" && this.options.retryDelayOnClusterDown > 0 ) { this.delayQueue.push("clusterdown", handlers.connectionClosed, { timeout: this.options.retryDelayOnClusterDown, callback: this.refreshSlotsCache.bind(this), }); } else if ( error.message === CONNECTION_CLOSED_ERROR_MSG && this.options.retryDelayOnFailover > 0 && this.status === "ready" ) { this.delayQueue.push("failover", handlers.connectionClosed, { timeout: this.options.retryDelayOnFailover, callback: this.refreshSlotsCache.bind(this), }); } else { handlers.defaults(); } } private resetOfflineQueue() { this.offlineQueue = new Deque(); } private clearNodesRefreshInterval() { if (this.slotsTimer) { clearTimeout(this.slotsTimer); this.slotsTimer = null; } } private resetNodesRefreshInterval() { if (this.slotsTimer || !this.options.slotsRefreshInterval) { return; } const nextRound = () => { const timer = setTimeout(() => { debug( 'refreshing slot caches... (triggered by "slotsRefreshInterval" option)' ); this.refreshSlotsCache(() => { if (this.slotsTimer === timer) { nextRound(); } }); }, this.options.slotsRefreshInterval); this.slotsTimer = timer; }; nextRound(); } /** * Change cluster instance's status */ private setStatus(status: ClusterStatus, arg?: unknown): void { debug("status: %s -> %s", this.status || "[empty]", status); this.status = status; process.nextTick(() => { this.emit(status, arg); }); } /** * Called when connect() gives up because the status changed while it was * resolving the startup nodes. connect() has not registered its "close" * listeners yet, so in two cases nothing else would move the cluster out of * its current status: * - "close": the late "end" of a node removed by a previous disconnection * made the pool emit "drain"; * - "disconnecting" with an empty pool: disconnect() or quit() was called, * and no node is left to emit "drain". */ private handleAbortedConnect(): void { if ( this.status === "disconnecting" && !this.connectionPool.getNodes().length ) { this.setStatus("close"); } if (this.status === "close") { this.invokeReadyDelayedCallbacks( new Error("None of startup nodes is available") ); this.handleCloseEvent(); } } /** * Called when closed to check whether a reconnection should be made */ private handleCloseEvent(reason?: Error): void { if (reason) { debug("closed because %s", reason); } let retryDelay: unknown; if ( !this.manuallyClosing && typeof this.options.clusterRetryStrategy === "function" ) { retryDelay = this.options.clusterRetryStrategy.call( this, ++this.retryAttempts, reason ); } if (typeof retryDelay === "number") { this.setStatus("reconnecting", retryDelay); this.reconnectTimeout = setTimeout(() => { this.reconnectTimeout = null; debug("Cluster is disconnected. Retrying after %dms", retryDelay); this.connect().catch(function (err) { debug("Got error %s when reconnecting. Ignoring...", err); }); }, retryDelay); } else { if (this.options.shardedSubscribers) { this.subscriberGroupEmitter?.removeAllListeners(); } this.setStatus("end"); this.flushQueue(new Error("None of startup nodes is available")); } } /** * Flush offline queue with error. */ private flushQueue(error: Error) { let item: OfflineQueueItem; while ((item = this.offlineQueue.shift())) { item.command.reject(error); } } private executeOfflineCommands() { if (this.offlineQueue.length) { debug("send %d commands in offline queue", this.offlineQueue.length); const offlineQueue = this.offlineQueue; this.resetOfflineQueue(); let item: OfflineQueueItem; while ((item = offlineQueue.shift())) { this.sendCommand(item.command, item.stream, item.node); } } } private natMapper(nodeKey: NodeKey | RedisOptions): RedisOptions { const key = typeof nodeKey === "string" ? nodeKey : `${nodeKey.host}:${nodeKey.port}`; let mapped = null; if (this.options.natMap && typeof this.options.natMap === "function") { mapped = this.options.natMap(key); } else if (this.options.natMap && typeof this.options.natMap === "object") { mapped = this.options.natMap[key]; } if (mapped) { debug("NAT mapping %s -> %O", key, mapped); return Object.assign({}, mapped); } return typeof nodeKey === "string" ? nodeKeyToRedisOptions(nodeKey) : nodeKey; } private getInfoFromNode(redis: Redis, callback: Callback) { if (!redis) { return callback(new Error("Node is disconnected")); } // Use a duplication of the connection to avoid // timeouts when the connection is in the blocking // mode (e.g. waiting for BLPOP). const duplicatedConnection = redis.duplicate({ enableOfflineQueue: true, enableReadyCheck: false, retryStrategy: null, maintNotifications: "disabled", protocol: 2, replyMapping: "legacy", stringNumbers: false, connectionName: getConnectionName( "refresher", this.options.redisOptions && this.options.redisOptions.connectionName ), }); // Ignore error events since we will handle // exceptions for the CLUSTER SLOTS command. duplicatedConnection.on("error", noop); duplicatedConnection.cluster( "SLOTS", timeout((err: Error, result) => { duplicatedConnection.disconnect(); if (err) { debug("error encountered running CLUSTER.SLOTS: %s", err); return callback(err); } if ( this.status === "disconnecting" || this.status === "close" || this.status === "end" ) { debug( "ignore CLUSTER.SLOTS results (count: %d) since cluster status is %s", result.length, this.status ); callback(); return; } const nodes: RedisOptions[] = []; debug("cluster slots result count: %d", result.length); for (let i = 0; i < result.length; ++i) { const items = result[i]; const slotRangeStart = items[0]; const slotRangeEnd = items[1]; const keys = []; for (let j = 2; j < items.length; j++) { if (!items[j][0]) { continue; } const node = this.natMapper({ host: items[j][0], port: items[j][1], }); node.readOnly = j !== 2; nodes.push(node); keys.push(node.host + ":" + node.port); } debug( "cluster slots result [%d]: slots %d~%d served by %s", i, slotRangeStart, slotRangeEnd, keys ); for (let slot = slotRangeStart; slot <= slotRangeEnd; slot++) { this.slots[slot] = keys; } } // Assign to each node keys a numeric value to make autopipeline comparison faster. this._groupsIds = Object.create(null); let j = 0; for (let i = 0; i < 16384; i++) { const target = (this.slots[i] || []).join(";"); if (!target.length) { this._groupsBySlot[i] = undefined; continue; } if (!this._groupsIds[target]) { this._groupsIds[target] = ++j; } this._groupsBySlot[i] = this._groupsIds[target]; } this.connectionPool.reset(nodes); // A refresh can reclassify an existing node without emitting "+node". this.subscriber.selectSubscriberIfNeeded(); if (this.options.shardedSubscribers) { this.shardedSubscribers .reset(this.slots, this.connectionPool.getNodes("all")) .catch((err) => { // TODO should we emit an error event here? debug("Error while starting subscribers: %s", err); }); } callback(); }, this.options.slotsRefreshTimeout) ); } private invokeReadyDelayedCallbacks(err?: Error) { for (const c of this._readyDelayedCallbacks) { process.nextTick(c, err); } this._readyDelayedCallbacks = []; } /** * Check whether Cluster is able to process commands */ private readyCheck(callback: Callback): void { this.cluster("INFO", (err, res) => { if (err) { return callback(err); } if (typeof res !== "string") { return callback(); } let state: string; const lines = res.split("\r\n"); for (let i = 0; i < lines.length; ++i) { const parts = lines[i].split(":"); if (parts[0] === "cluster_state") { state = parts[1]; break; } } if (state === "fail") { debug("cluster state not ok (%s)", state); callback(null, state); } else { callback(); } }); } private resolveSrv(hostname: string): Promise { return new Promise((resolve, reject) => { this.options.resolveSrv(hostname, (err, records) => { if (err) { return reject(err); } const self = this, groupedRecords = groupSrvRecords(records), sortedKeys = Object.keys(groupedRecords).sort( (a, b) => parseInt(a) - parseInt(b) ); function tryFirstOne(err?) { if (!sortedKeys.length) { return reject(err); } const key = sortedKeys[0], group = groupedRecords[key], record = weightSrvRecords(group); if (!group.records.length) { sortedKeys.shift(); } self.dnsLookup(record.name).then( (host) => resolve({ host, port: record.port, }), tryFirstOne ); } tryFirstOne(); }); }); } private dnsLookup(hostname: string): Promise { return new Promise((resolve, reject) => { this.options.dnsLookup(hostname, (err, address) => { if (err) { debug( "failed to resolve hostname %s to IP: %s", hostname, err.message ); reject(err); } else { debug("resolved hostname %s to IP %s", hostname, address); resolve(address); } }); }); } /** * Normalize startup nodes, and resolving hostnames to IPs. * * This process happens every time when #connect() is called since * #startupNodes and DNS records may chanage. */ private async resolveStartupNodeHostnames(): Promise { if (!Array.isArray(this.startupNodes) || this.startupNodes.length === 0) { throw new Error("`startupNodes` should contain at least one node."); } const startupNodes = normalizeNodeOptions(this.startupNodes); const hostnames = getUniqueHostnamesFromOptions(startupNodes); if (hostnames.length === 0) { return startupNodes; } const configs = await Promise.all( hostnames.map( (this.options.useSRVRecords ? this.resolveSrv : this.dnsLookup).bind( this ) ) ); const hostnameToConfig = zipMap(hostnames, configs); return startupNodes.map((node) => { const config = hostnameToConfig.get(node.host); if (!config) { return node; } if (this.options.useSRVRecords) { return Object.assign({}, node, config); } return Object.assign({}, node, { host: config }); }); } private createScanStream( command: string, { key, options = {} }: { key?: string; options?: ScanStreamOptions } ) { return new ScanStream({ objectMode: true, key: key, redis: this, command: command, ...options, }); } private createShardedSubscriberGroup() { this.subscriberGroupEmitter = new EventEmitter(); this.shardedSubscribers = new ClusterSubscriberGroup( this.subscriberGroupEmitter, this.options ); this.addSubscriberGroupListeners(); } private addSubscriberGroupListeners() { // Error handler used only for sharded-subscriber-triggered slots cache refreshes. // Normal (non-subscriber) connections are created with lazyConnect: true and can // become zombied. For sharded subscribers, a ClusterAllFailedError means // we have lost all nodes from the subscriber perspective and must tear down. const refreshSlotsCacheCallback = (err?: Error) => { // Disconnect only when refreshing the slots cache fails with ClusterAllFailedError if (err instanceof ClusterAllFailedError) { this.disconnect(true); } }; this.subscriberGroupEmitter.on("-node", (redis, nodeKey) => { this.emit("-node", redis, nodeKey); }); this.subscriberGroupEmitter.on("subscriberLost", () => { this.refreshSlotsCache(refreshSlotsCacheCallback); }); // One refresh repairs every failed subscriber, so failures while a retry // is pending join it instead of starting retry chains of their own. let connectFailedRetry: NodeJS.Timeout | null = null; this.subscriberGroupEmitter.on( "subscriberConnectFailed", ({ delay, error }) => { Redis.prototype.silentEmit.call(this, "error", error); if (connectFailedRetry) { return; } connectFailedRetry = setTimeout(() => { connectFailedRetry = null; this.refreshSlotsCache(refreshSlotsCacheCallback); }, delay); } ); this.subscriberGroupEmitter.on("resubscribeRetry", () => { this.refreshSlotsCache(refreshSlotsCacheCallback); }); // Same as autoResubscribe on a standalone client: a resubscribe that // cannot succeed is reported through the "error" event. this.subscriberGroupEmitter.on("resubscribeError", (error) => { Redis.prototype.silentEmit.call(this, "error", error); }); this.subscriberGroupEmitter.on("moved", () => { this.refreshSlotsCache(refreshSlotsCacheCallback); }); this.subscriberGroupEmitter.on("-subscriber", () => { this.emit("-subscriber"); }); this.subscriberGroupEmitter.on("+subscriber", () => { this.emit("+subscriber"); }); this.subscriberGroupEmitter.on("nodeError", (error, nodeKey) => { this.emit("nodeError", error, nodeKey); }); this.subscriberGroupEmitter.on("subscribersReady", () => { this.emit("subscribersReady"); }); for (const event of ["smessage", "smessageBuffer"]) { this.subscriberGroupEmitter.on(event, (arg1, arg2, arg3) => { this.emit(event, arg1, arg2, arg3); }); } } } // eslint-disable-next-line @typescript-eslint/no-unused-vars interface Cluster extends EventEmitter { on(event: "message", cb: (channel: string, message: string) => void): this; once(event: "message", cb: (channel: string, message: string) => void): this; on( event: "messageBuffer", cb: (channel: Buffer, message: Buffer) => void ): this; once( event: "messageBuffer", cb: (channel: Buffer, message: Buffer) => void ): this; on( event: "pmessage", cb: (pattern: string, channel: string, message: string) => void ): this; once( event: "pmessage", cb: (pattern: string, channel: string, message: string) => void ): this; on( event: "pmessageBuffer", cb: (pattern: string, channel: Buffer, message: Buffer) => void ): this; once( event: "pmessageBuffer", cb: (pattern: string, channel: Buffer, message: Buffer) => void ): this; on(event: "smessage", cb: (channel: string, message: string) => void): this; once(event: "smessage", cb: (channel: string, message: string) => void): this; on( event: "smessageBuffer", cb: (channel: Buffer, message: Buffer) => void ): this; once( event: "smessageBuffer", cb: (channel: Buffer, message: Buffer) => void ): this; on(event: "error", cb: (error: Error) => void): this; once(event: "error", cb: (error: Error) => void): this; on(event: "+node", cb: (node: Redis) => void): this; once(event: "+node", cb: (node: Redis) => void): this; on(event: "-node", cb: (node: Redis, nodeKey?: string) => void): this; once(event: "-node", cb: (node: Redis, nodeKey?: string) => void): this; on(event: "node error", cb: (error: Error, nodeKey: string) => void): this; once(event: "node error", cb: (error: Error, nodeKey: string) => void): this; on(event: "nodeError", cb: (error: Error, nodeKey: string) => void): this; once(event: "nodeError", cb: (error: Error, nodeKey: string) => void): this; on( event: "+subscriber" | "-subscriber" | "subscribersReady" | "refresh", cb: () => void ): this; once( event: "+subscriber" | "-subscriber" | "subscribersReady" | "refresh", cb: () => void ): this; on(event: ClusterStatus, cb: () => void): this; once(event: ClusterStatus, cb: () => void): this; // base method of EventEmitter on(event: string | symbol, listener: (...args: any[]) => void): this; once(event: string | symbol, listener: (...args: any[]) => void): this; } applyMixin(Cluster, EventEmitter); addTransactionSupport(Cluster.prototype); interface Cluster extends Transaction {} export default Cluster;