// SPDX-License-Identifier: Apache-2.0 import GrpcServiceError from "./grpc/GrpcServiceError.js"; import GrpcStatus from "./grpc/GrpcStatus.js"; import List from "./transaction/List.js"; import * as hex from "./encoding/hex.js"; import HttpError from "./http/HttpError.js"; import Status from "./Status.js"; import MaxAttemptsOrTimeoutError from "./MaxAttemptsOrTimeoutError.js"; /** * @typedef {import("./account/AccountId.js").default} AccountId * @typedef {import("./channel/Channel.js").default} Channel * @typedef {import("./channel/MirrorChannel.js").default} MirrorChannel * @typedef {import("./transaction/TransactionId.js").default} TransactionId * @typedef {import("./client/Client.js").ClientOperator} ClientOperator * @typedef {import("./Node.js").default} Node * @typedef {import("./Signer.js").Signer} Signer * @typedef {import("./PublicKey.js").default} PublicKey * @typedef {import("./logger/Logger.js").default} Logger */ /** * @enum {string} */ export const ExecutionState = { Finished: "Finished", Retry: "Retry", Error: "Error", }; export const RST_STREAM = /\brst[^0-9a-zA-Z]stream\b/i; /** * @abstract * @internal * @template RequestT * @template ResponseT * @template OutputT */ export default class Executable { constructor() { /** * The number of times we can retry the grpc call * * @internal * @type {?number} */ this._maxAttempts = null; /** * List of node account IDs for each transaction that has been * built. * * @internal * @type {List} */ this._nodeAccountIds = new List(); /** * List of the transaction node account IDs to check if * the node account ID of the request is in the list * * @protected * @type {Array} */ this.transactionNodeIds = []; /** * @internal */ this._signOnDemand = false; /** * This is the request's min backoff * * @internal * @type {number | null} */ this._minBackoff = null; /** * This is the request's max backoff. `null` until `_setupExecution` * copies the client's value, unless `setMaxBackoff` was called. * * @internal * @type {number | null} */ this._maxBackoff = null; /** * The operator that was used to execute this request. * The reason we save the operator in the request is because of the signing on * demand feature. This feature requires us to sign new request on each attempt * meaning if a client with an operator was used we'd need to sign with the operator * on each attempt. * * @internal * @type {ClientOperator | null} */ this._operator = null; /** * The complete timeout for running the `execute()` method * * @internal * @type {number | null} */ this._requestTimeout = null; /** * The grpc request timeout aka deadline. * * The reason we have this is because there were times that consensus nodes held the grpc * connection, but didn't return anything; not error nor regular response. This resulted * in some weird behavior in the SDKs. To fix this we've added a grpc deadline to prevent * nodes from stalling the executing of a request. * * @internal * @type {number | null} */ this._grpcDeadline = null; /** * Logger * * @protected * @type {Logger | null} */ this._logger = null; } /** * Get the list of node account IDs on the request. If no nodes are set, then null is returned. * The reasoning for this is simply "legacy behavior". * * @returns {?AccountId[]} */ get nodeAccountIds() { if (this._nodeAccountIds.isEmpty) { return null; } else { this._nodeAccountIds.setLocked(); return this._nodeAccountIds.list; } } /** * Set the node account IDs on the request * * @param {AccountId[]} nodeIds * @returns {this} */ setNodeAccountIds(nodeIds) { // Set the node account IDs, and lock the list. This will require `execute` // to use these nodes instead of random nodes from the network. this._nodeAccountIds.setList(nodeIds).setLocked(); return this; } /** * @deprecated * @returns {?number} */ get maxRetries() { console.warn("Deprecated: use maxAttempts instead"); return this.maxAttempts; } /** * @param {?number} maxRetries * @returns {this} */ setMaxRetries(maxRetries) { console.warn("Deprecated: use setMaxAttempts() instead"); return this.setMaxAttempts(maxRetries); } /** * Get the max attempts on the request * * @returns {?number} */ get maxAttempts() { return this._maxAttempts; } /** * Set the max attempts on the request * * @param {?number} maxAttempts * @returns {this} */ setMaxAttempts(maxAttempts) { this._maxAttempts = maxAttempts; return this; } /** * Get the grpc deadline * * @returns {?number} */ get grpcDeadline() { return this._grpcDeadline; } /** * Set the grpc deadline * * @param {number} grpcDeadline * @returns {this} */ setGrpcDeadline(grpcDeadline) { this._grpcDeadline = grpcDeadline; return this; } /** * Set the min backoff for the request * * @param {number} minBackoff * @returns {this} */ setMinBackoff(minBackoff) { // Honestly we shouldn't be checking for null since that should be TypeScript's job. // Also verify that min backoff is not greater than max backoff. if (minBackoff == null) { throw new Error("minBackoff cannot be null."); } else if (this._maxBackoff != null && minBackoff > this._maxBackoff) { throw new Error("minBackoff cannot be larger than maxBackoff."); } this._minBackoff = minBackoff; return this; } /** * Get the min backoff * * @returns {number | null} */ get minBackoff() { return this._minBackoff; } /** * Set the max backoff for the request * * @param {?number} maxBackoff * @returns {this} */ setMaxBackoff(maxBackoff) { // Honestly we shouldn't be checking for null since that should be TypeScript's job. // Also verify that max backoff is not less than min backoff. if (maxBackoff == null) { throw new Error("maxBackoff cannot be null."); } else if (this._minBackoff != null && maxBackoff < this._minBackoff) { throw new Error("maxBackoff cannot be smaller than minBackoff."); } this._maxBackoff = maxBackoff; return this; } /** * Get the max backoff * * @returns {number | null} */ get maxBackoff() { return this._maxBackoff; } /** * This method is responsible for doing any work before the executing process begins. * For paid queries this will result in executing a cost query, for transactions this * will make sure we save the operator and sign any requests that need to be signed * in case signing on demand is disabled. * * @abstract * @protected * @param {import("./client/Client.js").default} client * @returns {Promise} */ // eslint-disable-next-line @typescript-eslint/no-unused-vars _beforeExecute(client) { throw new Error("not implemented"); } /** * Create a protobuf request which will be passed into the `_execute()` method * * @abstract * @protected * @returns {Promise} */ _makeRequestAsync() { throw new Error("not implemented"); } /** * This name is a bit wrong now, but the purpose of this method is to map the * request and response into an error. This method will only be called when * `_getStatusAndExecutionState()` returned execution state `ExecutionState.Error` * * @abstract * @internal * @param {RequestT} request * @param {ResponseT} response * @param {AccountId} nodeId * @returns {Error} */ // eslint-disable-next-line @typescript-eslint/no-unused-vars _mapStatusError(request, response, nodeId) { throw new Error("not implemented"); } /** * Map the request, response, and the node account ID used for this attempt into a response. * This method will only be called when `_shouldRetry` returned `ExecutionState.Finished` * * @abstract * @protected * @param {ResponseT} response * @param {AccountId} nodeAccountId * @param {RequestT} request * @returns {Promise} */ // eslint-disable-next-line @typescript-eslint/no-unused-vars _mapResponse(response, nodeAccountId, request) { throw new Error("not implemented"); } /** * Perform a single grpc call with the given request. Each request has it's own * required service so we just pass in channel, and it's the request's responsibility * to use the right service and call the right grpc method. * * @abstract * @internal * @param {Channel} channel * @param {RequestT} request * @returns {Promise} */ // eslint-disable-next-line @typescript-eslint/no-unused-vars _execute(channel, request) { throw new Error("not implemented"); } /** * Return the current transaction ID for the request. All requests which are * use the same transaction ID for each node, but the catch is that `Transaction` * implicitly supports chunked transactions. Meaning there could be multiple * transaction IDs stored in the request, and a different transaction ID will be used * on subsequent calls to `execute()` * * FIXME: This method can most likely be removed, although some further inspection * is required. * * @abstract * @protected * @returns {TransactionId} */ _getTransactionId() { throw new Error("not implemented"); } /** * Return the log ID for this particular request * * Log IDs are simply a string constructed to make it easy to track each request's * execution even when multiple requests are executing in parallel. Typically, this * method returns the format of `[.]` * * Maybe we should deduplicate this using ${this.consturtor.name} * * @abstract * @internal * @returns {string} */ _getLogId() { throw new Error("not implemented"); } /** * Serialize the request into bytes * * @abstract * @param {RequestT} request * @returns {Uint8Array} */ // eslint-disable-next-line @typescript-eslint/no-unused-vars _requestToBytes(request) { throw new Error("not implemented"); } /** * Serialize the response into bytes * * @abstract * @param {ResponseT} response * @returns {Uint8Array} */ // eslint-disable-next-line @typescript-eslint/no-unused-vars _responseToBytes(response) { throw new Error("not implemented"); } /** * Determine if we should continue the execution process, error, or finish. * * FIXME: This method should really be called something else. Initially it returned * a boolean so `shouldRetry` made sense, but now it returns an enum, so the name * no longer makes sense. * * @abstract * @protected * @param {RequestT} request * @param {ResponseT} response * @returns {[Status, ExecutionState]} */ // eslint-disable-next-line @typescript-eslint/no-unused-vars _getStatusAndExecutionState(request, response) { throw new Error("not implemented"); } /** * Determine if we should error based on the gRPC status * * Unlike `shouldRetry` this method does in fact still return a boolean * * @protected * @param {Error} error * @returns {boolean} */ _shouldRetryExceptionally(error) { if (error instanceof GrpcServiceError) { return ( error.status._code === GrpcStatus.Timeout._code || error.status._code === GrpcStatus.DeadlineExceeded._code || error.status._code === GrpcStatus.Unavailable._code || error.status._code === GrpcStatus.ResourceExhausted._code || error.status._code === GrpcStatus.GrpcWeb._code || (error.status._code === GrpcStatus.Internal._code && RST_STREAM.test(error.message)) ); } else { // if we get to the 'else' statement, the 'error' is instanceof 'HttpError' // and in this case, we have to retry always return true; } } /** * @private * @param {Error} error * @param {number} attempt * @returns {boolean} */ _shouldRetryRequestError(error, attempt) { return ( (error instanceof GrpcServiceError || error instanceof HttpError) && this._shouldRetryExceptionally(error) && attempt <= /** @type {number} */ (this._maxAttempts) ); } /** * A helper method for setting the operator on the request * * @internal * @param {AccountId} accountId * @param {PublicKey} publicKey * @param {(message: Uint8Array) => Promise} transactionSigner * @returns {this} */ _setOperatorWith(accountId, publicKey, transactionSigner) { this._operator = { transactionSigner, accountId, publicKey, }; return this; } /** * Execute this request using the signer * * This method is part of the signature providers feature * https://hips.hedera.com/hip/hip-338 * * @param {Signer} signer * @returns {Promise} */ async executeWithSigner(signer) { return signer.call(this); } /** * @returns {boolean} * @abstract * @protected */ isBatchedAndNotBatchTransaction() { return false; } /** * @private * @returns {void} */ _validateTransactionNodeIds() { if (!this.transactionNodeIds.length) { return; } const nodeAccountIds = this._nodeAccountIds.list.map((nodeId) => nodeId.toString(), ); const hasValidNodes = this.transactionNodeIds.some((nodeId) => nodeAccountIds.includes(nodeId), ); if (hasValidNodes) { return; } const displayNodeAccountIds = nodeAccountIds.length > 2 ? `${nodeAccountIds.slice(0, 2).join(", ")} ...` : nodeAccountIds.join(", "); const isSingleNode = nodeAccountIds.length === 1; throw new Error( `Attempting to execute a transaction against node${ isSingleNode ? "" : "s" } ${displayNodeAccountIds}, ` + `which ${ isSingleNode ? "is" : "are" } not included in the Client's node list. Please review your Client configuration.`, ); } /** * @private * @template {Channel} ChannelT * @template {MirrorChannel} MirrorChannelT * @param {import("./client/Client.js").default} client * @param {number=} requestTimeout * @returns {Promise} */ async _setupExecution(client, requestTimeout) { if (this.isBatchedAndNotBatchTransaction()) { throw new Error( "Cannot execute batchified transaction outside of BatchTransaction", ); } // If the logger on the request is not set, use the logger in client // (if set, otherwise do not use logger) this._logger = this._logger ?? client._logger; // If the request timeout is set on the request we'll prioritize that instead // of the parameter provided, and if the parameter isn't provided we'll // use the default request timeout on client if (this._requestTimeout == null) { this._requestTimeout = requestTimeout || client.requestTimeout; } // If the grpc deadline is not set on the request, use the default value from client if (this._grpcDeadline == null) { this._grpcDeadline = client.grpcDeadline; } // If the max backoff on the request is not set, use the default value in client if (this._maxBackoff == null) { this._maxBackoff = client.maxBackoff; } // If the min backoff on the request is not set, use the default value in client if (this._minBackoff == null) { this._minBackoff = client.minBackoff; } if (this._maxAttempts == null) { this._maxAttempts = client.maxAttempts; } // Some request need to perform additional requests before the executing // such as paid queries need to fetch the cost of the query before // finally executing the actual query. await this._beforeExecute(client); } /** * @private * @template {Channel} ChannelT * @template {MirrorChannel} MirrorChannelT * @param {import("./client/Client.js").default} client * @returns {Node} */ _getExecutionNode(client) { if (this._nodeAccountIds.isEmpty) { const currentNode = client._network.getNode(); this._nodeAccountIds.setList([currentNode.accountId]); return currentNode; } // A pinned node account ID may have left the network map since the // request was built (address book refresh). Skip it while another // pinned ID still resolves; rethrow only when none of them do. for (let i = 0; ; i++) { try { return client._network.getNode(this._nodeAccountIds.current); } catch (error) { if (i + 1 >= this._nodeAccountIds.length) { throw error; } this._nodeAccountIds.advance(); } } } /** * This is used to skip the current node if the node account ID is not valid for the transaction. * @private * @param {AccountId} nodeAccountId * @returns {boolean} */ _shouldSkipAttemptForNodeAccountId(nodeAccountId) { if (!this.transactionNodeIds.length) { return false; } const isNodeAccountIdValid = this.transactionNodeIds.includes( nodeAccountId.toString(), ); if (isNodeAccountIdValid) { return false; } return true; } /** * @private * @param {Node} currentNode * @param {RequestT} request * @param {number} attempt * @param {boolean} isLocalNode * @returns {Promise} */ async _handleUnhealthyNode(currentNode, request, attempt, isLocalNode) { // Check if the request is a transaction receipt or record request // with a single node (traditional behavior), or if it's a local node. // For single-node receipt queries, we retry the same node with backoff. // For multi-node receipt queries (when failover is enabled), we allow // advancing to the next node like other queries. const isSingleNodeReceiptOrRecordRequest = isTransactionReceiptOrRecordRequest(request) && this._nodeAccountIds.length <= 1; if (isSingleNodeReceiptOrRecordRequest || isLocalNode) { await delayForAttempt( isLocalNode, attempt, /** @type {number} */ (this._minBackoff), /** @type {number} */ (this._maxBackoff), ); return; } const isLastNode = this._nodeAccountIds.index === this._nodeAccountIds.list.length - 1; if (isLastNode) { throw new Error( `Network connectivity issue: All nodes are unhealthy. Original node list: ${this._nodeAccountIds.list.join( ", ", )}`, ); } this._logger?.debug( `[${this._getLogId()}] Node is not healthy, trying the next node.`, ); this._nodeAccountIds.advance(); } /** * @private * @template {Channel} ChannelT * @template {MirrorChannel} MirrorChannelT * @param {import("./client/Client.js").default} client * @param {Node} currentNode * @param {AccountId} nodeAccountId * @returns {Promise} */ async _handleInvalidNodeAccountId(client, currentNode, nodeAccountId) { this._logger?.debug( `[${this._getLogId()}] node with accountId: ${nodeAccountId.toString()} and proxy IP: ${currentNode.address.toString()} has invalid node account ID, marking as unhealthy and updating network`, ); // Mark the node as unusable by increasing its backoff and removing it from the healthy nodes list client._network.increaseBackoff(currentNode); // Initiate addressbook query and update the client's network // This will make the SDK client have the latest node account IDs for subsequent transactions try { if (client.mirrorNetwork.length > 0) { await client.updateNetwork(); } else { this._logger?.warn( `[${this._getLogId()}] Cannot update address book: no mirror network configured. Retrying with existing network configuration.`, ); } } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); this._logger?.trace( `[${this._getLogId()}] failed to update client address book after INVALID_NODE_ACCOUNT_ID: ${errorMessage}`, ); } } /** * @private * @param {Channel} channel * @param {RequestT} request * @returns {Promise} */ async _executeRequestWithGrpcDeadline(channel, request) { // Race the execution promise against the grpc timeout to prevent grpc connections // from blocking this request const promises = []; /** @type {ReturnType | null} */ let deadlineTimer = null; // If a grpc deadline is set, we should race it, otherwise the only thing in the // list of promises will be the execution promise. if (this._grpcDeadline != null) { promises.push( new Promise((_, reject) => { deadlineTimer = setTimeout( () => reject( new GrpcServiceError( GrpcStatus.DeadlineExceeded, ), ), /** @type {number=} */ (this._grpcDeadline), ); }), ); } this._logger?.trace( `[${this._getLogId()}] sending protobuf ${hex.encode( this._requestToBytes(request), )}`, ); promises.push(this._execute(channel, request)); try { // eslint-disable-next-line @typescript-eslint/no-unsafe-return return /** @type {ResponseT} */ (await Promise.race(promises)); } finally { if (deadlineTimer != null) { clearTimeout(deadlineTimer); } } } /** * Execute the request using a client and an optional request timeout * * @template {Channel} ChannelT * @template {MirrorChannel} MirrorChannelT * @param {import("./client/Client.js").default} client * @param {number=} requestTimeout * @returns {Promise} */ async execute(client, requestTimeout) { await this._setupExecution(client, requestTimeout); const isLocalNode = client.isLocalNetwork; // Checks if has a valid nodes to which the TX can be sent this._validateTransactionNodeIds(); // Save the start time to be used later with request timeout const requestStartTime = Date.now(); // Saves each error we get so when we err due to max attempts exceeded we'll have // the last error that was returned by the consensus node let persistentError = null; // The retry loop for ( let attempt = 1; attempt <= /** @type {number} */ (this._maxAttempts); attempt += 1 ) { if ( this._requestTimeout != null && requestStartTime + this._requestTimeout <= Date.now() ) { throw new MaxAttemptsOrTimeoutError( persistentError != null ? `timeout exceeded with last error being: ${persistentError.toString()}` : "timeout exceeded", this._nodeAccountIds.isEmpty ? "No node account ID set" : this._nodeAccountIds.current.toString(), ); } if ( this._shouldSkipAttemptForNodeAccountId( this._nodeAccountIds.current, ) ) { console.error( `Attempting to execute a transaction against node ${this._nodeAccountIds.current.toString()}, which is not included in the Client's node list. Please review your Client configuration.`, ); this._nodeAccountIds.advance(); continue; } const executionNode = this._getExecutionNode(client); this._logger?.debug( `[${this._getLogId()}] Node AccountID: ${executionNode.accountId.toString()}, IP: ${executionNode.address.toString()}`, ); const channel = executionNode.getChannel(); // Set the gRPC deadline on the channel if this query has a custom deadline if (this._grpcDeadline != null) { channel.setGrpcDeadline(this._grpcDeadline); } const request = await this._makeRequestAsync(); if (!executionNode.isHealthy()) { await this._handleUnhealthyNode( executionNode, request, attempt, isLocalNode, ); continue; } this._nodeAccountIds.advance(); let response; try { response = await this._executeRequestWithGrpcDeadline( channel, request, ); } catch (err) { // If we received a grpc status error we need to determine if // we should retry on this error, or err from the request entirely. const error = GrpcServiceError._fromResponse( /** @type {Error} */ (err), ); // Save the error in case we retry persistentError = error; this._logger?.debug( `[${this._getLogId()}] received error ${JSON.stringify( error, )}`, ); // If the GRPC or HTTP request level error is retryable, we should retry the request if (this._shouldRetryRequestError(error, attempt)) { // Increase the backoff for the particular node and remove it from // the healthy node list this._logger?.debug( `[${this._getLogId()}] node with accountId: ${executionNode.accountId.toString()} and proxy IP: ${executionNode.address.toString()} is unhealthy`, ); if (executionNode.isHealthy()) { client._network.increaseBackoff(executionNode); } continue; } throw err; } this._logger?.trace( `[${this._getLogId()}] sending protobuf ${hex.encode( this._responseToBytes(response), )}`, ); // If we didn't receive an error we should decrease the current nodes backoff // in case it is a recovering node client._network.decreaseBackoff(executionNode); // Determine what execution state we're in by the response // For transactions this would be as simple as checking the response status is `OK` // while for _most_ queries it would check if the response status is `SUCCESS` // The only odd balls are `TransactionReceiptQuery` and `TransactionRecordQuery` const [status, shouldRetry] = this._getStatusAndExecutionState( request, response, ); const isError = status.toString() !== Status.Ok.toString() && status.toString() !== Status.Success.toString(); if (isError) { persistentError = status; } // Determine by the executing state what we should do switch (shouldRetry) { case ExecutionState.Retry: if (status === Status.InvalidNodeAccount) { await this._handleInvalidNodeAccountId( client, executionNode, executionNode.accountId, ); } await delayForAttempt( isLocalNode, attempt, /** @type {number} */ (this._minBackoff), /** @type {number} */ (this._maxBackoff), ); continue; case ExecutionState.Finished: return this._mapResponse( response, executionNode.accountId, request, ); case ExecutionState.Error: throw this._mapStatusError( request, response, executionNode.accountId, ); default: throw new Error( "(BUG) non-exhaustive switch statement for `ExecutionState`", ); } } // We'll only get here if we've run out of attempts, so we return an error wrapping the // persistent error we saved before. throw new MaxAttemptsOrTimeoutError( `max attempts of ${ /** @type {number} */ (this._maxAttempts).toString() } was reached for request with last error being: ${ persistentError != null ? persistentError.toString() : "" }`, this._nodeAccountIds.current.toString(), ); } /** * The current purpose of this method is to easily support signature providers since * signature providers need to serialize _any_ request into bytes. `Query` and `Transaction` * already implement `toBytes()` so it only made sense to make it available here too. * * @abstract * @returns {Uint8Array} */ toBytes() { throw new Error("not implemented"); } /** * Set logger * * @param {Logger} logger * @returns {this} */ setLogger(logger) { this._logger = logger; return this; } /** * Get logger if set * * @returns {?Logger} */ get logger() { return this._logger; } } /** * Checks if the request is a transaction receipt or record request * * @template T * @param {T} request - The request to check * @returns {boolean} - True if the request is a transaction receipt or record */ function isTransactionReceiptOrRecordRequest(request) { if (typeof request !== "object" || request === null) { return false; } return ( "transactionGetReceipt" in request || "transactionGetRecord" in request ); } /** * A simple function that returns a promise timeout for a specific period of time * * @param {boolean} isLocalNode * @param {number} attempt * @param {number} minBackoff * @param {number} maxBackoff * @returns {Promise} */ function delayForAttempt(isLocalNode, attempt, minBackoff, maxBackoff) { if (isLocalNode) { return new Promise((resolve) => setTimeout(resolve, minBackoff)); } // 0.1s, 0.2s, 0.4s, 0.8s, ... const ms = Math.min( Math.floor(minBackoff * Math.pow(2, attempt)), maxBackoff, ); return new Promise((resolve) => setTimeout(resolve, ms)); }