// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. import type { SpawnSyncReturns } from 'node:child_process'; import type { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { type ICredentialCacheEntry, CredentialCache } from '@rushstack/credential-cache'; import { Executable, Async, FileSystem, type FileSystemWriteStream, type FileSystemReadStream } from '@rushstack/node-core-library'; import type { ITerminal } from '@rushstack/terminal'; import { type ICloudBuildCacheProvider, type RushSession, EnvironmentConfiguration } from '@rushstack/rush-sdk'; import { WebClient, type IGetFetchOptions, type IFetchOptionsWithBody, type IWebClientResponse, type IWebClientStreamResponse } from '@rushstack/rush-sdk/lib/utilities/WebClient'; enum CredentialsOptions { Optional, Required, Omit } enum FailureType { None, Informational, Warning, Error, Authentication } export interface IHttpBuildCacheTokenHandler { exec: string; args?: string[]; } /** * @public */ export type UploadMethod = 'PUT' | 'POST' | 'PATCH'; /** * @public */ export interface IHttpBuildCacheProviderOptions { url: string; tokenHandler?: IHttpBuildCacheTokenHandler; uploadMethod?: UploadMethod; minHttpRetryDelayMs?: number; headers?: Record; cacheKeyPrefix?: string; isCacheWriteAllowed: boolean; pluginName: string; rushJsonFolder: string; } const MAX_HTTP_CACHE_ATTEMPTS: number = 3; const DEFAULT_MIN_HTTP_RETRY_DELAY_MS: number = 2500; const MAX_HTTP_RETRY_DELAY_MS: number = 60 * 1000; const RETRYABLE_HTTP_STATUS_CODES: ReadonlySet = new Set([ 408, 429, 500, 502, 503, 504, 520, 521, 522, 523, 524 ]); const RETRYABLE_ERROR_CODES: ReadonlySet = new Set([ 'ECONNRESET', 'ECONNREFUSED', 'ECONNABORTED', 'ETIMEDOUT', 'EPIPE', 'EAI_AGAIN', 'ENETUNREACH', 'EHOSTUNREACH' ]); export class HttpBuildCacheProvider implements ICloudBuildCacheProvider { readonly #pluginName: string; readonly #rushProjectRoot: string; readonly #environmentCredential: string | undefined; readonly #isCacheWriteAllowedByConfiguration: boolean; readonly #url: URL; readonly #uploadMethod: UploadMethod; readonly #headers: Record; readonly #cacheKeyPrefix: string; readonly #tokenHandler: IHttpBuildCacheTokenHandler | undefined; readonly #minHttpRetryDelayMs: number; readonly #webClient: WebClient = new WebClient(); #_credentialCacheId: string | undefined; public get isCacheWriteAllowed(): boolean { return EnvironmentConfiguration.buildCacheWriteAllowed ?? this.#isCacheWriteAllowedByConfiguration; } public constructor(options: IHttpBuildCacheProviderOptions, rushSession: RushSession) { const { pluginName, rushJsonFolder, isCacheWriteAllowed, url, uploadMethod = 'PUT', headers = {}, tokenHandler, cacheKeyPrefix = '', minHttpRetryDelayMs = DEFAULT_MIN_HTTP_RETRY_DELAY_MS } = options; this.#pluginName = pluginName; this.#rushProjectRoot = rushJsonFolder; this.#environmentCredential = EnvironmentConfiguration.buildCacheCredential; this.#isCacheWriteAllowedByConfiguration = isCacheWriteAllowed; this.#url = new URL(url.endsWith('/') ? url : url + '/'); this.#uploadMethod = uploadMethod; this.#headers = headers; this.#tokenHandler = tokenHandler; this.#cacheKeyPrefix = cacheKeyPrefix; this.#minHttpRetryDelayMs = minHttpRetryDelayMs; } public async tryGetCacheEntryBufferByIdAsync( terminal: ITerminal, cacheId: string ): Promise { try { const result: boolean | Buffer = await this.#makeHttpRequestAsync({ terminal, relUrl: `${this.#cacheKeyPrefix}${cacheId}`, method: 'GET', body: undefined, warningText: 'Could not get cache entry', readBody: true, maxAttempts: MAX_HTTP_CACHE_ATTEMPTS }); return Buffer.isBuffer(result) ? result : undefined; } catch (e) { terminal.writeWarningLine(`Error getting cache entry: ${e}`); return undefined; } } public async trySetCacheEntryBufferAsync( terminal: ITerminal, cacheId: string, objectBuffer: Buffer ): Promise { if (!this.#validateWriteAllowed(terminal, cacheId)) { return false; } try { const result: boolean | Buffer = await this.#makeHttpRequestAsync({ terminal, relUrl: `${this.#cacheKeyPrefix}${cacheId}`, method: this.#uploadMethod, body: objectBuffer, warningText: 'Could not write cache entry', readBody: false, maxAttempts: MAX_HTTP_CACHE_ATTEMPTS }); return result !== false; } catch (e) { terminal.writeWarningLine(`Error uploading cache entry: ${e}`); return false; } } public async tryDownloadCacheEntryToFileAsync( terminal: ITerminal, cacheId: string, localFilePath: string ): Promise { try { const result: IWebClientStreamResponse | false = await this.#makeHttpStreamRequestAsync({ terminal, relUrl: `${this.#cacheKeyPrefix}${cacheId}`, method: 'GET', body: undefined, warningText: 'Could not get cache entry', maxAttempts: MAX_HTTP_CACHE_ATTEMPTS }); if (result === false) { return false; } const writeStream: FileSystemWriteStream = await FileSystem.createWriteStreamAsync(localFilePath, { ensureFolderExists: true }); await pipeline(result.stream, writeStream); return true; } catch (e) { terminal.writeWarningLine(`Error getting cache entry: ${e}`); return false; } } public async tryUploadCacheEntryFromFileAsync( terminal: ITerminal, cacheId: string, localFilePath: string ): Promise { if (!this.#validateWriteAllowed(terminal, cacheId)) { return false; } try { const { size } = await FileSystem.getStatisticsAsync(localFilePath); const result: IWebClientStreamResponse | false = await this.#makeHttpStreamRequestAsync({ terminal, relUrl: `${this.#cacheKeyPrefix}${cacheId}`, method: this.#uploadMethod, body: (): FileSystemReadStream => FileSystem.createReadStream(localFilePath), headers: { 'Content-Length': `${size}` }, warningText: 'Could not write cache entry', maxAttempts: MAX_HTTP_CACHE_ATTEMPTS, credentialOptions: CredentialsOptions.Required }); if (result !== false) { // Drain the response body result.stream.resume(); } return result !== false; } catch (e) { terminal.writeWarningLine(`Error uploading cache entry: ${e}`); return false; } } public async updateCachedCredentialAsync(terminal: ITerminal, credential: string): Promise { await CredentialCache.usingAsync( { supportEditing: true }, async (credentialsCache: CredentialCache) => { credentialsCache.setCacheEntry(this.#credentialCacheId, { credential }); await credentialsCache.saveIfModifiedAsync(); } ); } public async updateCachedCredentialInteractiveAsync(terminal: ITerminal): Promise { if (!this.#tokenHandler) { throw new Error( `The interactive cloud credentials flow is not configured.\n` + `Set the 'tokenHandler' setting in 'common/config/rush-plugins/${this.#pluginName}.json' to a command that writes your credentials to standard output and exits with code 0 ` + `or provide your credentials to rush using the --credential flag instead. Credentials must be the ` + `'Authorization' header expected by ${this.#url.href}` ); } const cmd: string = `${this.#tokenHandler.exec} ${(this.#tokenHandler.args || []).join(' ')}`; terminal.writeVerboseLine(`Running '${cmd}' to get credentials`); const result: SpawnSyncReturns = Executable.spawnSync( this.#tokenHandler.exec, this.#tokenHandler.args || [], { currentWorkingDirectory: this.#rushProjectRoot } ); terminal.writeErrorLine(result.stderr); if (result.error) { throw new Error(`Could not obtain credentials. The command '${cmd}' failed.`); } const credential: string = result.stdout.trim(); terminal.writeVerboseLine('Got credentials'); await this.updateCachedCredentialAsync(terminal, credential); terminal.writeLine('Updated credentials cache'); } public async deleteCachedCredentialsAsync(terminal: ITerminal): Promise { await CredentialCache.usingAsync( { supportEditing: true }, async (credentialsCache: CredentialCache) => { credentialsCache.deleteCacheEntry(this.#credentialCacheId); await credentialsCache.saveIfModifiedAsync(); } ); } get #credentialCacheId(): string { if (!this.#_credentialCacheId) { const cacheIdParts: string[] = [this.#url.href]; if (this.#isCacheWriteAllowedByConfiguration) { cacheIdParts.push('cacheWriteAllowed'); } this.#_credentialCacheId = cacheIdParts.join('|'); } return this.#_credentialCacheId; } /** * Common validation for write operations. Returns `true` if writing is allowed, * `false` if it is not (and logs an error to the terminal). */ #validateWriteAllowed(terminal: ITerminal, cacheId: string): boolean { if (!this.isCacheWriteAllowed) { terminal.writeErrorLine('Writing to cache is not allowed in the current configuration.'); return false; } terminal.writeDebugLine('Uploading object with cacheId: ', cacheId); return true; } async #makeHttpRequestAsync(options: { terminal: ITerminal; relUrl: string; method: 'GET' | UploadMethod; body: Buffer | undefined; warningText: string; readBody: boolean; maxAttempts: number; credentialOptions?: CredentialsOptions; }): Promise { const { readBody } = options; // The stream: false flag ensures the response is an IWebClientResponse const response: IWebClientResponse | false = (await this.#makeHttpCoreRequestAsync({ ...options, stream: false })) as IWebClientResponse | false; if (response === false) { return false; } const result: Buffer | boolean = readBody ? await response.getBufferAsync() : true; options.terminal.writeDebugLine( `[http-build-cache] actual response: ${response.status} ${new URL(options.relUrl, this.#url).href} ${ result === true ? 'true' : result.length } bytes` ); return result; } async #makeHttpStreamRequestAsync(options: { terminal: ITerminal; relUrl: string; method: 'GET' | UploadMethod; body: (() => Readable) | undefined; headers?: Record; warningText: string; maxAttempts: number; credentialOptions?: CredentialsOptions; }): Promise { // The stream: true flag ensures the response is an IWebClientStreamResponse const response: IWebClientStreamResponse | false = (await this.#makeHttpCoreRequestAsync({ ...options, stream: true })) as IWebClientStreamResponse | false; if (response === false) { return false; } options.terminal.writeDebugLine( `[http-build-cache] stream response: ${response.status} ${new URL(options.relUrl, this.#url).href}` ); return response; } /** * Shared request core for both buffer-based and streaming HTTP requests. * Handles credentials resolution, header construction, retry logic, and failure reporting. */ async #makeHttpCoreRequestAsync(options: { terminal: ITerminal; relUrl: string; method: 'GET' | UploadMethod; body: Buffer | (() => Readable) | undefined; headers?: Record; warningText: string; maxAttempts: number; credentialOptions?: CredentialsOptions; stream: boolean; }): Promise { const { terminal, relUrl, method, body, headers: requestHeaders, warningText, credentialOptions, maxAttempts, stream } = options; const safeCredentialOptions: CredentialsOptions = credentialOptions ?? CredentialsOptions.Optional; const credentials: string | undefined = await this.#tryGetCredentialsAsync(safeCredentialOptions); const url: string = new URL(relUrl, this.#url).href; const headers: Record = {}; if (typeof credentials === 'string') { headers.Authorization = credentials; } for (const [key, value] of Object.entries(this.#headers)) { if (typeof value === 'string') { headers[key] = value; } } if (requestHeaders) { for (const [key, value] of Object.entries(requestHeaders)) { headers[key] = value; } } const bodyLengthDesc: string = Buffer.isBuffer(body) ? `${body.length} bytes` : 'unknown length'; for (let attempt: number = 1; ; attempt++) { terminal.writeDebugLine(`[http-build-cache] request: ${method} ${url} ${bodyLengthDesc}`); let requestBody: Buffer | Readable | undefined; let response: IWebClientResponse | IWebClientStreamResponse | undefined; let errorCode: string | undefined; try { requestBody = typeof body === 'function' ? body() : body; const fetchOptions: IGetFetchOptions | IFetchOptionsWithBody = { verb: method, headers, body: requestBody, redirect: 'follow', timeoutMs: 0 // Disable timeout for streaming transfers of large cache entries }; response = stream ? await this.#webClient.fetchStreamAsync(url, fetchOptions) : await this.#webClient.fetchAsync(url, fetchOptions); } catch (error) { const code: unknown = error && typeof error === 'object' && 'code' in error ? error.code : undefined; if (typeof code !== 'string' || !RETRYABLE_ERROR_CODES.has(code) || attempt >= maxAttempts) { throw error; } errorCode = code; } finally { if (requestBody && !Buffer.isBuffer(requestBody) && !response?.ok) { requestBody.destroy(); } } if (response) { if (response.ok) { return response; } if ('stream' in response) { response.stream.destroy(); } const isCredentialResponse: boolean = response.status === 401 || response.status === 403 || response.status === 407; if ( isCredentialResponse && !(method === 'GET' && response.redirected) && typeof credentials !== 'string' && safeCredentialOptions === CredentialsOptions.Optional ) { // A credential challenge is part of the same attempt, not a transient failure. return await this.#makeHttpCoreRequestAsync({ ...options, credentialOptions: CredentialsOptions.Required, maxAttempts: maxAttempts - attempt + 1 }); } if (!RETRYABLE_HTTP_STATUS_CODES.has(response.status) || attempt >= maxAttempts) { this.#reportFailure(terminal, method, response, false, warningText); return false; } } const retryDelay: number | undefined = this.#getRetryDelayMs( attempt, response?.headers?.['retry-after'] ); if (retryDelay === undefined && response) { terminal.writeVerboseLine( `[http-build-cache] Not retrying: Retry-After exceeds ${MAX_HTTP_RETRY_DELAY_MS} ms.` ); this.#reportFailure(terminal, method, response, false, warningText); return false; } if (retryDelay !== undefined) { const failure: string = response ? this.#describeResponse(response) : `${errorCode}`; terminal.writeVerboseLine( `[http-build-cache] ${method} attempt ${attempt}/${maxAttempts} failed: ${failure}. ` + `Retrying attempt ${attempt + 1}/${maxAttempts} in ${retryDelay} ms.` ); await Async.sleepAsync(retryDelay); } } } #getRetryDelayMs(attempt: number, retryAfterHeader: string | string[] | undefined): number | undefined { const retryAfter: string | undefined = ( Array.isArray(retryAfterHeader) ? retryAfterHeader[0] : retryAfterHeader )?.trim(); const retryAfterMs: number = retryAfter ? /^\d+$/.test(retryAfter) ? Number(retryAfter) * 1000 : Date.parse(retryAfter) - Date.now() : 0; // Do not retry earlier than requested, or stall a build for an unbounded server-specified delay. if (retryAfterMs > MAX_HTTP_RETRY_DELAY_MS) { return undefined; } const backoffMs: number = Math.min( MAX_HTTP_RETRY_DELAY_MS, Math.floor(this.#minHttpRetryDelayMs * 2 ** (attempt - 1) * (1 + Math.random())) ); return Math.max(backoffMs, Number.isFinite(retryAfterMs) ? retryAfterMs : 0); } #describeResponse(response: IWebClientResponse | IWebClientStreamResponse): string { const { status, statusText, headers } = response; const cfRay: string | string[] | undefined = headers?.['cf-ray']; return `HTTP ${status}: ${statusText}${cfRay ? ` (cf-ray: ${cfRay})` : ''}`; } async #tryGetCredentialsAsync(options: CredentialsOptions.Required): Promise; async #tryGetCredentialsAsync(options: CredentialsOptions.Optional): Promise; async #tryGetCredentialsAsync(options: CredentialsOptions.Omit): Promise; async #tryGetCredentialsAsync(options: CredentialsOptions): Promise; async #tryGetCredentialsAsync(options: CredentialsOptions): Promise { if (options === CredentialsOptions.Omit) { return; } let credentials: string | undefined = this.#environmentCredential; if (credentials === undefined) { credentials = await this.#tryGetCredentialsFromCacheAsync(); } if (typeof credentials !== 'string' && options === CredentialsOptions.Required) { throw new Error( [ `Credentials for ${this.#url.href} have not been provided.`, `In CI, verify that RUSH_BUILD_CACHE_CREDENTIAL contains a valid Authorization header value.`, ``, `For local developers, run:`, ``, ` rush update-cloud-credentials --interactive`, `` ].join('\n') ); } return credentials; } async #tryGetCredentialsFromCacheAsync(): Promise { let cacheEntry: ICredentialCacheEntry | undefined; await CredentialCache.usingAsync( { supportEditing: false }, (credentialsCache: CredentialCache) => { cacheEntry = credentialsCache.tryGetCacheEntry(this.#credentialCacheId); } ); if (cacheEntry) { const expirationTime: number | undefined = cacheEntry.expires?.getTime(); if (!expirationTime || expirationTime >= Date.now()) { return cacheEntry.credential; } } } #getFailureType( requestMethod: string, response: IWebClientResponse | IWebClientStreamResponse, isRedirect: boolean ): FailureType { if (response.ok) { return FailureType.None; } switch (response.status) { case 503: { // We select 503 specifically because this represents "service unavailable" and // "rate limit throttle" errors, which are transient issues. // // There are other 5xx errors, such as 501, that can occur if the request is // malformed, so as a general rule we want to let through other 5xx errors // so the user can troubleshoot. // Don't fail production builds with warnings for transient issues return FailureType.Informational; } case 401: case 403: case 407: { if (requestMethod === 'GET' && (isRedirect || response.redirected)) { // Cache misses for GET requests are not errors // This is a workaround behavior where a server can issue a redirect and we fail to authenticate at the new location. // We do not want to signal this as an authentication failure because the authorization header is not passed on to redirects. // i.e The authentication header was accepted for the first request and therefore subsequent failures // where it was not present should not be attributed to the header. // This scenario usually comes up with services that redirect to pre-signed URLS that don't actually exist. // Those services then usually treat the 404 as a 403 to prevent leaking information. return FailureType.None; } return FailureType.Authentication; } case 404: { if (requestMethod === 'GET') { // Cache misses for GET requests are not errors return FailureType.None; } } } // Let dev builds succeed, let Prod builds fail return FailureType.Warning; } #reportFailure( terminal: ITerminal, requestMethod: string, response: IWebClientResponse | IWebClientStreamResponse, isRedirect: boolean, message: string ): void { const failureMessage: string = `${message}: ${this.#describeResponse(response)}`; switch (this.#getFailureType(requestMethod, response, isRedirect)) { default: { terminal.writeErrorLine(failureMessage); break; } case FailureType.Warning: { terminal.writeWarningLine(failureMessage); break; } case FailureType.Informational: { terminal.writeLine(failureMessage); break; } case FailureType.None: { terminal.writeDebugLine(failureMessage); break; } case FailureType.Authentication: { throw new Error( [ `${this.#url.href} responded with ${this.#describeResponse(response)}.`, `Credentials may be misconfigured or have expired.`, `In CI, verify that RUSH_BUILD_CACHE_CREDENTIAL contains a valid Authorization header value.`, ``, `For local developers, run:`, ``, ` rush update-cloud-credentials --interactive`, `` ].join('\n') ); } } } }