'use strict'; // Shared WordPress REST transport core. // // This module is the single source of truth for the operational discipline a // WordPress reader must get right: auth header construction, URL building, // rate-limit throttling, and Retry-After-aware backoff on 429/503. It is // imported by: // - scripts/wp-discovery.js (discovery-time sampling), and // - the bulk-extract reader generated by rp-import-codegen, which vendors a // copy of this file into the migration project so the project stays // self-contained. // // Keep this file dependency-free (Node built-ins / global fetch only) so the // vendored copy needs no install step. const DEFAULT_TIMEOUT_MS = 60000; const DEFAULT_RATE_LIMIT_RPM = 120; const DEFAULT_MAX_RETRIES = 3; const MAX_BACKOFF_MS = 30000; // Shared throttle state. Every request routes through fetchJson, so a single // module-level throttle enforces the rate limit across the whole run. const rateState = { minIntervalMs: 0, lastRequestAt: 0, maxRetries: DEFAULT_MAX_RETRIES }; function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } // Configure the shared throttle. Call once before issuing requests. function configureRateLimit({ rateLimitRpm = DEFAULT_RATE_LIMIT_RPM, maxRetries = DEFAULT_MAX_RETRIES } = {}) { const rpm = Number.isFinite(rateLimitRpm) && rateLimitRpm >= 1 ? rateLimitRpm : DEFAULT_RATE_LIMIT_RPM; rateState.minIntervalMs = Math.ceil(60000 / rpm); rateState.maxRetries = Number.isFinite(maxRetries) && maxRetries >= 0 ? maxRetries : DEFAULT_MAX_RETRIES; } async function applyThrottle(progress) { if (rateState.minIntervalMs <= 0) { return; } const elapsed = Date.now() - rateState.lastRequestAt; const wait = rateState.minIntervalMs - elapsed; if (wait > 0) { if (progress) { await progress.withHeartbeat({ phase: 'discovery', step: 'rate-limit', message: 'Still waiting for WordPress rate limit' }, () => sleep(wait)); } else { await sleep(wait); } } rateState.lastRequestAt = Date.now(); } function retryAfterMs(responseHeaders) { const raw = responseHeaders?.['retry-after']; if (!raw) { return null; } const seconds = Number.parseInt(raw, 10); return Number.isFinite(seconds) ? seconds * 1000 : null; } function buildHeaders(args) { const headers = new Headers(); headers.set('accept', 'application/json'); for (const rawHeader of args.authHeaders || []) { const splitIndex = rawHeader.indexOf(':'); if (splitIndex === -1) { throw new Error(`Invalid --auth-header value: ${rawHeader}`); } const name = rawHeader.slice(0, splitIndex).trim(); const value = rawHeader.slice(splitIndex + 1).trim(); headers.set(name, value); } if (args.username && args.applicationPassword) { const credentials = Buffer.from(`${args.username}:${args.applicationPassword}`).toString('base64'); headers.set('authorization', `Basic ${credentials}`); } else if (args.apiKey) { const headerName = args.apiKeyHeader || 'Authorization'; const headerValue = headerName.toLowerCase() === 'authorization' && !/^bearer\s+/i.test(args.apiKey) ? `Bearer ${args.apiKey}` : args.apiKey; headers.set(headerName, headerValue); } return headers; } function normalizeBaseUrl(input) { return input.replace(/\/+$/, ''); } function buildApiUrl(baseUrl, routePath, query = {}) { const url = new URL(`${normalizeBaseUrl(baseUrl)}/wp-json${routePath}`); for (const [key, value] of Object.entries(query)) { if (value === undefined || value === null || value === '') { continue; } url.searchParams.set(key, String(value)); } return url; } async function fetchJson(baseUrl, routePath, { headers, method = 'GET', query, body, timeoutMs = DEFAULT_TIMEOUT_MS, progress = null, progressContext = {} }) { const url = buildApiUrl(baseUrl, routePath, query); // `body` is a small, generic escape hatch for profile-declared entities whose read path // is not a plain GET collection (see plugin-knowledge.js buildRequestOverrides) — it is // never set on the default GET-with-query sampling path. const requestInit = { method, headers, ...(body !== undefined ? { body } : {}), }; for (let attempt = 0; ; attempt += 1) { await applyThrottle(progress); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), timeoutMs); try { const response = progress ? await progress.withHeartbeat({ phase: 'discovery', step: 'wordpress-request', ...progressContext, message: `Still waiting on WordPress ${method} ${routePath || '/'}`, }, () => fetch(url, { ...requestInit, signal: controller.signal, })) : await fetch(url, { ...requestInit, signal: controller.signal, }); const text = await response.text(); const responseHeaders = Object.fromEntries(response.headers.entries()); // Back off and retry on throttling / transient unavailability, // honoring Retry-After when the server provides it. if ((response.status === 429 || response.status === 503) && attempt < rateState.maxRetries) { const backoff = retryAfterMs(responseHeaders) ?? Math.min(MAX_BACKOFF_MS, 1000 * 2 ** attempt); if (progress) { progress.warn(`WordPress ${response.status} response; backing off before retry`, { phase: 'discovery', step: 'wordpress-retry', ...progressContext, count: attempt + 1, total: rateState.maxRetries, unit: 'retries', }); await progress.withHeartbeat({ phase: 'discovery', step: 'wordpress-retry', ...progressContext, message: `Still backing off after WordPress ${response.status}`, }, () => sleep(backoff)); } else { await sleep(backoff); } continue; } let json; if (text) { try { json = JSON.parse(text); } catch { json = undefined; } } return { ok: response.ok, status: response.status, statusText: response.statusText, url: url.toString(), headers: responseHeaders, json, text, retries: attempt, }; } catch (error) { return { ok: false, status: 0, statusText: error.name === 'AbortError' ? 'Request Timeout' : error.message, url: url.toString(), headers: {}, json: undefined, text: '', error, retries: attempt, }; } finally { clearTimeout(timeout); } } } // Total record count for a collection. WordPress returns it in X-WP-Total; // X-WP-TotalPages carries the page count. Header names are lowercased by fetch's // Headers iterator, but accept the canonical casing too for resilience. function parseTotalHeader(responseHeaders, name = 'x-wp-total') { if (!responseHeaders) { return null; } // fetch's Headers iterator lowercases names, but match case-insensitively so a // plain object built with WordPress's canonical casing (X-WP-Total) also works. const target = name.toLowerCase(); let raw; for (const [key, value] of Object.entries(responseHeaders)) { if (key.toLowerCase() === target) { raw = value; break; } } if (raw === undefined || raw === null || raw === '') { return null; } const total = Number.parseInt(raw, 10); return Number.isFinite(total) ? total : null; } function parseTotalPagesHeader(responseHeaders) { return parseTotalHeader(responseHeaders, 'x-wp-totalpages'); } function shouldContinueCollectionPaging({ responseHeaders, page, perPage, itemCount }) { const totalPages = parseTotalPagesHeader(responseHeaders); if (totalPages !== null) { return Number(page) < totalPages; } if (Number.isFinite(perPage) && perPage > 0) { return Number(itemCount) >= perPage; } return false; } module.exports = { DEFAULT_TIMEOUT_MS, DEFAULT_RATE_LIMIT_RPM, DEFAULT_MAX_RETRIES, MAX_BACKOFF_MS, sleep, configureRateLimit, retryAfterMs, buildHeaders, normalizeBaseUrl, buildApiUrl, fetchJson, parseTotalHeader, parseTotalPagesHeader, shouldContinueCollectionPaging, };