import type { ClientRequest, IncomingMessage } from 'node:http'; import got from 'got'; import { HTTPError as GotHTTPError } from 'got'; import type { Agents as GotAgents, Response as GotResponse, Progress as GotProgress, RequestPromise as GotRequestPromise, } from 'got'; import * as cheerio from 'cheerio'; import ipaddr from 'ipaddr.js'; import type { IPv4, IPv6 } from 'ipaddr.js'; import type { GeneralScrapingOptions } from '@/general.js'; import { StatusError } from '@/utils/status-error.js'; import { detectEncoding, toUtf8 } from '@/utils/encoding.js'; export let agent: GotAgents = {}; export function setAgent(_agent: GotAgents) { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition agent = _agent || {}; } export type GotOptions = { url: string; method: 'GET' | 'POST' | 'HEAD'; body?: string; headers: Record; typeFilter?: RegExp; followRedirects?: boolean; responseTimeout?: number; operationTimeout?: number; contentLengthLimit?: number; contentLengthRequired?: boolean; }; export const DEFAULT_RESPONSE_TIMEOUT = 20 * 1000; export const DEFAULT_OPERATION_TIMEOUT = 60 * 1000; export const DEFAULT_MAX_RESPONSE_SIZE = 10 * 1024 * 1024; export const DEFAULT_BOT_UA = `SummalyBot/${_VERSION_} (+https://github.com/misskey-dev/summaly/blob/master/README.md)`; export function getGotOptions(url: string, opts?: GeneralScrapingOptions): Omit { return { url, headers: { 'accept': 'text/html,application/xhtml+xml', 'user-agent': opts?.userAgent ?? DEFAULT_BOT_UA, 'accept-language': opts?.lang ?? undefined, }, typeFilter: /^(text\/html|application\/xhtml\+xml)/, followRedirects: opts?.followRedirects, responseTimeout: opts?.responseTimeout, operationTimeout: opts?.operationTimeout, contentLengthLimit: opts?.contentLengthLimit, contentLengthRequired: opts?.contentLengthRequired, }; } export async function scpaping( url: string, opts?: GeneralScrapingOptions, ) { const args = getGotOptions(url, opts); const response = await getResponse({ ...args, method: 'GET', }); const encoding = detectEncoding(response.rawBody); const body = toUtf8(response.rawBody, encoding); const $ = cheerio.load(body); return { body, $, response, }; } export async function get(url: string) { const res = await getResponse({ url, method: 'GET', headers: { 'accept': '*/*', }, }); return res.body; } export async function head(url: string) { return await getResponse({ url, method: 'HEAD', headers: { 'accept': '*/*', }, }); } export async function getResponse(args: GotOptions) { const timeout = args.responseTimeout ?? DEFAULT_RESPONSE_TIMEOUT; const operationTimeout = args.operationTimeout ?? DEFAULT_OPERATION_TIMEOUT; const abort = new AbortController(); const req = got(args.url, { method: args.method, headers: args.headers, body: args.body, timeout: { lookup: timeout, connect: timeout, secureConnect: timeout, socket: timeout, // read timeout response: timeout, send: timeout, request: operationTimeout, // whole operation timeout }, followRedirect: args.followRedirects, agent, http2: false, retry: { limit: 0, }, signal: abort.signal, hooks: { // 応答ヘッダ受信時点のチェックでabortしても、gotはリダイレクト先へのリクエストを発行してしまうため、ここで止める beforeRedirect: [() => { abort.signal.throwIfAborted(); }], }, }); const res = await receiveResponse({ req, opts: args, abort }); // 念のため最終応答に対しても同じチェックを行う(通常はreceiveResponse内で弾かれている) if (!isPrivateIpAllowed(new URL(res.url).protocol) && res.ip != null) { assertPublicIp(res.ip); } // Check html const contentType = res.headers['content-type']; if (args.typeFilter && !contentType?.match(args.typeFilter)) { throw new Error(`Rejected by type filter ${contentType}`); } // 応答ヘッダでサイズチェック const contentLength = res.headers['content-length']; if (contentLength) { const maxSize = args.contentLengthLimit ?? DEFAULT_MAX_RESPONSE_SIZE; const size = Number(contentLength); if (size > maxSize) { throw new Error(`maxSize exceeded (${size} > ${maxSize}) on response`); } } else { if (args.contentLengthRequired) { throw new Error('content-length required'); } } return res; } /** * @param protocol 実際にリクエストを送るURLのプロトコル('http:' / 'https:')。リダイレクト時はリダイレクト先のもの */ function isPrivateIpAllowed(protocol: string) { // SUMMALY_ALLOW_PRIVATE_IPはテスト用 if (process.env.SUMMALY_ALLOW_PRIVATE_IP === 'true') return true; // そのプロトコル用のagentが指定されている場合、接続先はagent(プロキシ等)になり実際の宛先IPを検査できないため、 // 宛先の制限はagent側に任せる switch (protocol) { case 'http:': return agent.http != null; case 'https:': return agent.https != null; default: return false; } } function assertPublicIp(rawIp: string) { let ip: IPv4 | IPv6; try { ip = ipaddr.parse(rawIp); } catch { throw new StatusError(`Invalid IP ${rawIp}`, 500, 'Invalid IP'); } if (ip.kind() === 'ipv6' && (ip as IPv6).isIPv4MappedAddress()) { ip = (ip as IPv6).toIPv4Address(); } if (ip.range() !== 'unicast') { throw new StatusError(`Private IP rejected ${rawIp}`, 400, 'Private IP Rejected'); } } async function receiveResponse(args: { req: GotRequestPromise>, opts: GotOptions, abort: AbortController, }) { const req = args.req; const maxSize = args.opts.contentLengthLimit ?? DEFAULT_MAX_RESPONSE_SIZE; req.on('request', (clientRequest: ClientRequest) => { const allowPrivateIp = isPrivateIpAllowed(clientRequest.protocol); clientRequest.prependOnceListener('response', (res: IncomingMessage) => { try { const ip = res.socket.remoteAddress; if (!allowPrivateIp && ip != null) { assertPublicIp(ip); } // リダイレクトやエラー応答はここでは判定しない(ステータスコードによるエラーを優先する) const statusCode = res.statusCode ?? 0; if (statusCode < 200 || statusCode >= 300) return; const contentType = res.headers['content-type']; if (args.opts.typeFilter && !contentType?.match(args.opts.typeFilter)) { throw new Error(`Rejected by type filter ${contentType}`); } } catch (e) { args.abort.abort(e); } }); }); // 受信中のデータでサイズチェック req.on('downloadProgress', (progress: GotProgress) => { if (progress.transferred > maxSize && progress.percent !== 1) { args.abort.abort(`maxSize exceeded (${progress.transferred} > ${maxSize}) on response`); } }); // 応答取得 with ステータスコードエラーの整形 const res = await req.catch(e => { const abortReason = args.abort.signal.reason; if (args.abort.signal.aborted) { if (abortReason instanceof Error) { throw abortReason; } if (typeof abortReason === 'string' && abortReason.length > 0) { throw new Error(abortReason); } } if (e instanceof GotHTTPError) { throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage); } else { throw e; } }); return res; }