import titleExtractor from './title-extractor' import urlExtractor from './url-extractor' import { DOMParser } from 'linkedom' import defaultExtractors, { DefaultExtractors, TitleExtracted, UrlExtracted } from './default-extractors' import { DeepMerged, NestedPartial } from './utils/types' import { dedupe, deepMerge } from './utils/memoized-functions' import { Extractor, ExtractorExtracted } from './utils/extractors' import sanitizeHtml, { defaultSanitizeOptions } from './utils/sanitize-html' import sanitize from 'sanitize-html' import { absolutifyDocument } from './utils/urls' type MergedExtracted = NestedPartial> & TitleExtracted & UrlExtracted type DefaultExtracted = MergedExtracted> export { selectors as contentSelectors } from './content-extractor/selector-extractors' export interface ExtractOptions { /** * Url for the page. **may not be the final result** * @see urlExtractor */ url?: string /** * Extractors for extract. * @see defaultExtractors */ extractors?: Extractor[] /** * Options of sanitize-html * @see https://www.npmjs.com/package/sanitize-html */ sanitizeHtml?: sanitize.IOptions /** * For parse date * @see mapToNearestLanguage */ lang?: string } export async function extract( html: string | Document ): Promise export async function extract( html: string | Document, options: Omit, 'extractors'> ): Promise export async function extract( html: string | Document, options?: ExtractOptions ): Promise> export async function extract( html: string | Document, options: ExtractOptions = {} ) { const finalOptions = { extractors: []>defaultExtractors, sanitizeHtml: defaultSanitizeOptions, ...options } let document = typeof html === 'string' ? (new DOMParser().parseFromString( sanitizeHtml(html, finalOptions.sanitizeHtml), 'text/html' ) as unknown as Document) : html finalOptions.lang ??= document.documentElement.lang const title = titleExtractor.selector( titleExtractor.processor( // eslint-disable-next-line unicorn/no-useless-spread [ ...titleExtractor.operators .flatMap((it) => it[1](document)) .filter((it): it is string => !!it) ], finalOptions ), undefined, finalOptions ) const url = urlExtractor.selector( urlExtractor.processor( // eslint-disable-next-line unicorn/no-useless-spread [ ...urlExtractor.operators .flatMap((it) => it[1](document)) .filter((it): it is string => !!it) ], finalOptions ), title.title, finalOptions ) if (!document.baseURI && url?.url) { const base = document.createElement('base') base.setAttribute('href', url.url) document.head.append(base) } document = absolutifyDocument(document) const context = { ...finalOptions, url: url?.url } const results = await Promise.all(finalOptions.extractors).then((it) => it .map(({ operators, processor, selector }) => { const operated = operators .flatMap((it) => it[1](document, url.url)) .filter((it): it is string => !!it) const processed = dedupe(processor([...operated], context), (it) => it) if (processed.length > 0) return selector(processed, title.title, context) return }) .filter((it): it is T => !!it) ) const result = deepMerge(title, url, ...results) return & TitleExtracted & UrlExtracted>result }