/********************************************************************** node-feedparser - A robust RSS, Atom, RDF parser for node. http://github.com/danmactough/node-feedparser Copyright (c) 2011-2026 Dan MacTough and contributors http://mact.me **********************************************************************/ /** * Module dependencies. */ const sax = require('sax'); const addressparser = require('addressparser'); const indexOfObject = require('array-indexofobject'); const { inherits } = require('util'); const { Transform: TransformStream } = require('readable-stream'); const { HTML_URI_ATTRS } = require('./constants'); const _ = { ...require('./utils'), has: require('lodash.has'), assign: require('lodash.assign'), uniq: require('lodash.uniq') }; /** * Converts a feed date value to a valid Date or null. * @param {*} value Raw date value from the feed. * @returns {Date|null} Parsed date, or null for missing or invalid values. */ const parseDate = function parseDate (value) { if (!value) return null; const date = new Date(value); return isNaN(date.getTime()) ? null : date; }; /** * FeedParser constructor. * * Exposes a duplex (transform) stream to parse a feed. * * Each article/post in the feed will have the following keys: * - title {String} * - description {String} * - summary {String} * - date {Date} (or null) * - pubdate {Date} (or null) * - link {String} * - origlink {String} * - author {String} * - guid {String} * - comments {String} * - image {Object} * - categories {Array} * - source {Object} * - enclosures {Array} * - meta {Object} * - Object.keys(meta): * - #ns {Array} key,value pairs of each namespace declared for the feed * - #type {String} one of 'atom', 'rss', 'rdf' * - #version {String} * - title {String} * - description {String} * - date {Date} (or null) * - pubdate {Date} (or null) * - link {String} i.e., to the website, not the feed * - xmlurl {String} the canonical URL of the feed, as declared by the feed * - author {String} * - language {String} * - image {Object} * - favicon {String} * - copyright {String} * - generator {String} * - categories {Array} * * @this {FeedParserInstance} * @param {import('../index').Options} [options] */ function FeedParser (options) { if (!(this instanceof FeedParser)) return new FeedParser(options); TransformStream.call(this); this._readableState.objectMode = true; this._readableState.highWaterMark = 16; // max. # of output nodes buffered this.init(); // Parse options this.options = _.assign({}, options); if (!('strict' in this.options)) this.options.strict = false; if (!('normalize' in this.options)) this.options.normalize = true; if (!('addmeta' in this.options)) this.options.addmeta = true; if (!('guidlink' in this.options)) this.options.guidlink = true; if (!('resume_saxerror' in this.options)) this.options.resume_saxerror = true; // MAX_BUFFER_LENGTH is not part of the public API of sax, but we need to be // able to handle nodes that are larger than the 64K default if ('MAX_BUFFER_LENGTH' in this.options) { // @ts-expect-error - private API of sax sax.MAX_BUFFER_LENGTH = this.options.MAX_BUFFER_LENGTH; // set to Infinity to have unlimited buffers } else { // @ts-expect-error sax.MAX_BUFFER_LENGTH = 16 * 1024 * 1024; // 16M versus the 64K default } if (this.options.feedurl) this.xmlbase.unshift({ '#name': 'xml', '#': this.options.feedurl }); // See https://github.com/isaacs/sax-js for more info this.stream = sax.createStream(this.options.strict /* strict mode - no by default */, { lowercase: true, xmlns: true }); this.stream.on('error', this.handleSaxError.bind(this)); this.stream.on('processinginstruction', this.handleProcessingInstruction.bind(this)); this.stream.on('opentag', this.handleOpenTag.bind(this)); this.stream.on('closetag', this.handleCloseTag.bind(this)); this.stream.on('text', this.handleText.bind(this)); this.stream.on('cdata', this.handleText.bind(this)); this.stream.on('end', this.handleEnd.bind(this)); } inherits(FeedParser, TransformStream); /* * Initializes the SAX stream * * Initializes the class-variables */ /** @this {FeedParserInstance} */ FeedParser.prototype.init = function () { this.meta = { '#ns': [], '@': [], '#xml': {} }; this._namespaces = {}; this._emitted_meta = false; this.stack = []; this.xmlbase = []; this.in_xhtml = false; this.xhtml = {}; /* Where to store xhtml elements as associative array with keys: '#' (containing the text) and '#name' (containing the XML element name) */ this.errors = []; }; /** @this {FeedParserInstance} */ FeedParser.prototype.handleEnd = function () { // We made it to the end without throwing, but let's make sure we were actually // parsing a feed if (!(this.meta && this.meta['#type'])) { var e = new Error('Not a feed'); return this.handleError(e); } this.push(null); }; /** @this {FeedParserInstance} */ FeedParser.prototype.handleSaxError = function (e) { if (!this.options.resume_saxerror) { return this.handleError(e); } this.errors.push(e); this.resumeSaxError(); }; /** @this {FeedParserInstance} */ FeedParser.prototype.resumeSaxError = function () { if (this.stream._parser) { this.stream._parser.error = null; this.stream._parser.resume(); } }; /** @this {FeedParserInstance} */ FeedParser.prototype.handleError = function (e) { this.errors.push(e); this.emit('error', e); }; // parses the xml declaration, which looks like: // /** * @this {FeedParserInstance} * @param {SaxProcessingInstruction} node */ FeedParser.prototype.handleProcessingInstruction = function (node) { if (node.name === 'xml') { this.meta['#xml'] = node.body.trim().split(/\s+/).reduce(function (map, attr) { if (attr.indexOf('=') >= 0) { var parts = attr.split('='); map[parts[0]] = parts[1] && parts[1].length > 2 && parts[1].match(/^.(.*?).$/)[1]; } return map; }, this.meta['#xml']); } }; /** * @this {FeedParserInstance} * @param {import('sax').QualifiedTag} node */ FeedParser.prototype.handleOpenTag = function (node) { /** @type {ParsedNode} */ var n = {}; n['#name'] = node.name; // Avoid namespace collissions later... n['#prefix'] = node.prefix; // The current ns prefix n['#local'] = node.local; // The current element name, sans prefix n['#uri'] = node.uri; // The current ns uri n['@'] = {}; n['#'] = ''; if (Object.keys(node.attributes).length) { n['@'] = this.handleAttributes(node.attributes, n['#name']); } if (this.in_xhtml && this.xhtml['#name'] != n['#name']) { // We are in an xhtml node // This builds the opening tag, e.g.,
this.xhtml['#'] += '<' + n['#name']; Object.keys(n['@']).forEach((name) => { this.xhtml['#'] += ' ' + name + '="' + n['@'][name] + '"'; }); this.xhtml['#'] += '>'; } else if (this.stack.length === 0 && (n['#name'] === 'rss' || (n['#local'] === 'rdf' && _.nslookup([n['#uri']], 'rdf')) || (n['#local'] === 'feed' && _.nslookup([n['#uri']], 'atom')))) { Object.keys(n['@']).forEach((name) => { var o = {}; if (name != 'version') { o[name] = n['@'][name]; this.meta['@'].push(o); } }); switch (n['#local']) { case 'rss': this.meta['#type'] = 'rss'; this.meta['#version'] = n['@']['version']; break; case 'rdf': this.meta['#type'] = 'rdf'; this.meta['#version'] = n['@']['version'] || '1.0'; break; case 'feed': this.meta['#type'] = 'atom'; this.meta['#version'] = n['@']['version'] || '1.0'; break; } } this.stack.unshift(n); }; /** @this {FeedParserInstance} */ FeedParser.prototype.handleCloseTag = function (el) { var node = { '#name': el, '#prefix': '', '#local': '' } , stdEl , item , base , baseurl , isIllegallyNested = false ; var n = this.stack.shift(); // Parent nodes receive a string when this node is text-only; keep n as the parsed node. /** @type {ParsedNode|string} */ var nodeValue = n; el = el.split(':'); if (el.length > 1 && el[0] === n['#prefix']) { if (_.nslookup(n['#uri'], 'atom')) { node['#prefix'] = el[0]; node['#local'] = el.slice(1).join(':'); node['#type'] = 'atom'; } else if (_.nslookup(n['#uri'], 'rdf')) { node['#prefix'] = el[0]; node['#local'] = el.slice(1).join(':'); node['#type'] = 'rdf'; } else { node['#prefix'] = _.nsprefix(n['#uri']) || n['#prefix']; node['#local'] = el.slice(1).join(':'); } } else { node['#local'] = node['#name']; node['#type'] = _.nsprefix(n['#uri']) || n['#prefix']; } delete n['#name']; delete n['#local']; delete n['#prefix']; delete n['#uri']; if (this.xmlbase && this.xmlbase.length) { base = this.xmlbase[0]; baseurl = base['#']; } var mayHaveResolvableUrl = ( ( (node['#local'] === 'logo' || node['#local'] === 'icon') && node['#type'] === 'atom' ) || ( node['#local'] === 'link' // include rss:link, even though it should _never_ be a relative URL ) ); var mayHaveEmbeddedHtml = _.mayHaveEmbeddedHtml(node['#name'], n) || _.mayHaveEmbeddedHtml(node['#local'], n); if (baseurl && mayHaveResolvableUrl) { // Apply xml:base to these elements as they appear // rather than leaving it to the ultimate parser n['#'] = _.resolve(baseurl, n['#']); } if (baseurl && this.options.normalize && mayHaveEmbeddedHtml) { n['#'] = _.resolveHtmlUris(n['#'], baseurl); } if (this.xmlbase.length && (el == this.xmlbase[0]['#name'])) { void this.xmlbase.shift(); } if (this.in_xhtml) { if (node['#name'] == this.xhtml['#name']) { // The end of the XHTML // Add xhtml data to the container element n['#'] += this.xhtml['#'].trim(); // Clear xhtml nodes from the tree for (var key in n) { if (key != '@' && key != '#') { delete n[key]; } } this.xhtml = {}; this.in_xhtml = false; } else { // Somewhere in the middle of the XHTML this.xhtml['#'] += ''; } } if ('#' in n) { if (n['#'].match(/^\s*$/)) { // Delete text nodes with nothing by whitespace delete n['#']; } else { n['#'] = n['#'].trim(); if (Object.keys(n).length === 1) { // If there is only one text node, hoist it nodeValue = n['#']; } } } if (node['#name'] === 'item' || node['#name'] === 'entry' || (node['#local'] === 'item' && (node['#prefix'] === '' || node['#type'] === 'rdf')) || (node['#local'] == 'entry' && (node['#prefix'] === '' || node['#type'] === 'atom'))) { // We have an article! isIllegallyNested = ( (node['#name'] === 'item' && this.stack[0]['#name'] === 'item') || (node['#name'] === 'entry' && this.stack[0]['#name'] === 'entry') || ((node['#local'] === 'item' && (node['#prefix'] === '' || node['#type'] === 'rdf')) && this.stack[0]['#name'] === 'item') || ((node['#local'] == 'entry' && (node['#prefix'] === '' || node['#type'] === 'atom')) && this.stack[0]['#name'] === 'entry') ); if (isIllegallyNested) { return; } if (!this.meta.title) { // We haven't yet parsed all the metadata _.assign(this.meta, this.handleMeta(this.stack[0], this.meta['#type'], this.options)); if (!this._emitted_meta) { this.emit('meta', this.meta); this._emitted_meta = true; } } if (!baseurl && this.xmlbase && this.xmlbase.length) { // handleMeta was able to infer a baseurl without xml:base or options.feedurl n = _.reresolve(n, this.xmlbase[0]['#']); } item = this.handleItem(n, this.meta['#type'], this.options); if (this.options.addmeta) { item.meta = this.meta; } if (this.meta.author && !item.author) item.author = this.meta.author; this.push(item); } else if (node['#name'] === 'channel' || node['#name'] === 'feed' || (node['#local'] === 'channel' && (node['#prefix'] === '' || node['#type'] === 'rdf')) || (node['#local'] === 'feed' && (node['#prefix'] === '' || node['#type'] === 'atom'))) { _.assign(this.meta, this.handleMeta(n, this.meta['#type'], this.options)); if (!this._emitted_meta) { this.emit('meta', this.meta); this._emitted_meta = true; } } if (this.stack.length > 0) { if (node['#prefix'] && node['#local'] && !node['#type']) { stdEl = node['#prefix'] + ':' + node['#local']; } else if (node['#name'] && node['#type'] && node['#type'] !== this.meta['#type']) { stdEl = node['#name']; } else { stdEl = node['#local'] || node['#name']; } if (!Object.prototype.hasOwnProperty.call(this.stack[0], stdEl)) { this.stack[0][stdEl] = nodeValue; } else if (this.stack[0][stdEl] instanceof Array) { this.stack[0][stdEl].push(nodeValue); } else { this.stack[0][stdEl] = [this.stack[0][stdEl], nodeValue]; } } }; /** * @this {FeedParserInstance} * @param {string} text */ FeedParser.prototype.handleText = function (text) { if (this.in_xhtml) { this.xhtml['#'] += text; } else { if (this.stack.length) { if (this.stack[0] && '#' in this.stack[0]) { this.stack[0]['#'] += text; } else { this.stack[0]['#'] = text; } } } }; /** * @this {FeedParserInstance} * @param {Object.} attrs * @param {string} el * @returns {ParsedAttributes} */ FeedParser.prototype.handleAttributes = function handleAttributes (attrs, el) { /* * Using the sax.js option { xmlns: true } * attrs is an array of objects (not strings) having the following properties * name - e.g., xmlns:dc or href * value * prefix - the first part of the name of the attribute (before the colon) * local - the second part of the name of the attribute (after the colon) * uri - the uri of the namespace * */ var basepath = '' , simplifiedAttributes = /** @type {ParsedAttributes} */ ({}) ; if (this.xmlbase && this.xmlbase.length) { basepath = this.xmlbase[0]['#']; } Object.keys(attrs).forEach(/** @type (key: string) => void */ (key) => { var attr = attrs[key] , ns = {} , prefix = '' ; if (attr.prefix === 'xmlns') { if (!(attr.name in this._namespaces)) { ns[attr.name] = attr.value; this.meta['#ns'].push(ns); _.assign(this._namespaces, ns); } } // If the feed is using a non-default prefix, we'll use it, too // But we force the use of the 'xml' prefix if (attr.uri && attr.prefix && !_.nslookup(attr.uri, attr.prefix) || _.nslookup(attr.uri, 'xml')) { prefix = (_.nsprefix(attr.uri) || attr.prefix) + (attr.local ? ':' : ''); } if (basepath && HTML_URI_ATTRS.has(attr.local)) { // Apply xml:base to these elements as they appear // rather than leaving it to the ultimate parser attr.value = _.resolveHtmlAttributeValue(basepath, attr.local, attr.value); } else if (attr.local === 'base' && _.nslookup(attr.uri, 'xml')) { // Keep track of the xml:base for the current node if (basepath) { attr.value = _.resolve(basepath, attr.value); } // Per RFC 3986 ยง4.4, an empty or "#"-only xml:base is a same-document reference. // It does not change the effective base URI, so skip pushing it. if (attr.value && !/^#/.test(attr.value)) { this.xmlbase.unshift({ '#name': el, '#': attr.value }); } } else if (attr.name === 'type' && attr.value === 'xhtml') { this.in_xhtml = true; this.xhtml = { '#name': el, '#': '' }; } simplifiedAttributes[prefix + attr.local] = attr.value ? attr.value.trim() : ''; }); return simplifiedAttributes; }; /** * @this {FeedParserInstance} * @param {ParsedNode} node * @param {import('../index').Type} type * @param {import('../index').Options} options * @returns {Object} */ FeedParser.prototype.handleMeta = function handleMeta (node, type, options) { if (!type || !node) return {}; var meta = {} , normalize = !options || (options && options.normalize) ; if (normalize) { ['title', 'description', 'date', 'pubdate', 'pubDate', 'link', 'xmlurl', 'xmlUrl', 'author', 'language', 'favicon', 'copyright', 'generator'].forEach(function (property) { meta[property] = null; }); meta.cloud = {}; meta.image = {}; meta.categories = []; } Object.keys(node).forEach((name) => { var el = node[name]; if (name === 'item' || name === 'entry' || name.match(/:(item|entry)$/)) return; if (normalize) { switch (name) { case ('title'): meta.title = _.get(el); break; case ('description'): case ('subtitle'): meta.description = _.get(el); break; case ('pubdate'): case ('lastbuilddate'): case ('published'): case ('modified'): case ('updated'): case ('dc:date'): var date = parseDate(_.get(el)); if (!date) break; if (meta.pubdate === null || name == 'pubdate' || name == 'published') meta.pubdate = meta.pubDate = date; if (meta.date === null || name == 'lastbuilddate' || name == 'modified' || name == 'updated') meta.date = date; break; case ('link'): case ('atom:link'): case ('atom10:link'): if (Array.isArray(el)) { el.forEach((link) => { if (link['@']['href']) { // Atom if (_.get(link['@'], 'rel')) { if (link['@']['rel'] == 'alternate') { if (!meta.link) meta.link = link['@']['href']; } else if (link['@']['rel'] == 'self') { meta.xmlurl = meta.xmlUrl = link['@']['href']; if (_.isAbsoluteUrl(meta.xmlurl) && this.xmlbase && this.xmlbase.length === 0) { this.xmlbase.unshift({ '#name': 'xml', '#': meta.xmlurl }); this.stack[0] = _.reresolve(this.stack[0], meta.xmlurl); } else if (this.xmlbase && this.xmlbase.length > 0) { meta.xmlurl = meta.xmlUrl = _.resolve(_.get(this.xmlbase[0], '#'), meta.xmlurl); } } else if (link['@']['rel'] == 'hub' && !(meta.cloud.href || meta.cloud.domain)) { meta.cloud.type = 'hub'; meta.cloud.href = link['@']['href']; } } else { if (!meta.link) meta.link = link['@']['href']; } } else if (Object.keys(link['@']).length === 0) { // RSS meta.link = _.get(link); } if (_.isAbsoluteUrl(meta.link) && this.xmlbase && this.xmlbase.length === 0) { this.xmlbase.unshift({ '#name': 'xml', '#': meta.link }); this.stack[0] = _.reresolve(this.stack[0], meta.link); } else if (this.xmlbase && this.xmlbase.length > 0) { meta.link = _.resolve(_.get(this.xmlbase[0], '#'), meta.link); } }); } else { if (el['@']['href']) { // Atom if (_.get(el['@'], 'rel')) { if (el['@']['rel'] == 'alternate') { if (!meta.link) meta.link = el['@']['href']; } else if (el['@']['rel'] == 'self') { meta.xmlurl = meta.xmlUrl = el['@']['href']; if (_.isAbsoluteUrl(meta.xmlurl) && this.xmlbase && this.xmlbase.length === 0) { this.xmlbase.unshift({ '#name': 'xml', '#': meta.xmlurl }); this.stack[0] = _.reresolve(this.stack[0], meta.xmlurl); } else if (this.xmlbase && this.xmlbase.length > 0) { meta.xmlurl = meta.xmlUrl = _.resolve(_.get(this.xmlbase[0], '#'), meta.xmlurl); } } else if (el['@']['rel'] == 'hub' && !(meta.cloud.href || meta.cloud.domain)) { meta.cloud.type = 'hub'; meta.cloud.href = el['@']['href']; } } else { meta.link = el['@']['href']; } } else if (Object.keys(el['@']).length === 0) { // RSS if (!meta.link) meta.link = _.get(el); } if (_.isAbsoluteUrl(meta.link) && this.xmlbase && this.xmlbase.length === 0) { this.xmlbase.unshift({ '#name': 'xml', '#': meta.link }); this.stack[0] = _.reresolve(this.stack[0], meta.link); } else if (this.xmlbase && this.xmlbase.length > 0) { meta.link = _.resolve(_.get(this.xmlbase[0], '#'), meta.link); } } break; case ('managingeditor'): case ('webmaster'): case ('author'): var author = {}; if (name == 'author') { meta.author = _.get(el.name) || _.get(el.email) || _.get(el.uri); } else if (_.get(el)) { author = addressparser(_.get(el))[0]; if (author) { el['name'] = author.name; el['email'] = author.address; } if (meta.author === null || name == 'managingeditor') { meta.author = author.name || author.address || _.get(el); } } break; case ('cloud'): // I can't believe someone actually would put two cloud elements in their channel // but it happened // Nevertheless, there can be only one // This will ensure that rssCloud "wins" here. // If pubsubhubbub is also declared, it's still available in the link elements meta.cloud = {}; if (Array.isArray(el)) { Object.keys(el[0]['@']).forEach(function (attr) { if (_.has(el[0]['@'], attr)) { meta.cloud[attr] = el[0]['@'][attr]; } }); } else { Object.keys(el['@']).forEach(function (attr) { if (_.has(el['@'], attr)) { meta.cloud[attr] = el['@'][attr]; } }); } meta.cloud.type = 'rsscloud'; break; case ('language'): meta.language = _.get(el); break; case ('image'): case ('logo'): if (el.url) meta.image.url = _.get(el.url); if (el.title) meta.image.title = _.get(el.title); if (!meta.image.url && _.get(el)) meta.image.url = _.get(el); break; case ('icon'): meta.favicon = _.get(el); break; case ('copyright'): case ('rights'): case ('dc:rights'): meta.copyright = _.get(el); break; case ('generator'): meta.generator = _.get(el); if (_.get(el['@'], 'version')) meta.generator += (meta.generator ? ' ' : '') + 'v' + el['@'].version; if (_.get(el['@'], 'uri')) meta.generator += meta.generator ? ' (' + el['@'].uri + ')' : el['@'].uri; break; case ('category'): case ('dc:subject'): case ('itunes:category'): case ('media:category'): /* We handle all the kinds of categories within the switch loop because meta.categories * is an array, unlike the other properties, and therefore can handle multiple values */ var _category = '' , _categories = [] ; if (Array.isArray(el)) { el.forEach(function (category) { var _categoryValue; if ('category' == name && 'atom' == type) { if (category['@'] && (_categoryValue = _.safeTrim(_.get(category['@'], 'term')))) { meta.categories.push(_categoryValue); } } else if ('category' == name && 'rss' == type) { if ((_categoryValue = _.safeTrim(_.get(category)))) { meta.categories.push(_categoryValue); } } else if ('dc:subject' == name && (_categoryValue = _.safeTrim(_.get(category)))) { _categories = _categoryValue.split(' ').map(function (cat) { return cat.trim(); }); if (_categories.length) { meta.categories = meta.categories.concat(_categories); } } else if ('itunes:category' == name) { if (category['@'] && _.safeTrim(_.get(category['@'], 'text'))) _category = _.safeTrim(_.get(category['@'], 'text')); if (category[name]) { if (Array.isArray(category[name])) { category[name].forEach(function (subcategory) { var _subcategoryValue; if (subcategory['@'] && (_subcategoryValue = _.safeTrim(_.get(subcategory['@'], 'text')))) { meta.categories.push(_category + '/' + _subcategoryValue); } }); } else if (category[name]['@'] && (_categoryValue = _.safeTrim(_.get(category[name]['@'], 'text')))) { meta.categories.push(_category + '/' + _categoryValue); } } else if (_category) { meta.categories.push(_category); } } else if ('media:category' == name && (_categoryValue = _.safeTrim(_.get(category)))) { meta.categories.push(_categoryValue); } }); } else { if ('category' == name && 'atom' == type) { if ((_category = _.safeTrim(_.get(el['@'], 'term')))) { meta.categories.push(_category); } } else if ('category' == name && 'rss' == type) { if ((_category = _.safeTrim(_.get(el)))) { meta.categories.push(_category); } } else if ('dc:subject' == name && (_category = _.safeTrim(_.get(el)))) { _categories = _category.split(' ').map(function (cat) { return cat.trim(); }); if (_categories.length) { meta.categories = meta.categories.concat(_categories); } } else if ('itunes:category' == name) { if (el['@'] && _.safeTrim(_.get(el['@'], 'text'))) _category = _.safeTrim(_.get(el['@'], 'text')); if (el[name]) { if (Array.isArray(el[name])) { el[name].forEach(function (subcategory) { var _subcategoryValue; if (subcategory['@'] && (_subcategoryValue = _.safeTrim(_.get(subcategory['@'], 'text')))) { meta.categories.push(_category + '/' + _subcategoryValue); } }); } else if (el[name]['@'] && (_category = _.safeTrim(_.get(el[name]['@'], 'text')))) { meta.categories.push(_category + '/' + _category); } } else if (_category) { meta.categories.push(_category); } } else if ('media:category' == name && (_category = _.safeTrim(_.get(el)))) { meta.categories.push(_.get(el)); } } break; } // switch end } // Fill with all native other namespaced properties if (name.indexOf('#') !== 0) { if (~name.indexOf(':')) meta[name] = el; else meta[type + ':' + name] = el; } }); // forEach end if (normalize) { if (!meta.description) { if (node['itunes:summary']) meta.description = _.get(node['itunes:summary']); else if (node['tagline']) meta.description = _.get(node['tagline']); } if (!meta.author) { if (node['itunes:author']) meta.author = _.get(node['itunes:author']); else if (node['itunes:owner'] && node['itunes:owner']['itunes:name']) meta.author = _.get(node['itunes:owner']['itunes:name']); else if (node['dc:creator']) meta.author = _.get(node['dc:creator']); else if (node['dc:publisher']) meta.author = _.get(node['dc:publisher']); } if (!meta.language) { if (node['@'] && node['@']['xml:lang']) meta.language = _.get(node['@'], 'xml:lang'); else if (node['dc:language']) meta.language = _.get(node['dc:language']); } if (!meta.image.url) { if (node['itunes:image']) meta.image.url = _.get(node['itunes:image']['@'], 'href'); else if (node['media:thumbnail']) { if (Array.isArray(node['media:thumbnail'])) { node['media:thumbnail'] = node['media:thumbnail'][0]; } meta.image.url = _.get(node['media:thumbnail']['@'], 'url'); } } if (!meta.copyright) { if (node['media:copyright']) meta.copyright = _.get(node['media:copyright']); else if (node['dc:rights']) meta.copyright = _.get(node['dc:rights']); else if (node['creativecommons:license']) meta.copyright = _.get(node['creativecommons:license']); else if (node['cc:license']) { if (Array.isArray(node['cc:license']) && node['cc:license'][0]['@'] && node['cc:license'][0]['@']['rdf:resource']) { meta.copyright = _.get(node['cc:license'][0]['@'], 'rdf:resource'); } else if (node['cc:license']['@'] && node['cc:license']['@']['rdf:resource']) { meta.copyright = _.get(node['cc:license']['@'], 'rdf:resource'); } } } if (!meta.generator) { if (node['admin:generatoragent']) { if (Array.isArray(node['admin:generatoragent']) && node['admin:generatoragent'][0]['@'] && node['admin:generatoragent'][0]['@']['rdf:resource']) { meta.generator = _.get(node['admin:generatoragent'][0]['@'], 'rdf:resource'); } else if (node['admin:generatoragent']['@'] && node['admin:generatoragent']['@']['rdf:resource']) { meta.generator = _.get(node['admin:generatoragent']['@'], 'rdf:resource'); } } } if (meta.categories.length) { meta.categories = _.uniq(meta.categories); } if (!meta.link) { if (meta['atom:id'] && _.get(meta['atom:id']) && /^https?:/.test(_.get(meta['atom:id']))) { meta.link = _.get(meta['atom:id']); } } if (!meta.xmlurl && this.options.feedurl) { meta.xmlurl = meta.xmlUrl = this.options.feedurl; } meta.title = meta.title && _.stripHtml(meta.title); meta.description = meta.description && _.stripHtml(meta.description); } return meta; }; /** * @this {FeedParserInstance} * @param {ParsedNode} node * @param {import('../index').Type} type * @param {import('../index').Options} options * @returns {Object} */ FeedParser.prototype.handleItem = function handleItem (node, type, options) { if (!type || !node) return {}; var item = {} , normalize = !options || (options && options.normalize) ; if (normalize) { ['title', 'description', 'summary', 'date', 'pubdate', 'pubDate', 'link', 'guid', 'author', 'comments', 'origlink'].forEach(function (property) { item[property] = null; }); item.image = {}; item.source = {}; item.categories = []; item.enclosures = []; } Object.keys(node).forEach((name) => { var el = node[name] , attrs = _.get(el, '@') , enclosure; if (normalize) { switch (name) { case ('title'): item.title = _.get(el); break; case ('description'): case ('summary'): item.summary = _.get(el); if (!item.description) item.description = _.get(el); break; case ('content'): case ('content:encoded'): item.description = _.get(el); break; case ('pubdate'): case ('published'): case ('issued'): case ('modified'): case ('updated'): case ('dc:date'): var date = parseDate(_.get(el)); if (!date) break; if (item.pubdate === null || name == 'pubdate' || name == 'published' || name == 'issued') item.pubdate = item.pubDate = date; if (item.date === null || name == 'modified' || name == 'updated') item.date = date; break; case ('link'): if (Array.isArray(el)) { el.forEach(function (link) { if (link['@']['href']) { // Atom if (_.get(link['@'], 'rel')) { if (link['@']['rel'] == 'canonical') item.origlink = link['@']['href']; if (link['@']['rel'] == 'alternate') item.link = link['@']['href']; if (link['@']['rel'] == 'self' && !item.link) item.link = link['@']['href']; if (link['@']['rel'] == 'replies') item.comments = link['@']['href']; if (link['@']['rel'] == 'enclosure') { enclosure = /** @type {import('../index').Enclosure} */ ({}); enclosure.url = link['@']['href']; enclosure.type = _.get(link['@'], 'type'); enclosure.length = _.get(link['@'], 'length'); if (indexOfObject(item.enclosures, enclosure, ['url', 'type']) === -1) { item.enclosures.push(enclosure); } } } else { item.link = link['@']['href']; } } else if (Object.keys(link['@']).length === 0) { // RSS if (!item.link) item.link = _.get(link); } }); } else { if (el['@']['href']) { // Atom if (_.get(el['@'], 'rel')) { if (el['@']['rel'] == 'canonical') item.origlink = el['@']['href']; if (el['@']['rel'] == 'alternate') item.link = el['@']['href']; if (el['@']['rel'] == 'self' && !item.link) item.link = el['@']['href']; if (el['@']['rel'] == 'replies') item.comments = el['@']['href']; if (el['@']['rel'] == 'enclosure') { enclosure = /** @type {import('../index').Enclosure} */ ({}); enclosure.url = el['@']['href']; enclosure.type = _.get(el['@'], 'type'); enclosure.length = _.get(el['@'], 'length'); if (indexOfObject(item.enclosures, enclosure, ['url', 'type']) === -1) { item.enclosures.push(enclosure); } } } else { item.link = el['@']['href']; } } else if (Object.keys(el['@']).length === 0) { // RSS if (!item.link) item.link = _.get(el); } } if (!item.guid) item.guid = item.link; break; case ('guid'): case ('id'): item.guid = _.get(el); // http://cyber.law.harvard.edu/rss/rss.html#ltguidgtSubelementOfLtitemgt // If the guid element has an attribute named "isPermaLink" with a value // of true, the reader may assume that it is a permalink to the item, // that is, a url that can be opened in a Web browser, that points to // the full item described by the element. // isPermaLink is optional, its default value is true. If its value is // false, the guid may not be assumed to be a url, or a url to anything // in particular. if (item.guid && type == 'rss' && name == 'guid' && !(attrs.ispermalink && attrs.ispermalink.match(/false/i))) { item.permalink = item.guid; } break; case ('author'): var author = {}; if (_.get(el)) { // RSS author = addressparser(_.get(el))[0]; if (author) { el['name'] = author.name; el['email'] = author.address; item.author = author.name || author.address; } // addressparser failed else { item.author = _.get(el); } } else { item.author = _.get(el.name) || _.get(el.email) || _.get(el.uri); } break; case ('dc:creator'): item.author = _.get(el); break; case ('comments'): item.comments = _.get(el); break; case ('source'): if ('rss' == type) { item.source['title'] = _.get(el); item.source['url'] = _.get(el['@'], 'url'); } else if ('atom' == type) { if (el.title && _.get(el.title)) item.source['title'] = _.get(el.title); if (el.link && _.get(el.link['@'], 'href')) item.source['url'] = _.get(el.link['@'], 'href'); } if (item.source['url'] && !this.meta.xmlurl) { this.meta.xmlurl = this.meta.xmlUrl = item.source['url']; if (_.isAbsoluteUrl(item.source['url']) && this.xmlbase && this.xmlbase.length === 0) { this.xmlbase.unshift({ '#name': 'xml', '#': item.source['url'] }); this.stack[0] = _.reresolve(this.stack[0], item.source['url']); } else if (this.xmlbase && this.xmlbase.length > 0) { this.meta.xmlurl = this.meta.xmlUrl = item.source['url'] = _.resolve(_.get(this.xmlbase[0], '#'), item.source['url']); } } break; case ('enclosure'): if (Array.isArray(el)) { el.forEach(function (enc) { enclosure = /** @type {import('../index').Enclosure} */ ({}); enclosure.url = _.get(enc['@'], 'url'); enclosure.type = _.get(enc['@'], 'type'); enclosure.length = _.get(enc['@'], 'length'); if (~indexOfObject(item.enclosures, enclosure, ['url', 'type'])) { item.enclosures.splice(indexOfObject(item.enclosures, enclosure, ['url', 'type']), 1, enclosure); } else { item.enclosures.push(enclosure); } }); } else { enclosure = /** @type {import('../index').Enclosure} */ ({}); enclosure.url = _.get(el['@'], 'url'); enclosure.type = _.get(el['@'], 'type'); enclosure.length = _.get(el['@'], 'length'); if (~indexOfObject(item.enclosures, enclosure, ['url', 'type'])) { item.enclosures.splice(indexOfObject(item.enclosures, enclosure, ['url', 'type']), 1, enclosure); } else { item.enclosures.push(enclosure); } } break; case ('media:content'): var optionalAttributes = ['bitrate', 'framerate', 'samplingrate', 'duration', 'height', 'width']; if (Array.isArray(el)) { el.forEach(function (enc) { enclosure = /** @type {import('../index').Enclosure} */ ({}); enclosure.url = _.get(enc['@'], 'url'); enclosure.type = _.get(enc['@'], 'type') || _.get(enc['@'], 'medium'); enclosure.length = _.get(enc['@'], 'filesize'); var index = indexOfObject(item.enclosures, enclosure, ['url', 'type']); if (index !== -1) { enclosure = item.enclosures[index]; } optionalAttributes.forEach(function (attribute) { if (!enclosure[attribute] && _.get(enc['@'], attribute)) { enclosure[attribute] = _.get(enc['@'], attribute); } }); if (index === -1) { item.enclosures.push(enclosure); } }); } else { enclosure = /** @type {import('../index').Enclosure} */ ({}); enclosure.url = _.get(el['@'], 'url'); enclosure.type = _.get(el['@'], 'type') || _.get(el['@'], 'medium'); enclosure.length = _.get(el['@'], 'filesize'); var index = indexOfObject(item.enclosures, enclosure, ['url', 'type']); if (index !== -1) { enclosure = item.enclosures[index]; } optionalAttributes.forEach(function (attribute) { if (!enclosure[attribute] && _.get(el['@'], attribute)) { enclosure[attribute] = _.get(el['@'], attribute); } }); if (index === -1) { item.enclosures.push(enclosure); } } break; case ('enc:enclosure'): // Can't find this in use for an example to debug. Only example found does not comply with the spec -- can't code THAT! break; case ('category'): case ('dc:subject'): case ('itunes:category'): case ('media:category'): /* We handle all the kinds of categories within the switch loop because item.categories * is an array, unlike the other properties, and therefore can handle multiple values */ var _category = '' , _categories = [] ; if (Array.isArray(el)) { el.forEach(function (category) { if ('category' == name && 'atom' == type) { if (category['@'] && _.get(category['@'], 'term')) item.categories.push(_.get(category['@'], 'term')); } else if ('category' == name && _.get(category) && 'rss' == type) { item.categories.push(_.get(category).trim()); } else if ('dc:subject' == name && _.get(category)) { _categories = _.get(category).split(' ').map(function (cat) { return cat.trim(); }); if (_categories.length) item.categories = item.categories.concat(_categories); } else if ('itunes:category' == name) { if (category['@'] && _.get(category['@'], 'text')) _category = _.get(category['@'], 'text'); if (category[name]) { if (Array.isArray(category[name])) { category[name].forEach(function (subcategory) { if (subcategory['@'] && _.get(subcategory['@'], 'text')) item.categories.push(_category + '/' + _.get(subcategory['@'], 'text')); }); } else { if (category[name]['@'] && _.get(category[name]['@'], 'text')) item.categories.push(_category + '/' + _.get(category[name]['@'], 'text')); } } else { item.categories.push(_category); } } else if ('media:category' == name) { item.categories.push(_.get(category)); } }); } else { if ('category' == name && 'atom' == type) { if (_.get(el['@'], 'term')) item.categories.push(_.get(el['@'], 'term')); } else if ('category' == name && _.get(el) && 'rss' == type) { item.categories.push(_.get(el).trim()); } else if ('dc:subject' == name && _.get(el)) { _categories = _.get(el).split(' ').map(function (cat) { return cat.trim(); }); if (_categories.length) item.categories = item.categories.concat(_categories); } else if ('itunes:category' == name) { if (el['@'] && _.get(el['@'], 'text')) _category = _.get(el['@'], 'text'); if (el[name]) { if (Array.isArray(el[name])) { el[name].forEach(function (subcategory) { if (subcategory['@'] && _.get(subcategory['@'], 'text')) item.categories.push(_category + '/' + _.get(subcategory['@'], 'text')); }); } else { if (el[name]['@'] && _.get(el[name]['@'], 'text')) item.categories.push(_category + '/' + _.get(el[name]['@'], 'text')); } } else { item.categories.push(_category); } } else if ('media:category' == name) { item.categories.push(_.get(el)); } } break; case ('feedburner:origlink'): case ('pheedo:origlink'): if (!item.origlink) { item.origlink = _.get(el); } break; } // switch end } // Fill with all native other namespaced properties if (name.indexOf('#') !== 0) { if (~name.indexOf(':')) item[name] = el; else item[type + ':' + name] = el; } }); // forEach end if (normalize) { if (!item.description) { if (node['itunes:summary']) item.description = _.get(node['itunes:summary']); } if (!item.author) { if (node['itunes:author']) item.author = _.get(node['itunes:author']); else if (node['itunes:owner'] && node['itunes:owner']['itunes:name']) item.author = _.get(node['itunes:owner']['itunes:name']); else if (node['dc:publisher']) item.author = _.get(node['dc:publisher']); } if (!item.image.url) { if (node['itunes:image']) item.image.url = _.get(node['itunes:image']['@'], 'href'); else if (node['media:thumbnail']) { if (Array.isArray(node['media:thumbnail'])) { item.image.url = _.get(node['media:thumbnail'][0]['@'], 'url'); } else { item.image.url = _.get(node['media:thumbnail']['@'], 'url'); } } else if (node['media:content'] && node['media:content']['media:thumbnail']) item.image.url = _.get(node['media:content']['media:thumbnail']['@'], 'url'); else if (node['media:group'] && node['media:group']['media:thumbnail']) item.image.url = _.get(node['media:group']['media:thumbnail']['@'], 'url'); else if (node['media:group'] && node['media:group']['media:content'] && node['media:group']['media:content']['media:thumbnail']) item.image.url = _.get(node['media:group']['media:content']['media:thumbnail']['@'], 'url'); else if (node['g:image_link']) item.image.url = _.get(node['g:image_link']); } if (item.categories.length) { item.categories = _.uniq(item.categories); } if (!item.link && (!options || options.guidlink)) { if (item.guid && /^https?:/.test(item.guid)) { item.link = item.guid; } } item.title = item.title && _.stripHtml(item.title); } return item; }; // Naive Stream API /** @this {FeedParserInstance} */ FeedParser.prototype._transform = function (data, encoding, done) { try { this.stream.write(data); done(); } catch (e) { done(e); this.push(null); // Manually trigger and end, since we can't reliably do any more parsing } }; /** @this {FeedParserInstance} */ FeedParser.prototype._flush = function (done) { try { this.stream.end(); done(); } catch (e) { done(e); } }; /** * @typedef {Object.} ParsedAttributes */ /** * @typedef {{ * '#name'?: string, * '#prefix'?: string, * '#local'?: string, * '#uri'?: string, * '@'?: ParsedAttributes, * '#'?: string * }} ParsedNode * The internal accumulator object that handleOpenTag builds and pushes onto * this.stack. The parser-owned keys describe the current element: '#name', * '#prefix', '#local', '#uri', '@', and '#'. Parsed child elements are attached * using their element names as keys. For example, a channel node can have * `title`, `link`, `itunes:category`, and `item` keys; text-only children are * stored as strings, children with attributes or their own children are stored * as ParsedNodes, and repeated child names are stored as arrays. */ /** * @typedef {Object} XmlBaseEntry * An entry in the this.xmlbase stack. The '#name' key holds the element name * that established the base URL; the '#' key holds the xml:base URL value. */ /** * @typedef {Object} SaxProcessingInstruction * Payload of the sax 'processinginstruction' event. * @property {string} name - Processing instruction target, e.g. "xml" * @property {string} body - The rest of the processing instruction content */ /** * @typedef {Object} AddressParserResult * Shape of each item returned by the addressparser module. * @property {string} [name] * @property {string} [address] */ /** * @typedef {Object} FeedParserState * Instance properties set up by FeedParser.prototype.init and the constructor. * @property {Object} meta - Parsed feed metadata; shape evolves during parsing * @property {import('../index').Options} options * @property {Object.} _namespaces * @property {boolean} _emitted_meta * @property {Array.} stack * @property {Array.} xmlbase * @property {boolean} in_xhtml * @property {Object} xhtml * @property {Error[]} errors * @property {import('sax').SAXStream} stream - The underlying sax stream * @property {function(): void} init * @property {function(): void} handleEnd * @property {function(Error): void} handleSaxError * @property {function(): void} resumeSaxError * @property {function(Error): void} handleError * @property {function(SaxProcessingInstruction): void} handleProcessingInstruction * @property {function(import('sax').QualifiedTag): void} handleOpenTag * @property {function(string): void} handleCloseTag * @property {function(string): void} handleText * @property {function(Object., string): ParsedAttributes} handleAttributes * @property {function(ParsedNode, import('../index').Type, import('../index').Options): Object} handleMeta * @property {function(ParsedNode, import('../index').Type, import('../index').Options): Object} handleItem */ /** * @typedef {import('readable-stream').Transform & FeedParserState} FeedParserInstance */ /** @this {FeedParserInstance} */ FeedParser.prototype[Symbol.asyncIterator] = async function* () { var resolve = null; var error = null; var ended = false; function onReadable () { if (resolve) { resolve(); resolve = null; } } function onEnd () { ended = true; if (resolve) { resolve(); resolve = null; } } function onError (err) { error = err; if (resolve) { resolve(); resolve = null; } } this.on('readable', onReadable); this.on('end', onEnd); this.on('error', onError); try { while (true) { var item; while ((item = this.read()) !== null) { yield item; } if (ended) break; if (error) throw error; await new Promise(function (r) { resolve = r; }); if (error) throw error; } } finally { this.removeListener('readable', onReadable); this.removeListener('end', onEnd); this.removeListener('error', onError); if (!ended && !this.destroyed) { this.destroy(); } } }; exports = module.exports = FeedParser;