import zlib from 'node:zlib'; import { type FeedItem, default as FeedMe } from 'feedme'; import miniget from 'miniget'; import NewsEmitter from 'newsemitter'; import { TypedEmitter } from 'tiny-typed-emitter'; // Used for the skipdays tag. const DAYS = [ 'sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', ]; type Options = FeedSub.Options; type DefaultOptions = FeedSub.DefaultOptions; interface FeedSubEvents { error: (err: Error) => void; item: (item: FeedItem) => void; items: (items: FeedItem[]) => void; } class FeedSub extends TypedEmitter { static FeedSub = FeedSub; static default = FeedSub; public feed: string; public options: DefaultOptions; public news: NewsEmitter; public getOpts: miniget.Options & { headers: Record }; private _first: boolean; private _intervalid?: NodeJS.Timeout; constructor(feed: string, options?: Options) { super(); this.feed = feed; this.options = Object.assign( { interval: 10, forceInterval: false, autoStart: false, emitOnStart: false, lastDate: null, history: [], maxHistory: 10, skipHours: false, hoursToSkip: null, skipDays: false, daysToSkip: null, requestOpts: {}, }, options, ); // Create news emitter. this.news = new NewsEmitter({ maxHistory: this.options.maxHistory, manageHistory: true, identifier: (item: FeedItem[]) => { const feedItem = item[0]; return [ feedItem.title, feedItem.link, feedItem.pubdate, feedItem.published, feedItem.updated, ].join(','); }, }); this.news.addHistory('item', this.options.history); this._first = (this.news.history.get('item') as Set).size === 0; this.getOpts = Object.assign( { headers: {} as Record, acceptEncoding: { gzip: () => zlib.createGunzip(), deflate: () => zlib.createInflate(), }, }, this.options.requestOpts, ); if (this.options.autoStart) { this.start(true); } } /** * Start calling the read function on interval. */ start(readOnStart?: boolean) { this.stop(); const ms = this.options.interval * 60000; this._intervalid = setInterval(this.read.bind(this), ms); if (readOnStart) { this.read(); } } /** * Stop interval if any */ stop() { clearInterval(this._intervalid); } /** * Reads feed and determines if there are any new items * emits new items through event `item` * if there are new items, emits `items` event at end with all items * * if callback is given, calls callback with all new items * even when there are 0. */ read(callback?: (err: null | Error, items?: FeedItem[]) => void) { let ended = false; let req: miniget.Stream; const items: FeedItem[] = []; let newItems: FeedItem[] = []; let sortOrder = 0; const error = (err: Error) => { ended = true; this._first = false; if (typeof callback === 'function') { callback(err); } else { this.emit('error', err); } req.destroy(); }; const success = () => { if (ended) { return; } ended = true; if (sortOrder <= 0) { newItems.reverse(); } this.news.addHistory( 'item', newItems.map((item) => [item]), ); if (this._first && !this.options.emitOnStart) { newItems = []; } newItems.forEach((item) => { this.emit('item', item); }); this.emit('items', newItems); if (req) { req.destroy(); } if (typeof callback === 'function') { callback(null, newItems); } }; // If skipHours or skipDays are enabled and feed provides hours/days // to skip and it's currently one of those hours/days, abort. const now = new Date(); const shouldSkip = () => { if (this._first && this.options.emitOnStart) { return false; } if ( this.options.hoursToSkip && this.options.hoursToSkip.indexOf(now.getHours()) !== -1 ) { return true; } if ( this.options.daysToSkip?.some((day) => { return day.toLowerCase() === DAYS[now.getDay()]; }) ) { return true; } return false; }; if (shouldSkip()) { return success(); } req = miniget(this.feed, this.getOpts); req.on('response', (res) => { // Check if not modified code is sent back // in this case, the body will be empty. if (res.statusCode === 304) { return success(); } // Check headers for conditional get. if (res.headers['last-modified']) { this.getOpts.headers['If-Modified-Since'] = res.headers['last-modified']; } if (res.headers.etag) { this.getOpts.headers['If-None-Match'] = res.headers.etag; } // Save date. let date: string; const getdate = (text: string) => (date = text); // Create feed parser. const parser = new FeedMe(); parser.on('error', error); // Try to get date from one of the fields. parser.once('pubdate', getdate); parser.once('lastbuilddate', getdate); parser.once('updated', getdate); // Change interval time if ttl available. if (!this.options.forceInterval) { parser.once('ttl', (minutes) => { minutes = parseInt(minutes, 10); // Only update if ttl is longer than requested interval. if (minutes > this.options.interval) { this.options.interval = minutes; if (this.options.autoStart) { this.start(false); } } }); } // Listen for skipHours if enabled. if (this.options.skipHours && !this.options.hoursToSkip) { parser.once('skiphours', (data) => { this.options.hoursToSkip = [] .concat(data.hour) .map((h) => parseInt(h, 10)); }); } // Listen for skipDays if enabled. if (this.options.skipDays !== false && !this.options.daysToSkip) { parser.once('skipdays', (data) => { this.options.daysToSkip = [].concat(data.day); }); } // Compare date when first item is encountered. const firstitem = (item: FeedItem) => { if (shouldSkip()) { return success(); } // Continue if dates differ. if (date) { this.options.lastDate = date; } parser.on('item', getItem); getItem(item); }; parser.once('item', firstitem); const getItemDate = (item: FeedItem) => Date.parse(item.pubdate as string) || Date.parse(item.published as string) || 0; const getItem = (item: FeedItem) => { if (sortOrder === 0) { items.push(item); sortOrder = getItemDate(item) - getItemDate(items[0]); if (sortOrder < 0) { items.forEach(getOlderItem); } else if (sortOrder > 0) { items.forEach(getNewerItem); } } else if (sortOrder < 0) { getOlderItem(item); } else { getNewerItem(item); } }; const getOlderItem = (item: FeedItem) => { if (this._first) { newItems.push(item); } else if (!ended) { const emitted = this.news.emit('item', item); if (emitted) { newItems.push(item); } else { // Check if this item has already been read in previous requests // if it has, then stop parsing the rest of the document. parser.removeListener('item', getItem); success(); } } }; let foundPrevItem = false; const getNewerItem = (item: FeedItem) => { if (this._first) { newItems.push(item); } else if (!foundPrevItem && !this.news.emit('item', item)) { foundPrevItem = true; } else if (foundPrevItem && this.news.emit('item', item)) { newItems.push(item); } }; req.pipe(parser); parser.on('finish', () => { if (ended) { return; } // Process items in descending order if no order found at end. if (sortOrder === 0 && newItems.length === 0) { items.forEach(getOlderItem); } success(); this._first = false; }); }); req.on('error', error); } } namespace FeedSub { export interface Options { interval?: number; forceInterval?: boolean; autoStart?: boolean; emitOnStart?: boolean; lastDate?: null | string; history?: string[]; maxHistory?: number; skipHours?: boolean; hoursToSkip?: number[]; skipDays?: boolean; daysToSkip?: string[]; requestOpts?: miniget.Options; } export type DefaultOptions = Required; export type FeedItem = import('feedme').FeedItem; } // Statically recognizable exports for Node.js cjs-module-lexer (ESM named import support) module.exports = FeedSub; module.exports.default = FeedSub; module.exports.FeedSub = FeedSub; export = FeedSub;