true, // Maximum length of feed 'max_items' => 25, // Set to true to store feeds as RSS 2.0, false as Atom 'use_rss_format' => true, // Directory of feed files 'dir_feeds' => 'feeds', // Instance administrator contact 'instance_contact' => '', // Instance visual customization 'icon' => 'https://raw.githubusercontent.com/Warhammer40kGroup/wh40k-icon/master/src/svgs/librarius-02.svg', 'logo' => 'https://raw.githubusercontent.com/Warhammer40kGroup/wh40k-icon/master/src/svgs/librarius-02.svg', 'custom_xslt' => '', 'custom_css' => '', // Admin ID to attach instance messages to all feeds 'admin_id' => '', // An exported cookie file to access walled content 'cookie_file' => '', // Time before feeds are considered abandoned 'delete_abandoned_after' => 31536000, // 1 year -> 60*60*24*365 'delete_bogus_after' => 7776000, // 3 months -> 60*60*24*30*3 // External modules 'mod_custom_extractor' => __DIR__ . '/custom_extractor.php', 'mod_readability' => __DIR__ . '/vendor/autoload.php', ); // Create a unique id for users function make_id() { return hash('sha256', random_bytes(18)); } // Check if an id is the admin function is_admin($param_id) { global $g_config; return !empty($param_id) && !empty($g_config['admin_id']) && (hash_equals($g_config['admin_id'], $param_id)); } // Read configuration from JSON file function update_configuration() { global $g_config_file, $g_config; $json = @file_get_contents($g_config_file); if (!empty($json)) { $data = json_decode($json); foreach(array_keys($g_config) as $k) if (property_exists($data, $k)) $g_config[$k] = $data->$k; return; } if (empty($g_config['admin_id'])) $g_config['admin_id'] = make_id(); file_put_contents($g_config_file, json_encode($g_config, JSON_PRETTY_PRINT)); } // Helper to load readability and custom extraction modules function load_ext_modules() { global $g_config; $mods = ['mod_custom_extractor', 'mod_readability']; foreach ($mods as $mod) { try { if (is_file($g_config[$mod])) require_once $g_config[$mod]; } catch (\Throwable $e) { } } } // Sanitize a string to remove any invalid characters function sanitize_text($s) { if (empty($s)) return ''; $s = trim($s); // Remove control characters $s = preg_replace('/(?>[\x00-\x1F]|\xC2[\x80-\x9F]|\xE2[\x80-\x8F]{2}|\xE2\x80[\xA4-\xA8]|\xE2\x81[\x9F-\xAF])/', ' ', $s); // Reduce all multiple whitespace to a single space $s = preg_replace('/\s+/', ' ', $s); $s = html_entity_decode($s); $s = htmlspecialchars($s); return $s; } // Create the title of the RSS-Librarian instance function get_title($param_id) { return 'RSS-Librarian' . ((!empty($param_id)) ? (' (' . (is_admin($param_id) ? 'admin' : substr($param_id, 0, 4)) . ')') : ''); } // Get a simple local version number function get_version() { $f = basename(__FILE__); return 'two-' . date('Y-m-d', filemtime($f)) . '-' . substr(hash_file('sha256', $f), 0, 7); } // Produce path for local feed file function get_local_feed_file($param_id) { global $g_config; return $g_config['dir_feeds'] . '/' . $param_id . '.xml'; } // Produce URL for user feed function get_feed_url($param_id) { global $g_url_base; return $g_url_base . '/' . get_local_feed_file($param_id); } // Helper to attach new XML element to existing one function sxml_attach(SimpleXMLElement $to, SimpleXMLElement $from) { $toDom = dom_import_simplexml($to); $fromDom = dom_import_simplexml($from); $new_node = $toDom->ownerDocument->importNode($fromDom, true); $firstSibling = $toDom->getElementsByTagName('item')->item(0); $toDom->insertBefore($new_node, $firstSibling); } // Creates the base stub for an Atom feed function make_atom_feed($title, $subtitle, $personal_url, $feed_url, $ts_updated) { global $g_url_librarian, $g_config; return ' ' . (($g_config['custom_xslt'] !== "") ? '' : '') . ' ' . $title . ' ' . $subtitle . ' ' . $personal_url . ' ' . date('Y-m-d\TH:i:s\Z', $ts_updated) . ' RSS-Librarian RSS-Librarian ' . $g_config['icon'] .' ' . $g_config['logo'] .' '; } // Creates the base stub for an RSS feed function make_rss_feed($title, $subtitle, $personal_url, $feed_url, $ts_updated) { global $g_url_librarian, $g_config; return ' ' . (($g_config['custom_xslt'] !== "") ? '' : '') . ' ' . $title . ' ' . $subtitle . ' ' . $personal_url . ' RSS-Librarian '; } // Creates an XML element for a feed stub for the configured format (RSS or Atom) function make_feed($param_id) { global $g_config, $g_url_librarian; $title = get_title($param_id); $subtitle = 'A read-it-later service for RSS purists'; $personal_url = get_personal_url($param_id); $feed_url = get_feed_url($param_id); $ts_updated = time(); $feed_xml_str = ''; if ($g_config['use_rss_format']) $feed_xml_str = make_rss_feed($title, $subtitle, $personal_url, $feed_url, $ts_updated); else $feed_xml_str = make_atom_feed($title, $subtitle, $personal_url, $feed_url, $ts_updated); return simplexml_load_string($feed_xml_str); } // Creates an Atom feed entry: https://validator.w3.org/feed/docs/atom.html function make_atom_item($item) { $datef = date('Y-m-d\TH:i:s\Z', ($item['date'] ?? '') ?: time()); $author_element = !empty($item['author']) ? '' . sanitize_text($item['author']) . '' : ''; $enclosure_element = !empty($item['enclosure']) ? '' : ''; return ' ' . sanitize_text($item['title']) . ' ' . sanitize_text($item['url']) .' ' . $datef . ' ' . $datef . '' . $author_element . '' . sanitize_text($item['content']) . '' . $enclosure_element . ''; } // Creates an RSS feed entry: https://validator.w3.org/feed/docs/rss2.html, http://purl.org/dc/elements/1.1/ function make_rss_item($item) { $author_element = !empty($item['author']) ? '' . sanitize_text($item['author']) . '' : ''; $title_element = !empty($item['title']) ? '' . sanitize_text($item['title']) . '' : ''; $enclosure_element = !empty($item['enclosure']) ? '' : ''; return ' ' . sanitize_text($item['url']) . ' ' . $title_element . ' ' . sanitize_text($item['url']) .' ' . sanitize_text($item['content']) . ' ' . $author_element . ' ' . date('D, d M Y H:i:s O', ($item['date'] ?? '') ?: time()) . ' ' . $enclosure_element . ' '; } // Create XML element for an RSS item function make_feed_item($item) { global $g_config; $item_xml_str = ($g_config['use_rss_format']) ? make_rss_item($item) : make_atom_item($item); return simplexml_load_string($item_xml_str); } // Read RSS XML item and convert to internal item format function read_rss_item($xml_item) { $creator = $xml_item->xpath('dc:creator'); $author = empty($creator) ? sanitize_text($xml_item->author) : sanitize_text($creator[0][0]); $enclosure = empty($xml_item->enclosure) ? null : [$xml_item->enclosure->url, $xml_item->enclosure->type, $xml_item->enclosure->length]; return [ 'url' => $xml_item->guid, 'title' => sanitize_text($xml_item->title), 'content' => $xml_item->description, 'date' => strtotime($xml_item->pubDate) ?: time(), 'author' => $author, 'enclosure' => $enclosure, ]; } // Read Atom XML item and convert to internal item format function read_atom_item($xml_item) { $enclosure = null; foreach ($xml_item->link as $link) if (!empty($link->rel) && $link->rel == "enclosure") $enclosure = [$link->href, $link->type, $link->length]; return [ 'url' => $xml_item->id, 'title' => sanitize_text($xml_item->title), 'content' => $xml_item->content, 'date' => strtotime($xml_item->updated ?: $xml_item->published) ?: time(), 'author' => sanitize_text($xml_item->author->name), 'enclosure' => $enclosure, ]; } // Read feed into an array of internal feed items sorted by date function read_feed_file($param_id) { global $g_url_librarian; $local_feed_file = get_local_feed_file($param_id); $items_sorted = array(); // Try to open local subscriptions and copy items over $local_feed_text = @file_get_contents($local_feed_file); if (!empty($local_feed_text)) { $old_feed_xml = simplexml_load_string($local_feed_text); // Detect old feed file format $is_rss = $old_feed_xml->getName() == 'rss'; // Read into array of internal items if ($is_rss) { $old_feed_xml->registerXPATHNamespace('dc', 'http://purl.org/dc/elements/1.1/'); foreach ($old_feed_xml->channel->item as $xml_item) $items_sorted[] = read_rss_item($xml_item); } else { foreach ($old_feed_xml->entry as $xml_item) $items_sorted[] = read_atom_item($xml_item); } // Sort by date newest to oldest usort($items_sorted, function($a, $b) { return $b['date'] - $a['date']; }); } return $items_sorted; } // Write XML data to feed file function write_feed_file($param_id, $items) { global $g_config; // Check for subs dir if (!is_dir($g_config['dir_feeds'])) mkdir($g_config['dir_feeds']); $feed_xml = make_feed($param_id); // Re-attach foreach($items as $item) sxml_attach($g_config['use_rss_format'] ? $feed_xml->channel : $feed_xml, make_feed_item($item)); // Write formatted xml to feed file $local_feed_file = get_local_feed_file($param_id); $dom = new DOMDocument('1.0'); $dom->preserveWhiteSpace = false; $dom->formatOutput = true; $dom->loadXML($feed_xml->asXML()); file_put_contents($local_feed_file, $dom->saveXML()); } // Create a appropriate HTML tag for an enclosure embedding function make_enclosure_embed($enclosure) { if (empty($enclosure) || count($enclosure) != 3) return ''; if (str_contains($enclosure[1], "video/")) return '

'; else if (str_contains($enclosure[1], "audio/")) return '

'; else if (str_contains($enclosure[1], "image/")) return '

'; else if (str_contains($enclosure[1], "application/pdf")) return '

'; return '

' . sanitize_text($enclosure[0]) . '

'; } // Fetch all meta tags with into a lookup dictionary function get_all_meta_tags($html) { $result = []; if (preg_match_all('/ $key) { $val = trim($matches[2][$i]); array_key_exists($key, $result) ? $result[$key] .= '; ' . $val : $result[$key] = $val; } } return $result; } // Fetch content of a URL or peek into the header to fetch only the mimetype function fetch_url($url, $peek_header = false) { global $g_config; $cookie_file = ($g_config['cookie_file'] ?? '') ?: tempnam(sys_get_temp_dir(), 'rsslib_cookies_'); $ch = curl_init($url); curl_setopt_array($ch, [ CURLOPT_NOBODY => $peek_header, CURLOPT_RETURNTRANSFER => true, CURLOPT_FOLLOWLOCATION => true, CURLOPT_MAXREDIRS => 10, CURLOPT_USERAGENT => 'Mozilla/5.0', CURLOPT_ENCODING => '', CURLOPT_COOKIEFILE => $cookie_file, CURLOPT_COOKIEJAR => $cookie_file, CURLOPT_TIMEOUT => 15, CURLOPT_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS, CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS, ]) ; $content = curl_exec($ch); if (empty($g_config['cookie_file'])) @unlink($cookie_file); if ($content === false) return ["", "", 0]; $content_type = curl_getinfo($ch, CURLINFO_CONTENT_TYPE); $content_type = explode(';', $content_type)[0]; $content_length = curl_getinfo($ch, CURLINFO_CONTENT_LENGTH_DOWNLOAD); return [$content, $content_type, $content_length]; } // "Extract" binary files function extract_content_binary($url) { $html = ''; $enclosure = null; [$content_data, $content_type, $content_length] = fetch_url($url, true); if (str_contains($content_type, "text/")) return []; $enclosure = [$url, $content_type, $content_length]; $html = make_enclosure_embed($enclosure); $exp = explode('/', $url); return [ 'title' => end($exp), 'content' => $html, 'enclosure' => $enclosure, ]; } // Local custom extraction path function extract_content_custom($url) { global $g_config; if (function_exists('custom_extractor')) return custom_extractor($url); return []; } // Extract text from HTML/structured input using readability or metatags function extract_content_structured($url, $html = null) { global $g_config; if (empty($html)) [$html, $content_type, $content_length] = fetch_url($url); $meta = get_all_meta_tags($html); // Tidy up HTML if (function_exists('tidy_parse_string')) { $tidy = tidy_parse_string($html, array(), 'UTF8'); $tidy->cleanRepair(); $html = $tidy->value; } // Local Readability.php extraction path $rdata = []; if (class_exists('fivefilters\Readability\Readability')) { try { $readability = new Readability(fixRelativeURLs: true, originalURL: $url); $article = @$readability->parse($html); $rdata = [ 'title' => $article->title, 'content' => $article->content, 'author' => $article->byline, ]; } catch (ParseException $e) {} } // Fallback solution: Naive body extraction if (!isset($rdata['content']) || empty($rdata['content'])) { libxml_use_internal_errors(true); $doc = new DOMDocument(); $doc->loadHTML('' . $html, LIBXML_NOWARNING | LIBXML_NOERROR); libxml_clear_errors(); $bodyNodes = $doc->getElementsByTagName('body'); $body = $bodyNodes->item(0); $rdata['content'] = $doc->saveHTML($body) ?: ''; } return [ 'title' => ($rdata['title'] ?? '') ?: ($meta['og:title'] ?? '') ?: ($meta['twitter:title'] ?? '') ?: ($meta['title'] ?? '') ?: $url, 'content' => ($rdata['content'] ?? '') ?: ($meta['og:description'] ?? '') ?: ($meta['twitter:description'] ?? '') ?: ($meta['description'] ?? '') ?: '', 'author' => ($rdata['author'] ?? '') ?: ($meta['og:site_name'] ?? '') ?: ($meta['twitter:site'] ?? '') ?: ($meta['fediverse:creator'] ?? '') ?: ($meta['author'] ?? '') ?: '', ]; } // Extract content using the first successful method, otherwise just return URL function extract_content($url) { global $g_config; $item = []; if (empty($item)) $item = extract_content_binary($url); if (empty($item)) $item = extract_content_custom($url); if (empty($item)) $item = extract_content_structured($url); $item['url'] = $url; $item['date'] = time(); if (!$g_config['extract_content'] || empty($item['content'])) $item['content'] = 'No content available, please enable reader mode for this entry.'; return $item; } // Remove URL from personal feed function remove_url($param_id, $param_url) { $items = read_feed_file($param_id); foreach($items as $i => $item) { if ($item['url'] == $param_url) { unset($items[$i]); $items = array_values($items); write_feed_file($param_id, $items); return true; } } return false; } // Add stored article item to array of items if a personal feed function add_item($items, $item) { global $g_config; // Check max item count, remove anything beyond $c = count($items); while ($c >= $g_config['max_items']) { unset($items[$c-1]); $c--; } // Fetch content and add to items array_unshift($items, $item); return $items; } // Add a custom article text to a personal feed function add_custom_item($param_id, $message) { global $g_url_librarian; $items = read_feed_file($param_id); $item = [ 'url' => '', 'title' => 'RSS-Librarian Instance Notice ' . date("Y-m-d H:i"), 'content' => $message, 'date' => time(), 'author' => 'Admin', ]; $items = add_item($items, $item); write_feed_file($param_id, $items); } // Log into admin feed function add_admin_log_item($msg) { global $g_config; add_custom_item($g_config['admin_id'], $msg); } // Add URL to personal feed function add_url($param_id, $param_url) { $items = read_feed_file($param_id); // Turn parameter into fully qualified URL $parsed = parse_url($param_url); if (empty($parsed['scheme'])) $param_url = 'https://' . ltrim($param_url, '/'); // Check if item already exists foreach($items as $item) { if ($item['url'] == $param_url) return false; } $items = add_item($items, extract_content($param_url)); write_feed_file($param_id, $items); return true; } // Count number of feeds in feed directory function count_feeds() { global $g_config; $filecount = count(glob($g_config['dir_feeds'] . '/*.xml')); return $filecount; } // Fetch personal URL string function get_personal_url($param_id) { global $g_url_librarian; return $g_url_librarian . '?id=' . $param_id; } // Go through feeds directory and clean up likely abandoned feed files function run_maintenance($is_dry_run) { global $g_config; $num_removed = 0; $dir = new DirectoryIterator($g_config['dir_feeds']); $current_time = time(); $abandoned_feeds = array(); foreach ($dir as $fileinfo) { if ($fileinfo->isDot() || $fileinfo->getExtension() != "xml") continue; $age = $current_time - $fileinfo->getMTime(); $feed_file = $fileinfo->getPathname(); $feed_id = substr($fileinfo->getBasename(), 0, -4); // Skip admin feed if ($feed_id == $g_config['admin_id']) continue; // Delete abandoned files older than $g_delete_abandoned_after if ($age > $g_config['delete_abandoned_after']) $abandoned_feeds[] = $feed_id; // These are likely files created by accident (max one entry) else if ($age > $g_config['delete_bogus_after']) { $items = read_feed_file($feed_id); if (count($items) <= 1) $abandoned_feeds[] = $feed_id; } } if (!$is_dry_run) foreach($abandoned_feeds as $abandoned) unlink(get_local_feed_file($abandoned)); return $abandoned_feeds; } // Display a list of URLs that are stored in a personal feed and allow removal of items function show_saved_urls($param_id) { if (empty($param_id)) return; $items = read_feed_file($param_id); print('

Stored Feed Items

    '); foreach($items as $item) print('
  1. ❌ ' . sanitize_text($item['title']) . '
  2. '); print('
'); } // Print message with tools for RSS feed management and instance information function show_footer($param_id) { global $g_url_librarian; global $g_config; $personal_url = get_personal_url($param_id); $feed_url = get_feed_url($param_id); print(' '); } // Print header with personal urls function show_header($param_id) { global $g_url_librarian, $g_config; print('

' . get_title($param_id) . '

"Knowledge is power, store it well."

[Github]'); if (!empty($param_id)) print(' [Manage] [Subscribe] '); print('

'); } // Print main interface function show_interface($param_id, $param_url, $param_delete) { global $g_url_librarian, $g_config; // Adding URL for the first time, make sure user has saved their personal URLs! if (empty($param_id) && !empty($param_url)) { // Create new user id $param_id = make_id(); $personal_url = get_personal_url($param_id); $feed_url = get_feed_url($param_id); print('

You are about to create a new feed

Please confirm that you have bookmarked:

Your personal URL

and will subscribe with a feed reader to:

Your personal feed

'); } // Admin interface else if (!empty($param_id) && is_admin($param_id)) { if (empty($param_url) && empty($param_delete)) { $abandoned_feeds = run_maintenance(true); print('

Send a message to all feeds

Clean up abandoned feeds

There are currently ' . count($abandoned_feeds) . ' abandoned feeds.

'); } else if (!empty($param_url)) { // iterate over all feeds $feeds = glob($g_config['dir_feeds'] . '/*.xml'); foreach($feeds as $f) { $feed_id = basename($f, '.xml'); add_custom_item($feed_id, $param_url); } print('

Message sent to ' . count($feeds) . ' feeds.

'); add_admin_log_item('Message sent to ' . count($feeds) . ' feeds:
' . sanitize_text($param_url)); } else if (!empty($param_delete)) { $abandoned_feeds = run_maintenance(false); $num_cleaned = count($abandoned_feeds); if ($num_cleaned > 0) { print('

Cleaned up ' . $num_cleaned . ' abandoned feeds

'); add_admin_log_item('Cleaned up abandoned feeds:
' . implode('
', $abandoned_feeds)); } } } // Returning user view else { print('

Add a new URL to your feed

'); // Add or remove URL if (!empty($param_id) && !empty($param_url)) { if ($param_delete === true) { if(remove_url($param_id, $param_url)) print('

' . sanitize_text($param_url) . ' removed

'); } else { if(add_url($param_id, $param_url)) { print('

' . sanitize_text($param_url) . ' added

'); add_admin_log_item('User ' . $param_id . ' added:
' . sanitize_text($param_url) . ''); } else print('

URL already added!

'); } } print('
'); show_saved_urls($param_id); } } $param_url = filter_input(INPUT_GET, 'url'); $param_id = filter_input(INPUT_GET, 'id', FILTER_VALIDATE_REGEXP, [ 'options' => [ 'regexp' => '/^[0-9a-z]+$/' ]]); $param_delete = filter_input(INPUT_GET, 'delete', FILTER_VALIDATE_BOOL); update_configuration(); load_ext_modules(); ?> <?php print(get_title($param_id)) ?> '); } ?>