// ==UserScript== // @name Onward // @namespace https://github.com/SysAdminDoc/Onward_Userscript // @version 0.2.0 // @description Lean auto-pager. Finds the next page on paginated sites and appends it below the current one as you scroll. No rule database needed. // @author SysAdminDoc // @license MIT // @homepageURL https://github.com/SysAdminDoc/Onward_Userscript // @supportURL https://github.com/SysAdminDoc/Onward_Userscript/issues // @updateURL https://raw.githubusercontent.com/SysAdminDoc/Onward_Userscript/main/src/onward.user.js // @downloadURL https://raw.githubusercontent.com/SysAdminDoc/Onward_Userscript/main/src/onward.user.js // @match *://*/* // @grant GM_getValue // @grant GM_setValue // @grant GM_registerMenuCommand // @grant GM_xmlhttpRequest // @grant GM.getValue // @grant GM.setValue // @grant GM.xmlHttpRequest // @grant GM.registerMenuCommand // @connect self // @connect wedata.net // @connect hoothin.github.io // @connect cdn.jsdelivr.net // @run-at document-idle // @noframes // ==/UserScript== /* Onward is laid out as a factory so the detection code can be unit tested in * Node (module.exports) while the userscript manager simply boots it. */ (function (root, factory) { 'use strict'; const api = factory(root); if (typeof module === 'object' && module.exports) module.exports = api; else api.boot(); })(typeof window !== 'undefined' ? window : globalThis, function (win) { 'use strict'; const VERSION = '0.2.0'; const TAG = '[Onward]'; // --------------------------------------------------------------------------- // Localization // --------------------------------------------------------------------------- const STRINGS = { en: { pageLoaded: (n, c) => `Page ${n} loaded, ${c} ${c === 1 ? 'item' : 'items'}.`, noMore: 'No more pages.', noContent: 'no content found on the next page', allRepeats: 'The site returned a page we already have. End of results.', stoppedByYou: 'Stopped by you.', resume: 'Resume', loadPage: (n) => `Load page ${n}`, loading: (n, batch) => `Loading page ${n}${batch ? ` (${batch.i} of ${batch.n})` : ''}…`, noMoreItems: 'No more items.', skipFooter: 'Skip to footer', skipFooterMsg: (s) => `Loading waits ${s} s so you can reach the footer.`, stop: 'Stop loading', redrawing: 'This site keeps redrawing its list, so pages can’t be added here.', noGrowth: 'Pages were added but the page isn’t getting longer, so Onward stopped.', timedOut: 'The page didn’t respond.', retry: 'Retry', redirected: 'redirected to another site', enabled: (h) => `Enabled on ${h}`, disabled: (h) => `Disabled on ${h}`, settingsSaved: 'Settings saved and applied.', importDone: 'Settings imported. Reload the page to apply them.', diagCopied: 'Diagnostics copied.', diagFailed: 'Couldn’t copy. Select the text and copy it yourself.', notJson: 'Not valid JSON.', notSettings: 'Not a settings object.', cancel: 'Cancel', save: 'Save', exportBtn: 'Export', importBtn: 'Import', copyDiag: 'Copy diagnostics', updateLists: 'Update rule lists now', loadMore: (n) => `Load ${n} more pages`, runHere: 'Run Onward here anyway', pick: 'Pick next and items', settings: 'Settings', toggle: 'Toggle Onward on this site', }, 'zh-CN': { pageLoaded: (n, c) => `第 ${n} 页已加载,${c} 个项目。`, noMore: '没有更多页面了。', noContent: '下一页未找到内容', allRepeats: '网站返回了已有的页面。结果结束。', stoppedByYou: '已被您停止。', resume: '继续', loadPage: (n) => `加载第 ${n} 页`, loading: (n, batch) => `正在加载第 ${n} 页${batch ? `(第 ${batch.i} / ${batch.n})` : ''}…`, noMoreItems: '没有更多项目了。', skipFooter: '跳到页脚', skipFooterMsg: (s) => `加载将等待 ${s} 秒,以便您访问页脚。`, stop: '停止加载', redrawing: '此网站不断重绘其列表,因此无法在此处添加页面。', noGrowth: '页面已添加但页面未变长,Onward 已停止。', timedOut: '页面未响应。', retry: '重试', redirected: '被重定向到其他网站', enabled: (h) => `已在 ${h} 上启用`, disabled: (h) => `已在 ${h} 上禁用`, settingsSaved: '设置已保存并应用。', importDone: '设置已导入。重新加载页面以应用。', diagCopied: '诊断信息已复制。', diagFailed: '无法复制。请选择文本并自行复制。', notJson: '不是有效的 JSON。', notSettings: '不是设置对象。', cancel: '取消', save: '保存', exportBtn: '导出', importBtn: '导入', copyDiag: '复制诊断信息', updateLists: '立即更新规则列表', loadMore: (n) => `加载 ${n} 个更多页面`, runHere: '在此处运行 Onward', pick: '选取下一页和项目', settings: '设置', toggle: '在此网站切换 Onward', }, }; const lang = (typeof navigator !== 'undefined' && navigator.language || 'en').toLowerCase(); const t = STRINGS[lang] || STRINGS[lang.split('-')[0]] || STRINGS.en; // --------------------------------------------------------------------------- // Settings // --------------------------------------------------------------------------- const DEFAULTS = { threshold: 1.5, // start loading when less than N viewport heights remain maxPages: 40, // hard cap per visit spacing: 1000, // ms between page requests to a site, so paging never hammers it separators: true, // show a page bar between pages updateUrl: true, // replaceState to the page currently in view mode: 'auto', // auto | fetch | iframe loadPages: 'auto', // auto | click: load as you scroll, or only when you press the bar's button hostLoadPages: {}, // per-host override of loadPages: { host: 'auto' | 'click' } newTabLinks: false, // open links on added pages in a new tab prefetch: false, // fetch the next page's HTML as soon as the current one lands skipOffscreen: false, // content-visibility:auto on old pages, so the browser skips rendering them skipFooterMs: 30000, // how long "Skip to footer" holds loading off runOn: 'all', // all | listed (only the hosts in allowHosts) allowHosts: [], // Pages Onward stays off: appending pages broke checkout and account flows for other auto-pagers. // A path part that starts with one of these words ("/login.php", "/Cart-Show", "/my-account/", // "/password_reset/"), unless the path runs through a listing ("/tag/account-security" lists posts). skipPaths: '/(my[-_]?)?(checkout|cart|basket|log[-_]?in|sign[-_]?in|sign[-_]?up|register|account|password)([/._?-]|$)', disabledHosts: [], exclude: [ 'mail.google.com', 'docs.google.com', 'drive.google.com', 'calendar.google.com', 'www.youtube.com', 'x.com', 'twitter.com', 'www.facebook.com', 'www.instagram.com', 'outlook.live.com', 'outlook.office.com', 'web.whatsapp.com', 'discord.com', ], rules: [], // user rules (Onward format) sources: [], // URLs of rule lists (Onward or AutoPagerize/wedata JSON) sourceRules: [], // rules 0.1.0 cached flattened; used until every list has a packed copy sourceCache: {}, // each rule list's last good copy: { url: { at, count, hosts, generic, rules } } sourcesUpdated: 0, // last time every source updated cleanly sourcesTried: 0, // last refresh attempt, to back off after failures sourcesLock: 0, // { at, id } of the tab refreshing right now (expires after a minute) sourcesFormat: 0, // the LIST_FORMAT the lists were last all stored in (0: 0.1.0's, or never) schema: 1, // settings version; 0 or absent means 0.1.0 data }; // How rule lists are stored now: packed, indexed by host, general rules apart. // An older form is refreshed on the next page load instead of a week later. const LIST_FORMAT = 3; // The async GM.* API, where a manager offers only that (Userscripts for Safari). const gm4 = () => (typeof GM === 'object' && GM && typeof GM.getValue === 'function' ? GM : null); let asyncValues = null; // every value, read once at boot when only the async API exists const store = { get(key) { try { if (typeof GM_getValue === 'function') return GM_getValue(key, DEFAULTS[key]); if (asyncValues && key in asyncValues) return asyncValues[key]; } catch (e) { /* fall through to defaults */ } return DEFAULTS[key]; }, set(key, value) { try { if (typeof GM_setValue === 'function') GM_setValue(key, value); else if (gm4()) { if (asyncValues) asyncValues[key] = value; Promise.resolve(GM.setValue(key, value)).catch((e) => console.warn(TAG, 'could not save', key, e)); } } catch (e) { console.warn(TAG, 'could not save', key, e); } }, }; /** * With only the async API, read the values once, before anything needs one. * The rule lists are big, so they're only read where some are configured. */ async function loadAsyncValues() { if (typeof GM_getValue === 'function' || !gm4()) return; const read = (k) => Promise.resolve(GM.getValue(k, DEFAULTS[k])).catch(() => DEFAULTS[k]); const light = Object.keys(DEFAULTS).filter((k) => !HEAVY_KEYS.has(k)); const values = await Promise.all(light.map(read)); asyncValues = {}; light.forEach((k, i) => { asyncValues[k] = values[i]; }); if ((asyncValues.sources || []).length) for (const k of HEAVY_KEYS) asyncValues[k] = await read(k); } /** host is one of the listed hosts, or a subdomain of one. */ function hostListed(list, host) { return list.some((x) => host === x || host.endsWith('.' + x)); } /** The path is one Onward stays off (skipPaths, case-insensitive). A broken pattern skips nothing. */ function pathSkipped(pattern, path) { if (!pattern) return false; try { return new RegExp(pattern, 'i').test(path); } catch (e) { return false; } } /** * What toggling Onward on this host does: its new state, and both host lists. * Off takes the host off "Sites to run on" where it's listed by name, and * turns it off outright where it's still covered (every site, or a parent * domain on the list). On undoes that, listing the host when nothing covers it. */ function toggleLists(s, host) { const listed = s.runOn === 'listed'; const covered = (list) => !listed || hostListed(list, host); let allow = s.allowHosts.slice(); let off = s.disabledHosts.filter((h) => h !== host); const wasOn = !s.disabledHosts.includes(host) && covered(allow); if (wasOn) { if (listed) allow = allow.filter((h) => h !== host); if (covered(allow)) off = off.concat(host); } else if (!covered(allow)) { allow = allow.concat(host); } return { on: !wasOn, allowHosts: allow, disabledHosts: off }; } /** The page at href is one Onward stays off: its path, or a single-page app's #/route, matches the pattern. */ const LISTING_RE = /\/(tags?|topics?|categor(?:y|ies)|tagged|labels?|collections?|forums?|r|b|market)\//i; function pageSkipped(pattern, href) { let x; try { x = new URL(href); } catch (e) { return false; } const check = (path) => { if (!pathSkipped(pattern, path)) return false; const m = path.match(new RegExp(pattern, 'i')); if (!m) return true; const before = path.slice(0, m.index); return !LISTING_RE.test(before + '/'); }; if (check(x.pathname)) return true; return /^#!?\//.test(x.hash) && check(x.hash.replace(/^#!?/, '').split('?')[0]); } // Rule lists can be hundreds of kilobytes, so they're read once, where they're used. const HEAVY_KEYS = new Set(['sourceRules', 'sourceCache']); function loadSettings() { const s = {}; for (const key of Object.keys(DEFAULTS)) if (!HEAVY_KEYS.has(key)) s[key] = store.get(key); if (!s.schema) { s.schema = DEFAULTS.schema; store.set('schema', s.schema); } return s; } // --------------------------------------------------------------------------- // Text heuristics // --------------------------------------------------------------------------- const NEXT_WORDS = [ 'next', 'next page', 'next results', 'older', 'older posts', 'older entries', 'older articles', 'older results', 'continue', 'continue reading', 'next chapter', 'next post', 'next article', 'next ›', 'next »', 'suivant', 'suivante', 'page suivante', 'weiter', 'nächste', 'nächste seite', 'siguiente', 'página siguiente', 'próxima', 'próxima página', 'seguinte', 'successivo', 'successiva', 'pagina successiva', 'avanti', 'volgende', 'volgende pagina', 'nästa', 'neste', 'næste', 'seuraava', 'następna', 'następna strona', 'dalej', 'další', 'ďalšia', 'következő', 'următoarea', 'înainte', 'επόμενη', 'sonraki', 'sonraki sayfa', 'следующая', 'следующая страница', 'далее', 'вперёд', 'вперед', 'наступна', 'далі', 'напред', 'tiếp', 'tiếp theo', 'trang sau', 'trang tiếp', 'berikutnya', 'selanjutnya', 'ถัดไป', 'หน้าถัดไป', '下一页', '下一頁', '下页', '下頁', '后页', '後頁', '下一章', '下一篇', '下一张', '下一張', '次へ', '次のページ', '次', '次ページ', '다음', '다음 페이지', 'التالي', 'الصفحة التالية', 'הבא', 'הדף הבא', 'अगला', 'अगला पृष्ठ', ]; const MORE_WORDS = [ 'more', 'load more', 'show more', 'see more', 'view more', 'more results', 'more posts', 'mehr laden', 'mehr anzeigen', 'voir plus', 'charger plus', 'ver más', 'cargar más', 'mostrar mais', 'carregar mais', 'mostra altro', 'meer laden', 'показать ещё', 'показать еще', 'загрузить ещё', '加载更多', '載入更多', '查看更多', 'もっと見る', 'さらに表示', '더 보기', '더보기', ]; const ARROWS = /^[\s>›»→⟩❯▶▸⇒⇨≫]+$/; // Previous, First and Last as whole words ("Página anterior", "Newer posts", // not "Prevention" or "Backyard"). WordPress puts "Newer posts" in .nav-next, // and it leads back toward page 1. const PREV_WORD_RE = /(^|[^a-z])(prev|previous|back|newer|first|last|précédente?|precedente|zurück|vorherige[nrs]?|anterior|предыдущ|назад|попередн)(?![a-z])|上一|上页|上頁|前へ|前の|前页|首页|首頁|尾页|尾頁|末页|末頁|最後|最初|이전|처음|마지막/i; // A label that starts with one of these is Previous however long it is: "Previous page of results". const PREV_STRONG_RE = /^(?:(?:prev|previous|précédente?|precedente|vorherige[nrs]?|anterior)(?![a-z])|предыдущ|попередн|上一|上页|上頁|前へ|前の|前页|이전)/i; // A label that starts like this is a next link whatever follows: "Next (last page)". const NEXT_START_RE = /^(?:(?:next|older|continue|more|load more|show more)(?![a-z])|suivant|weiter|nächste|siguiente|próxima|successiv|volgende|nästa|следующ|далее|下一|下页|下頁|次|다음)/i; const BACK_ARROWS = /^[<«‹←⟨❮◀⇐⇦≪]+$/; // 下一页 / 下一頁 / 次ページ / 下一章 and friends const CJK_NEXT_RE = /^翻?[下后後次][一ー─1]?[页頁张張章话話节節篇]/; const NEXT_ATTR_RE = /(^|[^a-z])next([^a-z]|$)|nextpage|next_page|pagenext|page-next|pager-next|pagination-next|nav-next|pager-older/i; const PREV_ATTR_RE = /(^|[^a-z])prev(ious)?([^a-z]|$)|prevpage|page-prev/i; const PAGINATION_RE = /pag(e|in|ing)|pager|page-?numbers|page-?nav|nav-?links|wp-pagenavi|seite/i; const WIDGET_RE = /slick|swiper|carousel|slider|slideshow|banner|gallery|lightbox|owl-|glide|splide|flickity|tabs?-|modal|datepicker|calendar/i; const MORE_MULTI_RE = /\s|[぀-ヿ一-鿿가-힯]/; // multi-word or CJK "load more" phrases const JUNK_HREF_RE = /^\s*(javascript:|#|$)/i; // Onward fetches next pages with the reader's cookies, so a next link must never sign them out or delete something. const DANGER_URL_RE = /(^|[^a-z])(log[-_]?out|log[-_]?off|sign[-_]?out|sign[-_]?off|unsubscribe)([^a-z]|$)|[/=](delete|destroy|remove)([/?&#]|$)/i; const normalize = (s) => (s || '').replace(/\s+/g, ' ').trim().toLowerCase(); const stripDecor = (s) => s.replace(/^[\s<>›»→⟩❯▶▸«‹←⟨❮◀|\-–—:.()[\]]+|[\s<>›»→⟩❯▶▸«‹←⟨❮◀|\-–—:.()[\]]+$/g, '').trim(); const NEXT_SET = new Set(NEXT_WORDS); const MORE_SET = new Set(MORE_WORDS); function labelOf(el) { const parts = [el.textContent || '']; if (el.tagName === 'INPUT') parts.push(el.value || ''); for (const a of ['aria-label', 'title']) if (el.getAttribute(a)) parts.push(el.getAttribute(a)); const img = el.querySelector && el.querySelector('img[alt]'); if (img) parts.push(img.getAttribute('alt')); return parts.map(normalize).filter(Boolean); } /** What a button says, numbers aside ("Load 20 more" and "Load 15 more" are one button). */ const labelKey = (el) => labelOf(el).join(' ').replace(/\d[\d,.]*/g, '#').replace(/\b(\w{3,})s\b/gi, '$1'); function attrText(el) { return [el.id, el.getAttribute('class'), el.getAttribute('rel'), el.getAttribute('aria-label'), el.getAttribute('title'), el.getAttribute('data-testid')].filter(Boolean).join(' '); } // --------------------------------------------------------------------------- // DOM helpers // --------------------------------------------------------------------------- const isXPath = (sel) => /^(\(|\/|\.\/|id\()/.test(sel.trim()); function queryAll(doc, sel, ctx) { if (!sel) return []; ctx = ctx || doc; try { if (isXPath(sel)) { const out = []; const r = doc.evaluate(sel, ctx, null, 7 /* ORDERED_NODE_SNAPSHOT_TYPE */, null); for (let i = 0; i < r.snapshotLength; i++) { const n = r.snapshotItem(i); if (n.nodeType === 1) out.push(n); } return out; } return Array.from(ctx.querySelectorAll(sel)); } catch (e) { console.warn(TAG, 'bad selector', sel, e.message); return []; } } const hasLayout = (doc) => !!(doc.defaultView && doc.defaultView.innerHeight > 0 && doc.documentElement.clientHeight > 0); function isVisible(el, layout) { if (layout) { if (!el.getClientRects().length) return false; const cs = el.ownerDocument.defaultView.getComputedStyle(el); return cs.visibility !== 'hidden' && cs.opacity !== '0'; } // Parsed documents have no layout; fall back to what the markup says. for (let n = el, depth = 0; n && n.nodeType === 1 && depth < 8; n = n.parentElement, depth++) { if (n.hidden || n.getAttribute('aria-hidden') === 'true') return false; const style = (n.getAttribute('style') || '').replace(/\s/g, '').toLowerCase(); if (style.includes('display:none') || style.includes('visibility:hidden')) return false; } return true; } function inPagination(el) { for (let n = el.parentElement, depth = 0; n && depth < 5; n = n.parentElement, depth++) { if (n.tagName === 'NAV' || n.getAttribute('role') === 'navigation') return n; if (PAGINATION_RE.test(attrText(n))) return n; } return null; } function stripHash(u) { try { const x = new URL(u); x.hash = ''; return x.href; } catch (e) { return u; } } /** A #/route or #!/route hash names a view of a single-page app; #comments is just a spot on the page. */ const routeHash = (u) => /#!?\//.test(u); /** Two addresses of the same page, maybe scrolled to another spot on it. */ const samePage = (a, b) => a === b || (stripHash(a) === stripHash(b) && !routeHash(a) && !routeHash(b)); /** * Clicking it would take the tab to another page: a link with a real address. * A link to the page itself doesn't (a script-driven "Load more" often keeps * one for browsers without scripts). */ const navigates = (el, pageUrl) => { const href = el.getAttribute('href'); if (!href || JUNK_HREF_RE.test(href)) return false; const u = absUrl(href, pageUrl); return !(u && samePage(u, pageUrl)); }; /** * Where an address has a sign-out, unsubscribe or delete word: each path * segment with one (by its position) and each query pair with one. null for * a broken address. */ function dangerPlaces(u) { let x; try { x = new URL(u); } catch (e) { return null; } const words = new Set(); const add = (s) => { const m = s.match(DANGER_URL_RE); if (m) words.add((m[2] || m[4]).toLowerCase().replace(/[-_]/g, '')); }; for (const seg of x.pathname.split('/')) add('/' + seg + '/'); for (const [k, v] of x.searchParams) add('?' + k + '=' + v + '&'); return words; } /** * u would sign the reader out or delete something. A danger word the * page's own address also carries (anywhere) doesn't count: the next page * of /search/logout or ?q=logout isn't signing out, but /logout from * /about is. */ function dangerousUrl(u, pageUrl) { const words = dangerPlaces(u); if (!words) return true; if (!words.size) return false; const here = (pageUrl && dangerPlaces(pageUrl)) || new Set(); for (const w of words) if (!here.has(w)) return true; return false; } function absUrl(v, base) { try { return new URL(v, base).href; } catch (e) { return null; } } function formUrl(el, base) { const form = el.closest('form'); if (!form) return null; const method = (el.getAttribute('formmethod') || form.getAttribute('method') || 'GET').toUpperCase(); if (method !== 'GET') return null; const action = el.getAttribute('formaction') || form.getAttribute('action') || base; let u; try { u = new URL(action, base); } catch (e) { return null; } const data = new URLSearchParams(); for (const inp of form.elements) { if (inp.disabled || !inp.name) continue; if ((inp.type === 'checkbox' || inp.type === 'radio') && !inp.checked) continue; if (inp.type === 'submit' && inp !== el) continue; if (inp.type === 'image' || inp.type === 'file' || inp.type === 'reset') continue; data.append(inp.name, inp.value); } u.search = data.toString(); return u.href; } // --------------------------------------------------------------------------- // Next-page detection // --------------------------------------------------------------------------- /** * Find the link to the next page. * @returns {{url: string|null, el: Element, score: number, how: string}|null} * url is null for a button that has to be clicked (load-more style). */ /** * For links on the page at pageUrl: the address a next link leads to, or null * when Onward won't follow it (not http(s), this page or one already shown, * another origin, a sign-out or delete link). http becomes https on a secure page. */ function nextUrlChecker(pageUrl, seen) { const here = stripHash(pageUrl); // Same origin only (scheme, host and port): a next link elsewhere would be // fetched with the user's cookies into a page whose scripts can read it. const origin = safeOrigin(pageUrl); return (href) => { if (!href || JUNK_HREF_RE.test(href)) return null; let u = absUrl(href, pageUrl); if (!u || !/^https?:/.test(u)) return null; if (/^https:/.test(pageUrl) && /^http:/.test(u)) u = u.replace(/^http:/, 'https:'); if (stripHash(u) === here || (seen && seen.has(stripHash(u)))) return null; if (safeOrigin(u) !== origin) return null; if (dangerousUrl(u, pageUrl)) return null; return u; }; } function findNext(doc, pageUrl, opts) { opts = opts || {}; const layout = opts.layout !== undefined ? opts.layout : hasLayout(doc); const acceptUrl = nextUrlChecker(pageUrl, opts.seen); // 1. Explicit rule if (opts.rule && opts.rule.next) { for (const el of queryAll(doc, opts.rule.next)) { // The same classes often mark Previous on later pages; never take that. if (looksPrevious(el)) continue; const u = acceptUrl(el.getAttribute('href') || el.getAttribute('value')); if (u) return { url: u, el, score: 1000, how: 'rule' }; // A link Onward refused (another site, a page already shown) isn't clicked instead: that would leave the page. if (opts.rule.click && isVisible(el, layout) && !navigates(el, pageUrl)) return { url: null, el, score: 1000, how: 'rule-click' }; } return null; } // 2. in head is the strongest signal a page can give for (const el of doc.querySelectorAll('link[rel~="next" i][href]')) { const u = acceptUrl(el.getAttribute('href')); if (u) return { url: u, el, score: 500, how: 'link-rel' }; } // 3. Score clickable candidates const body = doc.body || doc.documentElement; const cands = body.querySelectorAll('a[href], button, [role="button"], [role="link"], input[type="button"], input[type="submit"]'); let best = null; // On a tie the later one wins (the pager under the list, as Vivaldi and // Pagetual do), unless it's a button and the earlier one is a link. const consider = (c) => { if (!best || c.score > best.score || (c.score === best.score && (c.url || !best.url))) best = c; }; const bumped = new Set(incrementUrls(pageUrl).map(stripHash)); // A site rule that supplied only the items: its own next link wasn't on the // page, so only a next page by address or rel counts ("Next thread" won't). const strict = !!(opts.rule && opts.rule.strictNext); for (const el of cands) { const labels = labelOf(el); const attrs = attrText(el); const rel = (el.getAttribute('rel') || '').toLowerCase().split(/\s+/); const relNext = rel.includes('next'); const longest = labels.reduce((m, s) => Math.max(m, s.length), 0); if (longest > 60 && !relNext) continue; if (looksPrevious(el, labels)) continue; if (isDisabledOrWidget(el, layout)) continue; let score = 0; if (relNext) score += 100; if (NEXT_ATTR_RE.test(attrs)) score += 45; let more = false; for (const raw of labels) { const t = stripDecor(raw); if (NEXT_SET.has(t) || (t.length <= 8 && CJK_NEXT_RE.test(t))) { score += 60; break; } if (t && MORE_SET.has(t)) { score += MORE_MULTI_RE.test(t) ? 45 : 20; more = true; break; } if (!t && ARROWS.test(raw) && raw.length <= 3) { score += 15; break; } } const href = el.getAttribute('href') || (el.type === 'submit' || el.tagName === 'BUTTON' ? formUrl(el, pageUrl) : null); const u = acceptUrl(href); if (strict && !relNext && !(u && nextByAddress(u, pageUrl))) continue; if (u && bumped.has(stripHash(u))) score += 45; // ?page=N+1 or /page/N+1 if (score === 0) continue; if (inPagination(el)) score += 20; else if (score < 45) continue; // a bare arrow or "more" outside pagination is too weak if (!isVisible(el, layout)) score -= 50; if (u) { consider({ url: u, el, score, how: more ? 'more-link' : 'text' }); } else if (layout && isVisible(el, layout) && (more || opts.allowButtons) && !navigates(el, pageUrl)) { // Load-more buttons only make sense on the live page. consider({ url: null, el, score: score - 5, how: 'button' }); } } // 4. Numbered pagination: current page N, link labelled N+1 const numbered = findNumberedNext(doc, acceptUrl, layout); if (numbered) consider(numbered); return best && best.score >= 40 ? best : null; } /** * A Previous (or First/Last) control rather than a next one. The evidence is * taken strongest first: rel, a label that starts like Next, a class that only * says next, a short label with a Previous word (longer labels are usually a * post's title and can say anything), a class that only says previous, and * last a bare back arrow (which points forward on right-to-left pages). */ function looksPrevious(el, raw) { const rel = (el.getAttribute('rel') || '').toLowerCase().split(/\s+/); if (rel.includes('prev')) return true; if (rel.includes('next')) return false; raw = raw || labelOf(el); const labels = raw.map(stripDecor).filter(Boolean); if (labels.some((t) => NEXT_SET.has(t) || MORE_SET.has(t) || NEXT_START_RE.test(t) || CJK_NEXT_RE.test(t))) return false; const attrs = attrText(el); const prevAttr = PREV_ATTR_RE.test(attrs); const nextAttr = NEXT_ATTR_RE.test(attrs); if (nextAttr && !prevAttr) return false; if (labels.some((t) => PREV_STRONG_RE.test(t) || (t.split(' ').length <= 3 && PREV_WORD_RE.test(t)))) return true; if (prevAttr && !nextAttr) return true; const dir = el.closest('[dir]'); return !(dir && /^rtl$/i.test(dir.getAttribute('dir'))) && raw.some((t) => BACK_ARROWS.test(t)); } function isDisabledOrWidget(el, layout) { if (el.disabled || el.getAttribute('aria-disabled') === 'true') return true; for (let n = el, depth = 0; n && n.nodeType === 1 && depth < 6; n = n.parentElement, depth++) { if (n.tagName === 'BLOCKQUOTE') return true; const cls = n.getAttribute('class') || ''; if (depth < 3 && /(^|\s)(disabled|is-disabled)(\s|$)/i.test(cls)) return true; const a = attrText(n); if (WIDGET_RE.test(a) && !PAGINATION_RE.test(a)) return true; } if (layout) { const cs = el.ownerDocument.defaultView.getComputedStyle(el); if (cs.cursor === 'not-allowed' || cs.pointerEvents === 'none') return true; } return false; } /** The URLs page N+1 would most likely have, given this page's URL. */ const PAGE_PARAM_RE = /^(p|page|pg|pn|paged|pagenum|pageno|page_no|pagenumber|seite)$/i; /** u is the page after pageUrl going by the address alone: N+1 of a numbered one, or page 2 of an unnumbered one. */ function nextByAddress(u, pageUrl) { if (incrementUrls(pageUrl).some((x) => stripHash(x) === stripHash(u))) return true; let a; let b; try { a = new URL(pageUrl); b = new URL(u); } catch (e) { return false; } if (a.origin !== b.origin) return false; if (a.pathname === b.pathname) { const added = [...b.searchParams.keys()].filter((k) => !a.searchParams.has(k)); const kept = [...a.searchParams.keys()].every((k) => b.searchParams.get(k) === a.searchParams.get(k)); return kept && added.length === 1 && PAGE_PARAM_RE.test(added[0]) && b.searchParams.get(added[0]) === '2'; } const base = a.pathname.replace(/\/$/, ''); return b.search === a.search && /^\/(page|p|seite|pagina|strona)[/-]?2\/?$/i.test(b.pathname.slice(base.length)) && b.pathname.startsWith(base + '/'); } function incrementUrls(pageUrl) { const out = []; let m = /^(.*[?&](?:p|page|pg|pn|paged|pagenum|pageno|page_no|pagenumber|seite)=)(\d{1,4})((?:[&#].*)?)$/i.exec(pageUrl); if (m) out.push(m[1] + (Number(m[2]) + 1) + m[3]); m = /^(.*\/(?:page|p|seite|pagina|strona)[/-]?)(\d{1,4})(\/?(?:\.s?html?)?(?:[?#].*)?)$/i.exec(pageUrl); if (m) out.push(m[1] + (Number(m[2]) + 1) + m[3]); return out; } function findNumberedNext(doc, acceptUrl, layout) { const marks = doc.querySelectorAll('[aria-current="page"], [aria-current="true"], .current, .active, .selected, .is-current, .is-active, .on, .cur, .curr, strong, em, b, span'); for (const m of marks) { const text = normalize(m.textContent); if (!/^\d{1,5}$/.test(text)) continue; const n = Number(text); const box = inPagination(m); if (box) { for (const a of box.querySelectorAll('a[href]')) { if (normalize(a.textContent) !== String(n + 1) || isDisabledOrWidget(a, layout)) continue; const u = acceptUrl(a.getAttribute('href')); if (u) return { url: u, el: a, score: 70, how: 'numbered' }; } continue; } // Unlabelled pagers: the N+1 link sits right after the marker, in a run of // consecutive numbers with at least two more links (a table cell holding 2 // next to a link to 3 is data, not a pager). const holder = m.tagName === 'A' ? m : m.closest('li, td') || m; const numberAt = (el) => { const a = el && (el.matches('a[href]') ? el : el.querySelector(':scope > a[href]')); const t = a && normalize(a.textContent); return t && /^\d{1,5}$/.test(t) ? { a, n: Number(t) } : null; }; const after = numberAt(holder.nextElementSibling); if (!after || after.n !== n + 1) continue; let more = 0; for (let s = holder.nextElementSibling.nextElementSibling, k = n + 2; s && (numberAt(s) || {}).n === k; s = s.nextElementSibling, k++) more++; for (let s = holder.previousElementSibling, k = n - 1; s && (numberAt(s) || {}).n === k; s = s.previousElementSibling, k--) more++; if (more < 2) continue; const u = acceptUrl(after.a.getAttribute('href')); if (u) return { url: u, el: after.a, score: 55, how: 'numbered' }; } return null; } function safeHost(u) { try { return new URL(u).hostname; } catch (e) { return ''; } } // --------------------------------------------------------------------------- // Content detection // --------------------------------------------------------------------------- const SKIP_TAGS = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'LINK', 'META', 'SVG', 'BR', 'HR']); const STATE_CLASS_RE = /\d|active|selected|current|hover|focus|odd|even|first|last|visible|hidden|loaded|lazy|show|open|in-view|animate/i; function signature(el) { const cls = (el.getAttribute('class') || '').split(/\s+/).filter((c) => c && !STATE_CLASS_RE.test(c)).sort().slice(0, 3); return el.tagName + (cls.length ? '.' + cls.join('.') : ''); } function inChrome(el) { for (let n = el; n && n.nodeType === 1; n = n.parentElement) { const tag = n.tagName; if (tag === 'HEADER' || tag === 'FOOTER' || tag === 'NAV' || tag === 'ASIDE') return true; const role = n.getAttribute('role'); if (role === 'navigation' || role === 'banner' || role === 'contentinfo' || role === 'menu' || role === 'menubar') return true; } return false; } /** * Find the element whose children are the page's repeated items (results, * posts, products, threads). Picks the container whose dominant group of * same-signature children covers the most area (or text, without layout). */ function findContent(doc, opts) { opts = opts || {}; const layout = opts.layout !== undefined ? opts.layout : hasLayout(doc); if (opts.rule && opts.rule.content) { const items = queryAll(doc, opts.rule.content); if (items.length) return { container: items[0].parentElement, items, how: 'rule' }; return null; } const nextEl = opts.nextEl || null; const body = doc.body; if (!body) return null; let best = null; const all = body.getElementsByTagName('*'); const limit = Math.min(all.length, 20000); for (let i = 0; i < limit; i++) { const box = all[i]; if (box.childElementCount < 3 || SKIP_TAGS.has(box.tagName)) continue; const groups = new Map(); for (const ch of box.children) { if (SKIP_TAGS.has(ch.tagName) || (nextEl && ch.contains(nextEl))) continue; const sig = signature(ch); if (!groups.has(sig)) groups.set(sig, []); groups.get(sig).push(ch); } let group = null; for (const g of groups.values()) if (!group || g.length > group.length) group = g; if (!group || group.length < 3) continue; if (inChrome(box)) continue; if (/^(OPTION|SELECT|TR)$/.test(box.tagName) || box.tagName === 'THEAD') continue; let weight = 0; let linkOnly = 0; let textLen = 0; for (const g of group) { const text = normalize(g.textContent); textLen += text.length; if (layout) { const r = g.getBoundingClientRect(); weight += r.width * r.height; } else { weight += Math.min(text.length, 2000) + 40 * g.getElementsByTagName('img').length; } if (text.length < 25 && g.getElementsByTagName('a').length <= 1 && !g.getElementsByTagName('img').length) linkOnly++; } if (weight <= 0) continue; let score = weight * Math.log2(group.length); if (linkOnly / group.length > 0.7) { if (textLen / group.length < 8) continue; // page numbers: the pager itself score *= 0.15; // menus, tag clouds } if (nextEl && (box.compareDocumentPosition(nextEl) & 2) && !box.contains(nextEl)) score *= 0.3; // below the pager if (!best || score > best.score) best = { container: box, score, how: 'auto' }; } if (best) { // Every child counts, not just the dominant group: lists often interleave // row types (title row + meta row). Only the pager is left out. best.items = Array.from(best.container.children).filter((c) => !isPagerChild(c, nextEl)); return best; } // Last resort: a conventional main-content element. const main = doc.querySelector('main, [role="main"], #content, #main, .content, article'); if (main && main.childElementCount) { return { container: main, items: Array.from(main.children).filter((c) => !SKIP_TAGS.has(c.tagName)), how: 'main' }; } return null; } // Describe an element's position so the same container can be found in a // freshly fetched copy of the site, where scripts haven't added state classes. function describePath(el) { const steps = []; for (let n = el; n && n.nodeType === 1 && n.tagName !== 'BODY' && n.tagName !== 'HTML'; n = n.parentElement) { const parent = n.parentElement; const same = parent ? Array.from(parent.children).filter((c) => c.tagName === n.tagName) : [n]; steps.unshift({ tag: n.tagName, id: n.id && !/\d{3,}/.test(n.id) ? n.id : '', cls: (n.getAttribute('class') || '').split(/\s+/).filter((c) => c && !STATE_CLASS_RE.test(c)), idx: same.indexOf(n), }); } return steps; } function resolvePath(doc, steps) { const withId = steps.map((s, i) => (s.id ? i : -1)).filter((i) => i >= 0); let start = 0; let node = doc.body; // Jump to the deepest ancestor with an id when the fetched page has it too. for (let k = withId.length - 1; k >= 0; k--) { const found = doc.getElementById(steps[withId[k]].id); if (found && found.tagName === steps[withId[k]].tag) { node = found; start = withId[k] + 1; break; } } for (let i = start; i < steps.length && node; i++) { const s = steps[i]; const kids = Array.from(node.children).filter((c) => c.tagName === s.tag); if (!kids.length) return null; let pick = s.id ? kids.find((c) => c.id === s.id) : null; if (!pick && s.cls.length) { let bestOverlap = 0; for (const c of kids) { const cl = c.classList; const overlap = s.cls.filter((x) => cl.contains(x)).length; if (overlap > bestOverlap) { bestOverlap = overlap; pick = c; } } } if (!pick) pick = kids[Math.min(s.idx, kids.length - 1)]; node = pick; } return node; } /** Pull the items to append out of a fetched page. */ function extractItems(doc, ctx, nextEl) { if (ctx.rule && ctx.rule.content) return queryAll(doc, ctx.rule.content); const box = resolvePath(doc, ctx.path); if (!box) return []; const kids = Array.from(box.children).filter((c) => !isPagerChild(c, nextEl)); if (!ctx.shape) return kids; // Keep what looks like the items we saw on page 1, and drop anything that // repeats verbatim from it (headings, sticky threads, "sort by" bars). const shaped = kids.filter((c) => matchesShape(c, ctx.shape) && !ctx.shape.texts.has(normalize(c.textContent))); return shaped.length ? shaped : kids; } function isPagerChild(c, nextEl) { if (SKIP_TAGS.has(c.tagName)) return true; if (c.matches('nav, [role="navigation"]')) return true; if (nextEl && (c === nextEl || c.contains(nextEl)) && normalize(c.textContent).length < 300) return true; if (!PAGINATION_RE.test(attrText(c)) || normalize(c.textContent).length >= 300) return false; // A class with "page" in it isn't enough (li.product-page-card): most links // have to be page numbers or next/previous. const links = Array.from(c.querySelectorAll('a')); if (links.length <= 2) return false; const pagerish = links.filter((a) => { const raw = normalize(a.textContent); const t = stripDecor(raw); return /^\d{1,5}$/.test(t) || NEXT_SET.has(t) || PREV_WORD_RE.test(t) || ARROWS.test(raw) || BACK_ARROWS.test(raw) || t === '…' || t === '...'; }); return pagerish.length * 2 >= links.length; } function itemShape(items) { const shape = { tags: new Set(), classes: new Set(), classless: false, texts: new Set() }; for (const it of items) { shape.tags.add(it.tagName); const stable = Array.from(it.classList).filter((c) => !STATE_CLASS_RE.test(c)); if (!stable.length) shape.classless = true; for (const c of stable) shape.classes.add(c); const t = normalize(it.textContent); if (t) shape.texts.add(t); } return shape; } function matchesShape(el, shape) { if (!shape.tags.has(el.tagName)) return false; const stable = Array.from(el.classList).filter((c) => !STATE_CLASS_RE.test(c)); if (!stable.length) return shape.classless; return stable.some((c) => shape.classes.has(c)); } // --------------------------------------------------------------------------- // Preparing fetched content // --------------------------------------------------------------------------- // Where lazy loaders keep the real address, full-size first (Pagetual's set // and a few more). Used only when src is missing or a placeholder. const LAZY_ATTRS = [ 'data-src', 'data-original', 'data-lazy-src', 'data-lazyload', 'data-lazyload-src', 'data-lazy-load-src', 'data-ks-lazyload', 'data-ks-lazyload-custom', 'data-defer-src', 'data-actualsrc', 'data-orig-file', 'data-hi-res-src', 'zoomfile', 'file', 'original', 'data-lazy', 'data-echo', 'data-url', 'data-imageurl', 'data-isrc', 'data-s', 'lazy-src', 'lazysrc', 'load-src', 'origin-src', 'real_src', 'imgsrc', 'src2', '_src', 'data-cover', 'data-thumb', 'data-placeholder', ]; const PLACEHOLDER_RE = /^data:|blank|placeholder|spacer|lazy|loading|grey|gray|transparent|1x1|pixel|(^|\/)none\.(gif|png)/i; const BG_ATTRS = ['data-bg', 'data-background-image']; // What an image address looks like, as against the flags and colours sites // also keep in data-bg ("dark", "#f5f5f5", "rgba(0,0,0,.5)", "1"): a URL or a // path, anything with a "/" or a "?", or a file name with an extension. const imageRef = (v) => !/^(#|rgba?\(|hsla?\(|var\(|oklch\(|oklab\(|lch\(|lab\(|color\(|hwb\()/i.test(v) && !/^[\d.\s%]+$/.test(v) && (/^(https?:|\/|\.\.?\/|data:image\/)/i.test(v) || /[/?]/.test(v) || /\.[a-z0-9]{2,5}([?#]|$)/i.test(v)); /** A srcset that only offers placeholders (data: URIs, blank.gif and the like). */ const placeholderSet = (set) => /^\s*data:/i.test(set) || set.split(',').every((part) => PLACEHOLDER_RE.test(part.trim().split(/\s+/)[0] || '')); /** root itself when it matches, then everything below it that does: items are often the or themselves. */ const selfAndBelow = (root, sel) => (root.matches && root.matches(sel) ? [root] : []).concat(Array.from(root.querySelectorAll(sel))); function fixLazyImages(root, base) { for (const img of selfAndBelow(root, 'img, source')) { const src = img.getAttribute('src') || ''; for (const a of LAZY_ATTRS) { const v = img.getAttribute(a); if (v && !/^data:/.test(v) && (!src || PLACEHOLDER_RE.test(src))) { img.setAttribute('src', v); break; } } const lazySet = img.getAttribute('data-srcset') || img.getAttribute('data-lazy-srcset'); const set = img.getAttribute('srcset'); if (lazySet && (!set || placeholderSet(set))) img.setAttribute('srcset', lazySet); } // Lazy background images:
. for (const el of selfAndBelow(root, BG_ATTRS.map((a) => '[' + a + ']').join(','))) { if (el.style.backgroundImage && !/^url\(["']?data:/.test(el.style.backgroundImage)) continue; const v = (BG_ATTRS.map((a) => el.getAttribute(a)).find((x) => x && x.trim()) || '').trim(); const inner = /^url\(/i.test(v) ? v.replace(/^url\(\s*["']?|["']?\s*\)$/gi, '') : v; if (!imageRef(inner)) continue; const u = base ? absUrl(inner, base) : inner; if (u) el.style.backgroundImage = 'url("' + u.replace(/["\\]/g, '\\$&') + '")'; } } function absolutize(root, base) { for (const [sel, attr] of [['[href]', 'href'], ['[src]', 'src'], ['[action]', 'action'], ['[poster]', 'poster']]) { for (const el of selfAndBelow(root, sel)) { const v = el.getAttribute(attr); if (!v || /^(#|javascript:|data:|mailto:|tel:)/i.test(v)) continue; const u = absUrl(v, base); if (u) el.setAttribute(attr, u); } } for (const el of selfAndBelow(root, '[srcset]')) { const fixed = el.getAttribute('srcset').split(',').map((part) => { const [u, ...rest] = part.trim().split(/\s+/); const a = u ? absUrl(u, base) : null; return a ? [a, ...rest].join(' ') : part.trim(); }).join(', '); el.setAttribute('srcset', fixed); } } // Markup that acts on the page instead of showing content: a refresh // navigates the tab, rebases every relative URL, srcdoc frames run // scripts, and