// Background: owns the hotkeys, the screenshot pipeline and the OCR queue. // A service worker on Chrome, an event page on Firefox. import { getSettings, saveSettings, getRegion, setRegion, getRun, setRun, getPages, setPages, thumbKey, cropKey, normalizeForCompare, advanceIsConfigured, withoutHistoryFields, } from './shared.js'; /* The OCR engine needs a canvas and a real Worker. Where that can live differs per browser, so the build picks the right host: an offscreen document on Chrome, the background event page itself on Firefox. */ import { runCommand as callOcr } from './ocr/host.js'; // Firefox exposes the promise-based APIs as `browser`; its `chrome` alias is // callback-style, which every `await` below would silently break on. const chrome = globalThis.browser ?? globalThis.chrome; const CONTENT_FILES = ['src/content/overlay.js']; /* ------------------------------------------------------------------ * * Small helpers * ------------------------------------------------------------------ */ const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); const newId = () => `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`; async function activeTab() { const [tab] = await chrome.tabs.query({ active: true, lastFocusedWindow: true }); if (!tab) throw new Error('No active tab.'); if (/^(chrome|edge|about|devtools|resource|view-source|chrome-extension|moz-extension):/i.test(tab.url || '')) { throw new Error('The browser blocks extensions on this page. Open a normal web page and try again.'); } return tab; } async function injectContent(tabId) { // Every frame gets it: sub-frames contribute the geometry auto-advance needs. try { await chrome.scripting.executeScript({ target: { tabId, allFrames: true }, files: CONTENT_FILES }); } catch { await chrome.scripting.executeScript({ target: { tabId }, files: CONTENT_FILES }); } } /** Fire-and-forget message to a tab; a missing content script must never break a capture. */ function tellTab(tabId, msg) { chrome.tabs.sendMessage(tabId, msg).catch(() => {}); } /** captureVisibleTab is rate limited to a couple of calls per second. */ async function captureVisible(windowId) { for (let attempt = 0; ; attempt++) { try { return await chrome.tabs.captureVisibleTab(windowId, { format: 'png' }); } catch (err) { const msg = String(err?.message || err); if (attempt >= 4 || !/max|quota|per second/i.test(msg)) throw err; await sleep(180 * (attempt + 1)); } } } /* Without a granted origin the extension runs on activeTab, which the browser hands over only when the user invokes it and revokes on navigation. Say so plainly instead of surfacing the browser's wording. */ function friendly(message) { const text = String(message ?? ''); return /cannot access contents|host permission|'' or 'activeTab'/i.test(text) ? 'The browser blocked access to this page. Open the OCR It popup and click “Allow on this site”, then try again.' : text; } async function setError(message) { message = message ? friendly(message) : message; await chrome.storage.local.set({ lastError: message ? { message, ts: Date.now() } : null }); await chrome.action.setBadgeBackgroundColor({ color: '#c0392b' }); await chrome.action.setBadgeText({ text: message ? '!' : '' }); } async function setBusyBadge() { const { lastError } = await chrome.storage.local.get('lastError'); if (lastError) return; const run = await getRun(); if (run.active) { await chrome.action.setBadgeBackgroundColor({ color: '#1f7a4d' }); await chrome.action.setBadgeText({ text: String(run.captured || 0) }); return; } const n = queue.length + (draining ? 1 : 0); await chrome.action.setBadgeBackgroundColor({ color: '#2d6cdf' }); await chrome.action.setBadgeText({ text: n > 0 ? String(n) : '' }); } /* chrome.storage has no read-modify-write primitive, so every mutation of the page list goes through one promise chain. */ let pagesLock = Promise.resolve(); function mutatePages(fn) { const run = pagesLock.then(async () => { const pages = await getPages(); const { next, value } = await fn(pages); if (next) await setPages(next); return value; }); pagesLock = run.then(() => {}, () => {}); return run; } async function patchPage(id, patch) { return mutatePages((pages) => { const i = pages.findIndex((p) => p.id === id); if (i === -1) return { next: null, value: null }; const updated = { ...pages[i], ...patch }; const next = [...pages]; next[i] = updated; return { next, value: updated }; }); } /* ------------------------------------------------------------------ * * Region picking * ------------------------------------------------------------------ */ async function startRegionPicker(tab) { await injectContent(tab.id); const region = await getRegion(); await chrome.tabs.sendMessage(tab.id, { type: 'ocrit:pick-region', current: region }); } async function startAdvancePicker(tab) { await injectContent(tab.id); await chrome.tabs.sendMessage(tab.id, { type: 'ocrit:pick-advance' }); } /* ------------------------------------------------------------------ * * Auto-advance * * The next-page control is stored as a point in top-frame viewport * coordinates rather than a CSS selector: a point survives the DOM * re-renders that break selectors, reaches into shadow roots, and lands * inside cross-origin iframes that no selector of ours could address. * ------------------------------------------------------------------ */ /** Runs in every frame; only the one that owns the point does anything. */ function advanceInPage(point, mode, key) { const off = window.__ocrItOffset?.(); if (!off) return { skipped: 'frame offset unknown' }; const cx = point.x - off.x; const cy = point.y - off.y; if (cx < 0 || cy < 0 || cx >= window.innerWidth || cy >= window.innerHeight) { return { skipped: 'point is outside this frame' }; } let el = document.elementFromPoint(cx, cy); if (!el) return { skipped: 'nothing at that point' }; for (let i = 0; el.shadowRoot && i < 20; i++) { const inner = el.shadowRoot.elementFromPoint(cx, cy); if (!inner || inner === el) break; el = inner; } if (/^(IFRAME|FRAME)$/.test(el.tagName)) return { skipped: 'a nested frame owns that point' }; if (/^(EMBED|OBJECT)$/.test(el.tagName)) { // A built-in PDF viewer lands here: no extension can reach inside it. return { skipped: 'an embedded viewer owns that point' }; } const describe = (n) => { let out = n.localName || 'element'; if (n.id) out += `#${n.id}`; else if (n.classList?.length) out += `.${n.classList[0]}`; const label = (n.getAttribute?.('aria-label') || n.textContent || '').trim().replace(/\s+/g, ' '); return label ? `${out} \u201c${label.slice(0, 24)}\u201d` : out; }; try { if (mode === 'key') { const codes = { ArrowRight: 39, ArrowLeft: 37, ArrowDown: 40, ArrowUp: 38, PageDown: 34, PageUp: 33, Enter: 13, Space: 32, ' ': 32, }; const keyCode = codes[key] ?? (key.length === 1 ? key.toUpperCase().charCodeAt(0) : 0); const active = document.activeElement; const target = active && active !== document.body ? active : (document.body || document.documentElement); for (const type of ['keydown', 'keyup']) { const ev = new KeyboardEvent(type, { key, code: key === ' ' ? 'Space' : key, bubbles: true, cancelable: true, composed: true, view: window, }); // Plenty of viewers still read the legacy numeric properties. for (const prop of ['keyCode', 'which']) Object.defineProperty(ev, prop, { get: () => keyCode }); target.dispatchEvent(ev); } return { ok: true, detail: `sent ${key} to ${describe(target)}` }; } // Reproduce the whole sequence a real click emits, so viewers that page on // pointerdown or mousedown react just like those listening for click. const control = el.closest?.('button, a[href], [role="button"], input[type="button"], input[type="submit"]') || el; if (control === document.body || control === document.documentElement) { // Better to say so than to "click" the background and report success. return { skipped: 'only the page background is at that point' }; } const r = control.getBoundingClientRect(); const base = { bubbles: true, cancelable: true, composed: true, view: window, button: 0, clientX: r.width ? r.left + r.width / 2 : cx, clientY: r.height ? r.top + r.height / 2 : cy, }; const pointer = { ...base, pointerId: 1, pointerType: 'mouse', isPrimary: true }; control.dispatchEvent(new PointerEvent('pointerdown', { ...pointer, buttons: 1 })); control.dispatchEvent(new MouseEvent('mousedown', { ...base, buttons: 1 })); control.dispatchEvent(new PointerEvent('pointerup', { ...pointer, buttons: 0 })); control.dispatchEvent(new MouseEvent('mouseup', { ...base, buttons: 0 })); control.click(); return { ok: true, detail: `clicked ${describe(control)}` }; } catch (err) { return { error: String(err?.message || err) }; } } /* Least-interesting reason last, so the toast blames the real problem. */ const SKIP_RANK = [ 'an embedded viewer owns that point', 'only the page background is at that point', 'nothing at that point', 'a nested frame owns that point', 'frame offset unknown', 'point is outside this frame', ]; async function reportAdvance(tabId, outcome) { await chrome.storage.local.set({ lastAdvance: { ...outcome, ts: Date.now() } }); if (!outcome.ok && tabId != null) { tellTab(tabId, { type: 'ocrit:toast', error: `Page not turned — ${outcome.detail}` }); } return outcome; } async function runAdvance(tabId, settings, region) { // In key mode the region itself says which viewer should receive the key. const point = settings.advanceMode === 'key' ? { x: Math.round(region.x + region.w / 2), y: Math.round(region.y + region.h / 2) } : settings.advancePoint; if (!point) { return reportAdvance(tabId, { ok: false, detail: 'no next-page control picked yet (open the popup and use “Pick control”)', }); } // Refresh cross-origin frame offsets, then let the configured delay run. await chrome.scripting.executeScript({ target: { tabId, allFrames: true }, func: () => window.__ocrItAnnounce?.(), }).catch(() => {}); await sleep(Math.max(0, settings.advanceDelay || 0)); let results; try { results = await chrome.scripting.executeScript({ target: { tabId, allFrames: true }, args: [point, settings.advanceMode, settings.advanceKey], func: advanceInPage, }); } catch (err) { return reportAdvance(tabId, { ok: false, detail: String(err?.message || err) }); } const verdicts = results.map((r) => r.result).filter(Boolean); const acted = verdicts.find((v) => v.ok); if (acted) return reportAdvance(tabId, { ok: true, detail: acted.detail }); const failure = verdicts.find((v) => v.error); const skipped = SKIP_RANK.find((reason) => verdicts.some((v) => v.skipped === reason)); return reportAdvance(tabId, { ok: false, detail: failure?.error || skipped || 'no frame handled that point', }); } /* ------------------------------------------------------------------ * * Capture pipeline * ------------------------------------------------------------------ */ async function capture(tab, { advance } = {}) { const settings = await getSettings(); const region = await getRegion(); if (!region) { await startRegionPicker(tab); throw new Error('No capture region yet — drag one out first.'); } await injectContent(tab.id); // Viewport metrics now, so zoom or a resize since the region was drawn is accounted for. const [{ result: vp }] = await chrome.scripting.executeScript({ target: { tabId: tab.id }, func: () => ({ iw: window.innerWidth, ih: window.innerHeight, dpr: window.devicePixelRatio }), }); // Our own HUD must be off-screen before the shot, or it lands inside the crop. await chrome.tabs.sendMessage(tab.id, { type: 'ocrit:hide-hud' }).catch(() => {}); const shot = await captureVisible(tab.windowId); if (settings.hud) { const run = await getRun(); tellTab(tab.id, { type: 'ocrit:flash', rect: region, run: run.active ? { active: true, captured: run.captured } : null, }); } // Screenshot pixels per CSS pixel — derived from the image itself, inside the OCR host. const { crop, thumb, width, height } = await callOcr('crop', { shot, region, viewport: vp, enhance: settings.enhance, }); const id = newId(); const page = await mutatePages((pages) => { const record = { id, n: pages.length + 1, status: 'queued', text: '', conf: null, dup: false, error: null, ts: Date.now(), w: width, h: height, }; return { next: [...pages, record], value: record }; }); await chrome.storage.local.set({ [thumbKey(id)]: thumb, [cropKey(id)]: crop }); // Turn the page before OCR runs — the screenshot is already safely captured. // Deliberately not awaited: the advance delay must not hold up the queue. const shouldAdvance = advance ?? settings.autoAdvance; const advancing = shouldAdvance ? runAdvance(tab.id, settings, region).catch((err) => ({ ok: false, detail: String(err?.message || err) })) : null; enqueue({ id, crop, tabId: tab.id, settings }); return { page, advance: advancing }; } /* ------------------------------------------------------------------ * * OCR queue (serial — one Tesseract worker, jobs never overlap) * ------------------------------------------------------------------ */ const queue = []; let draining = false; function enqueue(job) { queue.push(job); setBusyBadge(); drain(); } async function drain() { if (draining) return; draining = true; try { while (queue.length) { const job = queue.shift(); await runJob(job); await setBusyBadge(); } } finally { draining = false; await setBusyBadge(); } } async function runJob({ id, crop, tabId, settings }) { await patchPage(id, { status: 'running' }); try { const image = crop || (await chrome.storage.local.get(cropKey(id)))[cropKey(id)]; if (!image) throw new Error('Cropped image is gone — recapture this page.'); const { text, conf } = await callOcr('recognize', { image, lang: settings.lang, psm: settings.psm, }); const clean = (text || '').replace(/ /g, ' ').replace(/[ \t]+\n/g, '\n').trim(); let dup = false; if (settings.dedupeWarn && clean) { const pages = await getPages(); const idx = pages.findIndex((p) => p.id === id); const prev = pages.slice(0, Math.max(idx, 0)).reverse().find((p) => p.status === 'done' && p.text); dup = !!prev && normalizeForCompare(prev.text) === normalizeForCompare(clean); } const page = await patchPage(id, { status: 'done', text: clean, conf, dup, error: null }); await chrome.storage.local.remove(cropKey(id)); await setError(null); if (settings.hud && tabId != null) { tellTab(tabId, { type: 'ocrit:toast', n: page?.n, chars: clean.length, conf, dup, preview: clean.split('\n').find((l) => l.trim()) || '(no text found)', }); } } catch (err) { const message = String(err?.message || err); await patchPage(id, { status: 'error', error: message }); await setError(message); if (tabId != null) tellTab(tabId, { type: 'ocrit:toast', error: message }); } } /* ------------------------------------------------------------------ * * Automatic run: capture, turn, repeat, until something says stop * ------------------------------------------------------------------ */ const RUN_WATCHDOG = 'ocrit-run-watchdog'; /* Bumped on every start and stop; an in-flight loop whose token no longer matches exits at its next checkpoint, which is how stopping works. */ let runToken = 0; async function stopRun(reason) { reason = reason ? friendly(reason) : reason; runToken += 1; const before = await getRun(); const run = await setRun({ active: false, stopReason: reason || null, lastTick: Date.now() }); await chrome.alarms.clear(RUN_WATCHDOG); await setBusyBadge(); if (before.tabId != null) { tellTab(before.tabId, { type: 'ocrit:run', active: false, captured: run.captured, reason: reason || null }); } return run; } /** Wait for one page to come back from OCR, so the loop paces itself. */ async function waitForPage(id, token, timeoutMs = 60000) { const deadline = Date.now() + timeoutMs; while (runToken === token && Date.now() < deadline) { // Storage reads double as keep-alive: they reset the worker's idle timer. const page = (await getPages()).find((p) => p.id === id); if (page && (page.status === 'done' || page.status === 'error')) return page; await sleep(200); } return null; } async function runLoop(tab, token) { let dupes = 0; while (runToken === token) { const settings = await getSettings(); try { await chrome.tabs.get(tab.id); } catch { return stopRun('the tab was closed'); } // Counted before the shot so the on-page indicator includes this page // rather than trailing it by one. const run = await setRun({ captured: (await getRun()).captured + 1, lastTick: Date.now() }); const { page, advance } = await captureSerial(tab, { advance: true }); await setBusyBadge(); // Waiting for the read keeps the queue from running away and, more // importantly, is what makes "the page stopped changing" detectable. const finished = await waitForPage(page.id, token); if (runToken !== token) return null; if (!finished) return stopRun('OCR stopped responding'); if (finished.status === 'error') return stopRun(`OCR failed — ${finished.error}`); dupes = finished.dup ? dupes + 1 : 0; if (settings.runStopOnDupes && dupes >= settings.runStopOnDupes) { return stopRun(`the page stopped changing after ${run.captured} captures`); } if (run.captured >= settings.runMaxPages) { return stopRun(`reached the ${settings.runMaxPages}-page limit`); } const outcome = await advance; if (runToken !== token) return null; if (outcome && !outcome.ok) return stopRun(`could not turn the page — ${outcome.detail}`); await setRun({ lastTick: Date.now() }); await sleep(Math.max(0, settings.runInterval)); } return null; } async function startRun(tab) { const settings = await getSettings(); const region = await getRegion(); if (!region) { await startRegionPicker(tab); throw new Error('Draw a capture region first.'); } if (!advanceIsConfigured(settings)) { throw new Error('Pick a next-page control first — a run has to be able to turn the page.'); } runToken += 1; const token = runToken; await setRun({ active: true, tabId: tab.id, captured: 0, startedAt: Date.now(), lastTick: Date.now(), stopReason: null, }); await chrome.alarms.create(RUN_WATCHDOG, { periodInMinutes: 1 }); await setBusyBadge(); tellTab(tab.id, { type: 'ocrit:run', active: true, captured: 0 }); runLoop(tab, token).catch((err) => stopRun(String(err?.message || err))); return getRun(); } async function toggleRun(tab) { const run = await getRun(); return run.active ? stopRun('stopped by hand') : startRun(tab); } /* If the worker is recycled mid-run nothing else would restart the loop. */ chrome.alarms.onAlarm.addListener(async (alarm) => { if (alarm.name !== RUN_WATCHDOG) return; const run = await getRun(); if (!run.active) { await chrome.alarms.clear(RUN_WATCHDOG); return; } if (Date.now() - run.lastTick < 20000) return; // still ticking try { const tab = await chrome.tabs.get(run.tabId); runToken += 1; const token = runToken; await setRun({ lastTick: Date.now() }); runLoop(tab, token).catch((err) => stopRun(String(err?.message || err))); } catch { await stopRun('the tab went away'); } }); /** Pick up anything left mid-flight if the service worker was recycled. */ async function resumePending() { const pages = await getPages(); const settings = await getSettings(); for (const p of pages) { if (p.status === 'queued' || p.status === 'running') { enqueue({ id: p.id, crop: null, tabId: null, settings }); } } } /* ------------------------------------------------------------------ * * Wiring * ------------------------------------------------------------------ */ /* Hotkey mashing is the whole point of this extension, so captures queue up instead of racing each other. */ let captureLock = Promise.resolve(); function captureSerial(tab, opts) { const next = captureLock.then(() => capture(tab, opts), () => capture(tab, opts)); captureLock = next.then(() => {}, () => {}); return next; } chrome.commands.onCommand.addListener(async (command) => { try { const tab = await activeTab(); if (command === 'capture-region') await captureSerial(tab); else if (command === 'set-region') await startRegionPicker(tab); else if (command === 'toggle-run') await toggleRun(tab); } catch (err) { await setError(String(err?.message || err)); } }); chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => { if (!msg || msg.target === 'offscreen') return; // not ours (async () => { try { switch (msg.type) { case 'ocrit:region-picked': { const region = { ...msg.rect, origin: msg.origin || '', setAt: Date.now() }; await setRegion(region); await setError(null); sendResponse({ ok: true, region }); break; } case 'ocrit:advance-picked': { const settings = await saveSettings({ advancePoint: msg.point, advanceMode: 'click' }); sendResponse({ ok: true, settings }); break; } case 'ocrit:test-advance': { const tab = await activeTab(); await injectContent(tab.id); const settings = await getSettings(); const region = await getRegion(); const outcome = await runAdvance(tab.id, settings, region || { x: 0, y: 0, w: 0, h: 0 }); sendResponse({ ok: outcome.ok, outcome }); break; } case 'ocrit:capture': { const tab = await activeTab(); const { page } = await captureSerial(tab); sendResponse({ ok: true, page }); break; } case 'ocrit:toggle-run': { const run = await toggleRun(await activeTab()); sendResponse({ ok: true, run }); break; } case 'ocrit:stop-run': { const run = await stopRun(msg.reason || 'stopped by hand'); sendResponse({ ok: true, run }); break; } case 'ocrit:start-region-picker': { await startRegionPicker(await activeTab()); sendResponse({ ok: true }); break; } case 'ocrit:start-advance-picker': { await startAdvancePicker(await activeTab()); sendResponse({ ok: true }); break; } case 'ocrit:retry': { const settings = await getSettings(); enqueue({ id: msg.id, crop: null, tabId: null, settings }); sendResponse({ ok: true }); break; } case 'ocrit:clear-error': { await setError(null); await setBusyBadge(); sendResponse({ ok: true }); break; } case 'ocrit:warmup': { const settings = await getSettings(); await callOcr('warmup', { lang: settings.lang }); sendResponse({ ok: true }); break; } default: return; // leave unhandled messages alone } } catch (err) { const message = String(err?.message || err); await setError(message); sendResponse({ ok: false, error: message }); } })(); return true; // async response }); chrome.runtime.onStartup.addListener(async () => { const run = await getRun(); if (run.active) await stopRun('the browser restarted'); await resumePending(); }); chrome.runtime.onInstalled.addListener(async () => { // Pages captured by 0.2.0 still carry the url and title of the tab they came // from. New captures no longer store either; strip what is already there. await mutatePages((pages) => ({ next: withoutHistoryFields(pages), value: null })); await setBusyBadge(); resumePending(); });