/**
 * jd-page-fetch.mjs — read a JD out of the posting page's own bytes when no ATS
 * API covers the host.
 *
 * Template-driven career sites (Phenom, and most bespoke ones) render the job
 * body from <script> data: the page ships a schema.org JobPosting in
 * `application/ld+json`, and the visible HTML is title, location and Apply
 * chrome. Anything that converts the page to text drops script content, so it
 * sees ~270 characters and reports "partial page shell". WebFetch does exactly
 * that, which is how a live Robert Half posting (7 KB of JD in its JSON-LD) came
 * back from the web app as "Needs review".
 *
 * This is the same recovery the batch runner already does (batch/jd-rescue.mjs):
 * one unauthenticated GET, then offline parsing. The parsers are imported from
 * there, not copied, so the two paths cannot drift on what counts as a JD.
 *
 * The GET is the only network step, so it carries the egress guard every other
 * fetcher has, in three layers:
 *   1. a literal-host check (rejectPrivateOrInvalid) and a DNS pre-check
 *      (validateUrlSecurity) on the URL and on every redirect hop, which are
 *      followed by hand;
 *   2. each hop runs inside providerFetchContext, so the patched dns.lookup
 *      (providers/_dns-cache.mjs) validates the address the socket is actually
 *      dialled on. Layer 1 resolves once and layer 2 resolves again at connect
 *      time, so a name that answers public-then-private (DNS rebinding) is
 *      refused at the connection, not just at the pre-check;
 *   3. a body cap and an HTML content-type check.
 *
 * Three more refusals keep a wrong JD from reaching the evaluate worker, which
 * trusts what it is handed instead of fetching it:
 *   - LinkedIn is never fetched here (AGENTS.md "LinkedIn JD loading guard":
 *     one bounded browser attempt, no automatic extraction retry);
 *   - a posting whose `validThrough` is past is closed, so it is a miss and the
 *     worker's own closed-posting rule decides;
 *   - a page that holds more than one posting (a listing) is a miss, never the
 *     longest description on it.
 */

import { rejectPrivateOrInvalid, validateUrlSecurity } from '../liveness-browser.mjs';
import { DEFAULT_USER_AGENT } from '../user-agent.mjs';
import { asciiLower, fromLdJson, fromEmbeddedDescription, embeddedDescriptionStrings, ldJsonBlocks, wordCount, LATIN1_ENTITIES } from './jd-parse.mjs';
import { classifyLiveness } from '../liveness-core.mjs';
// Imported for its side effect: it installs the connect-time dns.lookup guard
// that providerFetchContext switches on. _http.mjs gets it the same way.
import { dnsLookupPatched } from '../providers/_dns-cache.mjs';
import { providerFetchContext } from '../providers/_ip-guard.mjs';

const MAX_REDIRECTS = 5;
const MAX_BODY_BYTES = 3 * 1024 * 1024;
// Same floor jd-rescue.mjs uses: below it the "JD" is a stub, not a posting.
export const MIN_PAGE_JD_WORDS = 80;

/** LinkedIn hosts, which this module never requests. */
export function isLinkedInHost(hostname) {
  const host = String(hostname || '').toLowerCase().replace(/\.$/, '');
  return host === 'linkedin.com' || host.endsWith('.linkedin.com');
}

async function readCapped(res, maxBytes) {
  const reader = res.body?.getReader();
  if (!reader) return '';
  const chunks = [];
  let total = 0;
  for (;;) {
    const { done, value } = await reader.read();
    if (done) break;
    total += value.byteLength;
    if (total > maxBytes) { await reader.cancel().catch(() => {}); return null; }
    chunks.push(value);
  }
  return Buffer.concat(chunks).toString('utf8');
}

/**
 * GET one HTML document, following redirects by hand so each hop is guarded.
 * Null on any failure: the caller falls through to its next source.
 *
 * @param {string} url
 * @param {number} timeoutMs
 * @returns {Promise<{html: string, finalUrl: string}|null>}
 */
async function fetchHtml(url, timeoutMs) {
  // The connect-time guard lives in the patched dns.lookup. With the patch opted
  // out (CAREER_OPS_NO_DNS_CACHE=1) a name could pass the pre-check and resolve
  // to a private address at connect time, so fail closed: a miss sends the
  // worker back to WebFetch.
  if (!dnsLookupPatched()) return null;
  // NODE_USE_ENV_PROXY=1 makes Node's own fetch go through HTTP(S)_PROXY, where
  // the address check above no longer sees the destination. Providers only use a
  // proxy when CAREER_OPS_TRUST_PROXY_EGRESS=1 says it blocks private addresses;
  // hold this fetch to the same rule and miss otherwise.
  if (/^(1|true)$/i.test(process.env.NODE_USE_ENV_PROXY ?? '') && process.env.CAREER_OPS_TRUST_PROXY_EGRESS !== '1') return null;
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    let current = url;
    for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
      if (rejectPrivateOrInvalid(current)) return null;
      if (isLinkedInHost(new URL(current).hostname)) return null;
      await validateUrlSecurity(current);
      const targetHost = new URL(current).hostname.replace(/^\[|\]$/g, '');
      // The whole hop, body read included, runs inside the provider-fetch
      // context: the socket's DNS lookup happens inside fetch(), after its
      // synchronous part returns, and the context has to still be entered then.
      const outcome = await providerFetchContext.run({ url: current, targetHost }, async () => {
        const res = await fetch(current, {
          headers: { accept: 'text/html,application/xhtml+xml', 'user-agent': DEFAULT_USER_AGENT },
          redirect: 'manual',
          signal: controller.signal,
        });
        if (res.status >= 300 && res.status < 400) {
          const location = res.headers.get('location');
          await res.body?.cancel().catch(() => {});
          return location ? { redirect: new URL(location, current).href } : null;
        }
        if (!res.ok) { await res.body?.cancel().catch(() => {}); return null; }
        const type = res.headers.get('content-type') || '';
        if (type && !/html|xml/i.test(type)) { await res.body?.cancel().catch(() => {}); return null; }
        const html = await readCapped(res, MAX_BODY_BYTES);
        return html == null ? null : { html, finalUrl: current };
      });
      if (!outcome) return null;
      if (outcome.redirect) { current = outcome.redirect; continue; }
      return outcome;
    }
    return null;
  } catch {
    return null;
  } finally {
    clearTimeout(timer);
  }
}

/**
 * Every schema.org JobPosting node in the page's JSON-LD blocks. A malformed
 * block is skipped, and a block may be an array or an @graph wrapper.
 *
 * @param {string} html
 * @returns {object[]}
 */
function jobPostingNodes(html) {
  const found = [];
  for (const block of ldJsonBlocks(html)) {
    let parsed;
    try { parsed = JSON.parse(block.trim()); } catch { continue; }
    const queue = [parsed];
    for (let qi = 0; qi < queue.length; qi += 1) {
      const node = queue[qi];
      if (!node || typeof node !== 'object') continue;
      if (Array.isArray(node)) { for (const child of node) queue.push(child); continue; }
      if (Array.isArray(node['@graph'])) for (const child of node['@graph']) queue.push(child);
      const type = node['@type'];
      if (Array.isArray(type) ? type.includes('JobPosting') : type === 'JobPosting') found.push(node);
    }
  }
  return found;
}

/** Is a posting's `validThrough` in the past? An absent or unparseable date is not evidence. */
function isPastValidThrough(node, now) {
  const raw = node?.validThrough;
  if (typeof raw !== 'string' || !raw.trim()) return false;
  let when = Date.parse(raw);
  // A date-only value parses as UTC midnight, but means "through that day".
  if (/^\d{4}-\d{2}-\d{2}$/.test(raw.trim())) when += 24 * 60 * 60 * 1000;
  return Number.isFinite(when) && when < now;
}

/** Distinct large description-like strings in a hydration payload (see fromEmbeddedDescription). */
function embeddedDescriptionCount(html) {
  const seen = new Set();
  for (const raw of embeddedDescriptionStrings(html, { lenient: true })) {
    if (wordCount(String(raw).replace(/<[^>]+>/g, ' ')) >= MIN_PAGE_JD_WORDS) seen.add(String(raw).replace(/\s+/g, ' ').trim());
  }
  return seen.size;
}

/**
 * Pick the JD out of already-downloaded page HTML. Pure — exported for tests.
 * Null when the page holds no single posting of at least MIN_PAGE_JD_WORDS
 * words. A posting past its `validThrough` date returns `{closed: true,
 * evidence}` instead, so the caller can pass the closure on rather than
 * letting a fallback fetch score a posting the page itself calls closed.
 *
 * @param {string} html
 * @param {number} [now] epoch ms, injectable for tests
 * @returns {{text: string, method: string, closed?: boolean, evidence?: string}|null}
 */
export function jdFromPageHtml(html, now = Date.now()) {
  return inspectPage(html, now).found;
}

/**
 * Like jdFromPageHtml, but says when the page is ambiguous (several postings or several
 * description-sized payloads): nothing on such a page, closure included, belongs to one URL.
 * @returns {{found: object|null, ambiguous: boolean}}
 */
const collapseWs = (v) => (typeof v === 'string' ? v.replace(/\s+/g, ' ').trim() : v);

/** Stable form of a JSON value: object keys sorted at every depth, strings whitespace-collapsed. */
function canon(v) {
  if (Array.isArray(v)) return v.map(canon);
  if (v && typeof v === 'object') return Object.fromEntries(Object.keys(v).sort().map((k) => [k, canon(v[k])]));
  return collapseWs(v);
}

// Tokens that identify a job in a posting URL: path segments containing a digit, and id-like query values.
function jobIdTokens(u) {
  try {
    const x = new URL(u);
    // SPA routes carry the job id in the fragment (/careers#/job/2), so read it like the path.
    const toks = `${x.pathname}/${x.hash.replace(/^#/, '').split('?')[0]}`.split('/').filter((seg) => /\d/.test(seg)).map((seg) => seg.toLowerCase());
    // id-like query values, from the real query and from a query inside the fragment (#/job?jobId=1).
    const fragQuery = x.hash.includes('?') ? x.hash.slice(x.hash.indexOf('?') + 1) : '';
    for (const params of [x.searchParams, new URLSearchParams(fragQuery)]) {
      for (const [k, val] of params) if (/(^|_)(id|jid|jobid|job_id|reqid|req)$/i.test(k) && val) toks.push(val.toLowerCase());
    }
    return toks;
  } catch { return []; }
}

// A bare year or version segment (/2024/, /v2/) is shared structure, not a job id.
const STRUCTURAL_TOKEN_RE = /^(?:(?:19|20)\d\d|v\d+(?:\.\d+)*)$/;

/**
 * Does this JobPosting's own URL clearly name a different job than the one requested? Only when both are
 * absolute URLs on the same host, both carry id-like tokens, and the tokens share nothing. Postings with no
 * URL, a different host (CDN or alias), or no identifying tokens are not rejected.
 */
function postingNamesOtherJob(node, requestedUrl) {
  if (!requestedUrl) return false;
  const own = typeof node?.url === 'string' ? node.url : (typeof node?.['@id'] === 'string' ? node['@id'] : '');
  if (!own) return false;
  try {
    const a = new URL(requestedUrl); const b = new URL(own);
    if (a.hostname.replace(/^www\./, '').toLowerCase() !== b.hostname.replace(/^www\./, '').toLowerCase()) return false;
  } catch { return false; }
  let want = jobIdTokens(requestedUrl); let got = jobIdTokens(own);
  // Compare without year/version segments when both sides still have a real id; otherwise the
  // raw tokens are the only identity there is (/job/2031 vs /job/2044 must still differ).
  const wantStrong = want.filter((t) => !STRUCTURAL_TOKEN_RE.test(t));
  const gotStrong = got.filter((t) => !STRUCTURAL_TOKEN_RE.test(t));
  if (wantStrong.length && gotStrong.length) { want = wantStrong; got = gotStrong; }
  if (!want.length || !got.length) return false;
  return !want.some((t) => got.includes(t));
}

export function inspectPage(html, now = Date.now(), requestedUrl = '') {
  const postings = jobPostingNodes(html);
  // A page that names more than one posting is a listing, or a page whose other
  // postings we cannot tell apart from the requested one. Nothing on it can be
  // attributed to this URL: not a JD, and not a closure either. Postings are
  // told apart by title, description, url, @id, identifier, employer and location, and an identical
  // duplicate representation of one posting does not count twice.
  const distinctLd = new Set(postings.map((node) => JSON.stringify(canon([node.title ?? '', node.description ?? '', node.url ?? '', node['@id'] ?? '', node.identifier ?? '', node.hiringOrganization ?? '', node.jobLocation ?? ''])))).size;
  if (distinctLd > 1) return { found: null, ambiguous: true };
  // A single posting that names a different job than the requested URL is that other job's, not ours.
  if (postings.length && postingNamesOtherJob(postings[0], requestedUrl)) return { found: null, ambiguous: true };

  // Closed per the page's own structured data: not a JD to score. A miss hands
  // the URL back to the worker's WebFetch path and the mode's closed-posting rule.
  const expired = postings.find((node) => isPastValidThrough(node, now));
  if (expired) return { found: { closed: true, evidence: `JobPosting validThrough ${String(expired.validThrough).slice(0, 40)}`, text: '', method: 'ld-json-expired' }, ambiguous: false };

  let ld = null;
  try { ld = fromLdJson(html); } catch { /* malformed JSON-LD fields: ignore, the ambiguity and closure checks still run */ }
  if (ld && wordCount(ld.text) >= MIN_PAGE_JD_WORDS) return { found: ld, ambiguous: false };

  // The embedded-payload fallback picks the LONGEST description on the page. It
  // is only trusted when the page holds one description-sized string (and, above,
  // at most one posting), so what it returns can only be this posting's.
  if (embeddedDescriptionCount(html) > 1) return { found: null, ambiguous: true };
  const embedded = fromEmbeddedDescription(html);
  return { found: embedded && wordCount(embedded.text) >= MIN_PAGE_JD_WORDS ? embedded : null, ambiguous: false };
}

/** Largest page the already-downloaded closure check will read (a 1-3 MB cap keeps hostile markup cheap). */
const PAGE_HTML_CAP = 3_000_000;

export const TRUNCATION_NOTICE = (cap) =>
  `[Job description truncated here: the posting is longer than ${cap} characters. This text is NOT the complete job description.]`;

/**
 * Cap a JD without splitting a surrogate pair, and say so when it was cut. A
 * silent cut would let a caller score and archive a partial JD as the whole
 * thing, so the notice is part of the text. Pure — exported for tests.
 */
export function capJdText(text, cap) {
  if (text.length <= cap) return text;
  let cut = text.slice(0, cap);
  // A UTF-16 slice can land between the halves of a surrogate pair.
  if (/[\ud800-\udbff]$/.test(cut)) cut = cut.slice(0, -1);
  // Prefer to end on a paragraph boundary when one is near the cap.
  const para = cut.lastIndexOf('\n\n');
  if (para > cap * 0.6) cut = cut.slice(0, para);
  return `${cut.trimEnd()}\n\n${TRUNCATION_NOTICE(cap)}`;
}

// Closure codes classifyLiveness can reach from page text alone. The softer
// codes (listing page, missing apply control) lean on a rendered DOM's apply
// buttons, which a plain download does not have, so they are not trusted here.
const VISIBLE_CLOSURE_CODES = new Set(['expired_url', 'expired_body', 'deadline_passed']);

const SKIP_ELEMENTS = new Set(['script', 'style', 'noscript', 'template']);
const VISIBLE_SCAN_CAP = 1_000_000;

/**
 * The text a reader of the page would see: no scripts, styles or markup.
 * One forward pass with indexOf, so hostile input (millions of `<`, `<!--` or
 * unclosed `<script`) costs linear time instead of the quadratic backtracking a
 * lazy regex would.
 */
function visibleText(html) {
  const src = String(html).slice(0, VISIBLE_SCAN_CAP);
  const lower = asciiLower(src);
  const out = [];
  let i = 0;
  let nextGt = -2;
  while (i < src.length) {
    const lt = src.indexOf('<', i);
    if (lt === -1) { out.push(src.slice(i)); break; }
    out.push(src.slice(i, lt));
    if (src.startsWith('<!--', lt)) {
      const end = src.indexOf('-->', lt + 4);
      i = end === -1 ? src.length : end + 3;
      out.push(' ');
      continue;
    }
    const m = /^<(\/?)([a-z][a-z0-9]*)/.exec(lower.slice(lt, lt + 24));
    // Cache the next '>' so a run of '<' with no '>' after it costs one scan, not one per '<'.
    if (nextGt !== -1 && nextGt <= lt) nextGt = src.indexOf('>', lt + 1);
    const gt = nextGt;
    if (!m || gt === -1) { out.push(' '); i = lt + 1; continue; }
    if (!m[1] && SKIP_ELEMENTS.has(m[2])) {
      const close = lower.indexOf(`</${m[2]}`, gt + 1);
      const closeEnd = close === -1 ? -1 : src.indexOf('>', close);
      i = closeEnd === -1 ? src.length : closeEnd + 1;
      out.push(' ');
      continue;
    }
    out.push(/^(br|p|div|li|h[1-6]|tr|section|article|ul|ol|table|header|footer|main)$/.test(m[2]) ? '\n' : ' ');
    i = gt + 1;
  }
  return decodeEntities(out.join('')).replace(/[ \t]+/g, ' ');
}

const NAMED_ENTITIES = { nbsp: ' ', amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", rsquo: '\u2019', lsquo: '\u2018', rdquo: '\u201d', ldquo: '\u201c', ndash: '\u2013', mdash: '\u2014' };

/** Decode numeric and the common named character references; unknown ones stay as written. */
function decodeEntities(text) {
  return text.replace(/&(?:#(\d{1,7})|#[xX]([0-9a-fA-F]{1,6})|([a-zA-Z]{2,8}));/g, (m, dec, hex, name) => {
    if (name) { if (Object.hasOwn(LATIN1_ENTITIES, name)) return LATIN1_ENTITIES[name]; const k = name.toLowerCase(); return Object.hasOwn(NAMED_ENTITIES, k) ? NAMED_ENTITIES[k] : m; }
    const cp = dec ? Number(dec) : parseInt(hex, 16);
    return cp > 0 && cp <= 0x10ffff && !(cp >= 0xd800 && cp <= 0xdfff) ? String.fromCodePoint(cp) : m;
  });
}

/**
 * Did the fetch land on the page that was asked for? A job URL that redirects to
 * a listing or another job would otherwise lend that page's one JobPosting (or
 * its closure) to the requested job. Host `www.` and a trailing slash are not a
 * different page; tracking parameters are ignored.
 */
export function sameDocument(requested, final) {
  try {
    const norm = (u) => {
      const x = new URL(u);
      const q = [...x.searchParams].filter(([k]) => !/^(utm_|gclid$|fbclid$)/i.test(k)).map(([k, v]) => `${k}=${v}`).sort().join('&');
      return `${x.hostname.toLowerCase().replace(/^www\./, '')}${x.pathname.replace(/\/+$/, '').toLowerCase()}?${q}`;
    };
    return norm(requested) === norm(final);
  } catch { return false; }
}

// A script-toggled expiry view that is always present in the static markup (Phenom).
const EXPIRE_TEMPLATE_RE = /\b(?:job-expire-view|expire-job-view|job-expire-block)\b/i;

/** The markup with script/style elements and comments removed (linear, indexOf-based). */
function withoutScripts(html) {
  const src = String(html).slice(0, VISIBLE_SCAN_CAP);
  const lower = asciiLower(src);
  const out = [];
  let i = 0;
  while (i < src.length) {
    const lt = src.indexOf('<', i);
    if (lt === -1) { out.push(src.slice(i)); break; }
    out.push(src.slice(i, lt));
    if (src.startsWith('<!--', lt)) {
      const end = src.indexOf('-->', lt + 4);
      i = end === -1 ? src.length : end + 3;
      continue;
    }
    const m = /^<(script|style)\b/.exec(lower.slice(lt, lt + 10));
    if (m) {
      const close = lower.indexOf(`</${m[1]}`, lt + 1);
      const closeEnd = close === -1 ? -1 : src.indexOf('>', close);
      i = closeEnd === -1 ? src.length : closeEnd + 1;
      continue;
    }
    out.push('<');
    i = lt + 1;
  }
  return out.join('');
}

const VOID_ELEMENTS = new Set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr']);

/**
 * Remove every element whose opening tag carries the expiry-view marker. One forward pass: the next
 * '>' is cached, tags are length-capped before the marker test, and nesting is a counter, so hostile
 * markup (a run of '<a', deep nesting) costs time linear in the page, never quadratic.
 */
function withoutExpiryViews(html) {
  const src = String(html);
  const lower = asciiLower(src);
  const out = [];
  let i = 0; let nextGt = -2; let skipName = ''; let depth = 0;
  while (i < src.length) {
    const lt = src.indexOf('<', i);
    if (lt === -1) { if (!skipName) out.push(src.slice(i)); break; }
    if (!skipName) out.push(src.slice(i, lt));
    if (nextGt !== -1 && nextGt <= lt) nextGt = src.indexOf('>', lt + 1);
    const gt = nextGt;
    const m = /^<(\/?)([a-z][a-z0-9]*)/.exec(lower.slice(lt, lt + 24));
    if (!m || gt === -1) { if (!skipName) out.push('<'); i = lt + 1; continue; }
    if (skipName) {
      if (m[2] === skipName) {
        if (m[1]) { if (--depth === 0) skipName = ''; } else depth++;
      }
      i = gt + 1;
      continue;
    }
    if (!m[1] && EXPIRE_TEMPLATE_RE.test(src.slice(lt, Math.min(gt, lt + 4096)))) {
      out.push(' ');
      // A void element (img, br, ...) or a self-closed tag has no end tag: drop just the tag.
      if (!VOID_ELEMENTS.has(m[2]) && lower[gt - 1] !== '/') { skipName = m[2]; depth = 1; }
    } else out.push(src.slice(lt, gt + 1));
    i = gt + 1;
  }
  return out.join('');
}

/**
 * Does the page's visible text say the posting is closed? JSON-LD often carries
 * no expiry, and a closure banner lives in the visible markup that the JD
 * extraction never reads, so check it with the repo's own classifier.
 *
 * @returns {string|null} evidence, or null when the page does not say so
 */
function visibleClosureEvidence(html, url, finalUrl, now = new Date()) {
  // Phenom career sites (observed on careers.roberthalf.com, every live posting) ship the
  // "the job you are trying to apply for has been filled" view inside the page markup and
  // show or hide it with script. Static HTML cannot say which, so the banner text proves
  // nothing there; only structured data (validThrough, an ATS flag) can still close it.
  // Only a marker in the page's own markup counts: one inside a script, style or comment
  // (a bundle that merely names the class) must not suppress a separate closure banner.
  // The expiry view itself is dropped; any other closure banner on the page still counts.
  const markup = withoutExpiryViews(withoutScripts(html));
  const verdict = classifyLiveness({ status: 200, requestedUrl: url, finalUrl, bodyText: visibleText(markup), now });
  return verdict.result === 'expired' && VISIBLE_CLOSURE_CODES.has(verdict.code)
    ? `page text: ${verdict.reason}`.slice(0, 200)
    : null;
}

/**
 * Closure evidence for a page already downloaded (the batch runner's static fetch): an
 * expired JobPosting validThrough, or closure text in the visible page. Null when the
 * page does not say it is closed.
 * @returns {string|null}
 */
export function pageClosureEvidence(html, url, finalUrl = url) {
  // Same gates as fetchJdFromPage: a redirect to another page is not this posting's closure,
  // an ambiguous page proves nothing, and the HTML is capped so hostile input stays cheap.
  if (finalUrl && !sameDocument(url, finalUrl)) {
    // Off the requested page: its text is not this posting's. Only the URL itself can still
    // say closed (a redirect to an error or "job not found" address).
    const verdict = classifyLiveness({ status: 200, requestedUrl: url, finalUrl, bodyText: '', now: new Date() });
    return verdict.result === 'expired' && verdict.code === 'expired_url' ? `redirect: ${verdict.reason}`.slice(0, 200) : null;
  }
  // A page past the cap cannot be checked whole: a second posting after the prefix could be hidden,
  // so nothing page-derived counts as closure (the URL check above and the ATS API still do).
  if (String(html).length > PAGE_HTML_CAP) return null;
  const capped = String(html);
  const { found, ambiguous } = inspectPage(capped, Date.now(), url);
  if (ambiguous) return null;
  if (found?.closed) return found.evidence;
  return visibleClosureEvidence(capped, url, finalUrl);
}

/**
 * ATS results are capped by browser-extract's compactText, which marks a cut
 * with a bare trailing `…`. Nothing downstream recognises that, so turn it into
 * the same explicit notice a cut page JD carries. Text that was not cut is
 * returned unchanged. Pure — exported for tests.
 *
 * @param {string} text
 * @param {number} cap the cap the text was compacted to
 */
export function markCappedAtsText(text, cap) {
  if (text.length !== cap + 1 || !text.endsWith('\u2026')) return text;
  return `${text.slice(0, -1).trimEnd()}\n\n${TRUNCATION_NOTICE(cap)}`;
}

/**
 * @param {string} url
 * @param {number} [textCap]
 * @param {number} [timeoutMs]
 * @returns {Promise<{url: string, title: string, text: string, method: string, closed?: boolean, evidence?: string}|null>}
 */
export async function fetchJdFromPage(url, textCap = 20_000, timeoutMs = 15_000) {
  let host;
  try { host = new URL(url).hostname; } catch { return null; }
  if (isLinkedInHost(host)) return null;
  const page = await fetchHtml(url, timeoutMs);
  if (!page) return null;
  if (page.finalUrl && !sameDocument(url, page.finalUrl)) return null;
  const { found, ambiguous } = inspectPage(page.html, Date.now(), url);
  if (!found) {
    // No extractable JD, but a closure banner is still an answer: exit 3, not a plain miss.
    // Not on an ambiguous page, though: its text may be a listing's, not this posting's.
    const only = ambiguous ? null : visibleClosureEvidence(page.html, url, page.finalUrl);
    return only ? { url, title: '', text: '', method: 'visible-closure', closed: true, evidence: only } : null;
  }
  if (found.closed) return { url, title: '', text: '', method: found.method, closed: true, evidence: found.evidence };
  const visibleClosure = visibleClosureEvidence(page.html, url, page.finalUrl);
  if (visibleClosure) return { url, title: '', text: '', method: 'visible-closure', closed: true, evidence: visibleClosure };
  return { url, title: '', text: capJdText(found.text, textCap), method: found.method };
}