/**
 * liveness-browser.mjs — Playwright-driven liveness check for a single URL.
 *
 * Shared by check-liveness.mjs (CLI tool) and scan.mjs (--verify flag).
 * Returns the same shape as classifyLiveness: { result, reason }.
 */

import { classifyLiveness, jobIdToken, JOB_ID_PATTERN } from './liveness-core.mjs';
import { BROWSER_LIKE_USER_AGENT } from './user-agent.mjs';
import { SINKHOLE_ADDRESSES } from './providers/_ip-guard.mjs';
import { browserLaunchOptions } from './lib/browser-launch.mjs';

const NAVIGATE_TIMEOUT_MS = 15_000;
// Single-page-app ATS render the posting after domcontentloaded, so the page
// is read repeatedly until the verdict stops being a "not rendered yet" code;
// a single early read would call a live posting insufficient_content.
// Measured over 386 loads of 188 posting URLs: the first decisive verdict
// arrived at p50 1785ms, p95 3071ms and at most 3308ms outside iCIMS (whose
// frames have their own poll below); 17 reads 250ms apart cover 4000ms. Bounded by a count rather than a clock so a
// page double whose waitForTimeout returns at once still terminates.
// Once a read is decisive no later read flipped a live posting to closed, so
// stopping at the first decisive read is safe. Stopping when the text stops
// changing is NOT: a spinner holds the text constant on live pages.
const HYDRATION_POLL_MS = 250;
const HYDRATION_MAX_POLLS = 16;
export const HYDRATING_CODES = new Set(['insufficient_content', 'no_apply_control']);
// Upper bound on the extra wait for same-origin child frames to populate, and
// the poll interval inside it. Only spent when the page poll ends undecided
// while such a frame is present.
const FRAME_CONTENT_TIMEOUT_MS = 6_000;
const FRAME_CONTENT_POLL_MS = 500;
// A failed navigation is not over when page.goto rejects: Chromium commits its
// error page (chrome-error://chromewebdata/) just after, 1-69 ms later over 36
// failed loads (a host with no address, four kinds of TLS failure). Playwright
// reports a goto that is still waiting when that commit lands as "interrupted
// by another navigation" (microsoft/playwright#35944), and Chromium goes on
// loading the URL that goto asked for, whose late commit then interrupts the
// goto after it. On the one page that check-liveness and scan --verify share,
// one such failure turned the checks after it into navigation_errors. So a
// check whose navigation failed waits for that commit before it returns, up to
// this long. A goto that timed out has nothing to wait for: no error page
// commits, and the next goto cancels it.
const FAILED_NAVIGATION_COMMIT_MS = 2_000;

// The query parameters that carry a posting's own id, from the providers that
// put it there: UltiPro's opportunityId, SchoolSpring's and Deutsche Bahn's
// jobId, Tencent's postId, AppliTrack's AppliTrackJobId, Meituan's jobUnionId,
// and gh_jid on a careers page that embeds Greenhouse. Any other query value,
// such as a campaign or tracking number, is not the posting's id.
const POSTING_ID_PARAMS = new Set(['opportunityid', 'jobid', 'postid', 'applitrackjobid', 'jobunionid', 'gh_jid']);

// BambooHR's client bundle can throw during first paint — a failed
// /globals/locale request followed by an uncaught TypeError reading
// `hasPasskey` on null — which halts the SPA on its bare loading spinner for
// the whole hydration poll, so a live posting reads as insufficient_content. The
// posting itself is untouched; a reload clears it. This is a shared
// front-end bug across every *.bamboohr.com tenant (not one company's
// board) and common enough to matter: rerun checkUrlLiveness in a loop
// against any live *.bamboohr.com posting URL with this branch disabled to
// see the current failure rate.
//
// Scoped to this one host on purpose, not a generic "flaky ATS" mechanism: no
// other provider has shown this failure signature, and retrying every
// insufficient-content verdict on every host would pay an extra page load on
// every genuinely dead posting for no evidence of benefit elsewhere. If a
// second ATS turns up the same symptom, generalize then.
const BAMBOOHR_HOSTS = [/(^|\.)bamboohr\.com$/];

function isBambooHrHost(hostname) {
  return BAMBOOHR_HOSTS.some((pattern) => pattern.test(hostname));
}

/**
 * Same-origin test used to decide whether a child frame is part of the posting
 * or somebody else's widget. Deliberately strict: about:blank, data: frames,
 * tag managers and ad iframes all fail it, so only the ATS's own embedded
 * document contributes text and apply controls.
 */
export function sameOrigin(frameUrl, pageUrl) {
  try {
    const a = new URL(frameUrl);
    const b = new URL(pageUrl);
    if (a.protocol !== 'http:' && a.protocol !== 'https:') return false;
    return a.origin === b.origin;
  } catch {
    return false;
  }
}

// The default Playwright headless UA contains "HeadlessChrome", which Cloudflare
// and similar WAFs flag — portals like pracuj.pl then serve a 403 challenge page
// instead of the posting. Presenting a normal desktop Chrome UA clears the wall
// headlessly (the scan parser scripts/parsers/pracuj-jobs.mjs relies on the same
// trick), so the common case never needs the slower headed-browser fallback.
export const LIVENESS_CONTEXT_OPTIONS = {
  userAgent: BROWSER_LIKE_USER_AGENT,
  locale: 'en-US',
};

// Open a page in a context that already presents a realistic UA. Both callers use
// this instead of browser.newPage() so headless checks aren't instantly bot-walled.
export async function newLivenessPage(browser) {
  const context = await browser.newContext(LIVENESS_CONTEXT_OPTIONS);
  return context.newPage();
}

export const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

// Throttle delay with jitter: a value in [baseMs, 2*baseMs). Spacing requests out
// (and randomizing the gap) keeps a bulk run under rate-based WAF thresholds —
// pracuj.pl's Cloudflare flags the session after ~2 rapid hits, after which even
// headed retries are blocked. A randomized gap also avoids a fixed-cadence
// fingerprint. Returns 0 for a non-positive base (throttling disabled).
export function jitteredDelayMs(baseMs) {
  if (!baseMs || baseMs <= 0) return 0;
  return baseMs + Math.floor(Math.random() * baseMs);
}

// Per-host spacing for a browser batch. A WAF rate limit is per site, so a run
// across many sites only needs to space out repeat visits to the SAME host
// (exact hostname, `www.` ignored; careers.x.com and jobs.x.com count as two):
// the gap since that host's last browser check ended counts toward its
// jittered delay, and a host seen for the first time waits nothing. With a global gap, 641 URLs
// over ~40 hosts cost 5-10 s each (~1.5 h); per host, only same-site runs wait.
const hostKey = (url) => {
  try { return new URL(url).hostname.toLowerCase().replace(/^www\./, ''); } catch { return String(url); }
};

export function createHostSpacer(baseMs, { now = Date.now, delay = jitteredDelayMs } = {}) {
  const lastVisit = new Map();
  return {
    /** Milliseconds to wait before a browser check of `url`. */
    waitBefore(url) {
      if (!baseMs || baseMs <= 0) return 0;
      const last = lastVisit.get(hostKey(url));
      if (last == null) return 0;
      return Math.max(0, delay(baseMs) - (now() - last));
    },
    /** Record that `url`'s host was just visited by the browser. */
    visited(url) {
      lastVisit.set(hostKey(url), now());
    },
  };
}

// Defensive guards: URLs come from ATS feeds (mostly trusted) but a misconfigured
// portals.yml entry or a hijacked feed shouldn't be able to point Playwright at
// internal infrastructure. Only allow http(s) and reject loopback/private/link-local.
//
// The hostname coming out of `new URL(...)` needs normalization before the regex
// pass, because the WHATWG URL parser surfaces several encodings that bypass a
// naive match against `parsed.hostname`:
//   1. IPv6 hosts are serialized with brackets — `new URL('http://[::1]/').hostname`
//      is `'[::1]'`, so a regex like `/^::1$/` never fires unless brackets are stripped.
//   2. FQDN trailing dot is preserved — `localhost.` reaches the network as
//      localhost, but `/^localhost$/` doesn't match it.
//   3. IPv4-mapped IPv6 (`::ffff:127.0.0.1` or the hex form `::ffff:7f00:1`)
//      routes to the embedded IPv4 in Chromium, so the embedded address must
//      also be matched against the IPv4 block list.
// `0.0.0.0` and the all-zeros IPv6 `::` both reach loopback on Linux and need
// explicit entries; the original list omitted them.
//
// IPv6 ULA is fc00::/7 (RFC 4193): both fc00::/8 and fd00::/8. IPv6
// link-local is fe80::/10 (RFC 4291): fe80–febf, not only the fe80: prefix.
const PRIVATE_HOST_PATTERNS = [
  /^localhost$/,
  /^localhost\.localdomain$/,
  /^0\.0\.0\.0$/,
  /^127\./,
  /^10\./,
  /^192\.168\./,
  /^172\.(1[6-9]|2\d|3[01])\./,
  /^169\.254\./,
  /^::1$/,
  /^::$/,
  /^f[cd][0-9a-f]{2}:/,
  /^fe[89ab][0-9a-f]:/,
];

// Lowercase, strip IPv6 brackets, strip FQDN trailing dot. The `hostname`
// returned by `new URL(...)` is already percent-decoded and IDNA-normalized,
// but it preserves brackets around IPv6 hosts and trailing dots on FQDNs.
function normalizeHost(rawHostname) {
  if (!rawHostname) return '';
  let h = String(rawHostname).toLowerCase();
  if (h.startsWith('[') && h.endsWith(']')) h = h.slice(1, -1);
  if (h.endsWith('.')) h = h.slice(0, -1);
  return h;
}

// IPv4-mapped IPv6 (RFC 4291 §2.5.5.2): `::ffff:0:0/96` routes to the embedded
// IPv4 address. Two textual forms — dotted (`::ffff:127.0.0.1`) and pure-hex
// (`::ffff:7f00:1`). Return the embedded IPv4 in dotted-decimal form, or null
// if `host` is not an IPv4-mapped IPv6.
function extractMappedIPv4(host) {
  const dotted = host.match(/^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/);
  if (dotted) return dotted[1];
  const hex = host.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
  if (hex) {
    const a = parseInt(hex[1], 16);
    const b = parseInt(hex[2], 16);
    return `${(a >> 8) & 0xff}.${a & 0xff}.${(b >> 8) & 0xff}.${b & 0xff}`;
  }
  return null;
}

// Returns null when the URL is safe to fetch, otherwise a structured guard
// result with a stable `code` (used for routing in scan.mjs) plus a human
// `reason`. Stable codes — not regex on reason strings — drive downstream
// dispatch so the wording can change freely without breaking callers.
//
// Exported for unit tests; the main entry point is checkUrlLiveness.
export function rejectPrivateOrInvalid(url) {
  let parsed;
  try {
    parsed = new URL(url);
  } catch {
    return { code: 'invalid_url', reason: 'invalid URL' };
  }
  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
    return { code: 'unsupported_protocol', reason: `unsupported protocol ${parsed.protocol}` };
  }
  const host = normalizeHost(parsed.hostname);
  const mappedIPv4 = extractMappedIPv4(host);
  const candidates = mappedIPv4 ? [host, mappedIPv4] : [host];
  for (const candidate of candidates) {
    if (PRIVATE_HOST_PATTERNS.some((pattern) => pattern.test(candidate))) {
      return { code: 'blocked_host', reason: `blocked host ${parsed.hostname}` };
    }
  }
  return null;
}

const dnsCache = new Map();
// Hold failures for 30 s, the same TTL as providers/_dns-cache.mjs, so an outage
// is not pinned for the whole process. Unlike that cache, this includes NXDOMAIN:
// resolveViaDns turns all lookup failures into an empty answer.
const DNS_NEGATIVE_TTL_MS = 30_000;

// Real DNS: resolve4 + resolve6 + lookup, each tolerant of its own failure, so a
// host that only answers on one of the three still yields an address list.
async function resolveViaDns(hostname) {
  const dns = await import('dns/promises');
  const [ipv4, ipv6, lookupList] = await Promise.all([
    dns.resolve4(hostname).catch(() => []),
    dns.resolve6(hostname).catch(() => []),
    dns.lookup(hostname, { all: true }).catch(() => [])
  ]);
  return Array.from(new Set([
    ...ipv4,
    ...ipv6,
    ...lookupList.map(item => item.address)
  ]));
}

let hostResolver = resolveViaDns;

/**
 * Swap the resolver the egress guard uses, returning a restore function.
 *
 * `dns/promises` is imported dynamically and the guard calls the ESM namespace
 * bindings, which are immutable — monkey-patching the module object has no
 * effect on them (#2386). Without this seam a test can only ever reach the
 * "host resolved to nothing" branch: the real resolver returns an empty list
 * for the synthetic hostname, the guard blocks on that, and the loopback
 * rejection the test exists to cover never runs. The memo cache is cleared on
 * every swap, in both directions, so a verdict computed under one resolver can
 * never be served to the next.
 *
 * @param {((hostname: string) => Promise<string[]>)|null} resolver - Resolver to
 *   install, or null to restore the real DNS one.
 * @returns {() => void} Restores the resolver in place before this call.
 */
export function setHostResolver(resolver) {
  const previous = hostResolver;
  hostResolver = resolver ?? resolveViaDns;
  dnsCache.clear();
  return () => {
    hostResolver = previous;
    dnsCache.clear();
  };
}

async function resolveDnsCached(hostname) {
  if (dnsCache.has(hostname)) {
    const cached = dnsCache.get(hostname);
    if (Array.isArray(cached)) return cached;
    if (Date.now() < cached.expiresAt) throw cached.error;
    dnsCache.delete(hostname);
  }
  try {
    const addresses = await hostResolver(hostname);
    if (addresses.length === 0) {
      // Tagged so the route guard can tell "this host does not exist" apart from
      // "this host resolves into private space". The first is a dead third-party
      // script, the second is an egress-guard hit. Only the second says anything
      // about the page being checked.
      const missing = new Error(`DNS resolution returned no addresses for ${hostname}`);
      missing.livenessCode = 'dns_no_addresses';
      throw missing;
    }
    dnsCache.set(hostname, addresses);
    return addresses;
  } catch (err) {
    if (!Array.isArray(dnsCache.get(hostname))) {
      dnsCache.set(hostname, { error: err, expiresAt: Date.now() + DNS_NEGATIVE_TTL_MS });
    }
    throw err;
  }
}

// Second layer of the egress guard: `rejectPrivateOrInvalid` only sees the
// literal host, so a public hostname that *resolves* to private space still
// gets through it. Resolve and re-check every address before the request is
// allowed out. Exported so other Playwright callers (archive-posting.mjs) wire
// up the same two-layer guard instead of growing a second implementation.
export async function validateUrlSecurity(urlString) {
  const url = new URL(urlString.endsWith('.') ? urlString.slice(0, -1) : urlString);
  const hostname = url.hostname;
  const host = normalizeHost(hostname);
  const addresses = await resolveDnsCached(host);
  // Every private address, not only the first: whether a DNS filter gave the
  // answer depends on all of them.
  const blocked = [];
  for (const ip of addresses) {
    const norm = normalizeHost(ip);
    const mapped = extractMappedIPv4(norm);
    const candidates = mapped ? [norm, mapped] : [norm];
    const hit = candidates.find((candidate) => PRIVATE_HOST_PATTERNS.some((pattern) => pattern.test(candidate)));
    if (hit) blocked.push({ ip, hit });
  }
  if (blocked.length > 0) {
    const denied = new Error(`Access denied: Egress guard blocked private target IP ${blocked[0].ip}`);
    if (blocked.every(({ hit }) => SINKHOLE_ADDRESSES.has(hit))) denied.livenessCode = 'dns_sinkhole';
    throw denied;
  }
}

// Rediscovery can navigate a newly created page before its first liveness
// check. Install the same URL/DNS guards before that search as well.
let suppressedSubresourceBlocks = 0;

/** How many subresource requests were aborted for having no usable address since the last call (then resets). */
export function takeSuppressedBlockCount() {
  const n = suppressedSubresourceBlocks;
  suppressedSubresourceBlocks = 0;
  return n;
}

export async function installLivenessRouteGuard(page) {
  if (page && typeof page.route === 'function' && !page._routeInterceptorRegistered) {
    page._routeInterceptorRegistered = true;
    await page.route('**/*', async (route) => {
      const requestUrl = route.request().url();
      const errGuard = rejectPrivateOrInvalid(requestUrl);
      if (errGuard) {
        console.warn(`Blocked request to restricted destination: ${requestUrl}`);
        page._blockedByGuard = errGuard;
        return route.abort('blockedbyclient');
      }
      try {
        await validateUrlSecurity(requestUrl);
        return route.continue();
      } catch (err) {
        // A host that resolves to nothing is a DEAD THIRD-PARTY SCRIPT, not a
        // statement about the posting. Measured 2026-08-14 over a 217-URL
        // recheck: 78 live postings were returned as `uncertain` because an
        // analytics or ad host on the page no longer exists — 53 on
        // personalisation.visitorqueue.com, 17 on s7.addthis.com (AddThis was
        // shut down in 2023), the rest on fluidads and cloudfront. One was
        // opened by hand to confirm: 11,178 characters of live posting and a
        // working apply control, called uncertain because of a dead tracker.
        //
        // A tracker the user's own DNS filter blocks says nothing about the
        // posting either. validateUrlSecurity tags that answer `dns_sinkhole`.
        // Measured 2026-10-11 over 35 postings on 19 boards, with Control D
        // as the resolver: 20 read blocked_host because of Google Analytics,
        // DoubleClick, Cloudflare Insights, LinkedIn's scripts and the like, and
        // 11 of the 20 had a visible apply control. AdGuard DNS: 12 blocked_host.
        //
        // The request is still aborted either way, so the egress guard loses
        // nothing. Only the VERDICT stops being poisoned, and only for a
        // subresource: if the main document itself has no usable address, that is a
        // real finding about the posting and still counts.
        //
        // It counts as `uncertain` under its DNS tag, though, not as `blocked_host`.
        // That code says the URL points into private space, and `scan --verify`
        // records it as skipped_blocked_host, for good. No usable address can just
        // as well be a DNS outage or the user's own filter, which say nothing lasting
        // about the URL. The catch in checkUrlLiveness treats a DNS failure as
        // transient for the same reason, and `--verify` keeps this verdict as it
        // keeps a navigation_error.
        //
        // Whether this was the main document or a subresource can only be asked
        // of a real Playwright request. Callers may pass a lighter route double
        // (the test suite does, with request() returning just a url()), and for
        // those the answer is unknowable — so default to TRUE, which keeps the
        // pre-existing behaviour of poisoning the verdict. The relaxation only
        // applies where we can positively establish it was a subresource.
        const request = typeof route?.request === 'function' ? route.request() : null;
        const canTell =
          typeof request?.isNavigationRequest === 'function' &&
          typeof request?.frame === 'function' &&
          typeof page?.mainFrame === 'function';
        const isMainDocument = canTell
          ? request.isNavigationRequest() && request.frame() === page.mainFrame()
          : true;
        const noUsableAddress = err?.livenessCode === 'dns_no_addresses' || err?.livenessCode === 'dns_sinkhole';
        // A subresource with no usable address is counted, not logged: with a DNS
        // filter on, every page logged a line per tracker request (440 lines over
        // 142 postings on 2026-10-16) and buried the verdicts. Everything that
        // bears on a verdict — the main document, a private address — still logs.
        if (noUsableAddress && !isMainDocument) suppressedSubresourceBlocks++;
        else console.warn(`Blocked request to restricted destination (DNS): ${requestUrl} - ${err.message}`);
        if (!noUsableAddress) {
          page._blockedByGuard = { code: 'blocked_host', reason: err.message };
        } else if (isMainDocument) {
          page._blockedByGuard = { code: err.livenessCode, reason: err.message };
        }
        return route.abort('blockedbyclient');
      }
    });
  }
}

// One navigation of the page (a goto or a reload). If it fails, wait for the
// main frame to commit Chromium's error page, so the page is settled for
// whatever navigates it next (see FAILED_NAVIGATION_COMMIT_MS). No other
// navigation ends the wait: a child frame's, or the old page changing its own
// URL (a fragment or history change fires framenavigated too), would let the
// next goto start before the error page commits and interrupts it. A lighter
// page double without waitForEvent skips the wait.
async function navigateSettled(page, navigate) {
  try {
    return await navigate();
  } catch (err) {
    const canWait = typeof page.waitForEvent === 'function' && typeof page.mainFrame === 'function';
    if (canWait && err?.name !== 'TimeoutError') {
      await page
        .waitForEvent('framenavigated', {
          predicate: (frame) => frame === page.mainFrame() && frame.url().startsWith('chrome-error:'),
          timeout: FAILED_NAVIGATION_COMMIT_MS,
        })
        .catch(() => {});
    }
    throw err;
  }
}

// A navigation that times out is retried once with a longer budget. A slow
// careers page (block.xyz took >15 s on a cold load) is live but was answered
// `uncertain` and stayed unresolved in the queue. Only a timeout earns the retry:
// a refused connection, a DNS failure or a guard block fails exactly the same way
// the second time, and the guard verdict must win as before.
export const NAVIGATE_RETRY_TIMEOUT_MS = 30_000;

export async function gotoWithTimeoutRetry(page, url, {
  timeoutMs = NAVIGATE_TIMEOUT_MS,
  retryTimeoutMs = NAVIGATE_RETRY_TIMEOUT_MS,
} = {}) {
  const go = (timeout) => navigateSettled(page, () => page.goto(url, { waitUntil: 'domcontentloaded', timeout }));
  try {
    return await go(timeoutMs);
  } catch (err) {
    if (err?.name !== 'TimeoutError' || page?._blockedByGuard || !(retryTimeoutMs > timeoutMs)) throw err;
    return go(retryTimeoutMs);
  }
}

export const extractApplyControls = ({ postingId, postingSegment, pageUrl, idPattern }) => {
  // Page chrome (nav, header, footer) is skipped because its links are the
  // site's own: Jobs, Careers, Sign in. A link to this posting's own page,
  // on the page's own origin, is not chrome, wherever it sits; a link to
  // another site says nothing about this page's posting, even with its id
  // in the path. Tech in Asia puts its only Apply control,
  // "APPLY FOR JOB" -> /jobs/<id>/apply, in the page header: measured
  // 2026-10-22, 20 of 20 live postings read no_apply_control without this,
  // and 40 of 40 closed ones answer "Job not found" with no such link. Only
  // the path counts, so a sign-in link that carries the posting in its
  // return address (?returnUrl=/jobs/<id>) is still chrome, and so is a
  // link back into this page under another query, fragment or trailing
  // slash, such as a skip link to #main-content. A sign-in or sign-up page
  // is chrome even under the posting's own path (/jobs/<id>/login): it is
  // the site's sign-in, not the posting. In a frame, the page the
  // frame sits in is this page too; its address is passed in, because a
  // frame sandboxed without allow-same-origin cannot read window.top. The
  // id has to be the link's own: its last path segment that carries an id,
  // whole, so posting 12345 claims neither /jobs/123456/apply nor
  // /jobs/job-12345/apply, nor /jobs/12345/related/67890/apply, whose own
  // posting is 67890. A posting whose own segment carries the id with more
  // around it claims that whole segment too: posting job-12345 claims
  // /jobs/job-12345/apply.
  const idBearing = new RegExp(idPattern, 'i');
  const signIn = /^(?:log-?in|sign[-_]?(?:in|up)|register)(?:\.\w+)?$/;
  const linksToPosting = (element) => {
    const href = postingId && element.getAttribute('href');
    if (!href) return false;
    try {
      const target = new URL(href, document.baseURI);
      const pageKey = (u) => u.origin + u.pathname.replace(/\/+$/, '');
      const thisPage = [location.href, pageUrl].map((u) => pageKey(new URL(u)));
      if (thisPage.includes(pageKey(target))) return false;
      if (target.origin !== new URL(pageUrl).origin) return false;
      const segments = target.pathname.toLowerCase().split('/');
      if (segments.some((s) => signIn.test(s))) return false;
      const own = segments.filter((s) => idBearing.test(s)).pop();
      return own === postingId || own === postingSegment;
    } catch {
      return false;
    }
  };

  // A design-system button is a custom element: UKG Pro (UltiPro) renders
  // Apply as <ukg-button>Apply now</ukg-button>, with no role on it. The
  // native <button> inside its shadow root has no text of its own (the
  // label is slotted in), so the element itself is the control to read.
  // No selector matches a tag-name suffix, hence the filter.
  const customButtons = Array.from(document.querySelectorAll('*'))
    .filter((element) => element.localName.endsWith('-button'));
  const candidates = Array.from(
    document.querySelectorAll('a, button, input[type="submit"], input[type="button"], [role="button"]')
  ).concat(customButtons);

  return candidates
    .filter((element) => {
      if (element.closest('nav, header, footer') && !linksToPosting(element)) return false;
      if (element.closest('[aria-hidden="true"]')) return false;

      const style = window.getComputedStyle(element);
      if (style.display === 'none' || style.visibility === 'hidden') return false;
      if (!element.getClientRects().length) return false;

      return Array.from(element.getClientRects()).some((rect) => rect.width > 0 && rect.height > 0);
    })
    .map((element) => {
      const label = [
        element.innerText,
        element.value,
        element.getAttribute('aria-label'),
        element.getAttribute('title'),
      ]
        .filter(Boolean)
        .join(' ')
        .replace(/\s+/g, ' ')
        .trim();

      return label;
    })
    .filter(Boolean);
};

// The posting's id, from the address of the page being read, so a posting
// reached through a redirect (/jobs/senior-engineer to /jobs/12345) is
// matched by the address it landed on. A page whose address lost the
// requested URL's id reads as off the posting whatever its controls (the
// redirect guard), and one whose own id is another posting's claims
// nothing in the chrome: /jobs/12345 that lands on
// /jobs/67890?returnUrl=/jobs/12345 passes the guard, but the page is
// posting 67890. The id in the last path segment, so a number in the
// query (?campaign=123456) is not taken for it. When that segment has
// none, the id in a query parameter that names the posting, and only then
// the last id in the path: on UltiPro,
// /JobBoard/{board}/OpportunityDetail?opportunityId={posting}, the path's
// only id is the board's, and every link under the board's path has it.
// The segment itself is kept only when the id came from it, so a generic
// last segment (/jobs/12345/details) claims nothing.
export const postingIdentity = (address) => {
  const postingUrl = new URL(address);
  const lastSegment = postingUrl.pathname.split('/').filter(Boolean).pop() ?? '';
  const queryId = [...postingUrl.searchParams]
    .filter(([name]) => POSTING_ID_PARAMS.has(name.toLowerCase()))
    .map(([, value]) => jobIdToken(value))
    .find(Boolean);
  return {
    postingId: jobIdToken(lastSegment) ?? queryId ?? jobIdToken(postingUrl.pathname),
    postingSegment: jobIdToken(lastSegment) ? lastSegment.toLowerCase() : null,
  };
};

export async function checkUrlLiveness(page, url, { extraSettleMs = 0, timeoutRetry = true } = {}) {
  const guardError = rejectPrivateOrInvalid(url);
  if (guardError) {
    return { result: 'uncertain', code: guardError.code, reason: guardError.reason };
  }
  if (page) {
    page._blockedByGuard = null;
  }
  await installLivenessRouteGuard(page);
  try {
    const response = await gotoWithTimeoutRetry(page, url, timeoutRetry ? {} : { retryTimeoutMs: 0 });
    const status = response?.status() ?? 0;

    // extraSettleMs is slack for the headed retry, where a JS anti-bot
    // interstitial needs a moment to clear before the first read.
    if (extraSettleMs > 0) await page.waitForTimeout(extraSettleMs);

    // Frame aggregation is an enhancement, never a requirement. Callers may pass
    // a lightweight page object that only implements goto/url/evaluate — the
    // test doubles in test-all.mjs do — and such a caller must keep getting the
    // top-level verdict rather than a navigation_error.
    const supportsFrames = typeof page?.frames === 'function' && typeof page?.mainFrame === 'function';

    const childFrames = (topUrl) =>
      !supportsFrames
        ? []
        : page.frames().filter((frame) => {
            if (frame === page.mainFrame()) return false;
            try {
              return sameOrigin(frame.url() || '', topUrl); // excludes about:blank, ads, tag managers
            } catch {
              return false;
            }
          });

    const requested = postingIdentity(url);

    // One reading of the page: the top-level document plus every same-origin
    // child frame, so a posting rendered inside a frame and one rendered inline
    // are judged the same way. A top-level read that throws propagates; a frame
    // that detaches or turns cross-origin mid-read is skipped.
    const readPage = async () => {
      const finalUrl = page.url();
      const landed = postingIdentity(finalUrl);
      const anotherPosting = requested.postingId && landed.postingId !== requested.postingId;
      const walkArgs = {
        ...(anotherPosting ? { postingId: null, postingSegment: null } : landed),
        pageUrl: finalUrl,
        idPattern: JOB_ID_PATTERN,
      };
      let bodyText = await page.evaluate(() => document.body?.innerText ?? '');
      let applyControls = await page.evaluate(extractApplyControls, walkArgs);
      for (const frame of childFrames(finalUrl)) {
        try {
          const text = await frame.evaluate(() => document.body?.innerText ?? '');
          if (text && text.trim()) bodyText += '\n' + text;
          applyControls = applyControls.concat(await frame.evaluate(extractApplyControls, walkArgs));
        } catch {
          // detached or cross-origin mid-read; the rest of the reading still stands
        }
      }
      return { finalUrl, bodyText, applyControls };
    };

    // Read the page until the verdict is decisive, the request guard has
    // already decided it, or HYDRATION_MAX_POLLS waits have passed. A frame
    // appearing does not end the poll: it may be a sign-in, chat or consent
    // widget rather than the posting, and the posting can still render on the
    // top level. A read that throws (the SPA rebuilding its DOM) is retried. If
    // the poll ends on a failed read while the last good reading still looked
    // unrendered, that reading is stale evidence, so the error propagates and
    // becomes a navigation_error instead of an expired verdict; the same
    // happens when no read succeeded at all.
    const pollPage = async (pageStatus) => {
      let reading = null;
      let hydrating = true;
      let lastError = null;
      for (let poll = 0; ; poll += 1) {
        try {
          reading = await readPage();
          lastError = null;
          const { code } = classifyLiveness({ status: pageStatus, requestedUrl: url, ...reading });
          hydrating = HYDRATING_CODES.has(code);
          if (!hydrating) break;
        } catch (err) {
          lastError = err;
        }
        if (page._blockedByGuard || poll >= HYDRATION_MAX_POLLS) break;
        await page.waitForTimeout(HYDRATION_POLL_MS);
      }
      if (lastError && hydrating) throw lastError;
      if (hydrating && childFrames(reading.finalUrl).length > 0) {
        await waitForFramesToFill(reading.finalUrl);
        reading = await readPage();
      }
      return reading;
    };

    // Some ATS render the whole posting inside a same-origin iframe and leave the
    // top-level document as an empty shell. iCIMS is the reference case: measured
    // 2026-08-14, the outer document of a LIVE posting held 13 characters and no
    // apply control, so classifyLiveness reached `insufficient_content` and called
    // it expired. 92 live postings were closed that way in one sweep, and a false
    // `expired` is the expensive direction — it is written to scan-history as
    // skipped_expired and dedup-filters the job out of every later scan.
    //
    // Reading same-origin child frames cannot resurrect a dead posting: a removed
    // iCIMS job answers HTTP 410 at the top level (verified on two fabricated job
    // ids and one genuinely dead posting), so it short-circuits on status long
    // before any content check, and its error frame carries zero apply controls.
    // The frame ATTACHES fast but FILLS late. Measured on iCIMS 2026-08-14: the
    // same-origin child frame is present at 2000ms with 0 characters and only
    // populates between 3000 and 4000ms, so reading it as soon as it attaches
    // gets an empty document and changes nothing. The page poll reads the frame
    // on every pass, so a frame that fills within the poll is caught there; a
    // poll that ends still undecided with a same-origin frame present waits for
    // the frames to fill, bounded, then takes one fresh reading. The cost is
    // only paid on pages that actually have a same-origin child frame, so the
    // ATS that render inline are unaffected.
    //
    // A 404/410 is decided on the first read: classifyLiveness checks the status
    // before any content, so the poll stops there and no frame wait is paid.
    const waitForFramesToFill = async (topUrl) => {
      const deadline = Date.now() + FRAME_CONTENT_TIMEOUT_MS;
      // Wait for EVERY qualifying frame, not merely the first one to fill: with
      // two same-origin frames the posting could otherwise be read while still
      // empty. Measured across five iCIMS tenants there is exactly one
      // qualifying frame per page, so in practice this is the same loop.
      for (;;) {
        let anyEmpty = false;
        for (const frame of childFrames(topUrl)) {
          try {
            const probe = await frame.evaluate(() => document.body?.innerText ?? '');
            if (!probe.trim()) anyEmpty = true;
          } catch {
            // detached mid-poll; try again on the next tick
          }
        }
        if (!anyEmpty || Date.now() >= deadline) break;
        await page.waitForTimeout(FRAME_CONTENT_POLL_MS);
      }
    };

    const reading = await pollPage(status);
    const { finalUrl } = reading;
    let verdict = classifyLiveness({ status, requestedUrl: url, ...reading });

    // See BAMBOOHR_HOSTS above. Only fires on the specific verdict this
    // render race produces (a short/empty body, not an explicit closure
    // banner or a 404/410, both of which are trusted as-is).
    if (verdict.code === 'insufficient_content' && typeof page.reload === 'function') {
      let host = '';
      try { host = new URL(finalUrl || url).hostname; } catch { /* leave empty, retry test below just fails closed */ }
      if (isBambooHrHost(host)) {
        try {
          const reloadResponse = await navigateSettled(page, () =>
            page.reload({ waitUntil: 'domcontentloaded', timeout: NAVIGATE_TIMEOUT_MS }));
          const reloadStatus = reloadResponse?.status() ?? status;
          const reloaded = await pollPage(reloadStatus);
          const reloadVerdict = classifyLiveness({ status: reloadStatus, requestedUrl: url, ...reloaded });
          const stillNotFound = reloadVerdict.code === 'insufficient_content' || reloadVerdict.code === 'listing_page';
          verdict = !stillNotFound
            ? { ...reloadVerdict, reason: `${reloadVerdict.reason} (after BambooHR reload retry)` }
            : {
                result: 'uncertain',
                code: 'bamboohr_render_retry_failed',
                reason: 'BambooHR page did not render content even after a reload retry — not trusted as evidence of removal',
              };
        } catch (err) {
          // Reload itself failed — still never let this surface as `expired`.
          verdict = {
            result: 'uncertain',
            code: 'bamboohr_render_retry_failed',
            reason: `BambooHR reload retry failed: ${err.message.split('\n')[0]}`,
          };
        }
      }
    }

    // A page still empty when the poll gives up (after the frame wait, if a
    // same-origin frame is present) has shown nothing: no posting, no closure
    // notice, no error page. AGENTS.md calls a loading placeholder unconfirmed,
    // not closed, and a false `expired` is the expensive direction (see the
    // iCIMS note above). Measured over 218 loads on 17 ATS: 18 were still empty
    // when the poll ended, 16 of them live postings. Only a body with no text at
    // all qualifies: a short page ("Page not found", a header and footer) keeps
    // insufficient_content, also `uncertain` but with a code that says the page
    // did render something.
    if (verdict.code === 'insufficient_content' && !reading.bodyText.trim()) {
      verdict = {
        result: 'uncertain',
        code: 'empty_page',
        reason: 'page still empty when the poll ended — not trusted as evidence of removal',
      };
    }

    if (page && page._blockedByGuard) {
      return { result: 'uncertain', code: page._blockedByGuard.code, reason: page._blockedByGuard.reason };
    }

    return verdict;
  } catch (err) {
    if (page && page._blockedByGuard) {
      return { result: 'uncertain', code: page._blockedByGuard.code, reason: page._blockedByGuard.reason };
    }
    // Transient failures (timeout, DNS, TLS, 5xx) shouldn't be treated as expired —
    // doing so would cause scan --verify to drop the URL and write it to scan-history,
    // permanently filtering it out on subsequent scans.
    return {
      result: 'uncertain',
      code: 'navigation_error',
      reason: `navigation error: ${err.message.split('\n')[0]}`,
    };
  }
}

// Anti-bot results that a headed browser may be able to get past. A real (headed)
// Chromium clears the JS/Cloudflare challenge that headless trips on (e.g. pracuj.pl).
const CHALLENGE_CODES = new Set(['bot_challenge', 'access_blocked']);

export function isChallengeResult(result) {
  return result?.result === 'uncertain' && CHALLENGE_CODES.has(result.code);
}

// Lazily owns a single headed browser/page, created only on first use and reused
// across URLs. Headed Chromium needs a display, so launch can fail in headless/CI
// environments — in that case get() returns null and callers degrade to the
// headless result (challenge stays uncertain, never falsely expired).
export function createHeadedPageProvider(chromium, browserContext = {}) {
  let browser = null;
  let page = null;
  let launchFailed = false;
  // A cached page is only reusable while its browser is still up. If the headed
  // Chromium goes away mid-run (the user closes the window, the process dies),
  // the cached handle stays non-null, so every later get() hands back a dead
  // page and each anti-bot retry fails with "Target page, context or browser has
  // been closed" instead of a real verdict. Guarded defensively because the
  // provider is handed a chromium in tests, not necessarily a real Playwright one.
  const isCachedPageUsable = () => {
    if (!page) return false;
    if (typeof page.isClosed === 'function' && page.isClosed()) return false;
    if (browser && typeof browser.isConnected === 'function' && !browser.isConnected()) return false;
    return true;
  };

  return {
    async get() {
      if (page && !isCachedPageUsable()) {
        // Drop the dead handles and fall through to a fresh launch below. If the
        // page went away but its Chromium is still up, tear that browser down
        // first: close() only knows the current handle, so a replacement launch
        // would otherwise leave the stale process running until exit.
        if (browser && (typeof browser.isConnected !== 'function' || browser.isConnected())) {
          try {
            await browser.close();
          } catch {
            // best-effort teardown
          }
        }
        page = null;
        browser = null;
      }
      if (page) return page;
      if (launchFailed) return null;
      try {
        browser = await chromium.launch(browserLaunchOptions({ headless: false }, browserContext));
        const context = await browser.newContext(LIVENESS_CONTEXT_OPTIONS);
        page = await context.newPage();
        return page;
      } catch {
        launchFailed = true;
        browser = null;
        page = null;
        return null;
      }
    },
    async close() {
      if (browser) {
        try {
          await browser.close();
        } catch {
          // best-effort teardown
        }
      }
      browser = null;
      page = null;
    },
  };
}

// Runs the headless check, then retries once in a headed browser if the page was
// blocked by an anti-bot wall. The headed result wins when it actually sees the
// page; if the retry is still blocked (or no headed page is available) the
// original uncertain result is kept — we never upgrade a block to expired.
//
// `robotsGate(url) -> { allowed, reason }` (robots-check.mjs robotsAllows in
// both real callers) is consulted only once the headless result is a
// challenge, and before a headed page is asked for, so a site whose robots.txt
// does not permit the path never gets the headed retry, which exists only to
// get past its bot wall (#5776). A skip keeps the headless verdict, result and
// code unchanged, with the reason extended: it can drop an escalation, never a
// posting, because `uncertain` is never recorded as expired. A gate that throws
// counts as a skip. With no gate passed, the retry runs as it always did.
export async function checkUrlLivenessWithFallback(page, url, { getHeadedPage, robotsGate } = {}) {
  const first = await checkUrlLiveness(page, url);
  if (!getHeadedPage || !isChallengeResult(first)) {
    return first;
  }
  if (robotsGate) {
    let gate;
    try {
      gate = await robotsGate(url);
    } catch (err) {
      gate = { allowed: false, reason: `check failed (${String(err?.message ?? err).split('\n')[0]})` };
    }
    if (gate?.allowed !== true) {
      return { ...first, reason: `${first.reason} (headed retry skipped: robots.txt ${gate?.reason ?? 'gave no verdict'})` };
    }
  }
  const headedPage = await getHeadedPage();
  if (!headedPage) {
    return first;
  }
  // The headed retry is for an anti-bot wall, which loaded; it does not also
  // earn the slow-page retry, which would push one URL past a minute.
  const second = await checkUrlLiveness(headedPage, url, { extraSettleMs: 3_000, timeoutRetry: false });
  if (isChallengeResult(second)) {
    return { ...second, reason: `${second.reason} (headed retry also blocked)` };
  }
  return second;
}