#!/usr/bin/env node
/**
 * browser-extract.mjs — headless Playwright reader for the scan / JD-extraction
 * path (the opt-in alternative to the browser MCP; see #1449).
 *
 * The token cost of the MCP path is `browser_snapshot` streaming a page's whole
 * accessibility tree back to the model on every navigate. This helper renders
 * the same page headlessly and returns COMPACT JSON — just the fields the agent
 * needs — so the model processes a small result instead of a full snapshot.
 *
 * STRICTLY READ-ONLY: it navigates and reads the DOM. No clicks, typing, or form
 * fills — that boundary is exactly what keeps this separate from `apply`.
 *
 * Usage:
 *   node browser-extract.mjs <url> [--mode jd|listing] [--max N] [--max-chars N] [--timeout MS]
 *
 * `--max-chars` overrides the jd-mode text cap (default 12000) — raise it when a
 * long JD would otherwise be truncated at the tail, at the cost of more tokens.
 *
 * Modes:
 *   jd (default) — one posting page → { url, title, text }. `text` is the main
 *                  visible text, whitespace-collapsed and length-capped. For the
 *                  pipeline / oferta / auto-pipeline JD-extraction step.
 *   listing      — a careers/board page → { url, jobs: [{ title, url }] }. Visible
 *                  anchors that look like individual postings, deduped. For scan
 *                  Level 1 (reading a company's open roles).
 *
 * Workday (`*.myworkdayjobs.com`) is read through its public CXS JSON endpoint
 * instead of the rendered page: Workday hydrates the JD into a virtualized DOM
 * that readDom() below cannot see, so scraping it returned a well-formed result
 * with an EMPTY `text` — indistinguishable, to a caller, from a posting that
 * genuinely has no content. Same API family scan.mjs already uses for Workday
 * boards (providers/workday.mjs); the per-job URL derivation is reused from
 * liveness-api.mjs so the two cannot drift.
 *
 * Output: compact JSON to stdout. Exit 0 on success; exit 1 on a hard error,
 * printing `{ "error": "...", "code": "..." }` (so a caller/mode can fall back
 * to the MCP path silently). An empty/near-empty jd-mode extraction is one of
 * those hard errors (`code: "empty_text"`) rather than a successful-looking
 * empty JD — the documented silent fallback only fires if the tool actually
 * reports failure. So is a Cloudflare challenge or block page served in place
 * of the posting (`code: "bot_challenge"`). Reuses liveness-browser.mjs's SSRF host guard and
 * realistic-UA context so it isn't instantly bot-walled.
 */

import { readFileSync, existsSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { loadYaml } from './lib/yaml-document.mjs';
import { LIVENESS_CONTEXT_OPTIONS, rejectPrivateOrInvalid } from './liveness-browser.mjs';
import { getCareerOpsRoot, getProfilePath } from './path-resolver.mjs';
import { resolveAtsApi, JD_TEXT_API_ATS } from './liveness-api.mjs';
import { classifyLiveness, cloudflareInterstitial } from './liveness-core.mjs';
import { decodeEntities } from './providers/_html-entities.mjs';
import { HTML_TAG_RE } from './providers/_html-to-text.mjs';
import { isWorkModelOnly } from './providers/greenhouse.mjs';
import { buildLocation } from './providers/hirehive.mjs';
import { isZohoRecordClosed, parseZohoDetailPage } from './providers/zoho-recruit.mjs';
import { extractCandidateOpportunityDetail, extractLocations } from './providers/ultipro.mjs';
import { DEFAULT_USER_AGENT } from './user-agent.mjs';
import { flagValue, hasFlag, validateFlags } from './lib/cli-flags.mjs';
import { isMainModule } from './lib/is-main-module.mjs';
import { browserLaunchOptions } from './lib/browser-launch.mjs';

const CAREER_OPS = getCareerOpsRoot();

const DEFAULT_TIMEOUT_MS = 15_000;
const HYDRATION_WAIT_MS = 2_000;
const JD_TEXT_CAP = 12_000;     // plenty for a JD; a fraction of a full snapshot
const DEFAULT_LISTING_MAX = 200;
const WORKDAY_TIMEOUT_MS = 10_000;

// Floor below which a jd-mode extraction is treated as failure, not content.
// A posting page whose main text is a couple of sentences is a render we
// missed (SPA shell, consent wall, bot interstitial), never a real JD; the
// cost of being wrong is one silent fallback to the MCP path, whereas the
// cost of NOT failing is an empty JD evaluated as if it were the posting.
// Only jd mode gets this guard: an empty `listing` result is legitimate — a
// company with no open roles.
const MIN_JD_TEXT_CHARS = 200;

// Anchor labels that are navigation chrome, not job postings. Kept small and
// lowercase; matched against the trimmed label.
const NAV_LABEL_STOPWORDS = new Set([
  'home', 'about', 'about us', 'contact', 'contact us', 'login', 'log in', 'sign in',
  'sign up', 'register', 'privacy', 'privacy policy', 'terms', 'cookies', 'cookie policy',
  'careers', 'jobs', 'search', 'menu', 'back', 'next', 'previous', 'apply', 'apply now',
  'learn more', 'read more', 'faq', 'blog', 'news', 'help', 'support', 'english',
]);

/**
 * Resolve the configured scan extractor: `cli` (this helper) or `mcp` (default).
 * Reads `scan.extractor` from config/profile.yml; anything unrecognized — or a
 * missing/unreadable file — yields `mcp` so behavior never breaks. Exported so
 * doctor.mjs reports the same value.
 * @param {string} [profilePath]
 * @returns {'cli'|'mcp'}
 */
export function resolveExtractorMode(profilePath = getProfilePath()) {
  try {
    if (!existsSync(profilePath)) return 'mcp';
    const raw = loadYaml(readFileSync(profilePath, 'utf-8')) || {};
    const v = raw?.scan?.extractor;
    return v === 'cli' ? 'cli' : 'mcp';
  } catch {
    return 'mcp';
  }
}

// Collapse runs of whitespace and cap length so the JD text stays compact.
export function compactText(s, cap = JD_TEXT_CAP) {
  const text = String(s ?? '').replace(/[ \t ]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
  return text.length > cap ? `${text.slice(0, cap)}…` : text;
}

/**
 * Shape a JD-mode result from the raw DOM read. Pure — exported for tests.
 * @param {{ title?: string, text?: string }} raw
 * @param {string} finalUrl
 */
export function normalizeJd(raw, finalUrl, textCap = JD_TEXT_CAP) {
  // A schema.org JobPosting, when the page ships one, beats the rendered DOM:
  // it is the posting body as the employer published it, not as their template
  // happened to lay it out. On Phenom boards (careers.roche.com) the rendered
  // <main> holds title, location and Apply chrome only — ~300 characters — while
  // the body never becomes visible text at all, so the DOM path emitted a JD
  // with no requirements in it and reported success. Same shape on other
  // template-driven corporate boards, which is why this keys off the standard
  // rather than off a host.
  //
  // Longer wins rather than JSON-LD always winning: plenty of pages carry a
  // stub JobPosting (a one-line summary, or just the title) alongside a fully
  // rendered body, and preferring the stub there would break the common case to
  // fix the rare one.
  const ldText = jdHtmlToText(raw?.jsonLdDescription || '');
  // Normalized length, not raw length: raw DOM text on a chrome-heavy page is
  // mostly repeated newlines ("Apply now\n\n\n\n\n\nSave job\n\n\n\n\n\n..."), which
  // can out-length a shorter-in-raw-HTML but substantive JSON-LD description
  // before either side is compacted. Comparing pre-collapse defeats the whole
  // point of "longer wins". No cap here (Infinity) — only the comparison needs
  // normalizing; the real textCap is still applied below.
  const domText = compactText(String(raw?.text || ''), Infinity);
  const text = ldText.length > domText.length ? ldText : domText;

  return {
    url: finalUrl,
    title: compactText(raw?.title || raw?.jsonLdTitle || '', 300),
    text: compactText(text, textCap),
    // For the liveness check in jdFailure() only. `text` has had nav/header/footer
    // stripped, and an expired-posting banner often sits in one of them. Never
    // emitted on the success path.
    bodyText: compactText(raw?.bodyText || '', 4100),
  };
}

/**
 * Map a `*.myworkdayjobs.com` posting URL to its public per-job CXS endpoint,
 * or null for any other URL.
 *
 * Derivation (tenant/shard/site/jobPath -> `/wday/cxs/{tenant}/{site}/job/{path}`)
 * is delegated to liveness-api.mjs's Workday provider so there is exactly one
 * copy of it, including its SSRF guard: every path segment taken from the input
 * URL is charset-validated and ".." -rejected before it reaches the fixed
 * `{tenant}.{shard}.myworkdayjobs.com` host template.
 *
 * @param {string} rawUrl
 * @returns {string|null}
 */
export function workdayCxsUrl(rawUrl) {
  const ats = resolveAtsApi(rawUrl);
  return ats && ats.ats === 'workday' ? ats.apiUrl : null;
}

// Tags whose CLOSE ends a block, so it becomes a line break. `li` is absent on
// purpose: its OPEN tag already emits the break plus a bullet below, and
// breaking on both ends double-spaces every list.
const BLOCK_END_RE = /<\/(p|div|ul|ol|h[1-6]|tr|section|article|blockquote)\s*>/gi;

/**
 * Description markup -> plain text, keeping block structure as newlines. Pure —
 * exported for tests.
 *
 * Not providers/_html-to-text.mjs's htmlToText: that one is tuned for scan
 * payloads and hard-caps at 4000 chars while collapsing ALL whitespace
 * (newlines included) into single spaces. A JD read for evaluation wants the
 * full body up to `--max-chars`, with its paragraph and bullet breaks intact.
 * The entity decoder and the tag matcher ARE shared, so the two cannot drift on
 * the thing that has actually drifted historically (#1555/#1639/#2623).
 *
 * Double-decode for the same reason htmlToText does: payloads often carry
 * entity-escaped markup (`&lt;p&gt;`), and text-level entities only become
 * decodable once the real tags are stripped.
 *
 * @param {unknown} html
 * @returns {string}
 */
export function jdHtmlToText(html) {
  if (typeof html !== 'string' || !html) return '';
  const stripped = decodeEntities(html)
    .replace(/<(script|style)[^>]*>[\s\S]*?<\/\1>/gi, ' ')
    .replace(/<br\s*\/?>/gi, '\n')
    .replace(/<li[^>]*>/gi, '\n- ')
    .replace(BLOCK_END_RE, '\n')
    .replace(HTML_TAG_RE, ' ');
  return decodeEntities(stripped)
    .replace(/[ \t\u00a0]+/g, ' ')
    .replace(/ *\n */g, '\n')
    .replace(/\n{3,}/g, '\n\n')
    .trim();
}

/**
 * Shape a jd-mode result from a Workday CXS job payload, into the SAME
 * `{ url, title, text }` contract as the scraped path so no caller has to know
 * which route produced it. Pure — exported for tests.
 *
 * Returns null when the payload isn't a job (`jobPostingInfo` missing) or
 * carries no description, so the caller can fall through to the browser rather
 * than emit a confidently empty JD.
 *
 * `url` is the POSTING url the user passed, not the CXS endpoint: it is what
 * ends up in reports and the tracker.
 *
 * The metadata header is prepended to `text` because each of those fields is
 * evaluation signal the rendered page shows and the description alone does not
 * — location for the location filter, `jobReqId` for the tracker's same-title
 * disambiguation rule, and `canApply: false` as a liveness signal on a posting
 * still served but no longer accepting applications.
 *
 * @param {any} json - parsed CXS response body
 * @param {string} postingUrl
 * @param {number} [textCap]
 */
export function normalizeWorkdayJob(json, postingUrl, textCap = JD_TEXT_CAP) {
  const info = json && typeof json === 'object' ? json.jobPostingInfo : null;
  if (!info || typeof info !== 'object') return null;

  const body = jdHtmlToText(info.jobDescription);
  if (!body) return null;

  const str = (v) => (typeof v === 'string' && v.trim() ? v.trim() : '');
  const locations = [
    str(info.location),
    ...(Array.isArray(info.additionalLocations) ? info.additionalLocations.map(str) : []),
  ].filter(Boolean);

  const meta = [];
  if (locations.length) meta.push(`Location: ${locations.join(' | ')}`);
  if (str(info.timeType)) meta.push(`Job type: ${str(info.timeType)}`);
  if (str(info.postedOn)) meta.push(`Posted: ${str(info.postedOn)}`);
  if (str(info.jobReqId)) meta.push(`Req ID: ${str(info.jobReqId)}`);
  if (info.canApply === false) meta.push('Applications closed (canApply: false)');

  return {
    url: postingUrl,
    title: compactText(str(info.title), 300),
    text: compactText([meta.join('\n'), body].filter(Boolean).join('\n\n'), textCap),
  };
}

/**
 * Read the body of a response no fetcher will use, dropping each chunk as it
 * arrives, and return null. Left unread, a body still arriving keeps its
 * socket open, and with it the process: fetch-jd.mjs exits once nothing is
 * left to run. Fetchers await this inside their try, so the read stays under
 * their abort timer and a body that never ends stops at the same timeout as
 * the rest of the request. Reading, not cancelling: these error bodies are
 * small, while a cancel aborts the request mid-body and undici then opens a
 * new connection that the process waits for (0.1 to 2.5 s on real Greenhouse
 * and Workday misses, Node 24).
 *
 * @param {Response} res
 */
async function discard(res) {
  try {
    const reader = res.body?.getReader();
    while (reader && !(await reader.read()).done) { /* drop the chunk */ }
  } catch { /* cut off by the abort timer, or unreadable */ }
  return null;
}

/**
 * Fetch + shape one Workday posting from its CXS endpoint. Returns null for
 * ANY inconclusive outcome (blocked host, redirect, non-200, unparseable or
 * unexpected body, timeout) so the caller falls through to the browser path —
 * where a removed posting still yields the real "this job is no longer
 * available" page rather than a hard error.
 *
 * @param {string} apiUrl
 * @param {string} postingUrl
 * @param {number} textCap
 * @param {number} timeoutMs
 */
async function fetchWorkdayJd(apiUrl, postingUrl, textCap, timeoutMs) {
  // Same host as the already-guarded input, but the guard is cheap and this is
  // the request that actually leaves the process.
  if (rejectPrivateOrInvalid(apiUrl)) return null;

  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), Math.max(timeoutMs, WORKDAY_TIMEOUT_MS));
  try {
    const res = await fetch(apiUrl, {
      headers: { accept: 'application/json', 'user-agent': DEFAULT_USER_AGENT },
      redirect: 'error', // a redirect off the derived host is not one to follow
      signal: controller.signal,
    });
    if (!res.ok) return await discard(res);
    return normalizeWorkdayJob(await res.json(), postingUrl, textCap);
  } catch {
    return null;
  } finally {
    clearTimeout(timer);
  }
}

/**
 * Shape one Ashby board entry into a jd-mode result. Pure, so the field
 * mapping is unit-testable without a network round trip.
 *
 * Ashby's public API is ORG-level (the whole board), not per-job, so the job
 * is selected here by a case-insensitive `id` compare — mirroring
 * classifyAshbyBoard's match logic without importing it, since that function
 * returns a liveness verdict, not a job record.
 *
 * `descriptionPlain` is already plain text, so it needs no jdHtmlToText pass.
 * The metadata header carries what the rendered page shows and the description
 * does not, same rationale as normalizeWorkdayJob.
 *
 * @param {any} json - parsed board response body
 * @param {string} jobId - the {jobId} path segment from the posting URL
 * @param {string} postingUrl
 * @param {number} [textCap]
 */
export function normalizeAshbyJob(json, jobId, postingUrl, textCap = JD_TEXT_CAP) {
  const jobs = Array.isArray(json?.jobs) ? json.jobs : [];
  const target = String(jobId ?? '').toLowerCase();
  const job = jobs.find((j) => typeof j?.id === 'string' && j.id.toLowerCase() === target);
  if (!job) return null;

  const str = (v) => (typeof v === 'string' && v.trim() ? v.trim() : '');
  const body = str(job.descriptionPlain);
  if (!body) return null;

  const meta = [];
  if (str(job.location)) meta.push(`Location: ${str(job.location)}`);
  if (Array.isArray(job.secondaryLocations)) {
    const extra = job.secondaryLocations.map((l) => str(l?.location)).filter(Boolean);
    if (extra.length) meta.push(`Additional locations: ${extra.join(' | ')}`);
  }
  if (str(job.employmentType)) meta.push(`Type: ${str(job.employmentType)}`);
  // Remote work model, same precedence providers/ashby.mjs:160 settled on:
  // `workplaceType` wins whenever present and `isRemote` is only the fallback.
  // The two disagree constantly — 52 of 60 sampled ramp postings carry
  // `isRemote: true` beside `workplaceType: "Hybrid"` — so trusting isRemote
  // alone labels office-anchored roles Remote. The fallback still earns its
  // place: `workplaceType` is absent on 41 of 60 sampled openai postings.
  const workplaceType = str(job.workplaceType);
  if (workplaceType) meta.push(`Work model: ${workplaceType}`);
  else if (job.isRemote === true) meta.push('Work model: Remote');
  if (job.isListed === false) meta.push('Not currently listed (isListed: false)');

  return {
    url: postingUrl,
    title: compactText(str(job.title), 300),
    text: compactText([meta.join('\n'), body].filter(Boolean).join('\n\n'), textCap),
    // isListed:false is NOT closure: Ashby serves an unlisted posting by direct
    // link and it can still take applications. The note in the text is the only
    // carrier; closure needs independent evidence.
  };
}

/**
 * Fetch + shape one Ashby posting from its org's job-board API. Returns null
 * for ANY inconclusive outcome (blocked host, non-200, unparseable body, job
 * not found on the board, timeout) so the caller falls through to the browser
 * path, same contract as fetchWorkdayJd.
 *
 * Ashby's public API is ORG-level (the whole board), not per-job — this fetches
 * the board once and picks out `jobId`, mirroring classifyAshbyBoard's match
 * logic (case-insensitive `id` compare) without importing it, since that
 * function returns a liveness verdict, not a job record.
 *
 * @param {string} apiUrl - the org board URL (`resolveAtsApi(url).apiUrl`)
 * @param {string} jobId - the {jobId} path segment from the original URL
 * @param {string} postingUrl
 * @param {number} textCap
 * @param {number} timeoutMs
 */
async function fetchAshbyJd(apiUrl, jobId, postingUrl, textCap, timeoutMs) {
  if (rejectPrivateOrInvalid(apiUrl)) return null;
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    const res = await fetch(apiUrl, {
      headers: { accept: 'application/json', 'user-agent': DEFAULT_USER_AGENT },
      redirect: 'error',
      signal: controller.signal,
    });
    if (!res.ok) return await discard(res);
    const json = await res.json();
    return normalizeAshbyJob(json, jobId, postingUrl, textCap);
  } catch {
    return null;
  } finally {
    clearTimeout(timer);
  }
}

/**
 * Shape one Greenhouse per-job payload into a jd-mode result. Pure, same
 * contract as normalizeWorkdayJob/normalizeAshbyJob.
 *
 * `content` is HTML (and HTML-escaped at that), so it goes through
 * jdHtmlToText exactly like Workday's `jobDescription`. `requisition_id` is
 * carried for the tracker's same-title disambiguation rule.
 *
 * `location.name` is the PRIMARY location field and `offices[]` is enrichment
 * — the same precedence providers/greenhouse.mjs:196 applies, reusing its
 * exported isWorkModelOnly so the two cannot drift. Some boards put the work
 * model ("Hybrid", "Distributed") in `location.name` and keep the actual city
 * only in `offices[]`; for exactly those the two are joined, and everywhere
 * else `location.name` stands alone rather than being shadowed by a
 * near-duplicate office entry (a live figma posting carries
 * `location.name: "Berlin, Germany"` beside `offices[0].name: "Berlin, DE "`).
 * Reading `offices[]` alone — as this did before — dropped the `Location:`
 * line entirely for any job whose offices array is empty, which is 7 of 20
 * sampled gitlab postings.
 *
 * The provider pays a separate /offices request for this because the boards
 * LIST endpoint omits offices; the per-job endpoint here returns them inline,
 * so the enrichment is free and can also serve as the fallback when a board
 * ships no `location.name` at all.
 *
 * @param {any} json - parsed boards-api response body
 * @param {string} postingUrl
 * @param {number} [textCap]
 */
export function normalizeGreenhouseJob(json, postingUrl, textCap = JD_TEXT_CAP) {
  const body = jdHtmlToText(json?.content);
  if (!body) return null;

  const str = (v) => (typeof v === 'string' && v.trim() ? v.trim() : '');
  const meta = [];
  const offices = Array.isArray(json?.offices) ? json.offices.map((o) => str(o?.name)).filter(Boolean) : [];
  let location = str(json?.location?.name);
  if (offices.length && (!location || isWorkModelOnly(location))) {
    location = [location, ...offices].filter(Boolean).join(' · ');
  }
  if (location) meta.push(`Location: ${location}`);
  if (str(json?.requisition_id)) meta.push(`Req ID: ${str(json.requisition_id)}`);

  return {
    url: postingUrl,
    title: compactText(str(json?.title), 300),
    text: compactText([meta.join('\n'), body].filter(Boolean).join('\n\n'), textCap),
  };
}

/**
 * Fetch + shape one Greenhouse posting from its per-job boards-api endpoint.
 * Same failure contract as fetchWorkdayJd/fetchAshbyJd.
 *
 * `content=true` is required — without it the endpoint omits the JD body
 * entirely (same query param providers/greenhouse.mjs uses for its list scan).
 * `content` comes back as HTML, so it goes through jdHtmlToText like Workday's
 * `jobDescription`.
 *
 * @param {string} apiUrl - `resolveAtsApi(url).apiUrl` (no query string)
 * @param {string} postingUrl
 * @param {number} textCap
 * @param {number} timeoutMs
 */
async function fetchGreenhouseJd(apiUrl, postingUrl, textCap, timeoutMs) {
  const withContent = `${apiUrl}?content=true`;
  if (rejectPrivateOrInvalid(withContent)) return null;
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    const res = await fetch(withContent, {
      headers: { accept: 'application/json', 'user-agent': DEFAULT_USER_AGENT },
      redirect: 'error',
      signal: controller.signal,
    });
    if (!res.ok) return await discard(res);
    const json = await res.json();
    return normalizeGreenhouseJob(json, postingUrl, textCap);
  } catch {
    return null;
  } finally {
    clearTimeout(timer);
  }
}

/**
 * Shape one Lever posting into a jd-mode result. Pure, same contract as the
 * other normalizers.
 *
 * `descriptionPlain` is already plain text but is only the first slice of the
 * JD. `lists` carries the labeled sections ("Requirements", "Nice to have")
 * as separate HTML blocks, and `additionalPlain` the trailing "Life at …" /
 * benefits / comp copy — both appended in the order the rendered page shows
 * them, or the JD loses its requirements and its comp entirely. The share is
 * not marginal: on a sampled gopuff posting `descriptionPlain` is 760 chars
 * against 938 in `additionalPlain`, and `additionalPlain` is never contained
 * in `descriptionPlain` (checked over 10 postings on two boards). Do NOT also
 * append `openingPlain` / `descriptionBodyPlain`: `descriptionPlain` already
 * equals their concatenation, so adding them duplicates the description.
 *
 * Order is the contract, since compactText truncates the TAIL — a lowered
 * `--max-chars` has to drop boilerplate, not requirements.
 *
 * There is no closed-state check. The public v0 endpoint 404s a closed posting
 * (liveness-api.mjs:91) and emits no `state` field at all on a live one
 * (verified over 10 postings on two boards), so the fetcher's non-OK guard is
 * the whole of it.
 *
 * @param {any} json - parsed postings-api response body
 * @param {string} postingUrl
 * @param {number} [textCap]
 */
export function normalizeLeverJob(json, postingUrl, textCap = JD_TEXT_CAP) {
  if (!json || typeof json !== 'object') return null;

  const str = (v) => (typeof v === 'string' && v.trim() ? v.trim() : '');
  const lists = Array.isArray(json.lists)
    ? json.lists
        .map((l) => {
          const heading = str(l?.text);
          const content = jdHtmlToText(l?.content);
          return content ? `${heading ? `${heading}\n` : ''}${content}` : '';
        })
        .filter(Boolean)
        .join('\n\n')
    : '';
  // Lever can leave descriptionPlain empty while description holds the HTML.
  const intro = str(json.descriptionPlain) || jdHtmlToText(json.description);
  const body = [intro, lists, str(json.additionalPlain)].filter(Boolean).join('\n\n');
  if (!body) return null;

  const meta = [];
  if (str(json?.categories?.location)) meta.push(`Location: ${str(json.categories.location)}`);
  if (str(json?.categories?.team)) meta.push(`Team: ${str(json.categories.team)}`);

  return {
    url: postingUrl,
    title: compactText(str(json.text), 300),
    text: compactText([meta.join('\n'), body].filter(Boolean).join('\n\n'), textCap),
  };
}

/**
 * Fetch + shape one Lever posting from its per-job postings endpoint. Same
 * failure contract as fetchWorkdayJd/fetchAshbyJd/fetchGreenhouseJd.
 *
 * `descriptionPlain` is already plain text (no HTML stripping needed, unlike
 * Greenhouse/Workday). `lists` carries the JD's labeled sections (e.g.
 * "Requirements", "Nice to have") as separate HTML blocks — appended after
 * the main description the same way the rendered page shows them.
 *
 * @param {string} apiUrl - `resolveAtsApi(url).apiUrl`
 * @param {string} postingUrl
 * @param {number} textCap
 * @param {number} timeoutMs
 */
async function fetchLeverJd(apiUrl, postingUrl, textCap, timeoutMs) {
  if (rejectPrivateOrInvalid(apiUrl)) return null;
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    const res = await fetch(apiUrl, {
      headers: { accept: 'application/json', 'user-agent': DEFAULT_USER_AGENT },
      redirect: 'error',
      signal: controller.signal,
    });
    if (!res.ok) return await discard(res);
    const json = await res.json();
    return normalizeLeverJob(json, postingUrl, textCap);
  } catch {
    return null;
  } finally {
    clearTimeout(timer);
  }
}

/**
 * Greenhouse posting reached through a company careers page (`?gh_jid=` only,
 * no board in the URL). The embed redirect names the board — the same
 * `followEmbed` hop liveness uses, which also pins the redirect target to
 * greenhouse.io — and the per-job boards-api endpoint then ships the JD.
 */
async function fetchGreenhouseEmbeddedJd(resolved, postingUrl, textCap, timeoutMs) {
  if (rejectPrivateOrInvalid(resolved.apiUrl)) return null;
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  let jobApiUrl;
  try {
    const res = await fetch(resolved.apiUrl, {
      headers: { 'user-agent': DEFAULT_USER_AGENT },
      redirect: 'manual',
      signal: controller.signal,
    });
    jobApiUrl = await resolved.followEmbed(res, resolved.parts);
    await discard(res); // only its status and Location header are read
  } catch {
    return null;
  } finally {
    clearTimeout(timer);
  }
  return jobApiUrl ? fetchGreenhouseJd(jobApiUrl, postingUrl, textCap, timeoutMs) : null;
}

/**
 * Shape one SmartRecruiters posting into a jd-mode result. Pure, same contract
 * as the other normalizers. The body is `jobAd.sections`, HTML blocks appended
 * in the order the rendered page shows them; the location object also carries
 * remote/hybrid flags, surfaced because they are often the only work-model
 * signal the posting has.
 *
 * @param {any} json - parsed postings-API response body
 * @param {string} postingUrl
 * @param {number} [textCap]
 */
export function normalizeSmartRecruitersJob(json, postingUrl, textCap = JD_TEXT_CAP) {
  const sections = json?.jobAd?.sections;
  if (!sections || typeof sections !== 'object') return null;

  const str = (v) => (typeof v === 'string' && v.trim() ? v.trim() : '');
  const blocks = ['companyDescription', 'jobDescription', 'qualifications', 'additionalInformation']
    .map((key) => {
      const body = jdHtmlToText(sections[key]?.text);
      return body ? [str(sections[key]?.title), body].filter(Boolean).join('\n') : '';
    })
    .filter(Boolean);
  if (!blocks.length) return null;

  const loc = json?.location || {};
  const meta = [];
  const where = str(loc.fullLocation) || [str(loc.city), str(loc.country)].filter(Boolean).join(', ');
  if (where) meta.push(`Location: ${where}`);
  if (loc.remote === true) meta.push('Work model: Remote');
  else if (loc.hybrid === true) meta.push('Work model: Hybrid');
  // A closed posting can still ship its ad. The marker carries `active: false`
  // with the text the way Workday carries `canApply: false`, so the caller
  // reads the closure instead of falling back to a page that hides it.
  if (json?.active === false) meta.push('Applications closed (active: false)');

  return {
    url: postingUrl,
    title: compactText(str(json?.name), 300),
    text: compactText([meta.join('\n'), ...blocks].filter(Boolean).join('\n\n'), textCap),
  };
}

async function fetchSmartRecruitersJd(apiUrl, postingUrl, textCap, timeoutMs) {
  if (rejectPrivateOrInvalid(apiUrl)) return null;
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    const res = await fetch(apiUrl, {
      headers: { accept: 'application/json', 'user-agent': DEFAULT_USER_AGENT },
      redirect: 'error',
      signal: controller.signal,
    });
    if (!res.ok) return await discard(res);
    const json = await res.json();
    return normalizeSmartRecruitersJob(json, postingUrl, textCap);
  } catch {
    return null;
  } finally {
    clearTimeout(timer);
  }
}

/**
 * Shape one Zoho Recruit posting page into a jd-mode result. Pure, same
 * contract as the other normalizers. The page embeds the posting record
 * (providers/zoho-recruit.mjs parseZohoDetailPage); its `Job_Description` is
 * HTML. The record must be THIS posting's: a page carrying another id, or no
 * record (a removed posting), returns null so the caller falls through.
 *
 * @param {any} html - the posting page's body
 * @param {string} id - the {id} path segment from the posting URL
 * @param {string} postingUrl
 * @param {number} [textCap]
 */
export function normalizeZohoJob(html, id, postingUrl, textCap = JD_TEXT_CAP) {
  const record = parseZohoDetailPage(html);
  if (!record || String(record.id) !== String(id)) return null;
  const body = jdHtmlToText(record.Job_Description);
  if (!body) return null;

  const str = (v) => (typeof v === 'string' && v.trim() ? v.trim() : '');
  const meta = [];
  const location = [str(record.City), str(record.State), str(record.Country)].filter(Boolean).join(', ');
  if (location) meta.push(`Location: ${location}`);
  if (record.Remote_Job === true) meta.push('Work model: Remote');
  if (str(record.Job_Type)) meta.push(`Job type: ${str(record.Job_Type)}`);
  if (str(record.Salary)) meta.push(`Salary: ${str(record.Salary)}`);
  if (str(record.Date_Opened)) meta.push(`Posted: ${str(record.Date_Opened)}`);
  if (isZohoRecordClosed(record)) meta.push('Applications closed (the career site shows the posting as filled)');

  return {
    url: postingUrl,
    title: compactText(str(record.Posting_Title) || str(record.Job_Opening_Name), 300),
    text: compactText([meta.join('\n'), body].filter(Boolean).join('\n\n'), textCap),
  };
}

async function fetchZohoJd(apiUrl, id, postingUrl, textCap, timeoutMs) {
  if (rejectPrivateOrInvalid(apiUrl)) return null;
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    const res = await fetch(apiUrl, {
      headers: { accept: 'text/html', 'user-agent': DEFAULT_USER_AGENT },
      redirect: 'error',
      signal: controller.signal,
    });
    if (!res.ok) return await discard(res);
    return normalizeZohoJob(await res.text(), id, postingUrl, textCap);
  } catch {
    return null;
  } finally {
    clearTimeout(timer);
  }
}

/**
 * Shape one HireHive per-posting API answer into a jd-mode result. Pure, same
 * contract as the other normalizers. The body must be THIS posting's: another
 * id returns null so the caller falls through.
 *
 * @param {any} json - parsed `/api/v1|v2/jobs/<id>` response body
 * @param {string} id - `job_<shortid>` (v2) or the posting number (v1)
 * @param {string} postingUrl
 * @param {number} [textCap]
 */
export function normalizeHireHiveJob(json, id, postingUrl, textCap = JD_TEXT_CAP) {
  if (!json || String(json.id) !== String(id)) return null;
  const str = (v) => (typeof v === 'string' && v.trim() ? v.trim() : '');
  const body = jdHtmlToText(json.description?.html) || str(json.description?.text);
  if (!body) return null;
  const meta = [];
  const location = buildLocation(json);
  if (location) meta.push(`Location: ${location}`);
  if (str(json.type?.name)) meta.push(`Job type: ${str(json.type.name)}`);
  const posted = str(json.published_date) || (Number.isFinite(json.publishedDate) ? new Date(json.publishedDate * 1000).toISOString() : '');
  if (posted) meta.push(`Posted: ${posted.slice(0, 10)}`);
  return {
    url: postingUrl,
    title: compactText(str(json.title), 300),
    text: compactText([meta.join('\n'), body].filter(Boolean).join('\n\n'), textCap),
  };
}

async function fetchHireHiveJd(apiUrl, id, postingUrl, textCap, timeoutMs) {
  if (rejectPrivateOrInvalid(apiUrl)) return null;
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    const res = await fetch(apiUrl, {
      headers: { accept: 'application/json', 'user-agent': DEFAULT_USER_AGENT },
      redirect: 'error',
      signal: controller.signal,
    });
    if (!res.ok) return await discard(res);
    return normalizeHireHiveJob(await res.json(), id, postingUrl, textCap);
  } catch {
    return null;
  } finally {
    clearTimeout(timer);
  }
}

// `JobLocationType` as the board labels it. The page maps the number in script
// it loads from elsewhere, so this was read off rendered pages: on 2026-10-29,
// of 103 postings read in a browser, the 41 with 1 showed "On-site", the 6
// with 2 "Remote" and the 8 with 0 "Hybrid"; the 48 without one showed none.
const ULTIPRO_WORK_MODELS = { 0: 'Hybrid', 1: 'On-site', 2: 'Remote' };

// The record's criteria lists, in the order the page's Qualifications section
// shows them, each item worded as the page words it.
const ULTIPRO_CRITERIA = [
  ['SkillCriteria', 'Skill', (c) => [c.SkillName, c.MinimumScaleValueCode && c.MinimumScaleValueCode !== 'NONE' ? c.MinimumScaleValueDescription : '', c.SkillDescription]],
  ['BehaviorCriteria', 'Behavior', (c) => [c.BehaviorName, c.BehaviorDescription]],
  ['MotivationCriteria', 'Motivation', (c) => [c.MotivationName, c.MotivationDescription]],
  ['EducationCriteria', 'Education', (c) => [c.DegreeName && `${c.DegreeName} or better${c.MajorName ? ` in ${c.MajorName}${c.Related ? ' or related field' : ''}` : ''}`]],
  ['WorkExperienceCriteria', 'Experience', (c) => [ultiproYears(c.MinimumRequiredYears, c.MaximumRequiredYears), c.Description]],
  ['LicenseAndCertificationCriteria', 'License or certification', (c) => [c.LicenseAndCertificationName]],
];

// The page reads "8-10 years" for a range and "5 years" for a minimum alone. No
// sampled posting had a maximum alone; "Up to" keeps it from reading as a floor.
function ultiproYears(min, max) {
  if (Number.isFinite(min) && Number.isFinite(max)) return `${min}-${max} years`;
  if (Number.isFinite(min)) return `${min} years`;
  return Number.isFinite(max) ? `Up to ${max} years` : '';
}

/**
 * Shape one UltiPro (UKG Pro Recruiting) detail page into a jd-mode result.
 * Pure, same contract as normalizeZohoJob: UKG has no per-posting JSON API
 * either, and the page embeds the posting record (providers/ultipro.mjs
 * extractCandidateOpportunityDetail), whose `Description` is HTML. The record
 * must be THIS posting's: the extractor throws on another id or on a page
 * that changed shape, and finds nothing on the board's empty page (a removed
 * posting). Each of those returns null so the caller falls through.
 *
 * The page shows more than the description: the header facts and a
 * Qualifications section built from the record's criteria lists. Both are kept,
 * so this text holds what a browser read of the page does, without its
 * buttons and the equal-opportunity notice: on 2026-10-29, of 103 postings
 * read both ways, every work model, pay line, travel line and criteria list
 * the page showed was here too. Qualifications go before the description
 * because compactText cuts the tail, and a long description must not push the
 * requirements out.
 *
 * A closed posting keeps its record, so its JD is still returned, with the
 * closure in the header, as normalizeWorkdayJob does for `canApply: false`.
 *
 * @param {any} html - the detail page's body
 * @param {string} id - the posting's opportunityId, lower-cased as the rung requests it
 * @param {string} postingUrl
 * @param {number} [textCap]
 */
export function normalizeUltiproJob(html, id, postingUrl, textCap = JD_TEXT_CAP) {
  let record;
  try {
    const found = extractCandidateOpportunityDetail(html, id);
    if (found.status !== 'ok') return null;
    record = found.detail;
  } catch {
    return null;
  }
  const body = jdHtmlToText(record.Description);
  if (!body) return null;

  const str = (v) => (typeof v === 'string' && v.trim() ? v.replace(/\s+/g, ' ').trim() : '');
  const meta = [];
  const location = extractLocations(record.Locations);
  if (location) meta.push(`Location: ${location}`);
  const workModel = ULTIPRO_WORK_MODELS[record.JobLocationType];
  if (workModel) meta.push(`Work model: ${workModel}`);
  if (typeof record.FullTime === 'boolean') meta.push(`Job type: ${record.FullTime ? 'Full time' : 'Part time'}`);
  if (str(record.JobCategoryName)) meta.push(`Job category: ${str(record.JobCategoryName)}`);
  const pay = record.PayRangeVisible === true ? record.PayRange : null;
  if (str(pay?.PayRangeMinimum) && str(pay?.PayRangeMaximum)) {
    meta.push(`Pay range: ${str(pay.PayRangeMinimum)} to ${str(pay.PayRangeMaximum)} ${str(record.PayRangeCurrencyCode)}`.trim());
  }
  const amount = record.CompensationAmount;
  if (str(amount?.Value)) {
    const salaried = record.Salaried === true;
    meta.push(`${salaried ? 'Salary' : 'Pay'}: ${[str(amount.Value), str(amount.Code), salaried ? 'per year' : ''].filter(Boolean).join(' ')}`);
  }
  if (typeof record.TravelRequired === 'boolean') {
    const travel = str(record.TravelDescription);
    meta.push(`Travel required: ${record.TravelRequired ? 'Yes' : 'No'}${travel ? ` (${travel})` : ''}`);
  }
  const posted = str(record.PostedDate).match(/^\d{4}-\d{2}-\d{2}/);
  if (posted) meta.push(`Posted: ${posted[0]}`);
  if (str(record.RequisitionNumber)) meta.push(`Req ID: ${str(record.RequisitionNumber)}`);
  if (record.OpportunityIsClosed === true) meta.push('Applications closed (OpportunityIsClosed: true)');

  const qualifications = [];
  for (const [key, label, words] of ULTIPRO_CRITERIA) {
    const items = Array.isArray(record[key]) ? record[key].filter((c) => c && typeof c === 'object') : [];
    // Required first, then preferred, as the page groups them.
    for (const required of [true, false]) {
      for (const c of items.filter((item) => (item.Required === true) === required)) {
        const text = words(c).map(str).filter(Boolean).join(': ');
        if (text) qualifications.push(`- ${label} (${required ? 'required' : 'preferred'}): ${text}`);
      }
    }
  }

  return {
    url: postingUrl,
    title: compactText(str(record.Title), 300),
    text: compactText([
      meta.join('\n'),
      qualifications.length ? `Qualifications:\n${qualifications.join('\n')}` : '',
      body,
    ].filter(Boolean).join('\n\n'), textCap),
  };
}

async function fetchUltiproJd(apiUrl, id, postingUrl, textCap, timeoutMs) {
  if (rejectPrivateOrInvalid(apiUrl)) return null;
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    const res = await fetch(apiUrl, {
      headers: { accept: 'text/html', 'user-agent': DEFAULT_USER_AGENT },
      redirect: 'error',
      signal: controller.signal,
    });
    if (!res.ok) return await discard(res);
    return normalizeUltiproJob(await res.text(), id, postingUrl, textCap);
  } catch {
    return null;
  } finally {
    clearTimeout(timer);
  }
}

/**
 * Shape one HiBob `/api/job-ad/{uuid}/application-form` answer into a jd-mode
 * result. Pure, same contract as the other normalizers. `data.jobAd` carries
 * the title, the site as the page shows it ("Berlin, Germany"), the work model
 * and the employment type; `jobDescription` keys each field by a path ending
 * in `/jobDescription/{field}` and wraps the value as `{ value }`. The four
 * text sections are HTML and keep the tenant's own section labels when it set
 * them.
 *
 * @param {any} json - parsed application-form response body
 * @param {string} postingUrl
 * @param {number} [textCap]
 */
export function normalizeHibobJob(json, postingUrl, textCap = JD_TEXT_CAP) {
  const ad = json?.data?.jobAd;
  if (!ad || typeof ad !== 'object') return null;
  const str = (v) => (typeof v === 'string' && v.trim() ? v.trim() : '');
  const fields = ad.jobDescription && typeof ad.jobDescription === 'object' ? ad.jobDescription : {};
  const field = (name) => {
    const key = Object.keys(fields).find((k) => k.endsWith(`/jobDescription/${name}`));
    return key ? fields[key]?.value : undefined;
  };
  const labels = field('sectionLabels') && typeof field('sectionLabels') === 'object' ? field('sectionLabels') : {};
  const blocks = [];
  for (const [key, label] of [['description', 'Description'], ['responsibilities', 'Responsibilities'], ['requirements', 'Requirements'], ['benefits', 'Benefits']]) {
    const body = jdHtmlToText(typeof field(key) === 'string' ? field(key) : '');
    if (body) blocks.push(`${str(labels[key]) || label}:\n${body}`);
  }
  if (!blocks.length) return null;

  const meta = [];
  if (str(ad.site)) meta.push(`Location: ${str(ad.site)}`);
  if (str(ad.workspaceType)) meta.push(`Work model: ${str(ad.workspaceType)}`);
  if (str(ad.employmentType)) meta.push(`Type: ${str(ad.employmentType)}`);
  const min = field('payTransparencyMinSalary');
  const max = field('payTransparencyMaxSalary');
  const range = [min, max].filter((v) => typeof v === 'number' && v > 0).map((v) => v.toLocaleString('en-US')).join(' – ');
  if (range) meta.push(`Salary: ${[range, str(field('payTransparencySalaryCurrency')), str(field('payTransparencySalaryPayPeriod'))].filter(Boolean).join(' ')}`);

  return {
    url: postingUrl,
    title: compactText(str(ad.title), 300),
    text: compactText([meta.join('\n'), ...blocks].filter(Boolean).join('\n\n'), textCap),
  };
}

async function fetchHibobJd(resolved, postingUrl, textCap, timeoutMs) {
  if (rejectPrivateOrInvalid(resolved.apiUrl)) return null;
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    const res = await fetch(resolved.apiUrl, {
      headers: { accept: 'application/json', 'user-agent': DEFAULT_USER_AGENT, ...resolved.headers },
      redirect: 'error',
      signal: controller.signal,
    });
    if (!res.ok) return await discard(res);
    return normalizeHibobJob(await res.json(), postingUrl, textCap);
  } catch {
    return null;
  } finally {
    clearTimeout(timer);
  }
}

/** Workable full-body JD, independent of liveness state and listing text caps. */
export function normalizeWorkableJob(json, shortcode, postingUrl, textCap = JD_TEXT_CAP) {
  if (!json || typeof json !== 'object' || json.shortcode !== shortcode ||
      typeof json.title !== 'string' || !json.title.trim()) return null;
  const title = compactText(jdHtmlToText(json.title), 300);
  if (!title) return null;
  const blocks = [];
  for (const [key, label] of [['description', 'Description'], ['requirements', 'Requirements'], ['benefits', 'Benefits']]) {
    const html = typeof json[key] === 'string' ? json[key] : '';
    // Decode before removing chrome: the API can entity-escape its HTML tags.
    const body = jdHtmlToText(decodeEntities(html).replace(/<(nav|header|footer)\b[^>]*>[\s\S]*?<\/\1>/gi, ' '));
    if (/[\p{L}\p{N}]/u.test(body)) blocks.push(`${label}:\n${body}`);
  }
  if (!blocks.length) return null;
  return {
    url: postingUrl,
    title,
    text: compactText(blocks.join('\n\n'), textCap),
  };
}

/** Reuse the resolver's guarded share-link hop; never follow API redirects. */
async function fetchWorkableJd(resolved, postingUrl, textCap, timeoutMs) {
  if (rejectPrivateOrInvalid(resolved.apiUrl)) return null;
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  const options = {
    headers: { accept: 'application/json', 'user-agent': DEFAULT_USER_AGENT },
    redirect: 'error',
    signal: controller.signal,
  };
  // The share link's redirect, of which only the status and Location header
  // are read. Its body is drained last: both requests share one timer, so a
  // slow body read first would spend the posting request's budget.
  let redirect;
  try {
    let apiUrl = resolved.apiUrl;
    if (resolved.followEmbed) {
      redirect = await fetch(apiUrl, { ...options, redirect: 'manual' });
      apiUrl = await resolved.followEmbed(redirect, resolved.parts);
      if (!apiUrl || rejectPrivateOrInvalid(apiUrl)) return null;
    }
    const res = await fetch(apiUrl, options);
    if (!res.ok) return await discard(res);
    return normalizeWorkableJob(await res.json(), resolved.parts.shortcode, postingUrl, textCap);
  } catch {
    return null;
  } finally {
    if (redirect) await discard(redirect);
    clearTimeout(timer);
  }
}

/**
 * ats id → fetcher. THE routing table `fetchJdViaKnownApi` dispatches through,
 * exported so the owned test can assert its key set IS `JD_TEXT_API_ATS`.
 *
 * A `switch` could not carry that guarantee: deleting `case 'lever'` leaves a
 * test that only re-derives ids from `resolveAtsApi` green, because such a
 * test never touches the dispatcher. A map's keys ARE the dispatcher, so the
 * same deletion reddens it.
 *
 * Every entry takes the same `(resolved, url, textCap, timeoutMs)` shape so
 * dispatch stays a lookup; the extra `parts` arguments (Ashby's `jobId`, Zoho
 * Recruit's, UltiPro's and HireHive's `id`) are unpacked inside their own
 * entries rather than widening the shared signature.
 */
export const JD_FETCHERS = {
  workable: fetchWorkableJd,
  'workable-short': fetchWorkableJd,
  workday: (resolved, url, textCap, timeoutMs) =>
    fetchWorkdayJd(resolved.apiUrl, url, textCap, timeoutMs),
  ashby: (resolved, url, textCap, timeoutMs) =>
    fetchAshbyJd(resolved.apiUrl, resolved.parts.jobId, url, textCap, timeoutMs),
  greenhouse: (resolved, url, textCap, timeoutMs) =>
    fetchGreenhouseJd(resolved.apiUrl, url, textCap, timeoutMs),
  lever: (resolved, url, textCap, timeoutMs) =>
    fetchLeverJd(resolved.apiUrl, url, textCap, timeoutMs),
  'greenhouse-embedded': (resolved, url, textCap, timeoutMs) =>
    fetchGreenhouseEmbeddedJd(resolved, url, textCap, timeoutMs),
  smartrecruiters: (resolved, url, textCap, timeoutMs) =>
    fetchSmartRecruitersJd(resolved.apiUrl, url, textCap, timeoutMs),
  'zoho-recruit': (resolved, url, textCap, timeoutMs) =>
    fetchZohoJd(resolved.apiUrl, resolved.parts.id, url, textCap, timeoutMs),
  ultipro: (resolved, url, textCap, timeoutMs) =>
    fetchUltiproJd(resolved.apiUrl, resolved.parts.id, url, textCap, timeoutMs),
  hirehive: (resolved, url, textCap, timeoutMs) =>
    fetchHireHiveJd(resolved.apiUrl, resolved.parts.id, url, textCap, timeoutMs),
  hibob: fetchHibobJd,
};

/**
 * Try every known JD-text-capable ATS API for one posting URL before anyone
 * reaches for a browser. This is the single dispatch point both this script's
 * own CLI (jd mode) and `fetch-jd.mjs` (the bash-callable front door) call —
 * one copy of the routing table, so the interactive and headless paths cannot
 * drift on which ATS is API-fetchable.
 *
 * @param {string} url
 * @param {number} [textCap]
 * @param {number} [timeoutMs]
 * @returns {Promise<{url: string, title: string, text: string, ats: string}|null>}
 *   null = not a JD_TEXT_API_ATS host, or the fetch/parse was inconclusive —
 *   caller should fall through to a browser-backed read.
 */
export async function fetchJdViaKnownApi(url, textCap = JD_TEXT_CAP, timeoutMs = WORKDAY_TIMEOUT_MS) {
  const resolved = resolveAtsApi(url);
  if (!resolved || !JD_TEXT_API_ATS.has(resolved.ats)) return null;
  const fetcher = JD_FETCHERS[resolved.ats];
  if (!fetcher) return null;

  // `resolveAtsApi` carries a per-ATS timeout for the APIs that need one
  // (Ashby: 20 s, liveness-api.mjs:113); without it a slow Ashby board that
  // liveness waits for falls through here on the generic 15 s both callers
  // pass. Same precedence liveness-api.mjs:357 applies to the same field —
  // the resolved value wins, the argument is the fallback for every ATS that
  // declares none.
  const budgetMs = resolved.timeoutMs || timeoutMs;
  const result = await fetcher(resolved, url, textCap, budgetMs);
  return result ? { ...result, ats: resolved.ats } : null;
}

/**
 * A Cloudflare challenge or block page served in place of the posting, as a
 * jd-mode failure, or null. Pure — exported for tests.
 *
 * jdFailure() never rejects a result above the length floor, and a
 * site-branded challenge body clears it (#6713: 838 characters, "Humans
 * only"), so the interstitial was emitted as the JD. Cloudflare titles the page
 * itself, and archive-posting.mjs refuses the same pages by that title on an
 * HTTP error (#6724). `bot_challenge` is the code liveness gives the same wall.
 *
 * @param {{ status?: number, pageTitle?: string, url?: string }} page
 */
export function challengeFailure({ status = 0, pageTitle = '', url = '' } = {}) {
  if (!(status >= 400)) return null;
  const kind = cloudflareInterstitial(pageTitle);
  if (!kind) return null;
  return { error: `HTTP ${status} Cloudflare ${kind} ("${String(pageTitle).trim()}"), not the posting`, code: 'bot_challenge', url };
}

/**
 * Write a jd-mode result to stdout, or fail with `empty_text` when the
 * extraction came back empty enough to be useless.
 *
 * A `--max-chars` below the floor is honored rather than made unsatisfiable: a
 * caller who asks for 50 chars gets 50, not a guaranteed failure.
 */
/**
 * Why a jd-mode result cannot be emitted, or null when it can. Pure — exported
 * for tests.
 *
 * A page whose cleaned text is below the floor is either unreadable (a client-
 * rendered shell: `empty_text`, worth another attempt) or gone (the posting
 * was taken down: `unavailable`, terminal). Both produce the same empty text,
 * because the "no longer available" banner usually sits in a header or footer
 * that readDom() strips — so the unstripped body is checked too. A result at or
 * above the floor is never reclassified: a full JD may quote such a phrase.
 */
export function jdFailure(result, maxChars) {
  const floor = Math.min(MIN_JD_TEXT_CHARS, maxChars);
  if (result.text.length >= floor) return null;
  // The shared liveness classifier, not a second phrase list. Only its
  // posting-specific expiry verdicts count: `insufficient_content` is what a
  // page that has not rendered yet looks like (liveness-browser treats it the
  // same way), so it stays empty_text and remains worth retrying.
  const verdict = classifyLiveness({
    requestedUrl: result.url,
    finalUrl: result.url,
    bodyText: `${result.text}\n${result.bodyText || ''}`,
  });
  if (verdict.result === 'expired' && (verdict.code === 'expired_body' || verdict.code === 'expired_body_soft')) {
    return { error: `posting is no longer available (${verdict.reason})`, code: 'unavailable', url: result.url };
  }
  return {
    error: `extracted ${result.text.length} chars of JD text (minimum ${floor}) — the page most likely renders its content client-side`,
    code: 'empty_text',
    url: result.url,
    body_text: String(result.bodyText || '').slice(0, 4000),
  };
}

function emitJd(result, maxChars) {
  const failure = jdFailure(result, maxChars);
  if (failure) {
    console.error(JSON.stringify(failure));
    process.exitCode = 1;
    return;
  }
  // bodyText exists only for jdFailure(); drop it so the documented success
  // shape `{ url, title, text }` is unchanged.
  const { bodyText: _bodyText, closed: _closed, evidence: _evidence, ...emitted } = result;
  process.stdout.write(JSON.stringify(emitted));
}

/**
 * Shape a listing-mode result: keep visible anchors that look like individual
 * job postings, deduped by resolved URL, capped at `max`. Pure — exported for
 * tests. Anchors are dropped when the label is empty/too short or a nav
 * stopword, or the href isn't a resolvable http(s) URL.
 * @param {Array<{ href?: string, label?: string }>} anchors
 * @param {string} finalUrl - the page URL, used as the base to resolve relatives
 * @param {number} [max]
 */
export function normalizeListing(anchors, finalUrl, max = DEFAULT_LISTING_MAX) {
  const jobs = [];
  const seen = new Set();
  for (const a of Array.isArray(anchors) ? anchors : []) {
    const label = String(a?.label ?? '').replace(/\s+/g, ' ').trim();
    if (label.length < 3 || NAV_LABEL_STOPWORDS.has(label.toLowerCase())) continue;

    let url;
    try {
      url = new URL(String(a?.href ?? ''), finalUrl).href;
    } catch {
      continue;
    }
    if (!/^https?:$/.test(new URL(url).protocol)) continue;
    if (seen.has(url)) continue;
    seen.add(url);
    jobs.push({ title: label, url });
    if (jobs.length >= max) break;
  }
  return { url: finalUrl, jobs };
}

const VALUE_FLAGS = ['--mode', '--max', '--max-chars', '--timeout'];
const KNOWN_FLAGS = [...VALUE_FLAGS, '--help', '-h'];

// One synopsis, used by both --help and the no_url error, so the two cannot
// drift apart: the error's own copy already omitted --timeout.
const USAGE_SYNOPSIS = 'browser-extract.mjs <url> [--mode jd|listing] [--max N] [--max-chars N] [--timeout MS]';

const USAGE = `Usage:
  node ${USAGE_SYNOPSIS}

  --mode jd|listing   jd (default) returns { url, title, text }; listing returns { url, jobs }
  --max N             listing: maximum postings to return (default ${DEFAULT_LISTING_MAX})
  --max-chars N       jd: text cap (default ${JD_TEXT_CAP}); raise it for a long JD
  --timeout MS        navigation timeout (default ${DEFAULT_TIMEOUT_MS})
  --help, -h          Show this help`;

/**
 * Parse CLI args into { url, mode, max, maxChars, timeout }.
 *
 * Value reads go through lib/cli-flags.mjs so BOTH accepted forms reach the
 * extractor. The hand-rolled loop this replaces matched tokens exactly against
 * its own `FLAGS` set, so `--max-chars=50000` was never recognized as a flag:
 * it fell to the `!tok.startsWith('--')` branch, was not the URL either, and
 * the run silently proceeded at the 12000 default — a JD truncated at the tail
 * for a caller who explicitly asked for more. Same silent-wrong-answer shape as
 * the `--from=…` class in #2401/#2402 that lib/cli-flags.mjs exists to end.
 *
 * The URL is still found positionally, and an explicit `0` is still honored
 * rather than silently replaced by the default.
 *
 * @param {string[]} argv - process.argv.slice(2)
 */
export function parseArgs(argv) {
  const args = Array.isArray(argv) ? argv : [];

  // A value token consumed by a space-separated flag is not the URL. Mirrors
  // validateFlags' own adjacency rule: only a token that does not itself start
  // with `--` is treated as a value, so `--mode --max 5` leaves `--max` to be
  // reported rather than swallowed as the mode.
  const consumed = new Set();
  args.forEach((a, i) => {
    if (VALUE_FLAGS.includes(a) && args[i + 1] !== undefined && !args[i + 1].startsWith('--')) {
      consumed.add(i + 1);
    }
  });

  let url;
  for (let i = 0; i < args.length; i++) {
    const tok = args[i];
    if (typeof tok !== 'string' || consumed.has(i)) continue;
    if (!tok.startsWith('-') && url === undefined) url = tok;
  }

  // Each numeric read keeps its own range rule: `--max` admits 0 (a listing
  // capped at nothing is a meaningful request), the other two do not.
  const num = (flag, ok, fallback) => {
    if (!hasFlag(args, flag)) return fallback;
    const n = Number(flagValue(args, flag));
    return Number.isInteger(n) && ok(n) ? n : fallback;
  };

  const modeVal = hasFlag(args, '--mode') ? flagValue(args, '--mode') : undefined;

  return {
    url,
    mode: modeVal == null ? 'jd' : modeVal,
    max: num('--max', (n) => n >= 0, DEFAULT_LISTING_MAX),
    maxChars: num('--max-chars', (n) => n > 0, JD_TEXT_CAP),
    timeout: num('--timeout', (n) => n > 0, DEFAULT_TIMEOUT_MS),
  };
}

// Read the raw DOM inside the page: title, main visible text, and visible
// anchors. Runs in the browser context; returns plain data only.
export async function readDom(page) {
  return page.evaluate(() => {
    const title = (document.querySelector('h1')?.innerText || document.title || '').trim();

    // Main text: prefer <main>/[role=main]/<article>, else body; strip nav chrome.
    // Keep the candidate with the most text left after stripping, not the first
    // match: a board can lead with a short <article> (a share widget) ahead of
    // the one holding the posting, and a <main> that is mostly menu would
    // otherwise outrank it.
    // Read innerText from the live element: on a detached clone it degrades to
    // textContent, so CSS-hidden text would count and could outrank the posting.
    // The chrome is hidden only for the read and its style attribute restored.
    // script/style/noscript are never rendered, so innerText already skips them.
    const strippedText = (el) => {
      if (el.getClientRects().length === 0 && getComputedStyle(el).display !== 'contents') return '';
      const chrome = Array.from(el.querySelectorAll('nav, header, footer'));
      const saved = chrome.map((n) => n.getAttribute('style'));
      // Attribute writes only: setting a property through n.style leaves style=""
      // behind after removeAttribute. The prefix is the browser's serialized
      // cssText, not the raw attribute, so an unterminated token in the original
      // (an open /* comment) cannot swallow the appended display:none.
      chrome.forEach((n) => n.setAttribute('style', `${n.style.cssText} display: none !important;`));
      try {
        return el.innerText || '';
      } finally {
        chrome.forEach((n, i) => (saved[i] == null ? n.removeAttribute('style') : n.setAttribute('style', saved[i])));
      }
    };
    const candidates = Array.from(document.querySelectorAll('main, [role="main"], article'));
    let text = '';
    for (const el of candidates) {
      const t = strippedText(el);
      if (t.length > text.length) text = t;
    }
    if (candidates.length === 0 && document.body) text = strippedText(document.body);

    const anchors = Array.from(document.querySelectorAll('a[href]'))
      .filter((el) => {
        if (el.closest('nav, header, footer')) return false;
        const style = window.getComputedStyle(el);
        if (style.display === 'none' || style.visibility === 'hidden') return false;
        return el.getClientRects().length > 0;
      })
      .map((el) => ({ href: el.getAttribute('href') || '', label: (el.innerText || '').trim() }));

    // schema.org JobPosting, if present. Returned as the raw description markup
    // so the shared jdHtmlToText() does the conversion, keeping one entity /
    // block-break implementation rather than a second one inside the page.
    let jsonLdDescription = '';
    let jsonLdTitle = '';
    for (const el of document.querySelectorAll('script[type="application/ld+json"]')) {
      let parsed;
      try {
        parsed = JSON.parse(el.textContent || 'null');
      } catch {
        continue; // a malformed block must not cost us the well-formed one after it
      }
      // Publishers ship a bare object, an array, or an @graph wrapper.
      const candidates = Array.isArray(parsed)
        ? parsed
        : Array.isArray(parsed?.['@graph'])
          ? parsed['@graph']
          : [parsed];
      // @type is the bare token in most feeds, but some publishers use the
      // absolute schema.org IRI instead — both forms occur in the wild.
      const isJobPostingType = (t) =>
        t === 'JobPosting' || t === 'http://schema.org/JobPosting' || t === 'https://schema.org/JobPosting';
      for (const node of candidates) {
        const type = node?.['@type'];
        const isJobPosting = Array.isArray(type) ? type.some(isJobPostingType) : isJobPostingType(type);
        if (!isJobPosting) continue;
        const desc = typeof node.description === 'string' ? node.description : '';
        if (desc.length > jsonLdDescription.length) {
          jsonLdDescription = desc;
          // Always reset, never carry over: a stub JobPosting's title must not
          // survive onto the description of a later, longer JobPosting node.
          jsonLdTitle = typeof node.title === 'string' ? node.title : '';
        }
      }
    }

    // The unstripped body: `text` above removes nav/header/footer, where an
    // expired-posting banner often lives.
    // Head and tail, so a banner in the footer is still in the sample when the
    // leading content fills the first 4000 characters.
    const fullBody = document.body?.innerText || '';
    const bodyText = fullBody.length > 4000 ? `${fullBody.slice(0, 2000)}\n…\n${fullBody.slice(-2000)}` : fullBody;

    // The document's own title, which `title` above passes over for an <h1>:
    // challengeFailure() reads it.
    return { title, text, anchors, jsonLdDescription, jsonLdTitle, bodyText, pageTitle: document.title || '' };
  });
}

async function main() {
  const args = process.argv.slice(2);

  // Before anything launches a browser, because each of these used to fail in a
  // way that named the wrong thing (measured on 764f20f8):
  //   `--max-char 5000 <url>`  the typo was skipped, `5000` became the URL and
  //                            the real one was discarded — reported as
  //                            `invalid URL`, which is not what was wrong.
  //   `<url> --bogus`          skipped entirely; the scan ran and exited 0.
  //   `--help`                 exit 1 with a `no_url` error, never usage.
  //   `-h`                     one dash, so it was read AS the URL: `invalid URL`.
  // requireOperand: this script has nothing more specific to say about a missing
  // operand than the shared message, and without it `--max-chars --help` prints
  // usage and exits 0 with the malformed flag never reported (the ordering
  // CodeRabbit caught on #2961).
  validateFlags(args, KNOWN_FLAGS, USAGE, { valueFlags: VALUE_FLAGS, requireOperand: true });

  const { url, mode, max, maxChars, timeout } = parseArgs(args);

  if (!url) {
    console.error(JSON.stringify({ error: `usage: ${USAGE_SYNOPSIS}`, code: 'no_url' }));
    process.exit(1);
  }
  if (mode !== 'jd' && mode !== 'listing') {
    console.error(JSON.stringify({ error: `unknown mode "${mode}" (expected jd|listing)`, code: 'bad_mode' }));
    process.exit(1);
  }

  const guard = rejectPrivateOrInvalid(url);
  if (guard) {
    console.error(JSON.stringify({ error: guard.reason, code: guard.code }));
    process.exit(1);
  }

  // Known-ATS API first: Workday, Ashby, Greenhouse, and Lever all ship the
  // full JD body in a public JSON endpoint (see fetchJdViaKnownApi), so a
  // browser is never needed for those hosts. Workday additionally hydrates its
  // JD into a virtualized DOM the readDom() below cannot see at all, so this
  // isn't just an optimization for it — it's the only path that works.
  // Inconclusive -> fall through to Playwright, unchanged.
  if (mode === 'jd') {
    const apiResult = await fetchJdViaKnownApi(url, maxChars, timeout);
    if (apiResult) {
      emitJd(apiResult, maxChars);
      return;
    }
  }

  let chromium;
  try {
    ({ chromium } = await import('playwright'));
  } catch {
    console.error(JSON.stringify({ error: 'playwright not installed', code: 'no_playwright' }));
    process.exit(1);
  }

  let browser;
  try {
    browser = await chromium.launch(browserLaunchOptions({ headless: true }));
    const context = await browser.newContext(LIVENESS_CONTEXT_OPTIONS);
    // Block every request (main navigation, redirect hop, or subresource) to a
    // private/loopback/link-local or non-http(s) host. Guarding only the initial
    // URL isn't enough once we return page CONTENT: a server-side redirect could
    // otherwise steer the browser at internal infrastructure (SSRF).
    await context.route('**/*', (route) => {
      if (rejectPrivateOrInvalid(route.request().url())) return route.abort('blockedbyclient');
      return route.continue();
    });
    const page = await context.newPage();
    const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout });
    await page.waitForTimeout(HYDRATION_WAIT_MS); // let SPAs hydrate

    // Belt-and-suspenders: never emit content read from a private final URL.
    const finalUrl = page.url();
    const finalGuard = rejectPrivateOrInvalid(finalUrl);
    if (finalGuard) {
      console.error(JSON.stringify({ error: `blocked final URL: ${finalGuard.reason}`, code: finalGuard.code }));
      process.exitCode = 1;
      return;
    }
    const raw = await readDom(page);

    if (mode === 'listing') {
      process.stdout.write(JSON.stringify(normalizeListing(raw.anchors, finalUrl, max)));
    } else {
      const challenge = challengeFailure({ status: response?.status() ?? 0, pageTitle: raw.pageTitle, url: finalUrl });
      if (challenge) {
        console.error(JSON.stringify(challenge));
        process.exitCode = 1;
      } else {
        emitJd(normalizeJd(raw, finalUrl, maxChars), maxChars);
      }
    }
  } catch (err) {
    console.error(JSON.stringify({ error: `navigation error: ${String(err.message).split('\n')[0]}`, code: 'navigation_error' }));
    process.exitCode = 1;
  } finally {
    if (browser) await browser.close().catch(() => {});
  }
}

// Only run main() when invoked directly, not when imported by tests.
if (isMainModule(import.meta.url)) {
  main();
}