#!/usr/bin/env node

/**
 * check-liveness.mjs — Playwright job link liveness checker
 *
 * Tests whether job posting URLs are still active or have expired.
 * Uses the same detection logic as scan.md step 7.5.
 * Zero Claude API tokens. Two rungs: a free public-API check first
 * (liveness-api.mjs, no browser), then Playwright for everything else.
 *
 * An `expired` verdict is recorded back into data/scan-history.tsv as a
 * `skipped_expired` row, so a posting confirmed dead once stops resurfacing
 * (#3891). See recordExpiredVerdicts below for why that is default-on.
 *
 * Usage:
 *   node check-liveness.mjs <url1> [url2] ...
 *   node check-liveness.mjs --file urls.txt
 *
 * Exit code: 0 if all active, 1 if any expired or uncertain
 */

import { readFile } from 'fs/promises';
import {
  checkUrlLivenessWithFallback,
  createHeadedPageProvider,
  newLivenessPage,
  createHostSpacer,
  takeSuppressedBlockCount,
  sleep,
} from './liveness-browser.mjs';
import { checkLivenessViaApi } from './liveness-api.mjs';
import { robotsAllows } from './robots-check.mjs';
import { planExpiredHistoryRows, trackerLivenessTargets, DEFAULT_TRACKER_STATUSES, planExpiredDiscards, postingClosedNote, DEFAULT_DISCARD_STATUSES, discardableExpiredUrls } from './liveness-core.mjs';
import { appendToScanHistory, seenJobKey, SCAN_HISTORY_PATH } from './scan.mjs';
import { withPipelineLock } from './pipeline-lock.mjs';
import { isMainModule } from './lib/is-main-module.mjs';
import { localToday } from './lib/local-today.mjs';
import { existsSync, readFileSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { flagValue, hasFlag, validateFlags } from './lib/cli-flags.mjs';
import { execFileSync } from 'child_process';
import { createHash } from 'crypto';
import { getCareerOpsRoot, resolveTrackerPath } from './path-resolver.mjs';
import { browserLaunchOptions } from './lib/browser-launch.mjs';

const USAGE = `Usage:
  node check-liveness.mjs [--no-fallback] [--throttle[=ms]] [--no-record] <url1> [url2] ...
  node check-liveness.mjs [--no-fallback] [--throttle[=ms]] [--no-record] --file urls.txt
  node check-liveness.mjs --tracker [--status=Evaluated,Applied]   # postings of tracker rows in these statuses
  node check-liveness.mjs --tracker --discard-expired [--apply]    # dry-run move expired Evaluated rows to Discarded; --apply writes
  node check-liveness.mjs --no-record             # check only; write nothing to scan-history
  node check-liveness.mjs --help                  # print this usage block and exit
  node check-liveness.mjs -h                      # alias for --help`;

const KNOWN_FLAGS = [
  '--no-fallback', '--throttle', '--no-record', '--file', '--tracker', '--status',
  '--discard-expired', '--apply', '--help', '-h',
];

/**
 * `--throttle` alone means the 5000ms default. `--throttle 1000` and
 * `--throttle=1000` both mean 1000. A non-numeric equals value stays on the
 * default rather than becoming a URL to check.
 *
 * @param {string[]} args
 * @returns {number}
 */
function throttleBaseMsFrom(args) {
  if (!hasFlag(args, '--throttle')) return 0;
  const eq = args.find((a) => typeof a === 'string' && a.startsWith('--throttle='));
  const raw = eq !== undefined ? eq.slice('--throttle='.length) : (() => {
    const idx = args.indexOf('--throttle');
    const next = idx === -1 ? undefined : args[idx + 1];
    return typeof next === 'string' && /^\d+$/.test(next) ? next : undefined;
  })();
  const n = Number(raw);
  return Number.isFinite(n) && n > 0 ? n : 5000;
}

/**
 * Tokens that are not flags and not a flag's operand. Those are the URLs.
 * `--dry-run` is not a flag this script accepts, so it never reaches here:
 * validateFlags rejects it first. A numeric `--throttle` operand is not a URL.
 *
 * @param {string[]} args
 * @returns {string[]}
 */
function urlPositionals(args) {
  const skip = new Set();
  for (let i = 0; i < args.length; i++) {
    const a = args[i];
    if (typeof a !== 'string') continue;
    if (a.startsWith('--file=') || a.startsWith('--status=') || a.startsWith('--throttle=')) {
      skip.add(i);
      continue;
    }
    if (a === '--file' || a === '--status') {
      skip.add(i);
      if (args[i + 1] !== undefined) skip.add(i + 1);
      continue;
    }
    if (a === '--throttle') {
      skip.add(i);
      const next = args[i + 1];
      if (typeof next === 'string' && /^\d+$/.test(next)) skip.add(i + 1);
      continue;
    }
    if (a === '-h' || a.startsWith('--')) skip.add(i);
  }
  return args.filter((_, i) => !skip.has(i));
}

/**
 * Is this run allowed to record its verdicts?
 *
 * Default-ON, opt-OUT. An opt-IN flag would be the bug wearing a fix: the
 * scheduled routines pass no flags, so a gated recording never runs from the
 * one caller that produces most of these verdicts — which is precisely how the
 * expensive `expired` result came to be computed and discarded (#3891).
 * `--no-record` covers the real need for the opposite: a read-only spot check
 * that leaves no trace.
 *
 * @param {string[]} args - process.argv.slice(2).
 * @returns {boolean}
 */
export function recordingEnabled(args = []) {
  return !args.includes('--no-record');
}

/**
 * Write the run's `expired` verdicts into data/scan-history.tsv.
 *
 * Everything about WHICH rows get written is decided by planExpiredHistoryRows
 * (only expired, only URLs the history knows, once each); this is the effect.
 * A history file that does not exist yet means no URL is known, so there is
 * nothing to retire and nothing to create.
 *
 * The row is stamped with localToday(), like every other scan-history writer:
 * a UTC day is the wrong day for most of the world for part of every day.
 *
 * The read and the append share ONE critical section. What to append is decided
 * by what the file already says, so reading outside the lock is a check-then-act:
 * a scanner appending the same `skipped_expired` row in between would leave two
 * death certificates for one posting. The lock is not reentrant, hence
 * `alreadyLocked` on the append rather than a second acquisition.
 *
 * @param {{url: string, result: string}[]} verdicts
 * @returns {Promise<number>} How many rows were written.
 */
export async function recordExpiredVerdicts(verdicts) {
  // This one check is deliberately OUTSIDE the lock, unlike the read below.
  // No history means no URL is known, so there is nothing to retire and no
  // reason to pay for a lock — and locking it would buy nothing, because the
  // scenario it looks like it prevents is not preventable. A scanner that
  // creates the file and surfaces this URL a microsecond after the check is
  // indistinguishable from one that does it a second after the whole function
  // returns: either way the posting was discovered after the verdict, and the
  // next sweep is what covers it. (The file is created by atomicWriteFile,
  // which renames into place, so this can never see a half-written history.)
  if (!existsSync(SCAN_HISTORY_PATH)) return 0;
  return withPipelineLock(SCAN_HISTORY_PATH, async () => {
    const rows = planExpiredHistoryRows(
      readFileSync(SCAN_HISTORY_PATH, 'utf-8'),
      verdicts,
      seenJobKey,
    );
    // The writer drops an alias when another spelling of that posting already
    // has a pinning skipped_expired row. The number below is rows that landed.
    if (rows.length > 0) return await appendToScanHistory(rows, localToday(), 'skipped_expired', { alreadyLocked: true });
    return 0;
  });
}

const CODE_ROOT = dirname(fileURLToPath(import.meta.url));

function runSetStatus(args, trackerPath) {
  try {
    return JSON.parse(execFileSync(process.execPath, [join(CODE_ROOT, 'set-status.mjs'), ...args], {
      encoding: 'utf-8',
      env: { ...process.env, CAREER_OPS_TRACKER: trackerPath },
      stdio: ['ignore', 'pipe', 'pipe'],
    }));
  } catch (err) {
    try {
      return JSON.parse(err.stdout);
    } catch {
      return { error: (typeof err.stderr === 'string' && err.stderr.trim()) || err.message };
    }
  }
}

/**
 * Move planned rows to Discarded through set-status.mjs (locked atomic write).
 * Dry-run when apply is false: resolve only, write nothing.
 *
 * The liveness sweep can take minutes, so each row is planned again against
 * the tracker as it is now, and set-status refuses the write under its lock
 * if the tracker changed after that re-plan.
 *
 * @param {{num: number, company: string, role: string, status: string, url: string}[]} rows
 * @param {{apply: boolean, note: string, trackerPath: string, reportsRoot: string}} options
 * @returns {{num: number, company: string, role: string, url: string, changed: boolean, dryRun: boolean, error?: string, skipped?: string}[]}
 */
export function discardExpiredRows(rows, { apply, note, trackerPath, reportsRoot }) {
  return rows.map((row) => {
    const base = { num: row.num, company: row.company, role: row.role, url: row.url, changed: false, dryRun: !apply };
    const current = readFileSync(trackerPath, 'utf-8');
    if (!planExpiredDiscards(current, [row.url], { reportsRoot }).some((r) => r.num === row.num)) {
      return { ...base, skipped: 'no longer Evaluated with this posting' };
    }
    const args = ['--row', String(row.num), 'Discarded', '--note', note, '--json',
      '--expect-tracker', createHash('sha256').update(current, 'utf8').digest('hex')];
    if (!apply) args.push('--dry-run');
    const out = runSetStatus(args, trackerPath);
    if (out.error) return { ...base, error: out.error };
    return { ...base, changed: Boolean(out.changed), dryRun: Boolean(out.dryRun) || !apply };
  });
}

async function main() {
  const args = process.argv.slice(2);

  // Unknown flags used to survive into the URL list: `--dry-run --file urls.txt`
  // checked three strings and never opened the file, and `--help` ran before
  // that filter so `--help --bogus` exited 0. validateFlags rejects the unknown
  // flag first, then prints help. --throttle's operand is optional (bare
  // `--throttle` is the 5000ms default), and --status keeps the message the
  // status-flag test locks, so the missing-operand check stays here rather
  // than in requireOperand. See tests/validate-flags-value-operand.test.mjs.
  validateFlags(args, KNOWN_FLAGS, USAGE, {
    valueFlags: ['--file', '--status', '--throttle'],
  });

  if (hasFlag(args, '--file')) {
    const filePath = flagValue(args, '--file');
    if (typeof filePath !== 'string' || !filePath.trim() || filePath.startsWith('--')) {
      console.error('Error: --file requires a value');
      process.exit(1);
    }
  }

  // Portals like pracuj.pl serve a Cloudflare anti-bot wall to headless Chromium.
  // On a challenge we retry once in a headed browser (which clears it); pass
  // --no-fallback to stay fully headless (e.g. on a machine with no display).
  const noFallback = args.includes('--no-fallback');
  // --throttle, --throttle <ms>, or --throttle=<ms>: wait base..2*base ms
  // (jittered) between browser checks of the same host (createHostSpacer in
  // liveness-browser.mjs) to stay under rate-based WAF limits. pracuj.pl's
  // Cloudflare flags the session after ~2 rapid hits, so a bulk run needs
  // spacing. Bare --throttle uses a 5000ms base. The space form used to be
  // ignored (the number was checked as a URL) and the base stayed 5000.
  const throttleBaseMs = throttleBaseMsFrom(args);
  const record = recordingEnabled(args);
  // `--status Rejected` as well as `--status=Rejected`. Only the `=` form was
  // read, so the space form, and any `--status` with no value, fell back to
  // the default statuses without a word, and `--discard-expired --apply`
  // then rewrote Evaluated and Applied rows the user never asked about.
  // A --status with no value is a usage error (the hasFlag + flagValue
  // pairing lib/cli-flags.mjs documents for this).
  const statusGiven = hasFlag(args, '--status');
  const statusValue = statusGiven ? flagValue(args, '--status') : undefined;
  if (statusGiven && (typeof statusValue !== 'string' || !statusValue.trim() || statusValue.startsWith('--'))) {
    console.error('--status needs a value, e.g. --status=Evaluated,Applied');
    process.exitCode = 1;
    return;
  }
  const fromTracker = args.includes('--tracker');
  const discardExpired = args.includes('--discard-expired');
  const applyDiscards = args.includes('--apply');
  const positional = urlPositionals(args);

  if (discardExpired && !fromTracker) {
    console.error('--discard-expired requires --tracker.');
    process.exitCode = 1;
    return;
  }

  if (positional.length === 0 && !fromTracker && !hasFlag(args, '--file')) {
    console.error(USAGE);
    process.exitCode = 1;
    return;
  }

  let urls;
  const labels = new Map();
  let trackerPath = null;
  let trackerText = null;
  let reportsRoot = null;
  if (fromTracker) {
    const root = getCareerOpsRoot();
    trackerPath = resolveTrackerPath(root);
    reportsRoot = join(root, 'reports');
    if (!existsSync(trackerPath)) {
      console.error(`No tracker at ${trackerPath}. Evaluate a role first.`);
      process.exitCode = 1;
      return;
    }
    trackerText = readFileSync(trackerPath, 'utf-8');
    const statuses = statusGiven ? statusValue.split(',').map((s) => s.trim()).filter(Boolean) : DEFAULT_TRACKER_STATUSES;
    const { targets, noUrl } = trackerLivenessTargets(trackerText, { statuses, reportsRoot });
    console.log(`Tracker rows with status ${statuses.join(' or ')}: ${targets.length} with a posting URL${noUrl.length ? `, ${noUrl.length} without (${noUrl.map((n) => `#${n}`).join(' ')})` : ''}`);
    for (const t of targets) labels.set(t.url, `#${t.num} ${t.company} · ${t.role} (${t.status})`);
    urls = targets.map((t) => t.url);
  } else if (hasFlag(args, '--file')) {
    const text = await readFile(flagValue(args, '--file'), 'utf-8');
    urls = text.split('\n').map(l => l.trim()).filter(l => l && !l.startsWith('#'));
  } else {
    urls = positional;
  }

  const notes = [
    noFallback ? null : 'headed fallback on challenge',
    throttleBaseMs ? `throttle ~${throttleBaseMs / 1000}-${(throttleBaseMs * 2) / 1000}s` : null,
    record ? null : 'no-record',
  ].filter(Boolean);
  console.log(`Checking ${urls.length} URL(s)...${notes.length ? ` (${notes.join(', ')})` : ''}\n`);

  // Lazy browser: the API rung resolves ATS postings with no browser at all, so we
  // only launch Playwright if a URL actually needs the fallback.
  let browser = null, page = null, headed = null;
  async function ensureBrowser() {
    if (browser) return;
    const { chromium } = await import('playwright');
    const launched = await chromium.launch(browserLaunchOptions({ headless: true }));
    try {
      page = await newLivenessPage(launched);
      headed = noFallback ? null : createHeadedPageProvider(chromium);
    } catch (err) {
      // Never leave a launched process behind when setup fails half way.
      await launched.close().catch(() => {});
      throw err;
    }
    browser = launched;
  }

  let active = 0, expired = 0, uncertain = 0, viaApi = 0;
  const verdicts = [];
  // --throttle spaces repeat browser visits to the same host; the API rung is
  // cheap and not WAF-rate-limited, so it never waits.
  const spacer = createHostSpacer(throttleBaseMs);
  // An Ashby board answers for every posting under its org: fetch each board
  // once per run, not once per posting.
  const boardCache = new Map();

  // Sequential — project rule: never Playwright in parallel
  for (let i = 0; i < urls.length; i++) {
    const url = urls[i];
    let result, reason, code;

    // Rung 1: zero-token ATS API check. A conclusive active/expired wins; otherwise fall through.
    const api = await checkLivenessViaApi(url, { boardCache });
    if (api) {
      ({ result, reason, code } = api);
      viaApi++;
    } else {
      // Rung 2: Playwright — handles non-ATS pages and inconclusive API results.
      // A browser that will not launch (missing binaries after a Playwright
      // update, no display) must not end the batch: every URL after it would
      // lose its verdict, including API-checkable ATS URLs that never needed
      // the browser. Report the URLs that needed it as uncertain with the
      // install hint and keep going; the exit code still reflects that they
      // were not confirmed.
      try {
        await ensureBrowser();
        const wait = spacer.waitBefore(url);
        if (wait) await sleep(wait);
        const getHeadedPage = headed ? () => headed.get() : undefined;
        try {
          ({ result, reason, code } = await checkUrlLivenessWithFallback(page, url, { getHeadedPage, robotsGate: robotsAllows }));
        } finally {
          // A failed visit counts too: that is when a WAF is most likely involved.
          spacer.visited(url);
        }
      } catch (err) {
        result = 'uncertain';
        code = 'browser_unavailable';
        const fix = err?.code === 'ERR_MODULE_NOT_FOUND' ? 'npm install' : 'npx playwright install chromium';
        reason = `browser unavailable (${String(err?.message ?? err).split('\n')[0]}); run \`${fix}\` and re-run`;
      }
    }

    const icon = { active: '✅', expired: '❌', uncertain: '⚠️' }[result];
    console.log(`${icon} ${result.padEnd(10)} ${api ? '(api) ' : '      '}${labels.has(url) ? `${labels.get(url)}  ` : ''}${url}`);
    if (result !== 'active') console.log(`           ${reason}`);
    if (result === 'active') active++;
    else if (result === 'expired') expired++;
    else uncertain++;
    verdicts.push({ url, result, code });
  }

  if (headed) await headed.close();
  if (browser) await browser.close();

  console.log(`\nResults: ${active} active  ${expired} expired  ${uncertain} uncertain  (${viaApi} via API, no browser)`);
  const quietBlocks = takeSuppressedBlockCount();
  if (quietBlocks) console.log(`(${quietBlocks} tracker/analytics requests skipped: their host had no usable address, e.g. a DNS ad-blocker; not counted against any posting)`);

  // Close the loop: the verdict is the expensive part of this run and it used
  // to die with the process. `--no-record` leaves no trace, so the write is
  // skipped entirely rather than computed and thrown away.
  if (record) {
    const recorded = await recordExpiredVerdicts(verdicts);
    if (recorded > 0) console.log(`Recorded ${recorded} expired posting(s) in ${SCAN_HISTORY_PATH}`);
  }

  if (discardExpired) {
    // Only closure EVIDENCE closes a row. An `insufficient_content` verdict
    // means the page could not be read, whether it arrives as `uncertain` or,
    // from an older verdict, as `expired`, and this path writes Discarded to
    // the tracker — see isClosureEvidence().
    const { closed: expiredUrls, unreadable } = discardableExpiredUrls(verdicts);
    if (unreadable.length > 0) {
      console.log(`\nDiscard expired: ${unreadable.length} page(s) could not be read and were left as they are `
        + '(an unreadable page is unconfirmed, not closed):');
      for (const url of unreadable) console.log(`  · ${url}`);
    }
    const planned = planExpiredDiscards(trackerText, expiredUrls, {
      statuses: DEFAULT_DISCARD_STATUSES,
      reportsRoot,
    });
    const note = postingClosedNote(localToday());
    if (planned.length === 0) {
      console.log('\nDiscard expired: no Evaluated rows matched an expired posting.');
    } else {
      const mode = applyDiscards ? 'apply' : 'dry-run';
      console.log(`\nDiscard expired (${mode}): ${planned.length} Evaluated row(s) → Discarded`);
      const results = discardExpiredRows(planned, { apply: applyDiscards, note, trackerPath, reportsRoot });
      for (const r of results) {
        if (r.skipped) {
          console.log(`  · #${r.num} ${r.company} · ${r.role} — skipped, ${r.skipped}`);
          continue;
        }
        if (r.error) {
          console.log(`  ❌ #${r.num} ${r.company} · ${r.role} — ${r.error}`);
          process.exitCode = 1;
          continue;
        }
        const verb = r.dryRun ? 'would discard' : (r.changed ? 'discarded' : 'already Discarded');
        console.log(`  ${r.dryRun ? '·' : '✅'} #${r.num} ${r.company} · ${r.role} — ${verb} (${note})`);
      }
      if (!applyDiscards) {
        console.log('Re-run with --apply to write. Uncertain and Applied rows are left alone.');
      }
    }
  }

  if (expired > 0 || uncertain > 0) process.exitCode = 1;
}

// Only run main() when invoked directly (`node check-liveness.mjs`), not when
// imported — the arg parsing above would otherwise read an importing test's argv.
if (isMainModule(import.meta.url)) {
  main().catch(err => {
    console.error('Fatal:', err.message);
    process.exitCode = 1;
  });
}