#!/usr/bin/env node
// Scores the detector study: reads sample.json + results.csv + limato.jsonl, prints FP/FN rates
// with Wilson 95% CIs, overall accuracy, a >=80-word subset cut, below-min-length counts, and
// coverage warnings for any missing detector x item cell.
//
// Usage:
//   node score.js                metrics from results.csv + limato.jsonl
//   node score.js --markdown     same tables as GitHub-flavoured markdown
//   node score.js --spread       min/median/max score per detector, human set and AI set
//   node score.js --selftest     metric-function fixtures, no file or network reads

const fs = require('fs');
const path = require('path');
const assert = require('assert');

const DIR = __dirname;
const SAMPLE_FILE = path.join(DIR, 'sample.json');
const RESULTS_FILE = path.join(DIR, 'results.csv');
const LIMATO_FILE = path.join(DIR, 'limato.jsonl');

// --- small parsers / helpers --------------------------------------------

// Minimal RFC4180-ish CSV line parser: handles quoted fields, commas inside quotes, "" escaping.
// No multi-line quoted fields — not needed for this file's one-line verdict strings.
function parseCsvLine(line) {
  const fields = [];
  let cur = '';
  let inQuotes = false;
  for (let i = 0; i < line.length; i++) {
    const c = line[i];
    if (inQuotes) {
      if (c === '"') {
        if (line[i + 1] === '"') {
          cur += '"';
          i++;
        } else {
          inQuotes = false;
        }
      } else {
        cur += c;
      }
    } else if (c === '"') {
      inQuotes = true;
    } else if (c === ',') {
      fields.push(cur);
      cur = '';
    } else {
      cur += c;
    }
  }
  fields.push(cur);
  return fields;
}

function parseCsv(text) {
  const lines = text.split(/\r?\n/).filter((l) => l.length > 0);
  if (!lines.length) return [];
  const header = parseCsvLine(lines[0]);
  return lines.slice(1).map((line) => {
    const fields = parseCsvLine(line);
    const obj = {};
    header.forEach((h, i) => (obj[h] = fields[i]));
    return obj;
  });
}

function parseNumber(value, label) {
  // Number('') and Number('  ') coerce to 0, which would silently turn a blank CSV cell into a
  // real reading — reject before that coercion happens.
  if (typeof value !== 'string' || value.trim() === '') throw new Error(`invalid ${label}: "${value}"`);
  const n = Number(value);
  if (!Number.isFinite(n)) throw new Error(`invalid ${label}: "${value}"`);
  return n;
}
const parseWords = (value) => parseNumber(value, 'words');

function rate(count, n) {
  if (n === 0) return null;
  return count / n;
}

// Wilson score interval — no dependency. z=1.96 -> ~95% CI.
function wilson(successes, n, z = 1.96) {
  if (n === 0) return { lower: 0, upper: 1 };
  const phat = successes / n;
  const z2 = z * z;
  const denom = 1 + z2 / n;
  const center = phat + z2 / (2 * n);
  const margin = z * Math.sqrt((phat * (1 - phat)) / n + z2 / (4 * n * n));
  return { lower: Math.max(0, (center - margin) / denom), upper: Math.min(1, (center + margin) / denom) };
}

const pctStr = (x) => (x === null ? 'n/a' : (x * 100).toFixed(1) + '%');
const ciStr = (ci) => (ci === null ? 'n/a' : `[${(ci.lower * 100).toFixed(1)}%, ${(ci.upper * 100).toFixed(1)}%]`);

function printTable(title, headers, rows, markdown) {
  console.log(`\n## ${title}\n`);
  if (!rows.length) {
    console.log('(no data)');
    return;
  }
  if (markdown) {
    console.log(`| ${headers.join(' | ')} |`);
    console.log(`|${headers.map(() => '---').join('|')}|`);
    for (const r of rows) console.log(`| ${r.join(' | ')} |`);
  } else {
    const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => String(r[i]).length)));
    const pad = (s, w) => String(s).padEnd(w);
    console.log(headers.map((h, i) => pad(h, widths[i])).join('  '));
    console.log(widths.map((w) => '-'.repeat(w)).join('  '));
    for (const r of rows) console.log(r.map((c, i) => pad(c, widths[i])).join('  '));
  }
}

// --- load phase ----------------------------------------------------------

function loadSample() {
  if (!fs.existsSync(SAMPLE_FILE)) {
    console.error(`No sample.json at ${SAMPLE_FILE}. Run "node build-sample.js" first.`);
    process.exit(1);
  }
  return JSON.parse(fs.readFileSync(SAMPLE_FILE, 'utf8'));
}

function loadResultsCsv() {
  if (!fs.existsSync(RESULTS_FILE)) return [];
  const raw = parseCsv(fs.readFileSync(RESULTS_FILE, 'utf8'));
  return raw.map((r) => ({
    item_id: r.item_id,
    kind: r.kind,
    words: parseWords(r.words),
    detector: r.detector,
    score_ai_pct: parseNumber(r.score_ai_pct, 'score_ai_pct'),
    verdict: r.verdict,
    below_min_length: parseNumber(r.below_min_length, 'below_min_length'),
    run_date: r.run_date,
  }));
}

function loadLimatoJsonl() {
  if (!fs.existsSync(LIMATO_FILE)) return [];
  return fs
    .readFileSync(LIMATO_FILE, 'utf8')
    .split('\n')
    .filter((l) => l.trim())
    .map((l) => {
      const r = JSON.parse(l);
      return {
        item_id: r.item_id,
        kind: r.kind,
        words: r.words,
        detector: 'limato',
        score_ai_pct: r.score,
        verdict: r.verdict,
        below_min_length: 0, // our own detector has no documented minimum input length
        run_date: r.run_date,
      };
    });
}

// Merges results.csv + limato.jsonl rows into one detector:item_id -> row map, warning on
// duplicates (last one wins) and on rows outside the sample (typo'd id/detector).
function buildRowMap(sample, rows) {
  const validIds = new Set([...sample.human.map((h) => h.id), ...sample.ai.map((a) => a.id)]);
  const validDetectors = new Set(sample.detectors);
  const map = new Map();
  for (const r of rows) {
    if (!validIds.has(r.item_id)) {
      console.warn(`WARNING: row for unknown item_id "${r.item_id}" (not in sample.json) — ignored`);
      continue;
    }
    if (!validDetectors.has(r.detector)) {
      console.warn(`WARNING: row for unknown detector "${r.detector}" (not in sample.json) — ignored`);
      continue;
    }
    const key = `${r.detector}:${r.item_id}`;
    if (map.has(key)) console.warn(`WARNING: duplicate row for ${key} — keeping the later one`);
    map.set(key, r);
  }
  return map;
}

// --- coverage --------------------------------------------------------------

function checkCoverage(sample, rowMap, items) {
  const total = items.length;
  let anyMissing = false;
  console.log('\n## Coverage\n');
  for (const detector of sample.detectors) {
    const missing = items.filter((it) => !rowMap.has(`${detector}:${it.id}`)).map((it) => it.id);
    const have = total - missing.length;
    if (missing.length) {
      anyMissing = true;
      console.warn(`WARNING: ${detector}: ${have}/${total} runs present. Missing: ${missing.join(', ')}`);
    } else {
      console.log(`${detector}: ${have}/${total} runs present`);
    }
  }
  if (anyMissing) {
    console.warn(
      '\nWARNING: the detector x item matrix is INCOMPLETE. Every table below is computed only from ' +
        'the rows that exist — n per detector reflects that, it is not silently padded to the full sample.'
    );
  }
  return anyMissing;
}

// --- metrics ---------------------------------------------------------------

// direction 'gte': flagged when score >= threshold (used for false positives on the human set).
// direction 'lt':  flagged when score <  threshold (used for false negatives on the ai set).
function ratesBySet(sample, rowMap, items, threshold, direction) {
  const out = [];
  for (const detector of sample.detectors) {
    let n = 0,
      count = 0;
    for (const item of items) {
      const row = rowMap.get(`${detector}:${item.id}`);
      if (!row) continue;
      n++;
      const flagged = direction === 'gte' ? row.score_ai_pct >= threshold : row.score_ai_pct < threshold;
      if (flagged) count++;
    }
    const r = rate(count, n);
    out.push({ detector, n, count, rate: r, ci: n ? wilson(count, n) : null });
  }
  return out;
}

function accuracyByDetector(sample, rowMap, allItems, threshold) {
  const out = [];
  for (const detector of sample.detectors) {
    let n = 0,
      correct = 0;
    for (const item of allItems) {
      const row = rowMap.get(`${detector}:${item.id}`);
      if (!row) continue;
      n++;
      const predictedAi = row.score_ai_pct >= threshold;
      const actualAi = item.kind === 'ai';
      if (predictedAi === actualAi) correct++;
    }
    out.push({ detector, n, correct, accuracy: rate(correct, n) });
  }
  return out;
}

function median(nums) {
  const s = [...nums].sort((a, b) => a - b);
  const mid = Math.floor(s.length / 2);
  return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
}

// min/median/max of score_ai_pct per detector, computed separately on the human set and the AI set.
function spreadBySet(sample, rowMap, items) {
  const out = [];
  for (const detector of sample.detectors) {
    const scores = items
      .map((item) => rowMap.get(`${detector}:${item.id}`))
      .filter(Boolean)
      .map((row) => row.score_ai_pct);
    if (!scores.length) {
      out.push({ detector, n: 0, min: null, median: null, max: null });
      continue;
    }
    out.push({ detector, n: scores.length, min: Math.min(...scores), median: median(scores), max: Math.max(...scores) });
  }
  return out;
}

function printSpreadTable(title, rows, markdown) {
  printTable(
    title,
    ['detector', 'n', 'min', 'median', 'max'],
    rows.map((r) => [r.detector, r.n, r.min === null ? 'n/a' : `${r.min}%`, r.median === null ? 'n/a' : `${r.median}%`, r.max === null ? 'n/a' : `${r.max}%`]),
    markdown
  );
}

function runSpread(markdown) {
  const sample = loadSample();
  const rows = [...loadResultsCsv(), ...loadLimatoJsonl()];
  const rowMap = buildRowMap(sample, rows);
  const humanItems = sample.human.map((h) => ({ ...h, kind: 'esl' }));
  const aiItems = sample.ai.map((a) => ({ ...a, kind: 'ai' }));
  printSpreadTable('Score spread, human set', spreadBySet(sample, rowMap, humanItems), markdown);
  printSpreadTable('Score spread, AI set', spreadBySet(sample, rowMap, aiItems), markdown);
}

function belowMinLengthCounts(sample, rowMap, allItems) {
  const out = [];
  for (const detector of sample.detectors) {
    let n = 0,
      flagged = 0;
    for (const item of allItems) {
      const row = rowMap.get(`${detector}:${item.id}`);
      if (!row) continue;
      n++;
      if (row.below_min_length) flagged++;
    }
    out.push({ detector, n, flagged });
  }
  return out;
}

function printRatesTable(title, rows, markdown) {
  printTable(
    title,
    ['detector', 'n', 'flagged', 'rate', '95% CI'],
    rows.map((r) => [r.detector, r.n, r.count, pctStr(r.rate), ciStr(r.ci)]),
    markdown
  );
}

function run(markdown) {
  const sample = loadSample();
  const rows = [...loadResultsCsv(), ...loadLimatoJsonl()];
  const rowMap = buildRowMap(sample, rows);

  const humanItems = sample.human.map((h) => ({ ...h, kind: 'esl' }));
  const aiItems = sample.ai.map((a) => ({ ...a, kind: 'ai' }));
  const allItems = [...humanItems, ...aiItems];

  checkCoverage(sample, rowMap, allItems);

  printRatesTable(
    `False positive rate on human set (score >= ${sample.thresholds.flag})`,
    ratesBySet(sample, rowMap, humanItems, sample.thresholds.flag, 'gte'),
    markdown
  );
  printRatesTable(
    `False positive rate on human set (score >= ${sample.thresholds.strict})`,
    ratesBySet(sample, rowMap, humanItems, sample.thresholds.strict, 'gte'),
    markdown
  );
  printRatesTable(
    `False negative rate on AI set (score < ${sample.thresholds.flag}, i.e. missed)`,
    ratesBySet(sample, rowMap, aiItems, sample.thresholds.flag, 'lt'),
    markdown
  );

  printTable(
    `Overall accuracy (decision threshold >= ${sample.thresholds.flag})`,
    ['detector', 'n', 'correct', 'accuracy'],
    accuracyByDetector(sample, rowMap, allItems, sample.thresholds.flag).map((r) => [
      r.detector,
      r.n,
      r.correct,
      pctStr(r.accuracy),
    ]),
    markdown
  );

  const longHuman = humanItems.filter((it) => it.words >= 80);
  printRatesTable(
    `False positive rate, >=80-word subset only (score >= ${sample.thresholds.flag})`,
    ratesBySet(sample, rowMap, longHuman, sample.thresholds.flag, 'gte'),
    markdown
  );
  printRatesTable(
    `False positive rate, >=80-word subset only (score >= ${sample.thresholds.strict})`,
    ratesBySet(sample, rowMap, longHuman, sample.thresholds.strict, 'gte'),
    markdown
  );

  printTable(
    'Runs below the detector\'s stated minimum input length',
    ['detector', 'n', 'below_min_length'],
    belowMinLengthCounts(sample, rowMap, allItems).map((r) => [r.detector, r.n, r.flagged]),
    markdown
  );
}

// --- selftest ------------------------------------------------------------

function selftest() {
  // CSV parser, including a quoted field containing a comma and an escaped quote
  assert.deepStrictEqual(parseCsvLine('a,b,c'), ['a', 'b', 'c']);
  assert.deepStrictEqual(parseCsvLine('id-1,esl,68,zerogpt,42,"AI-generated, likely",0,2026-08-14'), [
    'id-1',
    'esl',
    '68',
    'zerogpt',
    '42',
    'AI-generated, likely',
    '0',
    '2026-08-14',
  ]);
  assert.deepStrictEqual(parseCsvLine('a,"say ""hi""",c'), ['a', 'say "hi"', 'c']);
  assert.deepStrictEqual(parseCsv('h1,h2\nx,y\n'), [{ h1: 'x', h2: 'y' }]);
  assert.deepStrictEqual(parseCsv('h1,h2\n'), []); // header only, no data rows

  // word-count helper
  assert.strictEqual(parseWords('68'), 68);
  assert.strictEqual(parseWords('0'), 0);
  assert.throws(() => parseWords('sixty-eight'));
  assert.throws(() => parseWords(''));

  // rate math on a hand-computed fixture
  assert.strictEqual(rate(3, 10), 0.3);
  assert.strictEqual(rate(0, 5), 0);
  assert.strictEqual(rate(5, 5), 1);
  assert.strictEqual(rate(0, 0), null);

  // wilson() against known bounds: 0/30, 15/30 (symmetric around 0.5), 30/30
  const w0 = wilson(0, 30);
  assert.ok(w0.lower >= 0 && w0.lower < 0.001, `w0.lower ${w0.lower}`);
  assert.ok(Math.abs(w0.upper - 0.1135) < 0.002, `w0.upper ${w0.upper}`);

  const w15 = wilson(15, 30);
  assert.ok(w15.lower <= 0.5 && w15.upper >= 0.5, 'wilson CI must bracket the point estimate');
  assert.ok(Math.abs(w15.lower - 0.3315) < 0.002, `w15.lower ${w15.lower}`);
  assert.ok(Math.abs(w15.upper - 0.6685) < 0.002, `w15.upper ${w15.upper}`);
  assert.ok(Math.abs(0.5 - w15.lower - (w15.upper - 0.5)) < 1e-6, 'w15 CI must be symmetric around 0.5');

  const w30 = wilson(30, 30);
  assert.ok(w30.upper <= 1 && w30.upper > 0.999, `w30.upper ${w30.upper}`);
  assert.ok(Math.abs(w30.lower - 0.8865) < 0.002, `w30.lower ${w30.lower}`);

  for (const [s, n] of [[0, 30], [15, 30], [30, 30], [7, 45]]) {
    const { lower, upper } = wilson(s, n);
    const phat = s / n;
    assert.ok(lower >= 0 && upper <= 1, `wilson(${s},${n}) out of [0,1]`);
    assert.ok(lower <= phat && phat <= upper, `wilson(${s},${n}) does not bracket phat`);
  }

  console.log('selftest ok');
  process.exit(0);
}

// --- entry -----------------------------------------------------------------

if (process.argv.includes('--selftest')) {
  selftest();
} else if (process.argv.includes('--spread')) {
  runSpread(process.argv.includes('--markdown'));
} else {
  run(process.argv.includes('--markdown'));
}