#!/usr/bin/env node
/**
 * dedup-tracker.mjs — Remove duplicate entries from applications.md
 *
 * Groups by normalized company, then merges only rows whose full role title
 * matches exactly (case- and whitespace-normalized) and whose posting URLs, when
 * both rows carry one, do not conflict. Keeps entry with highest
 * score. If discarded entry had more advanced status, preserves that status,
 * and runs the same status hooks set-status.mjs runs (lib/status-hooks.mjs).
 * Merges notes.
 *
 * Run: node career-ops/dedup-tracker.mjs [--dry-run]
 */

import { readFileSync, copyFileSync, existsSync, mkdirSync } from 'fs';
import { dirname, join } from 'path';
import { fileURLToPath } from 'url';
import { getCareerOpsRoot, resolveTrackerPath } from './path-resolver.mjs';
import { relatePostingLanguageKeys, postingLanguageKey, normalizeUrl } from './url-key.mjs';
import {
  openTrackerTransaction, rebuildRow, normalizeCompany, corporateFormTokensMatch,
  MIN_STEM_CHARS, loadCanonicalStates, resolveCanonicalStateId, resolveCanonicalState,
  stripStatusDecoration,
  resolveWorkspaceRoot, containedReportFile, reportUrlToken,
} from './tracker-utils.mjs';
import { resolveColumns, parseTrackerRow, isUnsetUrlCell, normalizeVia, extractReqNumber, parseScore, normalizeTextKey } from './tracker-parse.mjs';
import { validateFlags } from './lib/cli-flags.mjs';

const CAREER_OPS = getCareerOpsRoot();
// Support both layouts: data/applications.md (boilerplate) and applications.md
// (original). CAREER_OPS_TRACKER lets tests point the script at an isolated
// fixture so the real user tracker is never touched.
const APPS_FILE = resolveTrackerPath(CAREER_OPS);
// Report links resolve against the same root merge-tracker.mjs uses, so the
// two tools read the same report for the same cell.
const REPORTS_ROOT = resolveWorkspaceRoot(APPS_FILE);

// ── CLI args ────────────────────────────────────────────────────────
// Same shape as scan-ats-full.mjs (#1633/PR #1635) and reply-watch.mjs
// (#2743): an unrecognized flag must fail fast, never silently fall through
// to the live-run default and write to the real tracker (#2744). Shared via
// lib/cli-flags.mjs's validateFlags() (#2775).
const KNOWN_FLAGS = ['--dry-run', '--help', '-h'];
const USAGE = `Usage: node dedup-tracker.mjs [--dry-run]`;

const cliArgs = process.argv.slice(2);

validateFlags(cliArgs, KNOWN_FLAGS, USAGE);
const DRY_RUN = process.argv.includes('--dry-run');

// Status advancement order (higher = more advanced in pipeline), keyed by
// states.yml id. Only the ORDER lives here. Every spelling of a state (label,
// alias, localized form) resolves to its id through states.yml, so a new alias
// needs no change in this file.
// Applied > Rejected because an active application outranks a terminal state.
const STATUS_RANK = {
  'skip': 0,
  'discarded': 0,
  'rejected': 1,
  'evaluated': 2,
  'applied': 3,
  'responded': 4,
  'assessment': 5,
  'interview': 6,
  'offer': 7,
  // Hired outranks everything: the accepted-job record must never lose a
  // dedup contest to a repost row.
  'hired': 8,
};

// templates/states.yml is system layer: it lives beside this script, not in the
// data root. It is the only place a status spelling is defined, so without it
// no row can be ranked, and a row ranked 0 is one this script may delete. Stop
// before touching the tracker rather than rank from a partial vocabulary.
//
// A file that parses but does not define one of the ranked states is the same
// hazard, only narrower: every spelling of that state resolves to nothing and
// ranks 0. `states: []` is the limiting case, where that is every row.
const STATES_FILE = join(dirname(fileURLToPath(import.meta.url)), 'templates', 'states.yml');
let CANONICAL_STATES;
try {
  CANONICAL_STATES = loadCanonicalStates(STATES_FILE);
  const defined = new Set(CANONICAL_STATES.map((s) => s.id.toLowerCase()));
  const undefinedIds = Object.keys(STATUS_RANK).filter((id) => !defined.has(id));
  if (undefinedIds.length > 0) throw new Error(`it defines no state with id ${undefinedIds.join(', ')}`);
} catch (err) {
  console.error(`Cannot rank statuses from ${STATES_FILE}: ${err.message}`);
  console.error('dedup-tracker ranks every status through that file and will not guess without it. Nothing was changed.');
  process.exit(1);
}

// Ensure the target tracker directory exists in both normal and fixture mode.
mkdirSync(dirname(APPS_FILE), { recursive: true });

/**
 * Convert a tracker status into its pipeline-advancement rank.
 *
 * Higher ranks represent states that carry more user intent and should not be
 * casually overwritten or removed. Unknown statuses rank as 0 so malformed data
 * is treated conservatively rather than promoted.
 *
 * The cell is resolved through states.yml (resolveCanonicalStateId), which also
 * strips the bold wrappers and trailing dates legacy trackers carry. An alias
 * ("Sent", "Aplicada", "Başvuruldu", "Mülakat"...) therefore ranks as the state
 * the dashboard and set-status.mjs read it as. Ranking one 0 made an Applied
 * row look like a throwaway and let a higher-scored Evaluated sibling delete it.
 *
 * @param {string} status - Raw status cell from applications.md.
 * @returns {number} Numeric rank from STATUS_RANK, or 0 for unknown statuses.
 */
function statusRank(status) {
  const id = resolveCanonicalStateId(status, CANONICAL_STATES);
  return id !== null && Object.hasOwn(STATUS_RANK, id) ? STATUS_RANK[id] : 0;
}

/**
 * Check whether a status represents a real application already in motion.
 *
 * Rows at Applied or later have user-visible history that dedup must preserve
 * unless the duplicate relationship is exact. This guard prevents fuzzy title
 * matches from silently deleting an active application record.
 *
 * @param {string} status - Raw status value from the tracker row.
 * @returns {boolean} True when the row is Applied, Responded, Assessment, Interview, or Offer.
 */
function isAdvancedStatus(status) {
  return statusRank(status) >= STATUS_RANK.applied;
}

/**
 * Extract the report number from a Markdown report link.
 *
 * Tracker report cells are normally written as links like
 * `[123](../reports/123-company-role-date.md)`. The bracketed number is the
 * stable report identity used to distinguish exact duplicates from merely
 * similar fuzzy-title matches.
 *
 * @param {string} reportStr - Raw report cell from applications.md.
 * @returns {number|null} Parsed report number, or null when no link number exists.
 */
function extractReportNum(reportStr) {
  const m = String(reportStr ?? '').match(/\[(\d+)\]/);
  return m ? parseInt(m[1]) : null;
}

/**
 * Determine whether two tracker rows point to the same exact report identity.
 *
 * Exact identity is stronger than fuzzy role matching: it may cluster rows
 * even when an advanced status is present. Matching bracketed report numbers
 * are that evidence. A shared tracker number alone is NOT — duplicate tracker
 * numbers are a known artifact of the old merge bug (verify-pipeline Check 12
 * exists because they never mean the same application), so a bare number match
 * only counts when the rows also carry the same exact role title.
 *
 * @param {object} a - First parsed applications.md row.
 * @param {object} b - Second parsed applications.md row.
 * @returns {boolean} True when both rows represent the same report identity.
 */
function sameReportIdentity(a, b) {
  const reportA = extractReportNum(a.report);
  const reportB = extractReportNum(b.report);
  if (reportA !== null && reportA === reportB) return true;
  return a.num === b.num && normalizeRole(a.role) === normalizeRole(b.role);
}

/**
 * Build a stable key for logging one protected same-title pair only once.
 *
 * The nested dedup loop can encounter a protected pair during cluster building.
 * Sorting the row numbers produces the same key regardless of comparison order,
 * which keeps the warning output readable and avoids repeated noise.
 *
 * @param {object} a - First parsed applications.md row.
 * @param {object} b - Second parsed applications.md row.
 * @returns {string} Stable pair key in ascending tracker-number order.
 */
function pairKey(a, b) {
  return [a.num, b.num].sort((x, y) => x - y).join(':');
}

const protectedTitlePairs = new Set();

/**
 * Normalize a role title into the key used for exact same-opening comparison.
 *
 * Deduplication must only collapse rows that describe the *same* opening, so
 * the comparison is exact on the meaningful title text. Only presentation noise
 * is removed — letter case and whitespace (leading, trailing, and repeated
 * internal spaces). Distinguishing words such as seniority ("Senior") or the
 * team suffix ("Data Infrastructure" vs "Agent Infrastructure") are preserved,
 * so sibling roles at one company are never merged.
 *
 * @param {string} role - Role title from an applications.md row.
 * @returns {string} Lowercase, whitespace-collapsed role key.
 */
function normalizeRole(role) {
  return String(role ?? '')
    .replace(/\s+/g, ' ')
    .trim()
    .toLowerCase();
}

/**
 * Check whether two tracker rows name two different postings.
 *
 * Only two present-and-different keys are evidence: a blank or placeholder URL
 * yields no key from normalizeUrl() and is UNKNOWN, never a conflict. Two
 * different known languages of one page are two postings (postingUrlRelation
 * in url-key.mjs).
 *
 * @param {object} a - First parsed applications.md row.
 * @param {object} b - Second parsed applications.md row.
 * @returns {boolean} True when both rows carry a URL and the URLs name two postings.
 */
function urlsConflict(a, b) {
  return relatePostingLanguageKeys(cachedLanguageKey(a), cachedLanguageKey(b)) === 'distinct';
}

const reportUrls = new Map();

/**
 * A row's posting URL for the URL guards: its URL cell, else the URL its report
 * names.
 *
 * The default nine-column tracker has no URL cell, so reading the cell alone
 * left both guards blind there: two same-company, same-title rows whose
 * reports name two different openings (`…/jobs/1001` and `…/jobs/2002`) looked
 * identical, and this tool deleted one of them. merge-tracker.mjs already
 * resolved the report's URL in exactly this case — its own comment says
 * "without the report the guards never see two same-titled postings as
 * distinct" — so `node merge-tracker.mjs` kept both rows and `node
 * dedup-tracker.mjs` then removed one. Both now read through
 * reportUrlToken(), with the same path containment. A placeholder such as
 * `N/A` counts as no cell: it has no key of its own, so stopping at it hid the
 * report and left the same two rows looking identical.
 *
 * Read lazily and memoised: only rows that already share a company reach a
 * guard.
 *
 * @param {object} row - A parsed applications.md row.
 * @returns {string}
 */
function postingUrl(row) {
  if (!isUnsetUrlCell(row.url)) return row.url;
  if (!reportUrls.has(row.report)) {
    reportUrls.set(row.report, reportUrlToken(containedReportFile(row.report, REPORTS_ROOT)));
  }
  return reportUrls.get(row.report);
}

// Company identity and posting-language keys are pure per row. The pair loop
// used to rebuild both on every comparison: normalizeTextKey for the company,
// and a URL parse for the posting. A bucket of a few thousand rows made that
// the whole command. Each row keeps one of each. The Maps are this run's
// rows, not a process-wide cache of every string the helpers have seen.
const compareStats = { normalizes: 0, urlKeys: 0, pairs: 0 };
const companyKeys = new Map();
const languageKeys = new Map();

function cachedCompany(entry) {
  let cached = companyKeys.get(entry);
  if (cached) return cached;
  cached = {
    key: normalizeCompany(entry.company),
    tokens: normalizeTextKey(entry.company, ' ').split(' ').filter(Boolean),
  };
  companyKeys.set(entry, cached);
  compareStats.normalizes++;
  return cached;
}

// Built only when a pair actually compares URLs. A row that never meets
// another must not pay for a report read it used to skip.
function cachedLanguageKey(row) {
  if (languageKeys.has(row)) return languageKeys.get(row);
  const key = postingLanguageKey(postingUrl(row));
  languageKeys.set(row, key);
  compareStats.urlKeys++;
  return key;
}

/**
 * Check whether two tracker rows show the same page with a language param on
 * one side only (`…/job/1` vs `…/job/1?lang=en`).
 *
 * Not proof of two postings, so the same report identity still merges them,
 * but not proof of one either: the bare row may be the German version and the
 * other its English one, so a role match alone must not delete either.
 *
 * @param {object} a - First parsed applications.md row.
 * @param {object} b - Second parsed applications.md row.
 * @returns {boolean}
 */
function languageOnOneSide(a, b) {
  return relatePostingLanguageKeys(cachedLanguageKey(a), cachedLanguageKey(b)) === 'one-sided-language';
}

/**
 * Check whether two tracker rows carry two different requisition ids in Notes.
 *
 * The rule merge-tracker.mjs applies as `reqNumDiffers` (#1524): an id missing
 * on either side is UNKNOWN, and only two present-and-different ids are evidence
 * that the rows are distinct openings. An employer posts one title once per city
 * or country, each copy with its own job id, and merge-tracker keeps those rows
 * apart on purpose; deleting one here would silently reverse that decision.
 *
 * @param {object} a - First parsed applications.md row.
 * @param {object} b - Second parsed applications.md row.
 * @returns {boolean} True when both rows carry a recognizable id and they differ.
 */
function reqNumsConflict(a, b) {
  const reqA = extractReqNumber(a.notes);
  const reqB = extractReqNumber(b.notes);
  return Boolean(reqA && reqB && reqA !== reqB);
}

/**
 * Decide whether two same-company tracker rows should be deduplicated.
 *
 * Rows merge only when they describe the same opening: either the exact same
 * report identity (same tracker number or bracketed report number), or an exact
 * role-title match after normalizing case and whitespace. Fuzzy title matching
 * is deliberately NOT used here — it collapsed distinct sibling roles at one
 * company (e.g. "Software Engineer, Data Infrastructure" vs "Senior Software
 * Engineer, Agent Infrastructure"), causing real data loss.
 *
 * When titles match exactly but either row is already Applied or later, dedup
 * still keeps both and warns: deleting an in-flight application would lose its
 * status, report link, and notes unless the rows are the exact same report
 * identity.
 *
 * Two present-and-different posting URLs veto every tier, report identity
 * included (#4562). That is the rule merge-tracker.mjs applies (`urlDiffers`),
 * through the same postingUrlRelation(): two rows cannot be one application while
 * naming two postings. A blank or placeholder URL yields no key and is
 * UNKNOWN, never a conflict, so rows without URLs behave exactly as before.
 * The same page with a language param on one URL only is not a conflict, so
 * report identity still merges it, but it blocks the role-title match.
 * Two present-and-different requisition ids in Notes (`Job id 8157736` vs
 * `Job id 8157738`, the #1524 rule) veto every tier the same way, so one title
 * posted once per city or country keeps one row per posting.
 * Blocking fails toward a duplicate the user can see; merging on a conflict
 * deletes a real application.
 *
 * @param {object} a - First parsed applications.md row.
 * @param {object} b - Second parsed applications.md row.
 * @returns {boolean} True when dedup may cluster the two rows as duplicates.
 */
function roleMatch(a, b) {
  if (urlsConflict(a, b)) return false;
  if (reqNumsConflict(a, b)) return false;

  if (sameReportIdentity(a, b)) return true;
  if (languageOnOneSide(a, b)) return false;
  if (normalizeRole(a.role) !== normalizeRole(b.role)) return false;

  // Exact-title duplicates that have entered the real application pipeline are
  // kept separate. A user may already have applied to one row; deleting it
  // because a higher-scored exact-title sibling exists would lose status,
  // report, and notes. Keep both unless the rows point to the exact same
  // report identity.
  if (isAdvancedStatus(a.status) || isAdvancedStatus(b.status)) {
    const key = pairKey(a, b);
    if (!protectedTitlePairs.has(key)) {
      protectedTitlePairs.add(key);
      console.warn(`⚠️  Keep #${a.num} and #${b.num}: exact-title match but advanced status requires exact report identity`);
    }
    return false;
  }

  return true;
}

/**
 * Parse one Markdown table row from applications.md into a tracker object.
 *
 * Header and separator rows return null because they either lack enough cells
 * or do not have a numeric tracker id. Valid data rows keep the raw line; the
 * caller attaches the physical line index after parsing so later updates and
 * removals never depend on tracker numbers being globally unique.
 *
 * @param {string} line - One line from applications.md.
 * @returns {object|null} Parsed tracker row, or null for non-application lines.
 */
function parseAppLine(line) {
  return parseTrackerRow(line, COLMAP);
}

// Read
if (!existsSync(APPS_FILE)) {
  console.log('No applications.md found. Nothing to dedup.');
  process.exit(0);
}

let trackerTransaction = null;
let statusHooks = null;
let localToday = null;
let committed = false;
const releaseClaims = [];
// Kept rows that take over a removed duplicate's more advanced status. Each is
// a real status change for that row number, so it gets the status hooks once
// the write has landed.
const promotions = [];
let COLMAP;
if (!DRY_RUN) {
  try {
    trackerTransaction = await openTrackerTransaction(APPS_FILE);
  } catch (err) {
    console.error(`Cannot acquire tracker lock: ${err.message}`);
    process.exit(1);
  }
  process.once('exit', () => {
    try { trackerTransaction.close(); } catch {}
  });
}
try {
const content = trackerTransaction ? trackerTransaction.read() : readFileSync(APPS_FILE, 'utf-8');
const lines = content.split('\n');
// Header-aware column map (tolerates an inserted Location column, etc.).
COLMAP = resolveColumns(lines);

// Parse all entries
const entries = [];

for (let i = 0; i < lines.length; i++) {
  if (!lines[i].startsWith('|')) continue;
  const app = parseAppLine(lines[i]);
  if (app && app.num > 0) {
    app.lineIdx = i;
    entries.push(app);
  }
}

console.log(`📊 ${entries.length} entries loaded`);

// Group by company+role. Unknown-employer rows (Company `?`, #1596) all
// normalize to the same empty key, so they group by their Via channel instead:
// the same agency re-blasting one listing IS a duplicate, while the same role
// via two different agencies is two real submissions and must never merge.
// The channel key is Unicode-aware (#1603/#2393): this file's own
// normalizeCompany() used to strip everything outside [a-z0-9], so distinct
// non-Latin agency names (リクルート, パーソル, …) all collapsed to the same empty
// key and one of two genuinely separate submissions was DELETED. Both keys are
// now Unicode-aware — normalizeCompany comes from tracker-utils.mjs (#2429), so
// the ordinary company path cannot regress the way this channel path did.
// normalizeVia() is the same key that
// merge-tracker.mjs uses for its cross-channel guard, so the two scripts
// cannot drift on agency identity. An absent Via (empty or `—`) still keys to
// '' and groups with other via-less blind rows, matching merge-tracker, whose
// guard does not reject a pair whose Via cells are both blank.
// The NUL prefix makes this key uncollidable with any real company name.
// It is written as the ESCAPE, never as a raw NUL byte in the source: a raw
// one makes grep classify this file as binary and report NO MATCH for any
// pattern in it, silently, with the same exit code as a genuine absence.
// Identical value at runtime, and the file stays greppable.
// Pinned by tests/source-no-nul-bytes.test.mjs.
const BLIND_KEY = '\u0000blind-via:';

// Pre-filter bucket for the pair-wise company gate below. Rows that could name
// one employer under a different corporate form ("Acme Widgets, APC" vs "Acme
// Widgets") must land together so they are compared at all — normalizeCompany()
// keeps the suffix, so keying groups on it split those pairs into two groups and
// a duplicate re-scan survived every cleanup run (#4421 defect 4).
// The key is the first MIN_STEM_CHARS characters of the normalized name, the
// narrowest width that provably holds every pair sameCompany() accepts: the
// corporate-form rule refuses a stem shorter than that, and an exact match
// shares the whole string. A first-TOKEN key is too narrow — "Foo-Bar" and
// "FooBar" are one employer under normalizeCompany() but tokenize to "foo" and
// "foobar", so tokenizing split two rows that used to group and made them
// unmergeable. Widening is free and narrowing is not: this key only decides
// which pairs get compared, never whether they merge, so an over-wide bucket
// costs a few extra comparisons while an over-narrow one leaves a duplicate
// row in applications.md forever.
function companyBucketKey(entry) {
  return cachedCompany(entry).key.slice(0, MIN_STEM_CHARS);
}

// True when two rows name the same employer: identical under normalizeCompany,
// or one is the other with a corporate-form tail ("Acme Widgets" / "Acme
// Widgets, APC"). corporateFormTokensMatch() is the token half of
// companiesMatchIgnoringCorporateForm(), OR-ed with the exact check because
// that rule returns false for two IDENTICAL names (equal token length) — that
// is the exact tier's business. Deletion is still decided by roleMatch + the
// status-rank guard; this only widens which pairs reach them. Both keys were
// built once per row, above.
function sameCompany(a, b) {
  const left = cachedCompany(a);
  const right = cachedCompany(b);
  return left.key === right.key || corporateFormTokensMatch(left.tokens, right.tokens);
}

const groups = new Map();
for (const entry of entries) {
  // Every row, including a blind `?` that groups by Via instead. The company
  // key is cheap next to the pair loop, and a later sameCompany() hit is a
  // map lookup. Posting keys stay lazy: see cachedLanguageKey().
  cachedCompany(entry);
  const key = String(entry.company).trim() === '?'
    ? BLIND_KEY + normalizeVia(entry.via || '')
    : companyBucketKey(entry);
  if (!groups.has(key)) groups.set(key, []);
  groups.get(key).push(entry);
}

// Two blind rows only count as the same listing when their evaluation dates
// sit within the re-post window (mirrors detect-reposts.mjs's 90 days).
// Unparseable dates never cluster — deleting a real application is worse than
// keeping a duplicate.
const BLIND_WINDOW_DAYS = 90;
function withinBlindWindow(a, b) {
  const ta = Date.parse(a);
  const tb = Date.parse(b);
  if (Number.isNaN(ta) || Number.isNaN(tb)) return false;
  return Math.abs(ta - tb) <= BLIND_WINDOW_DAYS * 86400000;
}

// Find duplicates
let removed = 0;
const linesToRemove = new Set();

for (const [company, companyEntries] of groups) {
  if (companyEntries.length < 2) continue;
  const isBlindGroup = company.startsWith(BLIND_KEY);

  // Within same company, find role matches
  const processed = new Set();
  for (let i = 0; i < companyEntries.length; i++) {
    if (processed.has(i)) continue;
    const cluster = [companyEntries[i]];
    processed.add(i);

    for (let j = i + 1; j < companyEntries.length; j++) {
      if (processed.has(j)) continue;
      compareStats.pairs++;
      // Named rows share a bucket, not an employer: gate the pair on the
      // corporate-form-aware identity so suffix variants merge but two distinct
      // employers sharing a prefix do not. The gate is against EVERY existing
      // cluster member, not just the anchor, because corporate-form matching is
      // not transitive: a bare "Acme" matches both "Acme Solutions" and "Acme
      // Technologies", so anchor-only gating pulled all three into one cluster
      // and deleted a real row even though the two tails never match each other.
      // Requiring the whole cluster to agree leaves "Acme Technologies" in its
      // own cluster, where it survives. Blind rows already share the Via bucket
      // and are gated on the re-post window instead.
      const companyOk = isBlindGroup
        ? withinBlindWindow(companyEntries[i].date, companyEntries[j].date)
        : cluster.every((m) => sameCompany(m, companyEntries[j]));
      // roleMatch() only compares against the seed row, so a seed with no URL
      // would admit two rows naming different postings and one would be
      // deleted. Check the candidate's URL and requisition id against every
      // member already in.
      if (companyOk && roleMatch(companyEntries[i], companyEntries[j])
          && !cluster.some(member => urlsConflict(member, companyEntries[j]) || reqNumsConflict(member, companyEntries[j]))) {
        cluster.push(companyEntries[j]);
        processed.add(j);
      }
    }

    if (cluster.length < 2) continue;

    // Keep the one with highest score
    cluster.sort((a, b) => parseScore(b.score) - parseScore(a.score));
    const keeper = cluster[0];

    // Check if any removed entry has more advanced status
    let bestStatusRank = statusRank(keeper.status);
    let bestStatus = keeper.status;
    for (let k = 1; k < cluster.length; k++) {
      const rank = statusRank(cluster[k].status);
      if (rank > bestStatusRank) {
        bestStatusRank = rank;
        bestStatus = cluster[k].status;
      }
    }

    // Update keeper's status if a removed entry had a more advanced one
    if (bestStatus !== keeper.status) {
      const lineIdx = keeper.lineIdx;
      if (lineIdx !== undefined) {
        // The donor's spelling, without its bold or trailing date. statusRank()
        // strips both before ranking, so a legacy `Applied 2026-01-02` cell can
        // win. Copied verbatim it put a date in the keeper's Status cell, and
        // resolveCanonicalState() then read the promoted status as unknown: the
        // ledger recorded `to: -`, which funnel-velocity replays as retracting
        // the row's last real transition, and the Applied/Interview hooks below
        // never ran.
        const promoted = stripStatusDecoration(bestStatus);
        const parts = lines[lineIdx].split('|').map(s => s.trim());
        parts[COLMAP.status] = promoted;
        lines[lineIdx] = rebuildRow(parts);
        const donorNum = cluster.find(e => e.status === bestStatus)?.num;
        console.log(`  📝 #${keeper.num}: status promoted to "${promoted}" (from #${donorNum})`);
        promotions.push({ keeper, from: keeper.status, to: promoted, donorNum });
      }
    }

    // Merge notes from removed entries
    let mergedNotes = String(keeper.notes || '').trim();
    const originalNotes = mergedNotes;
    for (let k = 1; k < cluster.length; k++) {
      const dupReport = String(cluster[k].report || '').trim();
      const dupReportNum = extractReportNum(dupReport);
      if (dupReportNum !== null && dupReportNum !== extractReportNum(keeper.report)) {
        const reportNote = `also evaluated in ${dupReport}`;
        if (!mergedNotes.includes(dupReport)) {
          mergedNotes = mergedNotes && mergedNotes !== 'N/A' && mergedNotes !== '-' ? `${mergedNotes}; ${reportNote}` : reportNote;
        }
      }
      const dupNotes = String(cluster[k].notes || '').trim();
      if (dupNotes && dupNotes !== 'N/A' && dupNotes !== '❌' && dupNotes !== 'pending' && dupNotes !== '-') {
        if (!mergedNotes.includes(dupNotes)) {
          mergedNotes = mergedNotes && mergedNotes !== 'N/A' && mergedNotes !== '-' ? `${mergedNotes}; ${dupNotes}` : dupNotes;
        }
      }
    }

    if (mergedNotes !== originalNotes) {
      const lineIdx = keeper.lineIdx;
      if (lineIdx !== undefined) {
        const parts = lines[lineIdx].split('|').map(s => s.trim());
        parts[COLMAP.notes] = mergedNotes;
        lines[lineIdx] = rebuildRow(parts);
        keeper.notes = mergedNotes;
        console.log(`  📝 #${keeper.num}: notes merged`);
      }
    }

    // Carry over a posting URL the keeper lacks. The URL cell is the
    // deterministic dedup key (#3516): left on a deleted row it went with it,
    // and the surviving row, often the Applied one whose status was just
    // promoted onto it, fell back to fuzzy company+role matching from then on.
    // Cluster members never name two different postings (urlsConflict vetoes
    // that), so the first URL found is this posting's. The donor's raw cell is
    // copied, so a `[label](href)` link stays a link.
    if (COLMAP.url != null && !normalizeUrl(keeper.url) && keeper.lineIdx !== undefined) {
      const donor = cluster.slice(1).find(e => e.lineIdx !== undefined && normalizeUrl(e.url));
      if (donor) {
        const parts = lines[keeper.lineIdx].split('|').map(s => s.trim());
        parts[COLMAP.url] = lines[donor.lineIdx].split('|').map(s => s.trim())[COLMAP.url];
        lines[keeper.lineIdx] = rebuildRow(parts);
        keeper.url = donor.url;
        console.log(`  🔗 #${keeper.num}: posting URL kept from #${donor.num}`);
      }
    }

    // Remove duplicates
    for (let k = 1; k < cluster.length; k++) {
      const dup = cluster[k];
      const lineIdx = dup.lineIdx;
      if (lineIdx !== undefined) {
        linesToRemove.add(lineIdx);
        removed++;
        console.log(`🗑️  Remove #${dup.num} (${dup.company} — ${dup.role}, ${dup.score}) → kept #${keeper.num} (${keeper.score})`);
      }
    }
  }
}

// Remove lines (in reverse order to preserve indices)
const sortedRemoveIndices = [...linesToRemove].sort((a, b) => b - a);
for (const idx of sortedRemoveIndices) {
  lines.splice(idx, 1);
}

console.log(`\n📊 ${removed} duplicates removed`);

if (!DRY_RUN && removed > 0) {
  const backupPath = `${APPS_FILE}.bak`;
  copyFileSync(APPS_FILE, backupPath);
  // Loaded only when a status moves, so a run that just drops duplicates keeps
  // the import closure it always had.
  if (promotions.length > 0) {
    statusHooks = await import('./lib/status-hooks.mjs');
    ({ localToday } = await import('./lib/local-today.mjs'));
    for (const p of promotions) releaseClaims.push(statusHooks.claimStatusActivity(p.keeper, `dedup-tracker → ${p.to}`));
  }
  trackerTransaction.replace(lines.join('\n'));
  console.log(`✅ Written to ${APPS_FILE} (backup: ${backupPath})`);
  // Inside the lock window, like set-status's own append. The removed row's
  // history stays under its old number; this row says where the kept row's
  // status came from. dedup-tracker did not observe the transition, it found
  // it, so funnel-velocity counts the row but keeps it out of day-math: dated
  // today, it would otherwise read as "applied today".
  // The keeper's own cell can be a dated legacy one too; strip it the same way
  // so a known prior state is not logged as unknown.
  const ledgerState = status => resolveCanonicalState(stripStatusDecoration(status), CANONICAL_STATES) ?? '-';
  for (const p of promotions) {
    statusHooks.appendStatusTransition(APPS_FILE, {
      num: p.keeper.num,
      date: localToday(),
      from: ledgerState(p.from),
      to: ledgerState(p.to),
      source: 'dedup-tracker',
      note: `from #${p.donorNum}`,
    });
  }
  committed = true;
} else if (DRY_RUN) {
  console.log('(dry-run — no changes written)');
} else {
  console.log('✅ No duplicates found');
}
} finally {
  trackerTransaction?.close();
}

// After the tracker lock is released: both hooks re-read the row this run
// wrote, and the follow-up seeder takes its own lock.
try {
  if (committed) {
    for (const p of promotions) {
      const to = resolveCanonicalState(p.to, CANONICAL_STATES);
      const keeper = { ...p.keeper, status: p.to };
      if (to === 'Applied') await statusHooks.seedFollowupOnApplied(keeper.num, { trackerPath: APPS_FILE });
      if (to === 'Interview') statusHooks.archiveJdOnInterview(keeper, { trackerPath: APPS_FILE });
    }
  }
} finally {
  for (const release of releaseClaims) release();
}

// Off unless a test asks. The counts are how many times the keys were built,
// so a pair loop that recomputes them fails the bound instead of only looking
// slower. Quiet on a normal run: stderr stays what callers already parse.
if (process.env.CAREER_OPS_DEDUP_STATS === '1') {
  console.error(`dedup-stats normalizes=${compareStats.normalizes} urlKeys=${compareStats.urlKeys} pairs=${compareStats.pairs}`);
}