#!/usr/bin/env node // Installed by the sync-pstack skill. // Usage: node .agents/pstack/proof.mjs --dir --name [--cwd ] [--timeout ] [--cores ] [--stress] [--force] -- [arg...] // Runs one proof command and writes its command line, the time the machine // last started, its output and its exit status to /-a.log, N // one past the highest attempt logged, so a log that a decision row cites is // never rewritten. One command argument runs in bash with pipefail; several run // directly. Exits with the command's status, or 1 when a timeout, memory stop, // signal or spawn error ended it. Without --stress, --timeout defaults to 3,600 // seconds and a longer one is refused. A run its timeout stopped logs that // before `exit=ETIMEDOUT`, whatever the command returned, and stops every // process still in the command's process group. On macOS it stops a command // the same way once the command's processes use more than half of the // machine's memory, or more than a tenth while under 15% of memory is free, and // logs the largest processes before `exit=EMEMORY`. It refuses a run whose last // attempt logged no exit and ran before the machine last started, unless // --force, because that attempt may have taken the machine down. // Before it starts, it asks this machine's core pool for half a core for each // of its --cores (default 1) workers. A --stress run asks for a whole core for // each, and is refused when its --cores is over half the machine's cores. // Every proof.mjs and land.mjs check of every session shares the pool, and // the pool counts the workers each one runs, so a command that runs // more workers than it asked for holds a share for each. A proof waits while // the pool is full or free memory is low, or while the pool still counts the // workers of a command that just started; a landing and its check never wait. // Waiters enter in the order they arrived, except that a small ask that fits // now enters past a larger waiter that has waited under five minutes. import { spawn, spawnSync } from 'node:child_process'; import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, realpathSync, rmSync, statSync, writeFileSync, writeSync } from 'node:fs'; import { availableParallelism, constants, freemem, tmpdir, totalmem } from 'node:os'; import { basename, dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; const quote = (arg) => (/^[\w@%+=:,./-]+$/u.test(arg) ? arg : `'${arg.replaceAll("'", "'\\''")}'`); function lastAttempt(folder, name) { const pattern = new RegExp(`^${basename(name).replace(/[.*+?^${}()|[\]\\]/gu, '\\$&')}-a(\\d+)\\.log$`, 'u'); return Math.max(0, ...(existsSync(folder) ? readdirSync(folder) : []).map((file) => Number(file.match(pattern)?.[1] ?? 0))); } function claim(dir, name) { const folder = dirname(join(dir, name)); mkdirSync(folder, { recursive: true }); let attempt = lastAttempt(folder, name) + 1; for (;;) { const path = join(folder, `${basename(name)}-a${attempt}.log`); try { return { fd: openSync(path, 'wx'), path }; } catch (error) { if (error.code !== 'EEXIST') throw error; attempt++; } } } const POOL = join(tmpdir(), 'pstack-slots'); const POOL_LOCK = join(POOL, '.lock'); const HELD = 'PSTACK_SLOT'; // land.mjs sets it in its check's environment, so the check's own share enters // the pool as a landing. export const LAND_CHECK = 'PSTACK_LAND_CHECK'; // A small ask that fits the free cores now enters past larger waiters until // the oldest of them has waited this long, so it never starves them. const BACKFILL_MS = 5 * 60_000; const MIN_FREE_MEMORY = 0.15; // The owner allows twice as many workers as cores (2026-10-26), because the // memory floor, not the core count, keeps the machine from freezing. Pool // files stay in cores, so a proof.mjs from an older checkout reads the same // fill. const CORES_PER_WORKER = 0.5; const STRESS_CORES_PER_WORKER = 1; const STRESS_MAX_CORE_SHARE = 0.5; const POLL_MS = 2000; const STALE_LOCK_MS = 10_000; // On a loaded machine a new command had started four fifths of its workers // within five seconds (measured 2026-11-14). Without this wait, commands that // start together all enter before the pool has counted the workers of any. const WATCH_MS = 6000; const tenths = (cores) => Math.round(cores * 10) / 10; const sleep = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); export const alive = (pid) => { try { process.kill(pid, 0); return true; } catch (error) { return error.code === 'EPERM'; } }; // macOS counts its file cache as used, so os.freemem() reads near zero on an // idle Mac; the kernel's own free-memory percentage is what its pressure uses. function memoryLevel() { if (process.platform !== 'darwin') return null; const level = Number(spawnSync('sysctl', ['-n', 'kern.memorystatus_level'], { encoding: 'utf8' }).stdout); return level > 0 ? level / 100 : null; } const freeMemory = () => memoryLevel() ?? freemem() / totalmem(); function lockPool() { try { mkdirSync(POOL_LOCK); return true; } catch (error) { if (error.code !== 'EEXIST') throw error; if (Date.now() - statSync(POOL_LOCK, { throwIfNoEntry: false })?.mtimeMs > STALE_LOCK_MS) rmSync(POOL_LOCK, { recursive: true, force: true }); return false; } } function readUnlessReleased(path) { try { return readFileSync(path, 'utf8'); } catch (error) { if (error.code === 'ENOENT') return null; throw error; } } // The holders (`.json`) or waiters (`.wait`) whose process still runs. function entries(suffix) { const live = []; for (const file of readdirSync(POOL).filter((name) => name.endsWith(suffix))) { const path = join(POOL, file); const text = readUnlessReleased(path); if (text === null) continue; const entry = JSON.parse(text); if (alive(entry.pid)) live.push(entry); else rmSync(path, { force: true }); } return live; } // Counts the workers that run under a process now. A process that is running // or ready to run is one worker, because on a loaded machine its CPU share // reads far under the core it would use, and a process whose threads use // several cores is that many. Where `ps` cannot run or hangs, nothing counts. // The count stops at another holder, such as the check a landing runs when // that check takes a share of its own, so no worker counts twice. function countWorkers() { const { stdout } = spawnSync('ps', ['-A', '-o', 'pid=', '-o', 'ppid=', '-o', 'pcpu=', '-o', 'state='], { encoding: 'utf8', env: { ...process.env, LC_ALL: 'C' }, timeout: 10_000 }); const children = new Map(); for (const line of (stdout ?? '').split('\n')) { const [pid, parent, pcpu, state] = line.trim().split(/\s+/u); if (!state) continue; const child = { pid: Number(pid), workers: Math.max(state.startsWith('R') ? 1 : 0, (Number(pcpu) || 0) / 100) }; if (!children.has(Number(parent))) children.set(Number(parent), []); children.get(Number(parent)).push(child); } return (root, holders) => { const seen = new Set(holders); let workers = 0; for (const under = [...(children.get(root) ?? [])]; under.length > 0; ) { const child = under.pop(); if (seen.has(child.pid)) continue; seen.add(child.pid); workers += child.workers; under.push(...(children.get(child.pid) ?? [])); } return workers; }; } // Holds the machine-wide pool's share for `workers` until the returned release // runs or this process exits; the release's `waited` is the milliseconds the // call waited. A process whose ancestor holds a share takes none, so a proof // started inside a proof never waits on its own parent. A waiter counts the // ask of every waiter ahead of it as held, so a small proof never takes the // cores a larger waiter ahead of it needs, unless its ask fits the free cores // now and the oldest waiter ahead has waited under BACKFILL_MS. A `first` // call, which land.mjs and its check make, enters at once, whatever the cores // and memory, because every session waits on its landing. It still holds a // share, so the pool counts its workers against the proofs that queue. A // landing from an older land.mjs can still wait, and no call without `first` // enters while one does, because a proof's workers can outgrow its ask. export function reserve(workers, label, { perWorker: coresPerWorker = CORES_PER_WORKER, first: asLanding = false } = {}) { const asked = Date.now(); if (process.env[HELD]) return Object.assign(() => {}, { waited: 0 }); const first = asLanding || process.env[LAND_CHECK] === '1'; const capacity = availableParallelism(); const want = Math.min(workers * coresPerWorker, capacity); mkdirSync(POOL, { recursive: true }); const mine = join(POOL, `${process.pid}.json`); const ticket = join(POOL, `${process.pid}.wait`); const rank = (waiter) => (waiter.first ? 0 : 1); let since = null; let slept = false; const reported = new Set(); for (;;) { const running = countWorkers(); if (lockPool()) { let waiting; try { const now = Date.now(); if (since === null) { since = now; writeFileSync(ticket, JSON.stringify({ pid: process.pid, cores: want, label, since, first })); } const holders = entries('.json'); const pids = holders.map((holder) => holder.pid); const live = holders.map((holder) => ({ ...holder, used: Math.min(capacity, Math.max(holder.cores, tenths(running(holder.pid, pids) * (holder.perWorker ?? CORES_PER_WORKER)))), })); const held = tenths(live.reduce((sum, holder) => sum + holder.used, 0)); const watched = live.find((holder) => Math.abs(now - holder.since) < WATCH_MS); const own = rank({ first }); const ahead = entries('.wait').filter( (waiter) => rank(waiter) < own || (rank(waiter) === own && (waiter.since < since || (waiter.since === since && waiter.pid < process.pid))) ); const queued = ahead.reduce((sum, waiter) => sum + waiter.cores, 0); const landing = first ? undefined : ahead.find((waiter) => waiter.first); const free = freeMemory(); const oldest = Math.min(...ahead.map((waiter) => waiter.since)); const fits = held + queued + want <= capacity || (held + want <= capacity && now - oldest < BACKFILL_MS); if (first || (fits && free >= MIN_FREE_MEMORY && !watched && !landing)) { writeFileSync(mine, JSON.stringify({ pid: process.pid, cores: want, label, since: now, perWorker: coresPerWorker })); rmSync(ticket, { force: true }); process.env[HELD] = '1'; const release = () => { rmSync(mine, { force: true }); delete process.env[HELD]; }; process.once('exit', release); return Object.assign(release, { waited: slept ? Date.now() - asked : 0 }); } const behind = ahead.length > 0 ? `, and ${queued} asked for by ${ahead.length} waiter${ahead.length === 1 ? '' : 's'} ahead` : ''; const share = (holder) => (holder.used > holder.cores ? `${holder.used} in use, asked ${holder.cores}` : holder.cores); if (!fits) waiting = { kind: 'cores', reason: `${held} of ${capacity} cores held by ${live.map((holder) => `${holder.label} (${holder.pid}, ${share(holder)})`).join(', ') || 'nobody'}${behind}` }; else if (free < MIN_FREE_MEMORY) waiting = { kind: 'memory', reason: `free memory is ${Math.round(free * 100)}%, under ${MIN_FREE_MEMORY * 100}%` }; else if (landing) waiting = { kind: 'landing', reason: `${landing.label} (${landing.pid}) waits for ${landing.cores} cores, and a waiting landing enters before any proof` }; else { const seconds = Math.round((now - watched.since) / 1000); waiting = { kind: 'count', reason: `${watched.label} (${watched.pid}) started ${seconds} second${seconds === 1 ? '' : 's'} ago, and the pool counts the workers of a new command for ${WATCH_MS / 1000} seconds before it admits the next`, }; } } finally { rmSync(POOL_LOCK, { recursive: true, force: true }); } if (!reported.has(waiting.kind)) console.error(`${label}: waiting for ${want} of this machine's cores; ${waiting.reason}`); reported.add(waiting.kind); } slept = true; sleep(POLL_MS); } } export const TIME_BUDGET_SECONDS = 3600; const SUPERVISE_GROUP = '--supervise-group'; const KILL_GRACE_MS = 5000; const MEMORY_SHARE = 0.5; const SHORT_MEMORY_SHARE = 0.1; // ps reads resident memory, which leaves out compressed and swapped pages. // footprint counts them as the kernel's memory pressure does, so the watch // reads footprint for a tree that could be near a limit, or for any tree while // memory runs short. const FOOTPRINT_FROM_SHARE = 0.05; const FOOTPRINT_UNDER_FREE = 0.25; const GB = 1024 ** 3; const gb = (bytes) => `${(bytes / GB).toFixed(1)} GB`; function processes() { const { stdout } = spawnSync('ps', ['-A', '-o', 'pid=', '-o', 'ppid=', '-o', 'pgid=', '-o', 'rss=', '-o', 'comm='], { encoding: 'utf8', env: { ...process.env, LC_ALL: 'C' }, timeout: 10_000 }); return (stdout ?? '').split('\n').flatMap((line) => { const fields = line.match(/^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(.+)$/u); return fields ? [{ pid: Number(fields[1]), parent: Number(fields[2]), group: Number(fields[3]), bytes: Number(fields[4]) * 1024, name: basename(fields[5].trim()) }] : []; }); } function tree(root, withGroup) { const all = processes(); const children = new Map(); for (const entry of all) { if (!children.has(entry.parent)) children.set(entry.parent, []); children.get(entry.parent).push(entry); } const found = new Map(all.filter((entry) => entry.pid === root || (withGroup && entry.group === root)).map((entry) => [entry.pid, entry])); for (const queue = [...found.values()]; queue.length > 0; ) { for (const child of children.get(queue.pop().pid) ?? []) { if (found.has(child.pid)) continue; found.set(child.pid, child); queue.push(child); } } return [...found.values()]; } function footprints(pids) { const { stdout } = spawnSync('footprint', ['-f', 'bytes', '--noCategories', ...pids.flatMap((pid) => ['-p', String(pid)])], { encoding: 'utf8', timeout: 30_000 }); return new Map([...(stdout ?? '').matchAll(/\[(\d+)\]: .*Footprint: (\d+) B/gu)].map(([, pid, bytes]) => [Number(pid), Number(bytes)])); } const signalAll = (pids, leader, signal) => { for (const pid of leader ? [-leader, ...pids] : pids) { try { process.kill(pid, signal); } catch (error) { if (error.code !== 'ESRCH' && error.code !== 'EPERM') throw error; } } }; /** * Every two seconds, adds up the memory of the process `pid` and every process * under it (with `group`, also every process in its process group). Past the * limit it stops them, this process excepted, with SIGTERM and, after a grace, * SIGKILL, and calls `stopped` once with a sentence that names the use, the * limit and the largest processes. Returns the function that ends the watch. * It watches nothing where the system gives no free-memory level. * @param {number} pid * @param {{ group?: boolean, stopped: (text: string) => void }} options * @returns {() => void} */ export function watchMemory(pid, { group = false, stopped }) { if (memoryLevel() === null) return () => {}; const total = totalmem(); const timer = setInterval(() => { const free = memoryLevel() ?? 1; const found = tree(pid, group); if (found.reduce((sum, entry) => sum + entry.bytes, 0) >= total * FOOTPRINT_FROM_SHARE || free < FOOTPRINT_UNDER_FREE) { const exact = footprints(found.map((entry) => entry.pid)); for (const entry of found) entry.bytes = exact.get(entry.pid) ?? entry.bytes; } const used = found.reduce((sum, entry) => sum + entry.bytes, 0); const short = free < MIN_FREE_MEMORY; const limit = total * (short ? SHORT_MEMORY_SHARE : MEMORY_SHARE); if (used <= limit) return; clearInterval(timer); const pids = found.map((entry) => entry.pid).filter((entry) => entry !== process.pid); const leader = group ? pid : null; signalAll(pids, leader, 'SIGTERM'); setTimeout(() => signalAll(pids, leader, 'SIGKILL'), KILL_GRACE_MS); const largest = found .toSorted((a, b) => b.bytes - a.bytes) .slice(0, 5) .map((entry) => `${entry.name} (${entry.pid}) ${gb(entry.bytes)}`); const when = short ? ` while ${Math.round(free * 100)}% of its memory was free` : ''; stopped(`used ${gb(used)} of this machine's ${gb(total)}${when}, over the ${gb(limit)} one command may use, so its processes were stopped; the largest: ${largest.join(', ')}`); }, 2000); timer.unref(); return () => clearInterval(timer); } function bootTime() { const seconds = process.platform === 'darwin' ? spawnSync('sysctl', ['-n', 'kern.boottime'], { encoding: 'utf8' }).stdout?.match(/sec = (\d+)/u)?.[1] : existsSync('/proc/stat') && readFileSync('/proc/stat', 'utf8').match(/^btime (\d+)$/mu)?.[1]; return seconds ? new Date(Number(seconds) * 1000).toISOString() : null; } const HEAD_BYTES = 64 * 1024; function unfinishedBoot(path) { const { size } = statSync(path); if (/^exit=\S+$/mu.test(readAt(path, Math.max(0, size - HEAD_BYTES), Math.min(size, HEAD_BYTES)))) return null; return readAt(path, 0, Math.min(size, HEAD_BYTES)).match(/^boot=(\S+)$/mu)?.[1] ?? null; } export function prove({ dir, name, command, cwd = process.cwd(), timeout, cores = 1, stress = false, force = false }) { if (!dir || !name || command.length === 0 || name.split('/').includes('..')) throw new Error('prove needs a run directory, a name without "..", and a command'); if (timeout !== undefined && !(Number.isFinite(timeout) && timeout > 0)) throw new Error(`--timeout takes a positive number of seconds, got ${timeout}`); if (!(Number.isInteger(cores) && cores > 0)) throw new Error(`--cores takes a positive whole number, got ${cores}`); if (stress) { const most = Math.max(1, Math.floor(availableParallelism() * STRESS_MAX_CORE_SHARE)); if (cores > most) throw new Error(`--stress with --cores ${cores} is over half of this machine's ${availableParallelism()} cores; run the command with at most ${most} workers and pass --cores ${most}`); } else { if (timeout > TIME_BUDGET_SECONDS) throw new Error(`--timeout ${timeout} is over the ${TIME_BUDGET_SECONDS}-second time budget; split the proof, or offer it as a suggested task whose session passes --stress`); timeout ??= TIME_BUDGET_SECONDS; } const boot = bootTime(); const folder = dirname(join(resolve(dir), name)); const last = lastAttempt(folder, name); if (boot && last > 0 && !force) { const previous = join(folder, `${basename(name)}-a${last}.log`); const ran = unfinishedBoot(previous); if (ran && ran !== boot) throw new Error(`${previous} logged no exit, and this machine has started again since that run (${ran}), so the run may have taken the machine down; read its log and fix the cause, or pass --force to run it again`); } const shell = command.length === 1; const line = shell ? command[0] : command.map(quote).join(' '); const { fd, path } = claim(resolve(dir), name); const header = `$ (${resolve(cwd)}) ${line}\n${boot ? `boot=${boot}\n` : ''}`; writeSync(fd, header); const [file, args] = shell ? ['bash', ['-o', 'pipefail', '-c', command[0]]] : [command[0], command.slice(1)]; const release = reserve(cores, `proof.mjs ${name}`, { perWorker: stress ? STRESS_CORES_PER_WORKER : CORES_PER_WORKER }); const result = spawnSync(process.execPath, [fileURLToPath(import.meta.url), SUPERVISE_GROUP, String(timeout ?? 0), file, ...args], { cwd, stdio: ['ignore', fd, fd, 'pipe'], timeout: timeout ? timeout * 1000 + 3 * KILL_GRACE_MS : undefined, killSignal: 'SIGKILL', }); release(); closeSync(fd); const report = JSON.parse(result.output?.[3]?.toString() || 'null') ?? { timedOut: result.error?.code === 'ETIMEDOUT', error: result.error?.code ?? 'error' }; const exit = report.timedOut ? 'ETIMEDOUT' : report.memory ? 'EMEMORY' : (report.code ?? report.signal ?? report.error ?? 'error'); const { size } = statSync(path); const stopped = report.timedOut ? `timed out after ${timeout} seconds\n` : ''; if (!report.logged) appendFileSync(path, `${size > Buffer.byteLength(header) && readAt(path, size - 1, 1) !== '\n' ? '\n' : ''}${stopped}exit=${exit}\n`); const shown = Math.min(size - Buffer.byteLength(header), OUTPUT_TAIL_BYTES); return { exit, path, output: readAt(path, size - shown, shown) }; } function superviseGroup([seconds, file, ...args]) { const child = spawn(file, args, { stdio: ['ignore', 'inherit', 'inherit'], detached: true }); const killGroup = (signal) => { try { process.kill(-child.pid, signal); } catch (error) { // macOS answers EPERM, not ESRCH, when every process left in the group is // a zombie. if (error.code !== 'ESRCH' && error.code !== 'EPERM') throw error; } }; // A stop signal sent to the whole group ends the parent proof.mjs too, and // its pipe can still take the report, so the log gets the exit line here. let stopped = null; let logged = false; const logExit = (cause, exit) => { if (logged) return; logged = true; writeSync(1, `\n${cause}\nexit=${exit}\n`); }; const report = (value) => { try { writeSync(3, JSON.stringify({ ...value, logged })); } catch (error) { if (error.code !== 'EPIPE') throw error; logExit('proof.mjs ended before its command', value.code ?? value.signal ?? value.error); } process.exit(0); }; let timedOut = false; let memory = false; watchMemory(child.pid, { group: true, stopped: (text) => { memory = true; writeSync(1, `\nproof.mjs: the command ${text}\n`); }, }); if (Number(seconds) > 0) { setTimeout(() => { timedOut = true; killGroup('SIGTERM'); setTimeout(() => killGroup('SIGKILL'), KILL_GRACE_MS); }, Number(seconds) * 1000); } for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) { process.on(signal, () => { stopped ??= signal; killGroup('SIGTERM'); setTimeout(() => { killGroup('SIGKILL'); logExit(`proof.mjs stopped by ${stopped}`, stopped); process.exit(128 + constants.signals[signal]); }, KILL_GRACE_MS); }); } child.on('error', (error) => report({ error: error.code })); child.on('exit', (code, signal) => { if (timedOut || memory) killGroup('SIGKILL'); if (stopped) logExit(`proof.mjs stopped by ${stopped}`, code ?? signal); report({ code, signal, timedOut, memory }); }); } const OUTPUT_TAIL_BYTES = 16 * 1024 * 1024; function readAt(path, position, length) { const fd = openSync(path, 'r'); const buffer = Buffer.alloc(length); readSync(fd, buffer, 0, length, position); closeSync(fd); return buffer.toString('utf8'); } function parse(argv) { const split = argv.indexOf('--'); const options = { command: split === -1 ? [] : argv.slice(split + 1) }; const flags = split === -1 ? argv : argv.slice(0, split); for (let index = 0; index < flags.length; index++) { const key = flags[index].replace(/^--/u, ''); if (key === 'stress' || key === 'force') { options[key] = true; continue; } if (!['dir', 'name', 'cwd', 'timeout', 'cores'].includes(key)) throw new Error(`unknown option ${flags[index]}`); options[key] = flags[++index]; if (key === 'timeout' && !(Number(options.timeout) > 0)) throw new Error(`--timeout takes a number of seconds, got ${options.timeout}`); if (key === 'timeout') options.timeout = Number(options.timeout); if (key === 'cores') options.cores = Number(options.cores); } return options; } const main = process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url); if (main && process.argv[2] === SUPERVISE_GROUP) superviseGroup(process.argv.slice(3)); else if (main) { try { const options = parse(process.argv.slice(2)); if (options.cwd && !existsSync(options.cwd)) throw new Error(`--cwd ${options.cwd} does not exist`); const { exit, path, output } = prove(options); const lines = output.split('\n'); console.log(lines.length > 60 ? `[${lines.length - 60} earlier lines in the log]\n${lines.slice(-60).join('\n')}` : output); console.log(`log: ${path} exit=${exit}`); process.exit(typeof exit === 'number' ? exit : 1); } catch (error) { console.error(`${error.message}\nUsage: node .agents/pstack/proof.mjs --dir --name [--cwd ] [--timeout ] [--cores ] [--stress] [--force] -- [arg...]`); process.exit(2); } }