/** * Runtime-state-first proxy liveness with identity checking. * * Historically `ensure`/`start` probed only `config.port` and accepted ANY 2xx /healthz: * a proxy that started on a fallback port was invisible (duplicate starts, Codex synced * back to a dead port), and a foreign app answering 200 on the configured port counted * as "our proxy". Liveness now (1) prefers the pid + runtime-port record and (2) requires * the /healthz body to identify as opencodex. * * Lives outside cli.ts (which dispatches argv at module top level) so tests can import it. */ import { loadConfig } from "../config"; import { isLoopbackHostname, isWildcardHostname } from "../codex/loopback-target"; import { normalizeBindHostname } from "./bind-host"; import { hasPublishedOwnershipRecords, isOcxCommandLine, readAlivePid, readProcessCommandLine, readRuntimePort, runtimeOwnerScope, sameRuntimeOwnerScope, verifyPidIdentity, type RuntimeOwnerScope, } from "../config/process-state"; import { createServer, isIP } from "node:net"; import { loopbackFamilyAvailability } from "../lib/loopback-family"; import { LOCAL_ATTESTATION_CHALLENGE_HEADER, LOCAL_ATTESTATION_PROOF_HEADER, createLocalAttestationChallenge, verifyLocalAttestationProof, } from "../lib/local-management-attestation"; import { directLocalHttpFetch } from "./direct-local-http"; export interface HealthzIdentity { service?: unknown; status?: unknown; version?: unknown; uptime?: unknown; pid?: unknown; port?: unknown; /** * Which listener answered: the standalone/hub server omits it, and the connected-client * machine listener reports `"client"` (src/client/machine-listener.ts). It is the only * on-the-wire way to tell those two apart, because both bind `config.port ?? 10100`. */ role?: unknown; restartCapability?: unknown; providerReloadCapability?: unknown; asideSyncCapability?: unknown; guiPairCapability?: unknown; /** Present on a package-tree-fenced 503: the version on disk an in-place respawn will run. */ installedVersion?: unknown; error?: unknown; } export type EndpointLiveness = "live" | "dead" | "unknown"; /** * Whether this process can bind `hostname:port` right now. A successful bind proves no process * owns that address, which no probe of it can: connecting to this machine's own non-loopback * address (a Tailscale IP, a LAN IP) with nothing listening TIMES OUT on some hosts instead of * being refused, and a timeout is indistinguishable from a slow owner. The listener is closed * immediately. Any bind error (EADDRINUSE, EADDRNOTAVAIL, EACCES) answers false, so an address * we cannot bind stays "unknown" and startup keeps refusing. */ const BIND_PROOF_TIMEOUT_MS = 1_500; export function canBindAddress(hostname: string, port: number, timeoutMs = BIND_PROOF_TIMEOUT_MS): Promise { if (!Number.isInteger(port) || port <= 0 || port > 65_535) return Promise.resolve(false); return new Promise(resolve => { const server = createServer(); let settled = false; const finish = (bound: boolean): void => { if (settled) return; settled = true; clearTimeout(timer); try { server.close(); } catch { /* never bound */ } resolve(bound); }; const timer = setTimeout(() => finish(false), timeoutMs); server.once("error", () => finish(false)); server.once("listening", () => finish(true)); try { server.listen({ host: hostname, port, exclusive: true }); } catch { finish(false); } }); } export interface LivenessIo { fetchFn?: typeof fetch; readPidFn?: () => number | null; /** * Full identity check of the passed candidate pid; must return the SAME pid or null. * Destructive callers only ever receive pids that passed this gate. */ verifyPidFn?: (candidatePid: number) => number | null; /** Fresh command-line evidence: undefined/empty is unreadable, never confirmed foreign. */ readCommandLineFn?: typeof readProcessCommandLine; loopbackAvailabilityFn?: typeof loopbackFamilyAvailability; /** * Test seam for the bind proof (see `canBindAddress`): whether `hostname:port` can be bound by * this process right now, which proves no process owns that address. `hostname` is the raw * socket address (IPv6 unbracketed) and `timeoutMs` the remaining discovery budget. */ bindProofFn?: (hostname: string, port: number, timeoutMs: number) => Promise; readRuntimeFn?: (pid?: number) => { pid?: number; port: number; hostname?: string; ownerScope?: unknown } | null; /** This process, as the self-record proof in {@link observeProxyLiveness} needs it (#8552). */ ownProcessFn?: () => OwnProcess; configFn?: () => { port?: number; hostname?: string }; timeoutMs?: number; /** * How many times to retry a probe that failed with a transport error (timeout / * connection refused). Definitive answers (non-OK HTTP, foreign /healthz body, pid * mismatch) do not retry. Default 1 = no retry. Stop paths should pass 2–3 (#764). */ attempts?: number; sleepFn?: (ms: number) => Promise; /** * Absolute wall-clock deadline for discovery. When set, each probe attempt aborts * once the remaining budget cannot cover another fetch — so multi-candidate * `findLiveProxy` under `SERVICE_STOP_LIVENESS` cannot overrun the stop-path * verification window (#764 / CodeRabbit). */ deadlineAt?: number; nowFn?: () => number; /** * Also accept a proxy whose `/healthz` is fenced by the package-tree guard (#5496). * * Opt-in, because such a proxy is ours but not healthy: `ocx restart` and the stop paths * must find it, while ensure, update health waits and replacement waits must not count it. * A fenced body is never trusted by itself. It is accepted only when this home's runtime * record names the same pid and port and the listener proves possession of that record's * attestation secret for a fresh challenge. */ acceptPackageTreeFenced?: boolean; /** Test seam for the fenced-identity challenge. */ createChallengeFn?: () => string; } /** * Operator override for the per-probe fetch ceilings below (`OCX_PROBE_TIMEOUT_MS`), * integer milliseconds in [1, MAX_PROBE_TIMEOUT_MS]. * * Some hosts put a security layer (content filter, EDR network extension) in front of * loopback TCP that adds a fixed per-connect cost, measured at about one second on an * affected macOS machine. The shipped 750 ms probe then aborts before a healthy proxy can * answer, and every CLI liveness consumer reports the proxy as down while a direct * `curl /healthz` succeeds. * * The override only raises: each ceiling keeps its shipped floor (750 ms for the shared * default, 1500 ms for the stop/start ownership budgets that guard against a duplicate * proxy, #764, #5004), so a small value can never shorten them. Values above the 30 s * ceiling are ignored rather than clamped: a stop multiplies its budget by the attempt * count, and a typo must not turn a stop into a wait of minutes or days. Parsed once at * module load; anything malformed falls back to the defaults and never breaks startup. */ export const MAX_PROBE_TIMEOUT_MS = 30_000; const SHARED_PROBE_FLOOR_MS = 750; const OWNERSHIP_PROBE_FLOOR_MS = 1500; export function parseProbeTimeoutOverrideMs(raw: string | undefined): number | undefined { const trimmed = raw?.trim(); if (!trimmed || !/^\d+$/.test(trimmed)) return undefined; const n = Number(trimmed); return n > 0 && n <= MAX_PROBE_TIMEOUT_MS ? n : undefined; } /** The ceiling for a probe whose shipped value is `floorMs`, raised by a valid override only. */ export function probeCeilingMs(floorMs: number, override: number | undefined): number { return Math.max(floorMs, override ?? 0); } const probeTimeoutOverrideMs = parseProbeTimeoutOverrideMs(process.env.OCX_PROBE_TIMEOUT_MS); /** Default per-probe fetch ceiling shared by liveness and readiness probes. */ export const DEFAULT_PROBE_TIMEOUT_MS = probeCeilingMs(SHARED_PROBE_FLOOR_MS, probeTimeoutOverrideMs); /** Default probe options for service stop / orphan cleanup — a just-bound proxy can miss a single 750ms probe. */ export const SERVICE_STOP_LIVENESS: Pick = { timeoutMs: probeCeilingMs(OWNERSHIP_PROBE_FLOOR_MS, probeTimeoutOverrideMs), attempts: 3, }; /** * Probe budget for a decision whose wrong answer starts a DUPLICATE proxy (#5004). * * `start` used the 750ms single-attempt default for both the pre-bind owner probe and * (implicitly) the busy-port question behind the ephemeral hop. On Windows that answered * "nothing is listening" for a proxy the previous command had just refused to shadow, and * the hop then spawned a second instance that took over this home's pid/runtime records * and re-pointed Codex at itself. A single unanswered probe is not evidence of absence * when the failure mode is a duplicate instance, so the start path borrows the numbers * the stop path already uses for the mirror-image decision. */ export const START_OWNERSHIP_LIVENESS: Pick = { timeoutMs: probeCeilingMs(OWNERSHIP_PROBE_FLOOR_MS, probeTimeoutOverrideMs), attempts: 3, }; type LivenessFetch = ( input: string | URL | Request, init?: RequestInit, ) => Promise; export interface LiveProxy { pid: number | null; port: number; /** Candidate bind scope, with the default loopback made explicit; never a probe alias. */ hostname?: string; /** Successful recovery probe address, when discovery checked both loopback families. */ probeHostname?: string; /** Whether the successful probe used runtime-port metadata or the configured listen port. */ source: "runtime" | "config"; /** * Version the live proxy reported on `/healthz`, when it reported one. * * Carried so a stale `ocx` on PATH can be detected without a second request: the * identity probe already parsed and validated this body. Absent for a legacy proxy whose * healthz body predates the field. */ version?: string; /** * Role the live listener reported on `/healthz`, when it reported one: `"client"` for the * connected-client machine listener, absent for a standalone or hub proxy. * * Liveness deliberately still ACCEPTS a client-role listener — see `isOpencodexHealthz`. * The role is carried so a caller that needs a management plane (src/cli/runtime-api.ts) * can refuse one, while `stop` and orphan cleanup keep finding the process they must act * on. Absent for a proxy whose healthz body predates the field. */ role?: string; /** * The proxy answered with an attested package-tree fence (#5496): it is this home's process, * but it is refusing traffic until it restarts. Only set for callers that opted in through * `LivenessIo.acceptPackageTreeFenced`. */ packageTreeFenced?: true; } /** * A /healthz identity proves only that a proxy holds the port. Before a destructive orphan stop, * require that listener to prove possession of this home's runtime-record secret as well. * * The verdict is three-valued on purpose. "proven" is the only answer that authorizes the * caller's action; "refuted" means the answer was definitive (no record, a pid/port * mismatch, or a failed proof); "indeterminate" means transport, deadline, or an * unattestable record left the question open, which a shared-write decision must treat * the same as a live owner it simply could not verify (#6198). */ export type HomeOwnershipProof = "proven" | "refuted" | "indeterminate"; export async function proveLiveProxyOwnedByHome(live: LiveProxy, io: LivenessIo = {}): Promise { if (live.pid === null) return "indeterminate"; return attestFencedIdentity( `http://${probeHostname(live.probeHostname ?? live.hostname)}:${live.port}/healthz`, live.port, live.pid, io, io.fetchFn ?? directLocalHttpFetch, io.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS, ); } /** * Host to probe for a given bind hostname: wildcards answer on IPv4 loopback, and raw * IPv6 addresses must be bracketed or the composed URL is invalid. * * The wildcard test is `isWildcardHostname`, not a list of spellings. This function used to * know exactly three (`0.0.0.0`, `::`, `[::]`) while the bind-scope predicate knew every * all-zero form, so `ocx` composed `http://0.0.0.0.:10100` or `http://*:10100` — unreachable * URLs — for a config the server itself treated as a wildcard bind. One predicate, both sides. */ export function probeHostname(hostname: string | undefined): string { const trimmed = (hostname ?? "").trim(); if (!trimmed || isWildcardHostname(trimmed)) return "127.0.0.1"; if (trimmed.startsWith("[") && trimmed.endsWith("]")) return trimmed; return trimmed.includes(":") ? `[${trimmed}]` : trimmed; } /** * True when a /healthz body identifies an opencodex proxy. Accepts the explicit * `service: "opencodex"` marker, plus the legacy `{status, version, uptime}` trio so a * still-running pre-identity proxy (e.g. right after `ocx update`) is not mistaken for a * foreign server and shadow-started over. * * The connected-client machine listener answers with the same `service: "opencodex"` marker * and an extra `role: "client"`, and it is accepted here on purpose. Liveness answers "is one * of our processes listening on this port", which is exactly what `ocx stop`, orphan cleanup, * and duplicate-start avoidance need: rejecting the client role would make those paths blind to * a real opencodex process and let them shadow-start over it. Callers that additionally need a * management plane discriminate on `LiveProxy.role` instead of narrowing this predicate. */ export function isOpencodexHealthz(body: HealthzIdentity | null): boolean { if (!body) return false; if (body.service === "opencodex") return true; if (body.service !== undefined) return false; return body.status === "ok" && typeof body.version === "string" && typeof body.uptime === "number"; } /** * True for the exact 503 body the package-tree guard serves on `/healthz`. Shape only: this is * what the listener CLAIMS, and whoever holds the port can claim it. Callers must attest it. */ export function isPackageTreeFencedHealthz(body: HealthzIdentity | null | undefined): boolean { if (!body || body.service !== "opencodex" || body.status !== "restart_required") return false; const error = body.error as { code?: unknown } | null | undefined; return typeof error === "object" && error !== null && error.code === "package_tree_changed" && typeof body.pid === "number" && Number.isSafeInteger(body.pid) && body.pid > 0; } /** * Prove that a fenced listener is the process this home's runtime record describes: the record * must name the same pid and port and carry an attestation secret, and the listener must answer a * fresh challenge with a proof bound to that secret, pid and port. The pid in the 503 body only * selects which record to check; it is never accepted on its own. */ async function attestFencedIdentity( url: string, port: number, pid: number, io: LivenessIo, fetchFn: LivenessFetch, timeoutMs: number, ): Promise { const readRuntimeFn = io.readRuntimeFn ?? readRuntimePort; let record: ReturnType>; try { record = readRuntimeFn(pid); } catch { return "indeterminate"; } // The typed seam omits the secret; the production record (readRuntimePort) carries it. const secret: unknown = record ? Reflect.get(record, "attestationSecret") : undefined; if (!record || record.pid !== pid || record.port !== port) return "refuted"; if (typeof secret !== "string" || secret.length === 0) return "indeterminate"; const challenge = (io.createChallengeFn ?? createLocalAttestationChallenge)(); // One proof failure is definitive and never retried; a transport failure only means // the listener did not answer yet, so it gets the same bounded retry the identity // probe uses — "did not answer" is not "not ours" (#6198). The challenge is minted // once: a retried attempt proves the same fresh nonce, not a replayed proof. const sleepFn = io.sleepFn ?? ((ms: number) => new Promise(r => setTimeout(r, ms))); const nowFn = io.nowFn ?? Date.now; const requestedAttempts = Math.trunc(io.attempts ?? 1); const attempts = Number.isNaN(requestedAttempts) ? 1 : Math.max(1, Math.min(requestedAttempts, 5)); for (let attempt = 1; attempt <= attempts; attempt++) { const remainingMs = io.deadlineAt === undefined ? timeoutMs : Math.min(timeoutMs, io.deadlineAt - nowFn()); if (remainingMs <= 0) return "indeterminate"; try { const res = await fetchFn(url, { headers: { [LOCAL_ATTESTATION_CHALLENGE_HEADER]: challenge }, signal: AbortSignal.timeout(remainingMs), }); const body = (await res.json().catch(() => null)) as HealthzIdentity | null; // The second answer must still be the same fenced (or by now healthy) process. if (!isOpencodexHealthz(body) && !isPackageTreeFencedHealthz(body)) return "refuted"; if (body?.pid !== pid) return "refuted"; return verifyLocalAttestationProof(secret, challenge, pid, port, res.headers.get(LOCAL_ATTESTATION_PROOF_HEADER)) ? "proven" : "refuted"; } catch { if (attempt >= attempts) return "indeterminate"; await sleepFn(100); } } return "indeterminate"; } /** A bounded version string safe to carry beyond the untrusted health response. */ export function isHealthzVersion(value: unknown): value is string { return typeof value === "string" && value.length <= 64 && /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/.test(value); } /** * "Nothing is listening" is narrower than "the probe failed". Only a connect-phase refusal * proves the endpoint is free; a timeout, reset, or other transport failure leaves the * question open. */ export function isConnectionRefused(error: unknown): boolean { const visit = (current: unknown, depth: number): boolean => { if (depth >= 4) return false; if (current === null || (typeof current !== "object" && typeof current !== "function")) return false; const record = current as { code?: unknown; cause?: unknown; errors?: unknown }; if (record.errors !== undefined) { // One connect attempt fanned out over several addresses reports a single AggregateError. // Only a unanimous refusal proves the endpoint is free: a bundle that mixes ECONNREFUSED // with a timeout means one address answered nothing at all, and an address whose state is // unreadable is unknown, not absence. Collapsing it to "refused" is how a second runtime // gets started on a port that already has one. return Array.isArray(record.errors) && record.errors.length > 0 && record.errors.every(error => visit(error, depth + 1)); } if (record.code === "ECONNREFUSED" || record.code === "ConnectionRefused") return true; if (typeof record.code === "string" && record.code.endsWith("ECONNREFUSED")) return true; return visit(record.cause, depth + 1); }; return visit(error, 0); } export async function classifyHealthz( url: string, fetchFn: LivenessFetch, timeoutMs: number, ): Promise { try { const response = await fetchFn(url, { signal: AbortSignal.timeout(timeoutMs) }); if (response.status !== 200) return "unknown"; const body = (await response.json().catch(() => undefined)) as HealthzIdentity | null | undefined; if (body === undefined) return "unknown"; return isOpencodexHealthz(body) ? "live" : "dead"; } catch (error) { return isConnectionRefused(error) ? "dead" : "unknown"; } } /** * Tri-state probe of one endpoint, the in-process counterpart of * `src/update/proxy-liveness-probe.mjs`. Only a connect-phase refusal or a clean 200 that is * not ours proves "dead"; a timeout, reset, non-200 or unreadable body leaves the question * open. Loopback endpoints are checked on both IPv4 and IPv6 because a listener may bind only * one family. Runs in-process because a compiled standalone binary cannot fork `execPath -e`. */ export async function probeEndpointLiveness( endpoint: { port: number; hostname?: string }, io: Pick = {}, ): Promise { if (!Number.isFinite(endpoint.port) || endpoint.port <= 0 || endpoint.port > 65535) return "dead"; const fetchFn = io.fetchFn ?? directLocalHttpFetch; const timeoutMs = io.timeoutMs ?? 1500; let sawUnknown = false; for (const hostname of loopbackProbeHosts(endpoint.hostname)) { const result = await classifyHealthz( `http://${hostname}:${endpoint.port}/healthz`, fetchFn, timeoutMs, ); if (result === "live") return "live"; if (result === "unknown") sawUnknown = true; } return sawUnknown ? "unknown" : "dead"; } /** Identity-checked /healthz probe; null when unreachable, non-OK, or not our proxy. */ export async function proxyIdentityAt( port: number, opts: { hostname?: string; expectedPid?: number } = {}, io: LivenessIo = {}, ): Promise<{ pid: number | null; version?: string; role?: string; packageTreeFenced?: true } | null> { return proxyIdentityAtObserved(port, opts, io); } /** Preserve negative evidence for recovery without changing identity-only callers. */ async function proxyIdentityAtObserved( port: number, opts: { hostname?: string; expectedPid?: number }, io: LivenessIo, observe?: (result: EndpointLiveness) => void, ): ReturnType { const negative = (result: "dead" | "unknown"): null => { observe?.(result); return null; }; const fetchFn = io.fetchFn ?? directLocalHttpFetch; const sleepFn = io.sleepFn ?? ((ms: number) => new Promise(r => setTimeout(r, ms))); const nowFn = io.nowFn ?? Date.now; const baseTimeoutMs = io.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS; const requestedAttempts = Math.trunc(io.attempts ?? 1); const attempts = Number.isNaN(requestedAttempts) ? 1 : Math.max(1, Math.min(requestedAttempts, 5)); let sawUnknown = false; for (let attempt = 1; attempt <= attempts; attempt++) { const remainingMs = io.deadlineAt === undefined ? baseTimeoutMs : io.deadlineAt - nowFn(); if (remainingMs <= 0) return negative("unknown"); const timeoutMs = Math.min(baseTimeoutMs, remainingMs); try { const url = `http://${probeHostname(opts.hostname)}:${port}/healthz`; const res = await fetchFn(url, { signal: AbortSignal.timeout(timeoutMs), }); if (!res.ok && io.acceptPackageTreeFenced && res.status === 503) { const fenced = (await res.json().catch(() => null)) as HealthzIdentity | null; if (!isPackageTreeFencedHealthz(fenced)) return negative("unknown"); const fencedPid = fenced!.pid as number; if (opts.expectedPid !== undefined && fencedPid !== opts.expectedPid) return negative("unknown"); const attestMs = io.deadlineAt === undefined ? baseTimeoutMs : Math.min(baseTimeoutMs, io.deadlineAt - nowFn()); if (attestMs <= 0) return negative("unknown"); if ((await attestFencedIdentity(url, port, fencedPid, io, fetchFn, attestMs)) !== "proven") return negative("unknown"); const fencedVersion = isHealthzVersion(fenced?.version) ? fenced!.version as string : undefined; return { pid: fencedPid, ...(fencedVersion === undefined ? {} : { version: fencedVersion }), packageTreeFenced: true, }; } if (!res.ok) return negative("unknown"); const body = (await res.json().catch(() => undefined)) as HealthzIdentity | null | undefined; if (!isOpencodexHealthz(body ?? null)) return negative(res.status === 200 && body !== undefined ? "dead" : "unknown"); const pid = typeof body?.pid === "number" ? body.pid : null; if (opts.expectedPid !== undefined && pid !== null && pid !== opts.expectedPid) return negative("unknown"); // Whoever holds the port controls this response. Only carry bounded semver text into // diagnostics; dropping anything else prevents terminal controls reaching human output. const version = isHealthzVersion(body?.version) ? body.version : undefined; // Same guard for the role, for the same reason: absent on a standalone/hub proxy and on // a legacy body, and never coerced from a non-string. const role = typeof body?.role === "string" ? body.role : undefined; return { pid, ...(version === undefined ? {} : { version }), ...(role === undefined ? {} : { role }), }; } catch (error) { // Transport failure (timeout / refused) — retry while budget remains; a proxy that // has only just begun listening can miss a single short probe (#764). if (!isConnectionRefused(error)) sawUnknown = true; if (attempt >= attempts) return negative(sawUnknown ? "unknown" : "dead"); if (io.deadlineAt !== undefined && io.deadlineAt - nowFn() <= 0) return negative("unknown"); await sleepFn(100); } } return negative("unknown"); } export type ProxyLivenessObservation = | { kind: "live"; live: LiveProxy } | { kind: "absent" | "unknown"; live: null }; export interface OwnProcess { pid: number; /** {@link hasPublishedOwnershipRecords}: this process has written ownership records itself. */ publishedOwnership: boolean; scope: RuntimeOwnerScope; } function currentOwnProcess(): OwnProcess { return { pid: process.pid, publishedOwnership: hasPublishedOwnershipRecords(), scope: runtimeOwnerScope() }; } /** Recovery additionally covers both available loopback families and fresh PID identity. */ export async function observeProxyLiveness(io: LivenessIo = {}): Promise { let alivePid: number | null = null, runtimePid: number | undefined, runtimeScope: unknown, unknown = false; const live = await discoverLiveProxy(io, { pid: pid => { alivePid = pid; }, runtimePid: (pid, ownerScope) => { runtimePid = pid; runtimeScope = ownerScope; }, endpoint: result => { if (result === "unknown") unknown = true; }, }); if (live) return { kind: "live", live }; // #8552: in a container the proxy is PID 1 on every start, so the record the previous container // left names this very process; signalling it succeeds and its command line is ocx. Such a // record is stale only when this process never published ownership itself (so the writer was an // earlier lifetime of the PID number) AND the runtime record's owner ran where this process runs // (same host and boot, plus the same network namespace or container). That scope check keeps a // RollingUpdate safe: the old pod is also PID 1 and its listener sits in another network // namespace, which the endpoint probe cannot see. A record without a scope (an older version, // which may still be serving during an upgrade) keeps the refusal. const own = (io.ownProcessFn ?? currentOwnProcess)(); const ownEarlierLifetime = !own.publishedOwnership && runtimePid === own.pid && sameRuntimeOwnerScope(runtimeScope, own.scope); const candidates = new Set(); if (alivePid !== null) candidates.add(alivePid); if (runtimePid !== undefined && Number.isSafeInteger(runtimePid) && runtimePid > 0) candidates.add(runtimePid); for (const pid of candidates) { if (unknown) break; if (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt) { unknown = true; break; } if (ownEarlierLifetime && pid === own.pid) continue; if (pid !== alivePid) { try { process.kill(pid, 0); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") continue; // Permission or an indeterminate probe is not proof that the process is dead. unknown = true; break; } } // A failed verifier alone conflates foreign with unreadable. Only fresh positive // evidence of a foreign command line releases a reused PID; never signal it. const commandLine = (io.readCommandLineFn ?? readProcessCommandLine)(pid); unknown = !commandLine?.trim() || isOcxCommandLine(commandLine); if (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt) unknown = true; } if (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt) unknown = true; return { kind: unknown ? "unknown" : "absent", live: null }; } /** * Locate the live proxy: pid file → runtime-port record → identity probe. Falls back to * the configured port ONLY when no runtime record answers, so a fallback-port proxy is * found and a foreign listener on the configured port is rejected. */ export async function findLiveProxy(io: LivenessIo = {}): Promise { return discoverLiveProxy(io); } async function discoverLiveProxy(io: LivenessIo, observer?: { pid: (pid: number | null) => void; runtimePid: (pid: number | undefined, ownerScope: unknown) => void; endpoint: (result: EndpointLiveness) => void; }): Promise { // Prefer the cheap alive-pid check: the Windows cmdline probe (WMIC/PowerShell) is too // expensive for waitForProxy's 150ms poll loop, and /healthz identity is the real trust gate. const readPidFn = io.readPidFn ?? readAlivePid; const verifyPidFn = io.verifyPidFn ?? verifyPidIdentity; const readRuntimeFn = io.readRuntimeFn ?? readRuntimePort; const configFn = io.configFn ?? loadConfig; const nowFn = io.nowFn ?? Date.now; const deadlineAt = io.deadlineAt; const probeIo: LivenessIo = io; const budgetExhausted = (): boolean => deadlineAt !== undefined && nowFn() >= deadlineAt; const expired = (): null => { observer?.endpoint("unknown"); return null; }; // The proof stays inside the discovery deadline: it is skipped once the budget is spent, its // own timeout is capped by what remains, and a proof that finishes past the deadline is // discarded, so an exhausted discovery still answers "unknown". A socket takes the raw // address, never the bracketed URL spelling `probeHostname` composes for IPv6. const bindProof = async (hostname: string, port: number): Promise => { const remainingMs = deadlineAt === undefined ? BIND_PROOF_TIMEOUT_MS : Math.min(BIND_PROOF_TIMEOUT_MS, deadlineAt - nowFn()); if (remainingMs <= 0) return false; const socketHost = hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname; // A hostname can resolve to different addresses for the probe and the bind. if (isIP(socketHost) === 0) return false; const bound = await (io.bindProofFn ?? canBindAddress)(socketHost, port, remainingMs); return bound && !budgetExhausted(); }; const probe = async (port: number, opts: { hostname?: string; expectedPid?: number }) => { // Missing/blank means an actual loopback bind, not a later config's bind scope. const bindHostname = opts.hostname?.trim() || "127.0.0.1"; const hostname = normalizeBindHostname(opts.hostname); if (!observer || (!recoveryLoopbackFamily(hostname) && !isWildcardHostname(hostname))) { // Recovery only (an observer is present): a transport miss on a specific non-loopback bind // address is "unknown", and a bind proof is the one thing that turns it into "dead". let verdict: EndpointLiveness | undefined; const identity = await proxyIdentityAtObserved(port, opts, probeIo, observer ? (r => { verdict = r; }) : undefined); if (observer && !identity && verdict !== undefined) { if (verdict === "unknown" && await bindProof(hostname, port)) verdict = "dead"; observer.endpoint(verdict); } return identity ? { ...identity, hostname: bindHostname, ...(observer ? { probeHostname: probeHostname(opts.hostname) } : {}) } : null; } // Parallel families share the existing per-probe attempts/timeout and absolute // deadline. A slow family cannot multiply the serial discovery budget. const hosts = recoveryProbeHosts(hostname).filter(host => (io.loopbackAvailabilityFn ?? loopbackFamilyAvailability)(host) !== "unavailable"); const identities = await Promise.all(hosts.map(async hostname => { const identity = await proxyIdentityAtObserved(port, { ...opts, hostname }, probeIo, observer.endpoint); return identity ? { ...identity, hostname: bindHostname, probeHostname: hostname } : null; })); return identities.find(identity => identity !== null) ?? null; }; // The cheap pid is discovery-only. Before it can appear in a returned (killable) result // it must pass the full identity check AND the verifier must echo the exact candidate — // a pidfile rewrite between discovery and verification can never swap in another process. const killablePid = (candidate: number | null): number | null => { if (candidate === null) return null; const verified = verifyPidFn(candidate); return verified === candidate ? verified : null; }; const verifiedReportedPid = (reported: number | null): number | null => { if (reported === null) return null; if (!Number.isSafeInteger(reported) || reported <= 0) return null; const verified = verifyPidFn(reported); return verified === reported ? verified : null; }; const pid = readPidFn(); observer?.pid(pid); let probedPort: number | null = null; if (pid) { const runtime = readRuntimeFn(pid); if (runtime?.port) { if (budgetExhausted()) return expired(); probedPort = runtime.port; const identity = await probe(runtime.port, { hostname: runtime.hostname, expectedPid: pid }); if (identity) { // healthz confirmed the pid itself → trusted; a pidless legacy body did not, // so the cheap pid must pass full identity verification before it is returned. const trusted = identity.pid === pid ? pid : killablePid(pid); return { pid: trusted, port: runtime.port, hostname: identity.hostname, ...(identity.probeHostname === undefined ? {} : { probeHostname: identity.probeHostname }), source: "runtime", ...(identity.version === undefined ? {} : { version: identity.version }), ...(identity.role === undefined ? {} : { role: identity.role }), ...(identity.packageTreeFenced ? { packageTreeFenced: true as const } : {}), }; } } } // Orphan recovery: the pid file can be lost/corrupt while the proxy is alive (crash of a // sibling command, manual deletion). The runtime record still says where it listens — // identity-probe it so ensure/update/stop see the live proxy instead of shadowing it. const record = readRuntimeFn(); observer?.runtimePid(record?.pid, record?.ownerScope); if (record?.port && record.port !== probedPort) { if (budgetExhausted()) return expired(); const expectedPid = typeof record.pid === "number" ? record.pid : undefined; const identity = await probe(record.port, { hostname: record.hostname, expectedPid }); // Only the healthz-reported pid is authoritative here. The record's pid may be stale // (its process dead, the port reused by a pidless legacy proxy) — synthesizing it // would hand destructive callers (stopProxy → kill fallback) a reusable pid. if (identity) { return { pid: verifiedReportedPid(identity.pid), port: record.port, hostname: identity.hostname, ...(identity.probeHostname === undefined ? {} : { probeHostname: identity.probeHostname }), source: "runtime", ...(identity.version === undefined ? {} : { version: identity.version }), ...(identity.role === undefined ? {} : { role: identity.role }), ...(identity.packageTreeFenced ? { packageTreeFenced: true as const } : {}), }; } } const config = configFn(); const port = config.port ?? 10100; if (budgetExhausted()) return expired(); // Port zero requests an OS-assigned port; it cannot name an existing listener. if (observer && port === 0) return null; const identity = await probe(port, { hostname: config.hostname }); if (identity) { return { pid: verifiedReportedPid(identity.pid) ?? killablePid(pid), port, hostname: identity.hostname, ...(identity.probeHostname === undefined ? {} : { probeHostname: identity.probeHostname }), source: "config", ...(identity.version === undefined ? {} : { version: identity.version }), ...(identity.role === undefined ? {} : { role: identity.role }), ...(identity.packageTreeFenced ? { packageTreeFenced: true as const } : {}), }; } return null; } // Recovery classification is intentionally local: it must not broaden admission or bind policy. function recoveryLoopbackFamily(hostname: string): 4 | 6 | null { if (isLoopbackHostname(hostname)) return hostname.includes(":") ? 6 : 4; const address = hostname.replace(/^\[(.*)\]$/, "$1"); const family = isIP(address); if (family === 4) return address.startsWith("127.") ? 4 : null; if (family !== 6) return null; // URL serializes validated IPv6 literals canonically, including dotted IPv4-mapped forms. const canonical = new URL(`http://[${address.split("%")[0]}]/`).hostname.slice(1, -1); if (canonical === "::1") return 6; return /^::ffff:7f[0-9a-f]{2}:[0-9a-f]{1,4}$/.test(canonical) ? 4 : null; } function recoveryProbeHosts(hostname: string): string[] { if (isWildcardHostname(hostname)) return ["127.0.0.1", "[::1]"]; const family = recoveryLoopbackFamily(hostname); if (family) return [probeHostname(hostname), family === 6 ? "127.0.0.1" : "[::1]"]; return [probeHostname(hostname)]; } /** * Loopback addresses to ask about a port whose holder must be identified before a * decision that is destructive when it answers "nobody". * * A listener and a probe can disagree about what "loopback" means, and on Windows they * do. `startServer` canonicalizes a `localhost` bind to 127.0.0.1 precisely because * Windows resolves the name IPv6-first (src/server/index.ts), while `probeHostname` * hands the literal name back and leaves the family choice to the resolver. A probe that * lands on `::1` while the listener holds `127.0.0.1` reports an empty port that another * process is demonstrably serving. Both literal addresses are therefore asked, cheapest * question first: the extra one costs a refused connection, and skipping it costs a * duplicate proxy. * * A non-loopback bind (a LAN address, a named host) gets exactly one candidate — the * address it was configured with. Guessing another interface for it would answer a * different question than the caller asked. */ export function loopbackProbeHosts(hostname: string | undefined): string[] { const normalized = normalizeBindHostname(hostname); const primary = probeHostname(normalized); if (isLoopbackHostname(normalized) || isWildcardHostname(normalized)) { return primary === "[::1]" ? ["[::1]", "127.0.0.1"] : ["127.0.0.1", "[::1]"]; } return [primary]; } /** * Identity-checked answer to "who holds this exact port", independent of the pid file * and the runtime-port record. * * `findLiveProxy` answers "is a proxy of this home alive", and it can only do that from * recorded state plus the configured port. This answers the narrower question a start * has to ask before it walks away from a busy port: an opencodex listening THERE, right * now, whatever this home's records say about it. Returns null only after every * candidate address has failed the identity check with the caller's full probe budget. */ export async function probePortOwner( port: number, opts: { hostname?: string } = {}, io: LivenessIo = {}, ): Promise<{ pid: number | null; hostname: string; version?: string; role?: string } | null> { for (const hostname of loopbackProbeHosts(opts.hostname)) { const identity = await proxyIdentityAt(port, { hostname }, io); if (identity) return { ...identity, hostname }; } return null; } // ───────────────────────────────────────────────────────────────────────────── // Readiness (/readyz) strict probe. // // Liveness (/healthz) confirms the process answers; readiness confirms the // post-startup Codex sync has settled. A readiness probe is identity-checked the // same way liveness is, AND additionally enforces the full /readyz contract so // an adversarial or malformed body can never count as ready: // // - HTTP 200 is required for status="ready"; HTTP 503 is required for pending // or failed. Any other HTTP/body-status pairing is an invalid contract. // - body.service must be exactly "opencodex". // - body.version must be a non-empty string. // - body.uptime must be a finite nonnegative number. // - body.pid must be a positive integer; when `expectedPid` is supplied it must // match exactly. // - body.port must be an integer in 1..65535 and equal the probed port. // - body.status must be exactly one of pending|ready|failed. // // Any unreachable, foreign, legacy, malformed, mismatched, or self-inconsistent // response returns `null` so callers can never treat an invalid identity/contract // as ready. // ───────────────────────────────────────────────────────────────────────────── interface ReadyzBody { service?: unknown; version?: unknown; uptime?: unknown; pid?: unknown; port?: unknown; status?: unknown; // Remote protocol metadata is intentionally additive here. Ordinary // readiness remains compatible with legacy standalone servers; `ocx connect` // validates these fields separately in src/remote/protocol.ts. protocol?: unknown; minimumClientProtocol?: unknown; managementUrl?: unknown; } export interface ReadinessProbeResult { /** True ONLY for a valid 200 + status="ready" body with a matching pid. */ ready: boolean; /** Fixed sanitized status. A foreign/unreadable body yields a `null` RESULT, never a `null` status. */ status: "ready" | "pending" | "failed"; /** Positive integer pid from a valid body. */ pid: number; /** Integer port from a valid body. */ port: number; } export interface ReadinessProbeIo { fetchFn?: typeof fetch; timeoutMs?: number; } const READYZ_STATUS_VALUES = new Set<"ready" | "pending" | "failed">(["ready", "pending", "failed"]); /** * Validate a parsed /readyz body against the strict contract. Returns the * sanitized probe result, or `null` when the body is foreign, legacy, * malformed, or fails the pid/port checks. Pure (no I/O) so it is fully * deterministic and unit-testable. */ export function validateReadyzBody( body: unknown, port: number, opts: { expectedPid?: number } = {}, ): ReadinessProbeResult | null { if (!body || typeof body !== "object") return null; const b = body as ReadyzBody; if (b.service !== "opencodex") return null; if (typeof b.version !== "string" || b.version.length === 0) return null; if (typeof b.uptime !== "number" || !Number.isFinite(b.uptime) || b.uptime < 0) return null; if (typeof b.pid !== "number" || !Number.isInteger(b.pid) || b.pid <= 0) return null; if ( typeof b.port !== "number" || !Number.isInteger(b.port) || b.port < 1 || b.port > 65535 || b.port !== port ) return null; if (typeof b.status !== "string" || !READYZ_STATUS_VALUES.has(b.status as "ready" | "pending" | "failed")) return null; const status = b.status as "ready" | "pending" | "failed"; if (opts.expectedPid !== undefined && b.pid !== opts.expectedPid) return null; return { ready: status === "ready", status, pid: b.pid, port: b.port }; } /** * Identity- and contract-checked /readyz probe. Returns `null` when the * endpoint is unreachable or the body fails the strict contract (foreign 200, * legacy health-only body, non-JSON, missing/malformed/mismatched fields, * wrong port/pid, or an HTTP/body-status inconsistency). Returns * `{ready:false, ...}` when the body is ours but pending or failed. Returns * `{ready:true, ...}` ONLY for a valid 200 body with `status:"ready"` and (when * requested) a matching pid. */ export async function probeReadiness( port: number, opts: { hostname?: string; expectedPid?: number } = {}, io: ReadinessProbeIo = {}, ): Promise { const fetchFn = io.fetchFn ?? directLocalHttpFetch; try { const res = await fetchFn(`http://${probeHostname(opts.hostname)}:${port}/readyz`, { signal: AbortSignal.timeout(io.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS), }); // Parse even on 503: /readyz returns JSON with a sanitized status while pending. const body = (await res.json().catch(() => null)) as unknown; const parsed = validateReadyzBody(body, port, opts); if (!parsed) return null; // HTTP/body-status consistency: ready requires 200; pending/failed require 503. if (parsed.status === "ready" && res.status !== 200) return null; if (parsed.status !== "ready" && res.status !== 503) return null; return parsed; } catch { return null; } }