import { deviceHubInputShadowedMessage, readDeviceHubInputShadowed } from "./device-hub-input"; import { axTreeHasSoftwareKeyboard, readHardwareKeyboardState, writeHardwareKeyboardState } from "./hardware-keyboard-state"; import { isModifierUsage } from "./keyboard-modifiers"; import { DuoRenderer, DuoRendererError, logDuoRendererFailure } from "./duo-renderer"; import { DuoPreview, type DuoRendererEvent } from "./duo-preview"; import { redactHome } from "./diagnostics-redact"; /** * In-process device session — the replacement for the spawned serve-sim-bin * helper. One session per booted simulator owns a NativeCapture + NativeHid and * serves the same wire endpoints the helper's HTTP server did, byte-for-byte: * * /stream.mjpeg multipart/x-mixed-replace JPEG fan-out (?raw=1 → octet-stream) * /stream.avcc length-prefixed AVCC envelopes (seed + decoder config replay) * /ws binary HID input protocol ([tag][JSON]) → NativeHid * /config { width, height, orientation } * /health { status: "ok" } * /ax axe-shaped accessibility JSON (one-shot) * /foreground { bundleId, pid } * * Replaces the helper's HTTP/client layer; the framing here mirrors the * original byte-for-byte so the existing browser client is unchanged. * Duo's 3D preview, fold clock, and rotation clock live on DuoPreview. */ import { DuoStateMonitor, type DuoState } from "./duo-state"; import type { IncomingMessage, ServerResponse } from "http"; import { displaySizeForPanel, duoPanel, resolveDevicePose, type DuoPanel } from "./device-pose"; import { NativeCapture, NativeHid, Orientation, axDescribeAsync, axFrontmostAsync, duoHidHelperPath, type MjpegFrame, } from "./native"; import { deviceSupportsHomeIndicatorFlick } from "./devicekit-chrome"; import { simulatorDeviceInfo } from "./device"; import { createReadStream } from "fs"; import type { Readable } from "stream"; import { mkdir, rm, stat } from "fs/promises"; import { tmpdir } from "os"; import { basename, join } from "path"; import { randomUUID } from "crypto"; import { MAX_RECORDING_MS, RECORDING_HOLD_MS, RECORDING_SUMMARY_HEADER, parseRecordingQuery, recordingFileName, recordingRotation, } from "./recording"; import { collectAxSnapshot } from "./ax"; import { AxAgent, AxAgentError } from "./ax-agent"; import { serveAxAgent, type AxAgentEndpoint } from "./ax-agent-http"; import { cliInputOrientation, displayTouchToRaw } from "./touch-remap"; import { RECORDING_INDEX_CONTENT_TYPE, RecordingEventCollector, recordingClock, recordingIndexEvents, recordingIndexLines, recordingIndexPath, splitRecordingSummary, type NativeRecordingIndex, type RecordingIndexEvent, type RecordingIndexSummary, } from "./recording-index"; import { eventLogEventForHidMessage, formatEventLogPoint, recordEventLogEvent, subscribeEventLog, updateEventLogEvent, type EventLogEntry, } from "./event-log"; import { RollingBufferSampler, parseRollingBufferEnv, parseRollingBufferQuery, type RollingBufferOptions, } from "./rolling-buffer"; /** * Minimal WebSocket surface the HID input channel needs. Satisfied by both the * `ws` library and the raw-socket adapter the middleware uses under Bun (where * `ws`'s server-side handshake doesn't flush). Messages arrive as binary * `[tag][JSON]` frames; `send` writes a binary frame. */ export interface HidSocket { send(data: Buffer): void; on(event: "message", cb: (data: Buffer) => void): void; on(event: "close" | "error", cb: () => void): void; close(): void; } // AVCC seed tag (StreamFormat.AVCCEnvelope.seedTag). description/keyframe/delta // envelopes are framed natively; only the on-connect JPEG seed is built here. const AVCC_SEED_TAG = 0x04; // WS server→client screen-config push (ClientManager.wsMsgConfig). const WS_MSG_CONFIG = 0x82; // WS server→client input acknowledgement: [0x84]{ ack, ok }. Sent only for // input frames whose JSON carries a numeric `ack` id (the CLI's // tap/button/gesture/type), so the browser viewer's protocol is unchanged (#136). const WS_MSG_INPUT_ACK = 0x84; const ACKABLE_INPUT_TAGS = new Set([0x03, 0x04, 0x05, 0x06, 0x0a, 0x0b]); type ShadowedInputResult = { ok: false; error: string }; function shadowedInputResult(udid: string): ShadowedInputResult { return { ok: false, error: deviceHubInputShadowedMessage(udid) }; } function isShadowedInputResult(result: unknown): result is ShadowedInputResult { return typeof result === "object" && result != null && (result as ShadowedInputResult).ok === false && typeof (result as ShadowedInputResult).error === "string"; } /** The CLI's `ack` id on an input frame, if it asked for one. */ function inputAckId(frame: Buffer): number | undefined { if (!ACKABLE_INPUT_TAGS.has(frame[0]!) || !frame.includes('"ack"', 1)) return undefined; try { const ack = (JSON.parse(frame.subarray(1).toString("utf8")) as { ack?: unknown }).ack; return typeof ack === "number" && Number.isFinite(ack) ? ack : undefined; } catch { return undefined; } } function withoutAck(message: T): Omit { const { ack: _ack, ...rest } = message; return rest; } const MJPEG_TRAILER = Buffer.from("\r\n", "ascii"); // Native capture is event-driven, so an unchanged screen can legitimately // produce no new JPEGs. Replay the already-encoded frame below the client's // 2s liveness window instead of re-encoding identical pixels in Swift. const MJPEG_IDLE_REPLAY_MS = 1_000; const TOUCH_TAP_MAX_DISTANCE = 0.004; type TouchGestureLog = { eventId?: number; startX: number; startY: number; lastX: number; lastY: number; moveCount: number; edge?: number; }; function touchGestureSummary(gesture: TouchGestureLog): string { return `Drag ${formatEventLogPoint(gesture.startX, gesture.startY)} -> ${formatEventLogPoint(gesture.lastX, gesture.lastY)}`; } function touchGestureMoved(gesture: TouchGestureLog): boolean { const dx = gesture.lastX - gesture.startX; const dy = gesture.lastY - gesture.startY; return Math.hypot(dx, dy) > TOUCH_TAP_MAX_DISTANCE; } function newTouchGesture(payload: { x: number; y: number; edge?: number }): TouchGestureLog { return { startX: payload.x, startY: payload.y, lastX: payload.x, lastY: payload.y, moveCount: 0, edge: payload.edge, }; } // One AX dump takes ~0.4 s; past this, fall back rather than stall the toggle. const SOFTWARE_KEYBOARD_PROBE_TIMEOUT_MS = 2_000; async function probeSoftwareKeyboardVisible(udid: string): Promise { try { return axTreeHasSoftwareKeyboard(JSON.parse(await axDescribeAsync(udid))); } catch { return null; } } function withTimeout(promise: Promise, ms: number): Promise { let timer: ReturnType | undefined; return Promise.race([ promise, new Promise((resolve) => { timer = setTimeout(() => resolve(null), ms); }), ]).finally(() => clearTimeout(timer)); } function mjpegHeader(jpegLength: number): Buffer { return Buffer.from(`--frame\r\nContent-Type: image/jpeg\r\nContent-Length: ${jpegLength}\r\n\r\n`, "ascii"); } function avccSeed(jpeg: Uint8Array): Buffer { const out = Buffer.allocUnsafe(5 + jpeg.length); out.writeUInt32BE(jpeg.length + 1, 0); // length covers the tag byte + payload out[4] = AVCC_SEED_TAG; out.set(jpeg, 5); return out; } const ORIENTATION_BY_NAME: Record = { portrait: Orientation.portrait, portrait_upside_down: Orientation.portraitUpsideDown, landscape_left: Orientation.landscapeLeft, landscape_right: Orientation.landscapeRight, }; function waitForDrain(res: ServerResponse): Promise { if (res.writableEnded || res.destroyed || !res.writableNeedDrain) return Promise.resolve(); return new Promise((resolve) => { const done = () => { cleanup(); resolve(); }; const cleanup = () => { res.off("drain", done); res.off("close", done); res.off("error", done); }; res.once("drain", done); res.once("close", done); res.once("error", done); }); } async function writeAndDrain( res: ServerResponse, write: () => boolean, ): Promise { await waitForDrain(res); if (res.writableEnded || res.destroyed) return false; const accepted = write(); if (!accepted || res.writableNeedDrain) await waitForDrain(res); return !res.writableEnded && !res.destroyed; } type CaptureTransport = Pick< NativeCapture, "start" | "stop" | "subscribeMjpeg" | "subscribeAvcc" | "subscribeSize" | "snapshotJpeg" | "setPreferredScreenSize" > & Partial>; export type StreamEndpoint = "mjpeg" | "avcc" | "3d"; type StreamHealth = { open: number; frames: number; last_frame_at: string | null; last_error: { at: string; message: string; stage?: string } | null; }; function newStreamHealth(): StreamHealth { return { open: 0, frames: 0, last_frame_at: null, last_error: null }; } /** 503 body for a Duo 3D stream that cannot produce a first frame. */ function duoRendererErrorBody(error: DuoRendererError): { error: "duo_renderer_unavailable"; reason: string; stage: string; details: Record } { return { error: "duo_renderer_unavailable", reason: redactHome(error.reason), stage: error.stage, details: redactHome(error.details) }; } /** Delivery-reporting sends resolve true only when the event reached the guest. */ type ReportsDelivery = F extends (...args: infer A) => Promise ? (...args: A) => Promise : never; type HidTransport = { [K in "touch" | "multiTouch" | "button" | "buttonHid" | "key" | "scroll" | "digitalCrown"]: ReportsDelivery; } & Pick< NativeHid, | "orientation" | "memoryWarning" | "softwareKeyboard" | "caDebug" | "hardwareKeyboard" | "pose" | "hinge" > & Partial>; export interface DeviceSessionDependencies { capture: CaptureTransport; hid: HidTransport; createDuoRenderer?: (onFailure: (error: DuoRendererError) => void) => Pick; /** `duoHelper` resolves the launchable guest helper (may run `codesign`); call it lazily. */ createDuoMonitor?: (udid: string, onState: (state: DuoState) => void, duoHelper: () => string) => Pick; /** Server-side recording cap; defaults to `MAX_RECORDING_MS`. */ maxRecordingMs?: number; /** How long a capped file is held for its owner; defaults to `RECORDING_HOLD_MS`. */ recordingHoldMs?: number; /** Per-boot hardware keyboard memory; defaults to files in `STATE_DIR`. */ hardwareKeyboardState?: { read: (udid: string) => boolean | null; write: (udid: string, enabled: boolean) => void; }; /** Whether the software keyboard is on screen, or null if unknown; defaults to an AX probe. */ softwareKeyboardVisible?: (udid: string) => Promise; /** Raw axe-shaped AX JSON; defaults to the native bridge (`axDescribeAsync`). */ axDescribe?: (udid: string) => Promise; /** * How long a finished recording's index is kept after its MP4 has been * sent; defaults to `RECORDING_INDEX_TTL_MS`. */ recordingIndexTtlMs?: number; /** Event-log subscription for indexed recordings; defaults to `subscribeEventLog`. */ subscribeEventLog?: (subscriber: (entry: EventLogEntry) => void) => () => void; /** Stream the finished MP4 to the client; defaults to `createReadStream`. */ openRecordingFile?: (path: string) => Readable; /** * Rolling buffer started with the session: options, or null for off. * Defaults to `SERVE_SIM_BUFFER` (`serve-sim --buffer`), which is unset/off. */ rollingBuffer?: RollingBufferOptions | null; /** * Guest Device Hub input-shadow flag (`com.apple.coredevice.dtuhidd.active`). * Defaults to a live `notifyutil` probe on production sessions. Tests inject * this so they never spawn simctl. */ readDeviceHubInputShadowed?: (udid: string) => Promise; /** Clock for the shadow-flag TTL; defaults to `Date.now`. */ now?: () => number; } type ActiveRecording = { path: string; startedAt: string; /** * Returned by start; only its holder may collect a file held at the cap, * until the hold expires. */ token: string; phase: "starting" | "recording" | "stopping"; /** Native finalize, started by `/recording/stop` or the duration cap. */ finalizing?: Promise; limitTimer?: ReturnType; /** When the duration cap's finalize settled (epoch ms); starts the hold. */ heldAt?: number; /** Set once the hold lapses: any client may collect, a new start discards. */ holdExpired?: boolean; holdTimer?: ReturnType; /** This device's HID events while recording; set only for `index=1` starts. */ events?: RecordingEventCollector; unsubscribeEvents?: () => void; }; /** A finished recording's index, served by `GET /recording/index?token=`. */ type StoredRecordingIndex = { fileName: string; summary: RecordingIndexSummary; index: NativeRecordingIndex; events: RecordingIndexEvent[]; eventsDropped: number; /** * Deletes this entry once the TTL lapses (unref'd). Armed when the MP4 * has finished streaming to the client, so a slow download doesn't eat it. */ expiry?: ReturnType; }; /** Finished indexes kept per session, and for how long. */ const MAX_STORED_RECORDING_INDEXES = 4; const RECORDING_INDEX_TTL_MS = 30 * 60_000; type Unsubscribe = () => void | Promise; export class DeviceSession { private readonly capture: CaptureTransport; private readonly hid: HidTransport; private unsubscribeMjpeg?: Unsubscribe; private unsubscribeSize?: Unsubscribe; private phase: "unstarted" | "starting" | "running" | "failed" | "stopped" = "unstarted"; private startPromise?: Promise; private width = 0; private height = 0; private orientation = "portrait"; /** Guest UI orientation from a confirmed 0x07 or the Duo monitor. Unset * until then — a default of "portrait" must not authorize a flick. */ private knownOrientation?: string; /** True only after chrome says Face ID and the device is not foldable. */ private homeIndicatorFlickSupported?: boolean; private latestJpegBuffer: Buffer | null = null; private latestJpegLength = 0; private readonly hidSockets = new Set(); private readonly duo: DuoPreview; private duoMonitor?: Pick; private followedCover?: boolean; private primaryDuoPanel?: DuoPanel; private poseQueue: Promise = Promise.resolve(); // The guest has no getter, and initial visibility after boot isn't // predictable from earlier boots. An explicit set lasts for the boot, so it // is remembered per boot on disk. Otherwise the AX tree tells whether the // software keyboard is on screen; forwarded typing is the last resort. private forwardedTypingKey = false; private hardwareKeyboardQueue: Promise = Promise.resolve(); private duoStateQueue: Promise = Promise.resolve(); private readonly streamResponses = new Set(); private touchGestureLog?: TouchGestureLog; private readonly streamHealth: Record = { mjpeg: newStreamHealth(), avcc: newStreamHealth(), "3d": newStreamHealth(), }; private deviceHubShadowed?: boolean; /** Epoch ms of the last shadow-flag probe; used to re-check with a short TTL. */ private deviceHubShadowedAt = 0; private deviceHubShadowWarned = false; /** Set once a probe has finished, even if `now()` was 0. */ private deviceHubShadowSettled = false; /** Single-flight probe; reused so a burst of events cannot spawn overlapping simctl. */ private deviceHubProbe?: Promise; /** Tests inject `hid`; production constructs `NativeHid` and must probe live. */ private readonly usesInjectedHid: boolean; /** Active screen recording (one per session), or a start/stop in flight. */ private recording?: ActiveRecording; /** Indexes of collected recordings, by start token (oldest first). */ private readonly recordingIndexes = new Map(); private rendererFailureEvent?: { id: number; key: string; count: number }; /** Ref session for `/ax/snapshot|act|settle`; created on first use. */ private axAgent?: AxAgent; /** Opt-in rolling frame buffer; created only when first started. */ private rollingBuffer?: RollingBufferSampler; constructor(public readonly udid: string, private readonly dependencies?: DeviceSessionDependencies) { this.usesInjectedHid = dependencies?.hid != null; this.hid = dependencies?.hid ?? new NativeHid(udid, { getOrientation: () => this.knownOrientation, isInputShadowed: () => this.deviceHubShadowed !== false, supportsHomeIndicator: () => this.homeIndicatorFlickSupported === true, }); if (!this.usesInjectedHid) void this.warmHomeIndicatorSupport(); this.capture = dependencies?.capture ?? new NativeCapture(udid); this.duo = new DuoPreview({ createRenderer: (onFailure) => dependencies?.createDuoRenderer ? dependencies.createDuoRenderer(onFailure) : new DuoRenderer(onFailure), onProjection: (frame) => { for (const ws of this.hidSockets) ws.send(frame); }, onStreamError: (res, error) => this.handleStreamError(res, error, "3d"), onRendererEvent: (event) => this.recordRendererEvent(event), onFrameDelivered: () => this.noteStreamFrame("3d"), }); } /** * Home button: the same path as `serve-sim button home`. Leaves a * foreground app via SpringBoardLauncher (#192 direct launch, #208 * timeout, then the launcher's simctl fallback). When SpringBoard is * already frontmost (App Library, folders, inner pages), uses a short * home-indicator flick because the launch is a no-op there. */ async pressHome(): Promise { return (await this.hid.button("home")) !== false; } /** * Begin capture and track the screen size. Idempotent. * * Only foldables keep a shared per-frame JPEG subscription (the Duo 3D * preview renders from it). Everything else encodes JPEGs only for MJPEG * viewers, and seeds new viewers from an on-demand snapshot: a shared * subscription JPEG-encodes every frame at full resolution on the CPU, which * cost ~6 ms per frame (p95 ~19 ms) and contended with the H.264 path even * when every viewer was on AVCC. */ start(): Promise { if (this.phase === "stopped") { return Promise.reject(new Error(`Device session ${this.udid} is closed`)); } if (this.startPromise) return this.startPromise; this.phase = "starting"; const startPromise = (async () => { await this.capture.start(); if (this.isStopped()) throw new Error(`Device session ${this.udid} closed while starting`); const unsubscribeSize = await this.capture.subscribeSize((size) => this.onScreenSize(size.width, size.height)); if (this.isStopped()) { await unsubscribeSize(); throw new Error(`Device session ${this.udid} closed while starting`); } this.unsubscribeSize = unsubscribeSize; this.phase = "running"; // Do not await: a wedged notifyutil must not delay the first stream frame. void this.refreshDeviceHubInput(); void this.warmHomeIndicatorSupport(); const bufferDefault = this.dependencies?.rollingBuffer !== undefined ? this.dependencies.rollingBuffer : parseRollingBufferEnv(); if (bufferDefault && this.supportsBuffer()) this.startBuffer(bufferDefault); if (await this.hid.isFoldable?.() && !this.isStopped()) { const unsubscribe = await this.capture.subscribeMjpeg((frame) => this.onSharedMjpegFrame(frame)); if (this.isStopped()) { await unsubscribe(); return; } this.unsubscribeMjpeg = unsubscribe; const createMonitor = this.dependencies?.createDuoMonitor ?? ((udid, onState, duoHelper) => new DuoStateMonitor(udid, onState, { duoHelper })); this.duoMonitor = createMonitor(this.udid, (state) => { void this.followDuoState(state).catch((error) => { console.error("[duo] State update failed:", error); }); }, duoHidHelperPath); } })().catch((error) => { // Failed DeviceSession instances are evicted by the registry below. Keep // this promise latched so concurrent endpoints all observe the same // failure instead of racing a second start on an instance being closed. if (this.phase === "starting") this.phase = "failed"; throw error; }); // A session is started eagerly by the registry, before an HTTP stream has // necessarily awaited it. Attach a rejection observer immediately so a // shutdown race can never become a process-fatal unhandled rejection. void startPromise.catch(() => {}); this.startPromise = startPromise; return startPromise; } private isStopped(): boolean { return this.phase === "stopped"; } close(): void { if (this.phase === "stopped") return; this.phase = "stopped"; this.duo.close(); this.duoMonitor?.close(); for (const ws of this.hidSockets) { this.runCleanup("HID socket close", () => ws.close()); } for (const res of this.streamResponses) { this.runCleanup("stream response close", () => { res.destroy(); }); } this.streamResponses.clear(); if (this.unsubscribeMjpeg) { this.runCleanup("shared MJPEG unsubscribe", this.unsubscribeMjpeg); } if (this.unsubscribeSize) { this.runCleanup("size unsubscribe", this.unsubscribeSize); } this.hidSockets.clear(); this.rollingBuffer?.stop(); // Capture stop cancels the native recorder; drop the partial temp file. // A stop in flight owns its file (it's streaming it or cleaning it up). if (this.recording) { const active = this.recording; this.recording = undefined; if (active.limitTimer) clearTimeout(active.limitTimer); if (active.holdTimer) clearTimeout(active.holdTimer); stopCollectingEvents(active); if (active.phase !== "stopping") { const finalized = active.finalizing?.catch(() => {}) ?? Promise.resolve(); this.runCleanup("recording cleanup", () => finalized.then(() => rm(active.path, { force: true }))); } } for (const stored of this.recordingIndexes.values()) clearTimeout(stored.expiry); this.recordingIndexes.clear(); this.runCleanup("HID close", () => this.hid.close?.()); this.runCleanup("capture stop", () => this.capture.stop()); } private followDuoState(state: DuoState): Promise { const update = this.duoStateQueue.then(() => this.applyDuoState(state)); this.duoStateQueue = update.catch(() => {}); return update; } private activePanel(): DuoPanel { return duoPanel({ primaryPanel: this.primaryDuoPanel, hingeDegrees: this.duo.hinge }); } private async applyDuoState(state: DuoState): Promise { if (this.isStopped()) return; if (state.hingeDegrees != null) this.duo.noteHinge(state.hingeDegrees); if (state.primaryPanel) this.primaryDuoPanel = state.primaryPanel; const panel = this.activePanel(); const size = displaySizeForPanel(panel); if ((state.hingeDegrees != null || state.primaryPanel) && (panel === "cover") !== this.followedCover) { await this.capture.setPreferredScreenSize(size.width, size.height); this.followedCover = panel === "cover"; } const orientation = state.orientations[panel]; if (orientation) { this.orientation = orientation; this.knownOrientation = orientation; } this.duo.setPanel(panel); this.broadcastConfig(); this.duo.requestFrame(); } // ── Frame handling ─────────────────────────────────────────────────────── private onScreenSize(width: number, height: number): void { if (width === this.width && height === this.height) return; this.width = width; this.height = height; this.broadcastConfig(); } private async onSharedMjpegFrame(frame: MjpegFrame): Promise { const { width, height, data: jpeg } = frame; this.onScreenSize(width, height); if (!this.latestJpegBuffer || this.latestJpegBuffer.length < jpeg.length) { const currentCapacity = this.latestJpegBuffer?.length ?? 0; this.latestJpegBuffer = Buffer.allocUnsafe(Math.max(jpeg.length, currentCapacity * 2)); } this.latestJpegBuffer.set(jpeg, 0); this.latestJpegLength = jpeg.length; const cached = this.latestJpeg(); if (cached) this.duo.onCapturedFrame(cached); } private latestJpeg(): Buffer | null { if (!this.latestJpegBuffer) return null; return this.latestJpegBuffer.subarray(0, this.latestJpegLength); } /** A private copy of the shared JPEG cache, kept current only on foldables. */ private sharedJpegCopy(): Buffer | null { const cached = this.unsubscribeMjpeg ? this.latestJpeg() : null; return cached ? Buffer.from(cached) : null; } /** * A private JPEG of the current screen for a new AVCC viewer: the shared * cache on foldables, else a one-shot native snapshot (copy + CPU encode). */ private async seedJpeg(): Promise { const cached = this.sharedJpegCopy(); if (cached) return cached; try { return await this.capture.snapshotJpeg(); } catch { return null; } } /** Write a multipart JPEG part (header + shared frame + boundary) without copying the JPEG. */ private writeMjpegFrame(res: ServerResponse, jpeg: Uint8Array): boolean { const headerAccepted = res.write(mjpegHeader(jpeg.length)); const frameAccepted = res.write(jpeg); const trailerAccepted = res.write(MJPEG_TRAILER); return headerAccepted && frameAccepted && trailerAccepted; } // ── HTTP handlers ──────────────────────────────────────────────────────── handleDuoMjpeg(req: IncomingMessage, res: ServerResponse): void { void (async () => { await this.start(); if (res.destroyed || res.writableEnded) return; if (!await this.hid.isFoldable?.()) { this.sendJson(res, 400, { error: "This device has no Duo model" }); return; } const raw = new URL(req.url ?? "", "http://x").searchParams.get("raw") === "1"; if (!this.trackStreamResponse(res, "3d")) return; this.duo.setPanel(this.activePanel()); res.once("close", () => this.duo.detach(res)); // Headers wait for the first PNG. Model lookup, worker spawn, and the // first render can all fail; with headers unsent each failure becomes a // 503 with a reason instead of an empty socket (ERR_EMPTY_RESPONSE). this.duo.attach(res, { onFirstFrame: () => { res.writeHead(200, { "Content-Type": raw ? "application/octet-stream" : "multipart/x-mixed-replace; boundary=frame", "Cache-Control": "no-store" }); }, }); })().catch((error) => this.handleStreamError(res, error, "3d")); } handleMjpeg(req: IncomingMessage, res: ServerResponse): void { void this.serveMjpeg(req, res).catch((error) => this.handleStreamError(res, error, "mjpeg")); } private async serveMjpeg(req: IncomingMessage, res: ServerResponse): Promise { await this.start(); if (res.writableEnded || res.destroyed) return; const raw = new URL(req.url ?? "", "http://x").searchParams.get("raw") === "1"; res.writeHead(200, { "Content-Type": raw ? "application/octet-stream" : "multipart/x-mixed-replace; boundary=frame", "Cache-Control": "no-cache, no-store", Connection: "keep-alive", }); if (!this.trackStreamResponse(res, "mjpeg")) return; // The native subscriber reuses its frame buffer after this callback // resolves. Keep the callback pending through HTTP backpressure so the // bytes cannot be mutated before res.write consumes them. CaptureEngine's // bufferingNewest(1) policy drops stale frames on the native side while a // slow client drains, so this neither interleaves parts nor grows a queue. let writeInFlight = false; let lastSentAt = 0; // This viewer's last frame, replayed while the screen is static. let replay: Buffer | null = null; let replayLength = 0; const remember = (jpeg: Uint8Array) => { if (!replay || replay.length < jpeg.length) replay = Buffer.allocUnsafe(Math.max(jpeg.length, (replay?.length ?? 0) * 2)); replay.set(jpeg, 0); replayLength = jpeg.length; }; const writeFrame = async (jpeg: Uint8Array): Promise => { // This can only overlap when the idle replay timer wins the event-loop // turn immediately before a native frame. Prefer the in-flight write and // let the next native frame provide the freshest image. if (writeInFlight || res.writableEnded || res.destroyed) return; writeInFlight = true; try { if (await writeAndDrain(res, () => this.writeMjpegFrame(res, jpeg))) { lastSentAt = Date.now(); this.noteStreamFrame("mjpeg"); } } catch (error) { if (!res.destroyed) { res.destroy(error instanceof Error ? error : new Error(String(error))); } } finally { writeInFlight = false; } }; // Subscribing forces an immediate native frame, so only foldables (whose // shared cache is already encoded) send a seed. The copy is private, safe // to hold across backpressure; subscriber frames below remain zero-copy. const seed = this.sharedJpegCopy(); if (seed) { remember(seed); await writeFrame(seed); } const unsubscribe = await this.capture.subscribeMjpeg(async (frame) => { remember(frame.data); await writeFrame(frame.data); }); const idleReplay = setInterval(() => { if (res.writableEnded || res.destroyed) return; if (Date.now() - lastSentAt < MJPEG_IDLE_REPLAY_MS) return; // The replay buffer is reused in place as fresh frames arrive. Copy only // this low-frequency replay so it stays immutable across backpressure. if (replay) void writeFrame(Buffer.from(replay.subarray(0, replayLength))); }, MJPEG_IDLE_REPLAY_MS); this.bindSubscription(res, async () => { clearInterval(idleReplay); await unsubscribe(); }); } handleAvcc(_req: IncomingMessage, res: ServerResponse): void { void this.serveAvcc(res).catch((error) => this.handleStreamError(res, error, "avcc")); } private async serveAvcc(res: ServerResponse): Promise { await this.start(); if (res.writableEnded || res.destroyed) return; res.writeHead(200, { "Content-Type": "application/octet-stream", "Cache-Control": "no-cache, no-store", Connection: "keep-alive", }); if (!this.trackStreamResponse(res, "avcc")) return; // Seed with the current screen while the per-client native AVCC // subscription (which forces an immediate frame) starts its own decoder // config and keyframe. The seed is fetched concurrently and dropped if // H.264 got there first — painting it then would rewind the picture. let streaming = false; void this.seedJpeg().then((seed) => { if (seed && !streaming && !res.writableEnded && !res.destroyed) res.write(avccSeed(seed)); }); const unsubscribe = await this.capture.subscribeAvcc(async (frame) => { streaming = true; await writeAndDrain(res, () => res.write(frame.data)); this.noteStreamFrame("avcc"); }); this.bindSubscription(res, unsubscribe); } handleConfig(_req: IncomingMessage, res: ServerResponse): void { this.sendJson(res, 200, this.screenConfig()); } handleHealth(_req: IncomingMessage, res: ServerResponse): void { this.sendJson(res, 200, { status: "ok" }); } handleAx(_req: IncomingMessage, res: ServerResponse): Promise { return this.serveAxJson(res, () => axDescribeAsync(this.udid), "ax_unavailable"); } /** `/ax/snapshot`, `/ax/act`, `/ax/settle` — see ax-agent-http.ts. */ handleAxAgent(req: IncomingMessage, res: ServerResponse, endpoint: AxAgentEndpoint): Promise { this.axAgent ??= new AxAgent({ collect: () => collectAxSnapshot(this.udid, this.dependencies?.axDescribe ?? ((udid) => axDescribeAsync(udid, "agent"))), contentFrames: () => this.contentFrameCount(), tap: (point, holdMs) => this.injectTap(point, holdMs), }); return serveAxAgent(this.axAgent, req, res, endpoint); } /** * Native count of captured frames whose IOSurface content changed (idle * re-emits excluded); null when the addon predates it or capture is down. * Polled every 20-50 ms while settling, so it reads only the counter, * never the doctor diagnostics payload. */ private async contentFrameCount(): Promise { if (!this.capture.contentChangeCount || this.phase !== "running") return null; try { const count = this.capture.contentChangeCount(); return typeof count === "number" ? count : null; } catch { return null; } } /** * Tap at normalized display coords through the same HID path (and event * log) as a `/ws` touch, remapped like `serve-sim tap` for rotated UIs. */ private async injectTap(point: { x: number; y: number }, holdMs: number): Promise { await this.start(); if (await this.legacyInputShadowed()) { throw new AxAgentError(503, "input_shadowed", deviceHubInputShadowedMessage(this.udid)); } if (this.width === 0 || this.height === 0) throw new Error("Screen size is not known yet; retry in a moment."); const orientation = cliInputOrientation(this.screenConfig()); const touch = async (type: "begin" | "end") => { const raw = displayTouchToRaw({ type, x: point.x, y: point.y }, orientation); this.recordTouchEvent(raw); await this.hid.touch(type, raw.x, raw.y, this.width, this.height, 0); }; await touch("begin"); await new Promise((resolve) => setTimeout(resolve, holdMs)); await touch("end"); } handleForeground(_req: IncomingMessage, res: ServerResponse): Promise { return this.serveAxJson(res, () => axFrontmostAsync(this.udid), "foreground_unavailable"); } /** Run a native AX probe and stream its JSON, or 503 with `errorCode` if it's not ready. */ private async serveAxJson(res: ServerResponse, probe: () => Promise, errorCode: string): Promise { try { const json = await probe(); if (res.writableEnded) return; this.sendJsonString(res, 200, json); } catch (err) { if (res.writableEnded) return; this.sendJson(res, 503, { error: errorCode, message: err instanceof Error ? err.message : String(err), }); } } // ── Rolling buffer ─────────────────────────────────────────────────────── private supportsBuffer(): boolean { return typeof this.capture.thumbnailJpeg === "function" && this.capture.supportsThumbnails !== false; } private startBuffer(options: RollingBufferOptions): RollingBufferSampler { this.rollingBuffer ??= new RollingBufferSampler({ device: this.udid, contentChangeCount: () => { if (!this.capture.contentChangeCount) return null; const count = this.capture.contentChangeCount(); return typeof count === "number" ? count : null; }, thumbnail: (maxDimension, quality) => this.capture.thumbnailJpeg!(maxDimension, quality), subscribeEventLog: this.dependencies?.subscribeEventLog ?? subscribeEventLog, }); this.rollingBuffer.start(options); return this.rollingBuffer; } private bufferState(): { enabled: false } | { enabled: true; options: RollingBufferOptions; startedAt: string } { const sampler = this.rollingBuffer; const store = sampler?.store; if (!sampler || !store) return { enabled: false }; return { enabled: true, options: store.options, startedAt: new Date(sampler.startedAt!).toISOString() }; } /** * `GET /buffer` (status + recent frames/events), `POST /buffer/start`, * `POST /buffer/stop`, `GET /buffer/frame` (one JPEG). See rolling-buffer.ts. */ async handleBuffer(req: IncomingMessage, res: ServerResponse, action: "status" | "start" | "stop" | "frame"): Promise { const method = (req.method ?? "GET").toUpperCase(); const wantsPost = action === "start" || action === "stop"; if (wantsPost ? method !== "POST" : method !== "GET" && method !== "HEAD") { res.setHeader("Allow", wantsPost ? "POST" : "GET, HEAD"); this.sendJson(res, 405, { error: "method_not_allowed" }); return; } const params = new URL(req.url ?? "", "http://x").searchParams; try { if (action === "start") return await this.startBufferRequest(params, res); if (action === "stop") { this.rollingBuffer?.stop(); this.sendJson(res, 200, { enabled: false, stopped: true }); return; } if (action === "frame") return this.sendBufferFrame(params, res); return this.sendBufferView(params, res); } catch (error) { if (res.headersSent || res.writableEnded) { res.destroy(); return; } this.sendJson(res, 500, { error: "buffer_failed", message: redactHome(error instanceof Error ? error.message : String(error)) }); } } private async startBufferRequest(params: URLSearchParams, res: ServerResponse): Promise { if (!this.supportsBuffer()) { this.sendJson(res, 501, { error: "buffer_unsupported", message: "This native build has no thumbnails; rebuild serve-sim" }); return; } const options = parseRollingBufferQuery(params); if ("error" in options) { this.sendJson(res, 400, { error: "invalid_buffer_options", message: options.error }); return; } if (this.respondIfBufferRunning(options, res)) return; await this.start(); if (this.isStopped()) { this.sendJson(res, 409, { error: "session_closed", message: "The device session closed" }); return; } // While start() was pending, the session's env default or another // request may have started it: compare again so a caller never gets // other options than it asked for. if (this.respondIfBufferRunning(options, res)) return; this.startBuffer(options); this.sendJson(res, 200, this.bufferState()); } /** If the buffer is on, answer 200 `alreadyRunning` for the same options or 409 for others; false when off. */ private respondIfBufferRunning(options: RollingBufferOptions, res: ServerResponse): boolean { const state = this.bufferState(); if (!state.enabled) return false; const same = (Object.keys(options) as Array).every((k) => options[k] === state.options[k]); if (same) this.sendJson(res, 200, { ...state, alreadyRunning: true }); else this.sendJson(res, 409, { error: "buffer_running", message: "The buffer is already running with other options; stop it first", ...state }); return true; } private sendBufferView(params: URLSearchParams, res: ServerResponse): void { const store = this.rollingBuffer?.store; const sinceRaw = params.get("since"); const since = sinceRaw == null ? undefined : Number(sinceRaw); if (since !== undefined && (sinceRaw!.trim() === "" || !Number.isFinite(since) || since < 0)) { this.sendJson(res, 400, { error: "invalid_since", message: "since must be a number of seconds >= 0" }); return; } if (!store) { this.sendJson(res, 200, { enabled: false }); return; } if (params.get("stats") === "1") { this.sendJson(res, 200, { ...this.bufferState(), stats: { ...store.stats(), maxBytes: store.options.maxBytes } }); return; } const now = Date.now(); const view = store.view(now, since === undefined ? {} : { sinceMs: now - since * 1000 }); const { events, ...rest } = view; this.sendJson(res, 200, { ...this.bufferState(), stats: { ...store.stats(), maxBytes: store.options.maxBytes }, ...rest, ...(params.get("events") === "0" ? {} : { events }), }); } private sendBufferFrame(params: URLSearchParams, res: ServerResponse): void { const idRaw = params.get("id"); const agoRaw = params.get("ago"); const id = idRaw == null ? undefined : Number(idRaw); const ago = agoRaw == null ? undefined : Number(agoRaw); if ( (id !== undefined && ago !== undefined) || (id !== undefined && (!Number.isInteger(id) || id < 1)) || (ago !== undefined && (agoRaw!.trim() === "" || !Number.isFinite(ago) || ago < 0)) ) { this.sendJson(res, 400, { error: "invalid_frame_query", message: "Pass id= or ago== 0>, not both" }); return; } const store = this.rollingBuffer?.store; if (!store) { this.sendJson(res, 409, { error: "buffer_off", message: "The rolling buffer is off; start it with `serve-sim buffer start`" }); return; } const frame = id !== undefined ? store.frame(id) : ago !== undefined ? store.frameAt(Date.now() - ago * 1000) : store.latest(); if (!frame) { this.sendJson(res, 404, { error: "frame_not_found", message: id !== undefined ? `No frame ${id} in the buffer` : "No frame in the buffer at that time" }); return; } const header = JSON.stringify({ id: frame.id, at: new Date(frame.atMs).toISOString(), width: frame.width, height: frame.height, sourceWidth: frame.sourceWidth, sourceHeight: frame.sourceHeight, }); res.writeHead(200, { "Content-Type": "image/jpeg", "Content-Length": String(frame.data.length), "Cache-Control": "no-store", "X-Serve-Sim-Buffer-Frame": header, }); res.end(frame.data); } // ── Screen recording ───────────────────────────────────────────────────── /** * `limitReached`: the duration cap finalized the file; it stays reported as * recording until a `/recording/stop` collects it. `holdExpiresAt`: when * the start's token stops being required to collect it. */ recordingStatus(): { recording: boolean; startedAt?: string; limitReached?: boolean; holdExpiresAt?: string } { const active = this.recording; if (!active) return { recording: false }; const held = active.finalizing && active.phase === "recording"; return { recording: true, startedAt: active.startedAt, ...(held ? { limitReached: true } : {}), ...(held && active.heldAt != null ? { holdExpiresAt: new Date(active.heldAt + this.recordingHoldMs()).toISOString() } : {}), }; } private recordingHoldMs(): number { return this.dependencies?.recordingHoldMs ?? RECORDING_HOLD_MS; } /** Drop a capped file whose hold expired, freeing the device to record. */ private async discardExpiredHold(): Promise { const active = this.recording; if (!active || !active.holdExpired || active.phase !== "recording") return; this.recording = undefined; console.error(`[record] ${this.udid}: discarding the uncollected recording held at the duration limit`); await active.finalizing?.catch(() => {}); await rm(active.path, { force: true }).catch(() => {}); } /** * `GET /recording` (status), `POST /recording/start`, `POST /recording/stop`. * Stop streams the finished MP4 back as an attachment and deletes the temp * file, so the browser (or `serve-sim record`) decides where it's saved. */ async handleRecording(req: IncomingMessage, res: ServerResponse, action: "status" | "start" | "stop" | "index"): Promise { const method = (req.method ?? "GET").toUpperCase(); if (action === "status") { this.sendJson(res, 200, this.recordingStatus()); return; } if (action === "index") { if (method !== "GET" && method !== "HEAD") { res.setHeader("Allow", "GET, HEAD"); this.sendJson(res, 405, { error: "method_not_allowed" }); return; } await this.sendRecordingIndex(req, res); return; } if (method !== "POST") { res.setHeader("Allow", "POST"); this.sendJson(res, 405, { error: "method_not_allowed" }); return; } try { if (action === "start") await this.startRecording(req, res); else await this.stopRecording(req, res); } catch (error) { if (res.headersSent || res.writableEnded) { res.destroy(); return; } this.sendJson(res, 500, { error: "recording_failed", message: redactHome(error instanceof Error ? error.message : String(error)), }); } } private async startRecording(req: IncomingMessage, res: ServerResponse): Promise { if (!this.capture.startRecording || !this.capture.stopRecording || this.capture.supportsRecording === false) { this.sendJson(res, 501, { error: "recording_unsupported", message: "This native build cannot record" }); return; } const parsed = parseRecordingQuery(new URL(req.url ?? "", "http://x").searchParams); if ("error" in parsed) { this.sendJson(res, 400, { error: "invalid_recording_options", message: parsed.error }); return; } await this.discardExpiredHold(); if (this.recording) { this.sendJson(res, 409, { error: "already_recording", ...this.recordingStatus() }); return; } const dir = join(tmpdir(), "serve-sim-recordings"); const path = join(dir, `${this.udid}-${randomUUID()}.mp4`); const startedAt = new Date().toISOString(); const token = randomUUID(); const active: ActiveRecording = { path, startedAt, token, phase: "starting" }; this.recording = active; if (parsed.index) { // Collect from before the native start: its first frame (pts 0) can be // captured before startRecording resolves. The window filter drops // anything earlier. const events = new RecordingEventCollector(this.udid); active.events = events; active.unsubscribeEvents = (this.dependencies?.subscribeEventLog ?? subscribeEventLog)((entry) => events.add(entry)); } const closedMeanwhile = () => { stopCollectingEvents(active); // close() ran during the start: capture stop cancelled the recorder. this.runCleanup("recording cleanup", () => rm(path, { force: true })); this.sendJson(res, 409, { error: "session_closed", message: "The device session closed while the recording was starting" }); }; try { await this.start(); await mkdir(dir, { recursive: true }); await this.capture.startRecording(path, { ...parsed, rotation: recordingRotation(this.screenConfig()) }); } catch (error) { if (this.recording !== active) return closedMeanwhile(); this.recording = undefined; stopCollectingEvents(active); await rm(path, { force: true }).catch(() => {}); throw error; } if (this.recording !== active) return closedMeanwhile(); active.phase = "recording"; const maxMs = this.dependencies?.maxRecordingMs ?? MAX_RECORDING_MS; active.limitTimer = setTimeout(() => { if (this.recording !== active || active.phase !== "recording") return; console.error(`[record] ${this.udid}: reached the ${Math.round(maxMs / 60_000)} min limit; finalizing`); this.finalizeRecording(active).catch((error) => { console.error(`[record] ${this.udid}: finalize at the duration limit failed:`, error instanceof Error ? error.message : error); }).finally(() => this.holdForOwner(active)); }, maxMs); active.limitTimer.unref?.(); this.sendJson(res, 200, { recording: true, startedAt, token }); } /** * The cap finalized `active`: keep it for the start's token holder for a * bounded time. The token lives only in that client's memory (a tab, * `serve-sim record`), so if it went away the hold lapses rather than * stranding the device: then any client may collect the file, and a new * start discards it. */ private holdForOwner(active: ActiveRecording): void { if (this.recording !== active || active.phase !== "recording") return; active.heldAt = Date.now(); active.holdTimer = setTimeout(() => { active.holdTimer = undefined; if (this.recording !== active || active.phase !== "recording") return; active.holdExpired = true; console.error(`[record] ${this.udid}: the recording held at the duration limit is now available to any client`); }, this.recordingHoldMs()); active.holdTimer.unref?.(); } /** Finalize natively once, whether triggered by stop or the duration cap. */ private finalizeRecording(active: ActiveRecording): Promise { if (active.limitTimer) clearTimeout(active.limitTimer); active.limitTimer = undefined; stopCollectingEvents(active); active.finalizing ??= this.capture.stopRecording!(); return active.finalizing; } private async stopRecording(req: IncomingMessage, res: ServerResponse): Promise { const active = this.recording; if (!active || active.phase !== "recording" || !this.capture.stopRecording) { this.sendJson(res, 409, { error: "not_recording", ...this.recordingStatus() }); return; } // A running recording can be stopped from anywhere (any tab, the CLI), // but a file held at the duration cap goes only to the client that // started it, so another tab's poll can't take the CLI's file. The hold // is bounded (`holdForOwner`) so a vanished owner can't strand it. if ( active.finalizing && !active.holdExpired && new URL(req.url ?? "", "http://x").searchParams.get("token") !== active.token ) { this.sendJson(res, 409, { error: "not_owner", message: "This recording hit the time limit and is held for the client that started it" + (active.heldAt != null ? ` until ${new Date(active.heldAt + this.recordingHoldMs()).toLocaleTimeString()}` : ""), ...this.recordingStatus(), }); return; } if (active.holdTimer) clearTimeout(active.holdTimer); active.holdTimer = undefined; active.phase = "stopping"; // Finalizing a long recording takes a moment; notice if the client // leaves. Runtimes differ in which of these fire for an aborted request. // (Not req "close": Node also emits it once a request completes normally.) let clientGone = req.aborted || res.destroyed; const onGone = () => { clientGone = true; }; req.once("aborted", onGone); res.once("close", onGone); const stopWatching = () => { req.off("aborted", onGone); res.off("close", onGone); }; let summary: string; let size: number; try { summary = await this.finalizeRecording(active); ({ size } = await stat(active.path)); } catch (error) { await rm(active.path, { force: true }).catch(() => {}); throw error; } finally { stopWatching(); if (this.recording === active) this.recording = undefined; } if (clientGone || req.aborted || res.destroyed || req.socket?.destroyed) { await rm(active.path, { force: true }).catch(() => {}); return; } const fileName = recordingFileName(); let publicSummary: string; if (active.events) { // Indexed: parse the (large) summary once for both the index and the header. const split = splitRecordingSummary(summary); publicSummary = publicRecordingSummary(split.summary); // The index is a side-channel: it must never cost the MP4. try { this.storeRecordingIndex(active, active.events, split, fileName); } catch (error) { console.error(`[record] ${this.udid}: could not keep the recording index:`, error); } } else { publicSummary = publicRecordingSummary(summary); } res.writeHead(200, { "Content-Type": "video/mp4", "Content-Length": String(size), "Content-Disposition": `attachment; filename="${fileName}"`, "Cache-Control": "no-store", [RECORDING_SUMMARY_HEADER]: publicSummary, }); try { await new Promise((resolve) => { const file = (this.dependencies?.openRecordingFile ?? createReadStream)(active.path); const done = () => { file.destroy(); void rm(active.path, { force: true }).finally(resolve); }; file.once("error", () => { res.destroy(); done(); }); res.once("close", done); file.pipe(res); }); } finally { // The index's TTL counts from the end of the MP4 transfer. if (active.events) this.armRecordingIndexExpiry(active.token); } } /** * Keep a collected recording's index (native per-sample metadata plus the * HID events in its window) for `GET /recording/index?token=`. The stop * response itself is unchanged; the index is fetched separately. */ private storeRecordingIndex( active: ActiveRecording, collector: RecordingEventCollector, { summary, index }: ReturnType, fileName: string, ): void { if (!index) return; const clock = recordingClock(index); const durationSec = typeof summary.duration === "number" ? summary.duration : 0; const events = clock ? recordingIndexEvents(collector.entries(), clock, { durationSec }) : []; this.recordingIndexes.set(active.token, { fileName, summary: summary as RecordingIndexSummary, index, events, eventsDropped: collector.dropped, }); while (this.recordingIndexes.size > MAX_STORED_RECORDING_INDEXES) { const [oldest, stored] = this.recordingIndexes.entries().next().value!; clearTimeout(stored.expiry); this.recordingIndexes.delete(oldest); } } /** Start the TTL for a kept index (once its MP4 has been sent). */ private armRecordingIndexExpiry(token: string): void { const stored = this.recordingIndexes.get(token); if (!stored || stored.expiry) return; const expiry = setTimeout(() => { if (this.recordingIndexes.get(token) === stored) this.recordingIndexes.delete(token); }, this.dependencies?.recordingIndexTtlMs ?? RECORDING_INDEX_TTL_MS); expiry.unref?.(); stored.expiry = expiry; } /** `GET /recording/index?token=[&file=]` → JSONL. */ private async sendRecordingIndex(req: IncomingMessage, res: ServerResponse): Promise { const params = new URL(req.url ?? "", "http://x").searchParams; const token = params.get("token"); if (!token) { this.sendJson(res, 400, { error: "missing_token", message: "Pass the token /recording/start returned" }); return; } const stored = this.recordingIndexes.get(token); if (!stored) { this.sendJson(res, 404, { error: "index_not_found", message: "No index for this token (not collected yet, expired, or recorded by a build without indexes)", }); return; } const file = sanitizeFileName(params.get("file")) ?? stored.fileName; res.writeHead(200, { "Content-Type": RECORDING_INDEX_CONTENT_TYPE, "Content-Disposition": `attachment; filename="${basename(recordingIndexPath(file))}"`, "Cache-Control": "no-store", }); if ((req.method ?? "GET").toUpperCase() === "HEAD") { res.end(); return; } const { summary, index, events, eventsDropped } = stored; for (const chunk of recordingIndexLines({ file, summary, index, events, eventsDropped })) { if (res.destroyed) return; if (!res.write(chunk)) { await new Promise((resolve) => { const done = () => { res.off("drain", done); res.off("close", done); resolve(); }; res.once("drain", done); res.once("close", done); }); } } res.end(); } // ── HID WebSocket ──────────────────────────────────────────────────────── attachHidSocket(ws: HidSocket): void { this.hidSockets.add(ws); const cfg = this.configFrame(); const projection = this.duo.projectionFrame(); if (projection) ws.send(projection); if (cfg) ws.send(cfg); // seed dimensions/orientation, replacing the old poll ws.on("message", (data: Buffer) => { const frame = Buffer.isBuffer(data) ? data : Buffer.from(data); if (frame[0] === 0x0e) { this.poseQueue = this.poseQueue.then(async () => { if (this.isStopped()) return; let ok = false; try { ok = await this.handleHidMessage(frame) === true; } catch (error) { console.error("[hid] Pose failed:", error); } if (this.hidSockets.has(ws)) { ws.send(Buffer.concat([Buffer.from([0x0e]), Buffer.from(JSON.stringify({ ok }))])); } }).catch((error) => { console.error("[hid] Pose reply failed:", error); }); } else { const ack = inputAckId(frame); const handled = this.handleHidMessage(frame); if (ack === undefined) { void handled.catch((error) => { console.error("[hid] Input failed:", error); }); } else { void handled .then((result) => { if (isShadowedInputResult(result)) return { ok: false as const, error: result.error }; return { ok: result === true }; }, (error) => { console.error("[hid] Input failed:", error); return { ok: false as const }; }) .then((body) => { if (this.hidSockets.has(ws)) { ws.send(Buffer.concat([Buffer.from([WS_MSG_INPUT_ACK]), Buffer.from(JSON.stringify({ ack, ...body }))])); } }); } } }); ws.on("close", () => this.hidSockets.delete(ws)); ws.on("error", () => this.hidSockets.delete(ws)); } private async handleHidMessage(data: Buffer): Promise { if (data.length < 1) return; const tag = data[0]; const body = data.length > 1 ? data.subarray(1) : null; const json = (): T | null => { if (!body) return null; try { return JSON.parse(body.toString("utf8")) as T; } catch { return null; } }; const W = this.width; const H = this.height; switch (tag) { // Touch and button resolve whether the injector delivered the event; the // socket handler turns that into an ack when the frame asked for one. case 0x03: { const raw = json<{ type: string; x: number; y: number; edge?: number; ack?: number }>(); if (!raw) return false; if (await this.legacyInputShadowed()) return shadowedInputResult(this.udid); const m = withoutAck(raw); this.recordTouchEvent(m); return this.hid.touch(m.type as "begin" | "move" | "end", m.x, m.y, W, H, m.edge ?? 0); } case 0x04: { const raw = json<{ button: string; page?: number; usage?: number; phase?: string; ack?: number }>(); if (!raw) return false; const m = withoutAck(raw); if (m.page != null && m.usage != null) { if (await this.legacyInputShadowed()) return shadowedInputResult(this.udid); this.recordHidEvent(tag, m); return this.hid.buttonHid(m.page, m.usage, (m.phase as "down" | "up" | "press") ?? "press"); } // SpringBoardLauncher home still reaches the guest when Device Hub // has disconnected legacy mouse/keyboard HID, so it is recorded even // while other buttons are refused. When SpringBoard is frontmost, // NativeHid stays on the launch path if input is shadowed (a flick // would be dropped) rather than acknowledging a touch that was not // delivered. if (m.button !== "home" && await this.legacyInputShadowed()) return shadowedInputResult(this.udid); this.recordHidEvent(tag, m); return this.hid.button(m.button); } case 0x05: { const m = json<{ type: string; x1: number; y1: number; x2: number; y2: number }>(); if (!m) break; if (await this.legacyInputShadowed()) return shadowedInputResult(this.udid); this.recordHidEvent(tag, m); return (await this.hid.multiTouch(m.type as "begin" | "move" | "end", m.x1, m.y1, m.x2, m.y2, W, H)) !== false; } case 0x06: { const m = json<{ type: string; usage: number }>(); if (!m) break; if (await this.legacyInputShadowed()) return shadowedInputResult(this.udid); this.recordHidEvent(tag, m); const delivered = (await this.hid.key(m.type as "down" | "up", m.usage)) !== false; if (delivered && m.type === "down" && !isModifierUsage(m.usage)) this.forwardedTypingKey = true; return delivered; } case 0x07: { const m = json<{ orientation: string }>(); if (!m) break; const value = ORIENTATION_BY_NAME[m.orientation]; if (value != null && await this.hid.orientation(value)) { this.recordHidEvent(tag, m); if (m.orientation !== this.orientation || m.orientation !== this.duo.viewOrientationName) { // Duo physical orientation and the active panel's UI orientation // differ (the inner panel's natural axis is rotated). Let the // monitor report the guest UI; never overwrite it with view state. if (this.duo.hinge == null) { this.orientation = m.orientation; this.knownOrientation = m.orientation; } this.duo.animateRotation(m.orientation); this.duoMonitor?.refreshOrientation(); this.broadcastConfig(); this.duo.requestFrame(); } } break; } case 0x08: { const m = json<{ option: string; enabled: boolean }>(); if (m) { this.recordHidEvent(tag, m); this.hid.caDebug(m.option, m.enabled); } break; } case 0x09: this.recordHidEvent(tag, {}); this.hid.memoryWarning(); break; case 0x0a: { const m = json<{ delta: number }>(); if (!m) break; if (await this.legacyInputShadowed()) return shadowedInputResult(this.udid); this.recordHidEvent(tag, m); return (await this.hid.digitalCrown(m.delta)) !== false; } case 0x0b: { // Payload deltas are a fraction of the display; scale to device pixels. const m = json<{ dx: number; dy: number; x?: number; y?: number }>(); if (!m) break; if (await this.legacyInputShadowed()) return shadowedInputResult(this.udid); this.recordHidEvent(tag, m); return (await this.hid.scroll(m.dx * W, m.dy * H, W, H, m.x, m.y)) !== false; } case 0x0c: this.recordHidEvent(tag, {}); this.hid.softwareKeyboard(); break; case 0x0f: { // Serialized so rapid presses each toggle from the settled state. // A same-boot explicit value flips. Otherwise: software keyboard // visible -> connect (hides it); hidden -> disconnect (lets it show). // If AX can't tell, assume forwarded typing implicitly connected. const state = this.dependencies?.hardwareKeyboardState ?? { read: (udid: string) => readHardwareKeyboardState(udid), write: (udid: string, enabled: boolean) => writeHardwareKeyboardState(udid, enabled), }; const probe = this.dependencies?.softwareKeyboardVisible ?? probeSoftwareKeyboardVisible; const toggle = this.hardwareKeyboardQueue.then(async () => { const saved = state.read(this.udid); let next: boolean; if (saved != null) { next = !saved; } else { const visible = await withTimeout(probe(this.udid), SOFTWARE_KEYBOARD_PROBE_TIMEOUT_MS).catch(() => null); next = visible ?? !this.forwardedTypingKey; } if (!await this.hid.hardwareKeyboard(next)) return; state.write(this.udid, next); this.recordHidEvent(tag, { enabled: next }); }); this.hardwareKeyboardQueue = toggle.catch(() => {}); await toggle; break; } case 0x0d: { const m = json<{ width?: number; height?: number }>(); if (!m) break; const width = Number(m.width); const height = Number(m.height); if (!Number.isFinite(width) || !Number.isFinite(height)) break; this.recordHidEvent(tag, m); await this.capture.setPreferredScreenSize(Math.round(width), Math.round(height)); break; } case 0x0e: { const m = json<{ pose?: string; hinge?: number }>(); if (!m) break; this.recordHidEvent(tag, m); if (typeof m.pose === "string" && m.pose.trim()) { const spec = resolveDevicePose(m.pose); if (!spec) break; const from = this.duo.displayAngle(); this.duo.beginPose(from, spec.hingeDegrees); const accepted = await this.hid.pose(m.pose, from); if (!accepted) { this.duo.cancelPose(); break; } this.duo.finishPose(spec.hingeDegrees); await this.followDuoState({ hingeDegrees: spec.hingeDegrees, orientations: {} }); this.duoMonitor?.refreshOrientation(); this.broadcastConfig(); this.duo.requestFrame(); return true; } else if (typeof m.hinge === "number" && Number.isFinite(m.hinge) && m.hinge >= 0 && m.hinge <= 180) { if (!await this.hid.hinge(m.hinge)) break; this.duo.cancelPose(); await this.followDuoState({ hingeDegrees: m.hinge, orientations: {} }); this.duoMonitor?.refreshOrientation(); this.broadcastConfig(); this.duo.requestFrame(); return true; } break; } } } private recordTouchEvent(payload: { type: string; x: number; y: number; edge?: number }): void { if (payload.type === "begin") { this.touchGestureLog = newTouchGesture(payload); return; } if (payload.type === "move") { let gesture = this.touchGestureLog; if (!gesture) { gesture = newTouchGesture(payload); this.touchGestureLog = gesture; } gesture.lastX = payload.x; gesture.lastY = payload.y; gesture.moveCount++; if (payload.edge != null) gesture.edge = payload.edge; if (touchGestureMoved(gesture)) { if (gesture.eventId == null) { const entry = recordEventLogEvent({ device: this.udid, source: "hid", kind: "drag", action: "drag", summary: touchGestureSummary(gesture), details: this.touchGestureDetails(gesture, "drag", "move"), }); gesture.eventId = entry.id; } else { // Keep the stored drag current without streaming every touchmove to the browser. const patch = { kind: "drag", action: "drag", summary: touchGestureSummary(gesture), details: this.touchGestureDetails(gesture, "drag", "move"), }; const updated = updateEventLogEvent(gesture.eventId, patch, { notify: false }); // Silent updates skip subscribers; an indexed recording still // wants the drag's latest point (and when it arrived), even once // the drag has fallen out of the global log. if (updated) { this.recording?.events?.add(updated); this.rollingBuffer?.noteEvent(updated); } else { this.recording?.events?.update(gesture.eventId, patch); this.rollingBuffer?.updateEvent(gesture.eventId, patch); } } } return; } if (payload.type === "end") { const gesture = this.touchGestureLog; if (gesture) { gesture.lastX = payload.x; gesture.lastY = payload.y; if (payload.edge != null) gesture.edge = payload.edge; if (gesture.moveCount > 0 && touchGestureMoved(gesture)) { if (gesture.eventId == null) { recordEventLogEvent({ device: this.udid, source: "hid", kind: "drag", action: "drag", summary: touchGestureSummary(gesture), details: this.touchGestureDetails(gesture, "drag", "end"), }); } else { const patch = { kind: "drag", action: "drag", summary: touchGestureSummary(gesture), details: this.touchGestureDetails(gesture, "drag", "end"), }; // A notified update reaches the collector via its subscription; // one the global log already evicted has to be patched directly. if (!updateEventLogEvent(gesture.eventId, patch)) { this.recording?.events?.update(gesture.eventId, patch); this.rollingBuffer?.updateEvent(gesture.eventId, patch); } } } else { recordEventLogEvent({ device: this.udid, source: "hid", kind: "tap", action: "tap", summary: `Tap ${formatEventLogPoint(payload.x, payload.y)}`, details: this.touchGestureDetails(gesture, "tap"), }); } this.touchGestureLog = undefined; return; } } this.recordHidEvent(0x03, payload); } private eventLogScreen(): { width: number; height: number } | undefined { return this.width > 0 && this.height > 0 ? { width: this.width, height: this.height } : undefined; } private touchGestureDetails( gesture: TouchGestureLog, type: "drag" | "tap", phase?: "move" | "end", ): Record { return { type, ...(phase ? { phase } : {}), start: { x: gesture.startX, y: gesture.startY }, current: { x: gesture.lastX, y: gesture.lastY }, moveCount: gesture.moveCount, ...(gesture.edge != null ? { edge: gesture.edge } : {}), ...(this.eventLogScreen() ? { screen: this.eventLogScreen() } : {}), }; } private recordHidEvent(tag: number, payload: Record): void { const event = eventLogEventForHidMessage( this.udid, tag, payload, this.eventLogScreen(), ); if (event) recordEventLogEvent(event); } // ── Config ─────────────────────────────────────────────────────────────── screenConfig(): { width: number; height: number; orientation: string; hingeDegrees?: number; duoViewOrientation?: string } { const hingeDegrees = this.duo.hinge; return { width: this.width, height: this.height, orientation: this.orientation, ...(hingeDegrees == null ? {} : { hingeDegrees, duoViewOrientation: this.duo.viewOrientationName }), }; } private configFrame(): Buffer | null { if (this.width === 0 && this.height === 0) return null; return Buffer.concat([Buffer.from([WS_MSG_CONFIG]), Buffer.from(JSON.stringify(this.screenConfig()))]); } private broadcastConfig(): void { const frame = this.configFrame(); if (!frame) return; for (const ws of this.hidSockets) ws.send(frame); } private bindSubscription(res: ServerResponse, unsubscribe: Unsubscribe): void { let cleanedUp = false; const cleanup = () => { if (cleanedUp) return; cleanedUp = true; this.streamResponses.delete(res); this.runCleanup("stream unsubscribe", unsubscribe); }; if (res.writableEnded || res.destroyed) { cleanup(); return; } res.once("close", cleanup); res.once("error", cleanup); } private trackStreamResponse(res: ServerResponse, endpoint: StreamEndpoint): boolean { if (this.isStopped() || res.writableEnded || res.destroyed) { if (!res.writableEnded && !res.destroyed) res.destroy(); return false; } this.streamResponses.add(res); this.streamHealth[endpoint].open++; const release = () => { res.off("close", release); res.off("error", release); if (this.streamResponses.delete(res)) this.streamHealth[endpoint].open--; }; res.once("close", release); res.once("error", release); return true; } private runCleanup(operation: string, cleanup: Unsubscribe): void { try { void Promise.resolve(cleanup()).catch((error) => this.logCleanupError(operation, error)); } catch (error) { this.logCleanupError(operation, error); } } private logCleanupError(operation: string, error: unknown): void { console.error( `[capture] ${operation} failed for ${this.udid}:`, error instanceof Error ? error.message : error, ); } private noteStreamFrame(endpoint: StreamEndpoint): void { const health = this.streamHealth[endpoint]; health.frames++; health.last_frame_at = new Date().toISOString(); } /** Coalesce repeated identical renderer failures (client retries) into one counted entry. */ private recordRendererEvent(event: DuoRendererEvent): void { if (event.phase === "ready") { this.rendererFailureEvent = undefined; recordEventLogEvent({ device: this.udid, source: "capture", kind: "duo.renderer", action: "ready", status: "ok", summary: `Duo renderer ready (${event.firstFrameMs} ms to first frame)`, details: { phase: "ready", "duo.first_frame_ms": event.firstFrameMs }, }); return; } const { error } = event; logDuoRendererFailure(error); const key = `${error.stage}\u0000${error.reason}`; const reason = redactHome(error.reason); const details = { phase: "failed", stage: error.stage, reason, ...redactHome(error.details) }; const previous = this.rendererFailureEvent; if (previous?.key === key && updateEventLogEvent(previous.id, { details: { ...details, count: previous.count + 1 } })) { previous.count++; return; } const entry = recordEventLogEvent({ device: this.udid, source: "capture", kind: "duo.renderer", action: "failed", status: "error", summary: `Duo renderer ${error.stage} failed: ${reason}`, details: { ...details, count: 1 }, }); this.rendererFailureEvent = { id: entry.id, key, count: 1 }; } private static readonly DEVICE_HUB_SHADOW_RECHECK_MS = 2_000; private now(): number { return this.dependencies?.now?.() ?? Date.now(); } private deviceHubInputReader(): ((udid: string) => Promise) | null { if (this.dependencies?.readDeviceHubInputShadowed) return this.dependencies.readDeviceHubInputShadowed; // Tests inject a fake `hid` and must not spawn simctl on dummy UDIDs. if (this.usesInjectedHid) return null; return readDeviceHubInputShadowed; } /** Single-flight probe. Callers that already have a reading must not await this. */ /** Resolve chrome and foldable off the Home hot path. Unknown stays false. */ private async warmHomeIndicatorSupport(): Promise { if (this.homeIndicatorFlickSupported != null || this.usesInjectedHid) return; try { if (await this.hid.isFoldable?.()) { this.homeIndicatorFlickSupported = false; return; } const info = await simulatorDeviceInfo(this.udid); if (this.isStopped()) return; this.homeIndicatorFlickSupported = info != null && deviceSupportsHomeIndicatorFlick(info); } catch { this.homeIndicatorFlickSupported = false; } } private refreshDeviceHubInput(): Promise { if (this.deviceHubProbe) return this.deviceHubProbe; const read = this.deviceHubInputReader(); if (!read) { this.deviceHubShadowedAt = this.now(); this.deviceHubShadowSettled = true; return Promise.resolve(); } // Assign the promise before it can settle so a sync throw in `read` // cannot leave a leftover in-flight probe. Attach `finally` after the // assignment so it actually clears. const probe = (async () => { try { const shadowed = await read(this.udid); this.deviceHubShadowedAt = this.now(); this.deviceHubShadowSettled = true; // A null (unreadable) reading keeps the last known value, failing // closed if we have already seen the flag as true. if (shadowed == null) return; this.deviceHubShadowed = shadowed; if (shadowed && !this.deviceHubShadowWarned) { this.deviceHubShadowWarned = true; console.warn(`[hid] ${deviceHubInputShadowedMessage(this.udid)}`); } if (!shadowed) this.deviceHubShadowWarned = false; } catch { this.deviceHubShadowedAt = this.now(); this.deviceHubShadowSettled = true; } })(); this.deviceHubProbe = probe; void probe.finally(() => { if (this.deviceHubProbe === probe) this.deviceHubProbe = undefined; }); return probe; } private maybeRefreshDeviceHubInput(): void { if (this.deviceHubProbe) return; if (this.deviceHubShadowSettled && this.now() - this.deviceHubShadowedAt < DeviceSession.DEVICE_HUB_SHADOW_RECHECK_MS) return; void this.refreshDeviceHubInput(); } /** True when Device Hub has disconnected legacy HID and the event must not be acked as delivered. */ private async legacyInputShadowed(): Promise { // Await only when we have no reading yet, and then only the shared probe, // so a burst of events cannot spawn overlapping simctl or complete out of order. if (!this.deviceHubShadowSettled) await this.refreshDeviceHubInput(); else this.maybeRefreshDeviceHubInput(); return this.deviceHubShadowed === true; } /** JSON-safe live state for `/api/diagnostics` / `serve-sim doctor`. */ async diagnostics(): Promise> { let native: unknown = null; if (this.capture.diagnostics) { try { native = JSON.parse(await this.capture.diagnostics()); } catch (error) { native = { error: error instanceof Error ? error.message : String(error) }; } } else { native = { error: "native capture diagnostics unavailable in this build" }; } return { udid: this.udid, phase: this.phase, screen: this.screenConfig(), streams: structuredClone(this.streamHealth), native, duo: { ...this.duo.diagnostics(), primary_panel: this.primaryDuoPanel ?? null, active_panel: this.activePanel(), followed_cover: this.followedCover ?? null, }, hid: { device_hub_input_shadowed: this.deviceHubShadowed ?? null, sockets: this.hidSockets.size, }, }; } private handleStreamError(res: ServerResponse, error: unknown, endpoint?: StreamEndpoint): void { const message = redactHome(error instanceof Error ? error.message : String(error)); if (endpoint) { this.streamHealth[endpoint].last_error = { at: new Date().toISOString(), message, ...(error instanceof DuoRendererError ? { stage: error.stage } : {}), }; } if (res.writableEnded || res.destroyed) return; if (error instanceof DuoRendererError && !res.headersSent) { this.sendJson(res, 503, duoRendererErrorBody(error)); return; } if (res.headersSent) { res.destroy(error instanceof Error ? error : new Error(String(error))); return; } this.sendJson(res, 503, { error: "capture_unavailable", message, }); } private sendJson(res: ServerResponse, status: number, body: unknown): void { this.sendJsonString(res, status, JSON.stringify(body)); } private sendJsonString(res: ServerResponse, status: number, json: string): void { const buf = Buffer.from(json, "utf8"); res.writeHead(status, { "Content-Type": "application/json", "Cache-Control": "no-cache, no-store", "Content-Length": String(buf.length), }); res.end(buf); } } // ── Registry ───────────────────────────────────────────────────────────── const sessions = new Map(); /** * Get (lazily creating + starting) the in-process session for `udid`. Capture * failures are surfaced by stream endpoints and evict the session so the next * request can retry. The session otherwise lives until `closeDeviceSession`. */ /** Live sessions only; never creates or starts one (diagnostics must not open capture). */ export function liveDeviceSessions(udid?: string | null): DeviceSession[] { if (udid) { const session = sessions.get(udid); return session ? [session] : []; } return [...sessions.values()]; } export function getDeviceSession(udid: string): DeviceSession { let session = sessions.get(udid); if (!session) { const created = new DeviceSession(udid); session = created; sessions.set(udid, created); void created.start().catch(() => { // A reconnect can race `simctl shutdown`: remove the failed session so a // later Start creates a fresh native capture instead of reusing a rejected // promise. The rejection is observed here and by the endpoint that awaited // it, so it cannot terminate Node/Bun as an unhandled rejection. if (sessions.get(udid) !== created) return; sessions.delete(udid); created.close(); }); } return session; } export function closeDeviceSession(udid: string): void { const session = sessions.get(udid); if (session) { session.close(); sessions.delete(udid); } } /** * The summary header: the native JSON (or, for an indexed recording, its * already-parsed form with `index` split off) without the server path. */ function publicRecordingSummary(summary: string | Record): string { let parsed: Record; if (typeof summary === "string") { try { parsed = JSON.parse(summary) as Record; } catch { return "{}"; } } else { parsed = { ...summary }; } delete parsed.path; // Header-safe: JSON.stringify escapes control chars; drop anything non-ASCII. return JSON.stringify(parsed).replace(/[^\x20-\x7e]/g, ""); } function stopCollectingEvents(active: ActiveRecording): void { active.unsubscribeEvents?.(); active.unsubscribeEvents = undefined; active.events?.stop(); } /** A header-safe bare file name from a `file=` query param, or null. */ function sanitizeFileName(raw: string | null): string | null { if (!raw) return null; const name = basename(raw).replace(/[^\x20-\x7e]|["\\]/g, "").trim(); return name && name !== "." && name !== ".." ? name.slice(0, 200) : null; }