//! OpenScreen Linux capture helper. //! //! WHY THIS EXISTS. Two reasons, and they turned out to be the same reason. //! //! `screen.getCursorScreenPoint()` returns {0,0} under Wayland, so //! `TelemetryRecordingSession` produced recordings whose every cursor sample sat //! in the screen's top-left corner while looking perfectly well-formed. The only //! source of truth for the pointer on Wayland is the ScreenCast portal's //! METADATA cursor mode. //! //! And in that same mode the compositor keeps the cursor OUT of the captured //! pixels — which is what the editor needs in order to draw its own. Chromium's //! `getDisplayMedia` gives the opposite: a frame with the pointer already //! painted in, and no way to know where it was. One portal session answers both, //! and it has to be one session because SelectSources may only be called once. //! //! SHAPE. A stdio sidecar, like the macOS and Windows helpers: JSON request in //! argv[1], NDJSON events on stdout, `stop`/`pause`/`resume` (or EOF) on stdin. //! With `outputPath` it also encodes H.264 and writes an MP4; without it, it is //! the cursor-only session Stage 1 shipped, which is what //! `PipeWireCursorRecordingSession` still uses. //! //! MOUSE BUTTONS. Not from the portal — Wayland exposes no portal for input //! events. The one source left is evdev (/dev/input/event*), which is root:input //! and so needs the user in the `input` group. When that permission exists the //! helper reads left-button presses and tags the coinciding sample "click"; when //! it does not, every sample is a "move", as before. See `input.rs` for the //! reader and its deliberately narrow scope (BTN_LEFT only, never keystrokes). mod bitmap; mod capture; mod dmabuf_import; mod encoder; mod events; mod ffmpeg; mod input; mod portal; mod shim; mod x11; use std::collections::HashSet; use std::io::{BufRead, Write}; use std::os::fd::OwnedFd; use std::path::PathBuf; use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender}; use std::sync::Arc; use std::time::{Duration, Instant}; use serde::Deserialize; use capture::Capture; use events::{timestamp_ms, CursorAsset, Emitter, Event}; use shim::{FrameMailbox, StreamEvent}; /// Matches the macOS helper's default and the Electron side's sampleIntervalMs. const DEFAULT_SAMPLE_INTERVAL_MS: u64 = 33; const MIN_SAMPLE_INTERVAL_MS: u64 = 8; const DEFAULT_FPS: i32 = 30; #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "camelCase", default)] struct VideoRequest { fps: Option, bitrate: Option, } /// Mirrors the `audio` block of `NativeWindowsRecordingRequest` /// (src/lib/nativeWindowsRecording.ts) so the three platforms take the same /// request shape. #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "camelCase", default)] struct AudioRequest { system: SystemAudioRequest, microphone: MicrophoneRequest, } #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "camelCase", default)] struct SystemAudioRequest { enabled: bool, } #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "camelCase", default)] struct MicrophoneRequest { enabled: bool, /// A PipeWire node name (`node.name`), not the browser device id the UI /// carries: the two namespaces are unrelated and there is no mapping /// between them. Absent means the session default source. device_name: Option, /// Linear multiplier the UI applies to microphone level. gain: Option, } const DEFAULT_AUDIO_BITRATE: i64 = 128_000; /// How long a window stream may go without a crop rectangle before the helper /// gives up waiting and records the whole stream, starting from the last frame. /// /// mutter records one frame synchronously when the stream is enabled, which can /// land before the picked window is mapped and therefore carry an empty rect; /// its real crop follows within a few frames. KWin never fills the crop at all /// (it sizes a window stream to the window) and sends one frame when the stream /// starts, then only when the window repaints. That is why this is a duration /// and not a frame count: counting frames waited forever on a still window, /// whose only frame had been thrown away, and the start timed out. const MAX_CROP_WAIT: Duration = Duration::from_millis(500); /// How much audio may queue before the oldest is discarded. Generous: the drain /// runs every loop tick, so reaching this means the encoder stopped entirely. const AUDIO_RING_SECONDS: usize = 2; /// How many dmabuf imports may fail in a row before the recording gives up. One /// failure is a recoverable dropped frame (the previous is held forward), but this /// many in a row means the import path is broken — typically a render node that /// cannot map the compositor's GPU buffers — and the file would otherwise be empty /// behind a wall of `frame-dropped` warnings. ~1–2 s at 30–60 fps. const MAX_CONSECUTIVE_IMPORT_FAILURES: u32 = 60; #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "camelCase", default)] struct Request { /// Explicit source chosen by the app on X11. Wayland ignores this and /// continues to obtain its source exclusively through the portal. source: Option, sample_interval_ms: Option, /// Where to write the MP4. Absent means cursor-only: no pixels are mapped, /// no encoder is opened, and the helper behaves exactly as Stage 1 did. output_path: Option, /// `"metadata"` (default) keeps the pointer out of the pixels so the editor /// can draw its own; `"embedded"` asks the compositor to paint it in. These /// are the two halves of the HUD's cursor-mode toggle. cursor_mode: Option, video: Option, audio: Option, /// Negotiate the portal, report `source-selected`, then WAIT for `record` on /// stdin before connecting to PipeWire. /// /// Splits "the user has chosen what to share" from "pixels are flowing", so a /// caller can put a countdown between them instead of counting down before /// the picker has even appeared. Defaults to false, which keeps the /// single-shot behaviour every existing caller relies on. defer_start: bool, /// Emit `ready` and exit, without ever calling the portal's `Start()`. /// /// Everything up to that point is non-interactive; `Start()` is the single /// call that raises the compositor's source picker. This flag is what makes /// the dlopen path, the portal connection and the cursor-mode check testable /// — in CI or by hand — without hijacking someone's screen. probe_only: bool, } impl Request { fn cursor_mode(&self) -> Result { match self.cursor_mode.as_deref() { None | Some("") | Some("metadata") => Ok(portal::CursorMode::Metadata), Some("embedded") => Ok(portal::CursorMode::Embedded), Some("hidden") => Ok(portal::CursorMode::Hidden), Some(other) => Err(format!( "cursorMode must be one of metadata, embedded, hidden — got {other:?}" )), } } fn fps(&self) -> i32 { self.video .as_ref() .and_then(|video| video.fps) .filter(|fps| *fps > 0 && *fps <= 240) .unwrap_or(DEFAULT_FPS) } /// An explicit override, or `None` to let the encoder derive one from the /// size the compositor negotiates. `None` is the normal case: on Wayland the /// app cannot know the capture resolution in advance, so a bitrate chosen /// caller-side is a guess about a picture nobody has seen yet. fn bitrate(&self) -> Option { self.video .as_ref() .and_then(|video| video.bitrate) .filter(|bitrate| *bitrate > 0) } /// The audio streams to open, in the order they become MP4 tracks. /// /// Empty when no output file was requested: a cursor-only session has /// nothing to mux audio into, and opening a capture stream would show the /// user a recording indicator for a recording that is not happening. fn audio_sources(&self) -> Vec { if self.output_path.is_none() { return Vec::new(); } let Some(audio) = self.audio.as_ref() else { return Vec::new(); }; let mut sources = Vec::new(); if audio.system.enabled { sources.push(AudioSourceConfig { label: "system", target: None, // The default SINK's monitor: what is being played, not what a // microphone hears. capture_sink: true, gain: 1.0, bitrate: DEFAULT_AUDIO_BITRATE, }); } if audio.microphone.enabled { sources.push(AudioSourceConfig { label: "microphone", target: audio.microphone.device_name.clone(), capture_sink: false, gain: audio.microphone.gain.filter(|gain| *gain > 0.0).unwrap_or(1.0), bitrate: DEFAULT_AUDIO_BITRATE, }); } sources } } struct AudioSourceConfig { label: &'static str, target: Option, capture_sink: bool, gain: f32, bitrate: i64, } enum Message { Portal(Box>), Stream(StreamEvent), /// A left mouse-button press observed on evdev, carrying the press time in /// milliseconds so the emitted click sample is stamped when it happened /// rather than at the next throttled sample. See [`input`] for why this is /// the only way to see a button on Wayland, and for its permission and /// privacy model. PointerButton(u64), /// A pointer reader thread stopped on a device error (typically an unplug); /// the string is the OS error. Surfaced as a warning so click capture going /// quiet mid-recording is not silent. PointerDeviceLost(String), /// Arm a deferred session: connect to PipeWire and start encoding. Record, Pause, Resume, Stop, } fn main() { let debug = std::env::var("OPENSCREEN_PIPEWIRE_DEBUG") .map(|value| !matches!(value.as_str(), "" | "0" | "false")) .unwrap_or(false); let mut emitter = Emitter::new(std::io::stdout(), debug); let request = match parse_request() { Ok(request) => request, Err(message) => { fail(&mut emitter, "invalid-arguments", &message); } }; let cursor_mode = match request.cursor_mode() { Ok(mode) => mode, Err(message) => fail(&mut emitter, "invalid-arguments", &message), }; let output_path = request.output_path.as_deref().map(PathBuf::from); let forced_encoder = match encoder::forced_backend_from_env() { Ok(backend) => backend, Err(message) => fail(&mut emitter, "invalid-arguments", &message), }; // The loop has to serve two clocks: cursor samples at the requested interval // and video frames at the video frame rate. It ticks at whichever is // shorter, so neither is ever late by more than the other's period. let sample_interval = Duration::from_millis( request .sample_interval_ms .unwrap_or(DEFAULT_SAMPLE_INTERVAL_MS) .max(MIN_SAMPLE_INTERVAL_MS), ); let tick = match output_path { Some(_) => sample_interval.min(Duration::from_nanos( 1_000_000_000 / request.fps().max(1) as u64, )), None => sample_interval, }; // The portal first: it is what gives dmabuf frames and GPU encoding. Direct // X11 capture is only for an X11 session where nothing answers on the // ScreenCast interface (Xfce, i3...). Probed before `shim::load` because an // X11 desktop without a portal often has no PipeWire either. let offered_cursor_modes = pollster::block_on(portal::offered_cursor_modes()); // A source means the app already chose X11, and the take follows it even if a // portal has appeared since. Otherwise only a portal that is missing, or that // refuses every session (#1491), falls back: one that fails later keeps its // error. So does a version 1 portal, which offers no cursor modes (#1537): // direct X11 reports the cursor, that portal cannot. if x11::is_x11_session() && (request.source.is_some() || match &offered_cursor_modes { Ok(modes) if modes.is_empty() => true, Ok(_) => pollster::block_on(portal::refuses_sessions()), Err(error) => matches!(error, portal::PortalError::Unavailable(_)), }) { if let Err(message) = x11::run(&request, &mut emitter) { fail(&mut emitter, "x11-capture-failed", &message); } return; } if let Err(message) = shim::load() { fail(&mut emitter, "pipewire-unavailable", &message); } // Probed before `ready` so the event can report it, and so an unsupported // compositor fails immediately instead of after a pointless picker. Only // fatal when METADATA is the mode actually requested — a recording that // wants the system cursor painted in has no use for it. let offered_cursor_modes = match offered_cursor_modes { Ok(modes) => modes, Err(error) => fail(&mut emitter, error.code(), &error.message()), }; let cursor_metadata_supported = offered_cursor_modes.contains(&portal::CursorMode::Metadata); // Resolved before anything reads it, so the portal, the click readers and // `ready` all see the mode that is really negotiated. let cursor_mode = cursor_mode.resolve(&offered_cursor_modes); if !cursor_metadata_supported && cursor_mode.reports_cursor() { let error = portal::PortalError::CursorMetadataUnsupported; fail(&mut emitter, error.code(), &error.message()); } let _ = emitter.emit(&Event::Ready { timestamp_ms: timestamp_ms(), pipewire_version: shim::library_version(), cursor_metadata_supported, cursor_mode: cursor_mode.as_str(), backend: "pipewire", }); if request.probe_only { std::process::exit(0); } let (sender, receiver) = mpsc::channel::(); spawn_stdin_reader(sender.clone()); spawn_portal(sender.clone(), cursor_mode); // Left-button telemetry from evdev. Gated on the cursor mode FIRST: a session // that paints no cursor emits no samples to tag, so opening every pointer node // and blocking a thread on each would buy nothing. Warn only when the devices // are unreadable — never when the operator opted out through the env var, // which would recommend the permission they just declined — so a "clicks do // nothing on Linux" report is answerable from the log alone rather than // looking like a capture bug. if cursor_mode.reports_cursor() && input::spawn_readers(&sender) == input::ClickCapture::NoDevice { let _ = emitter.emit(&Event::Warning { code: "click-capture-unavailable".to_owned(), message: "no readable /dev/input pointer device — grant this user access (a udev \ uaccess rule for mice, or the 'input' group) to record click telemetry; \ cursor samples will otherwise all be moves" .to_owned(), }); } let session = RunConfig { tick, sample_interval, output_path, fps: request.fps(), bitrate: request.bitrate(), forced_encoder, cursor_mode, audio: request.audio_sources(), defer_start: request.defer_start, }; let exit_code = run(&mut emitter, receiver, sender, session); std::process::exit(exit_code); } struct RunConfig { /// How often the loop wakes when nothing arrives. tick: Duration, /// Minimum spacing between cursor samples on stdout. sample_interval: Duration, /// `None` for a cursor-only session. output_path: Option, fps: i32, /// `None` lets the encoder derive one from the negotiated size. bitrate: Option, forced_encoder: Option, cursor_mode: portal::CursorMode, audio: Vec, /// Wait for `record` on stdin before connecting to PipeWire. See /// [`Request::defer_start`]. defer_start: bool, } /// Opens every requested audio stream, returning the live sessions (which must /// outlive the loop) and the sources to hand to the muxer. /// /// A stream that fails to open is a warning, not a failure: a recording with /// picture and no system audio is worth far more to the user than no recording /// at all, and the most likely cause — a sandbox without the PipeWire socket — /// is not something the helper can fix. fn start_audio( emitter: &mut Emitter, configs: &[AudioSourceConfig], ) -> (Vec, Vec) { let mut sessions = Vec::new(); let mut sources = Vec::new(); // Enumerated ONCE for the whole recording, not per stream: it opens its own // PipeWire connection and runs a loop to completion, so doing it per source // would pay that cost twice for no new information. let graph = if configs.iter().any(|c| c.target.is_some()) { match shim::list_audio_sources() { Ok(list) => list, Err(message) => { let _ = emitter.emit(&Event::Warning { code: "audio-enumeration-failed".to_owned(), message: format!( "the PipeWire graph could not be listed, so the requested microphone \ cannot be resolved and the session default will be used instead: {message}" ), }); Vec::new() } } } else { Vec::new() }; for config in configs { let ring = Arc::new(shim::AudioRing::new( AUDIO_RING_SECONDS, encoder::AUDIO_SAMPLE_RATE as usize, encoder::AUDIO_CHANNELS, )); let label = config.label; // A label we cannot place resolves to None, which means "no target" — // PipeWire then links to the session default source. That is the old // behaviour and the right fallback: a default microphone beats none. let resolved = config .target .as_deref() .and_then(|wanted| resolve_microphone_node(wanted, &graph)); let _ = emitter.emit(&Event::AudioSource { role: label.to_owned(), requested: config.target.clone(), node: resolved.clone(), }); if let Some(wanted) = config.target.as_deref() { if resolved.is_none() { let _ = emitter.emit(&Event::Warning { code: "microphone-not-found".to_owned(), message: format!( "no PipeWire capture node matches {wanted:?}, so the session default \ source will be recorded instead — which is often not the microphone \ the user picked." ), }); } } let session = shim::AudioSession::start( resolved.as_deref(), config.capture_sink, encoder::AUDIO_SAMPLE_RATE as u32, encoder::AUDIO_CHANNELS as u32, ring.clone(), Box::new(move |state, error| { if let Some(error) = error { eprintln!("[audio:{label}] stream error in state {state}: {error}"); } }), ); match session { Ok(session) => { sessions.push(session); sources.push(capture::AudioSource { label: config.label, ring, gain: config.gain, bitrate: config.bitrate, }); } Err(message) => { let _ = emitter.emit(&Event::Warning { code: "audio-unavailable".to_owned(), message: format!( "the {} audio stream could not be opened, so the recording will \ have no {} track: {message}", config.label, config.label ), }); } } } (sessions, sources) } fn parse_request() -> Result { match std::env::args().nth(1) { None => Ok(Request::default()), Some(raw) => serde_json::from_str(&raw) .map_err(|error| format!("could not parse the request argument as JSON: {error}")), } } fn fail(emitter: &mut Emitter, code: &str, message: &str) -> ! { let _ = emitter.emit(&Event::Error { code: code.to_owned(), message: message.to_owned(), }); std::process::exit(1); } /// EOF counts as `stop`: if the parent dies, so do we. /// /// The command vocabulary is the Windows and macOS helpers': `stop`, `pause`, /// `resume`, one per line on stdin (see the `proc.stdin.write("pause\n")` calls /// in electron/ipc/handlers.ts). Unknown lines are ignored rather than fatal — /// a newer parent talking to an older helper should degrade, not crash. fn spawn_stdin_reader(sender: Sender) { std::thread::spawn(move || { let stdin = std::io::stdin(); for line in stdin.lock().lines() { let Ok(line) = line else { break }; match line.trim() { "stop" => break, // Arms a `deferStart` session. Ignored by a session that was not // deferred, and unknown verbs already fall through to `continue`, // so an older helper receiving this degrades instead of dying. "record" => { if sender.send(Message::Record).is_err() { return; } } "pause" => { if sender.send(Message::Pause).is_err() { return; } } "resume" => { if sender.send(Message::Resume).is_err() { return; } } _ => continue, } } let _ = sender.send(Message::Stop); }); } /// The portal runs on its own thread because `Start()` blocks on the user for /// an unbounded time, and a `stop` arriving during the picker must still be /// honoured. fn spawn_portal(sender: Sender, cursor_mode: portal::CursorMode) { std::thread::spawn(move || { let result = pollster::block_on(portal::negotiate(cursor_mode)); let _ = sender.send(Message::Portal(Box::new(result))); }); } /// Latest known cursor state, resent on the heartbeat so a motionless pointer /// still produces the steady sample cadence the editor's cursor track expects. struct CursorState { x: i32, y: i32, asset_id: Option, } /// Connects the PipeWire consumer to a grant the portal already made. /// /// Split out because it runs from two places: straight off the portal reply in /// the single-shot path, and from `record` in the deferred path where the grant /// has been sitting in `pending_portal` while the caller ran its countdown. #[allow(clippy::too_many_arguments)] fn begin_stream( emitter: &mut Emitter, sender: &Sender, frames: &Option>, session: &mut Option, portal_stream: &mut Option, granted_kind: &mut Option, stream: portal::PortalStream, prefer_dmabuf: bool, shared_memory_only: bool, shared_memory_fallback: &mut Option, ) -> Result<(), ()> { let portal::PortalStream { fd, fallback_fd, node_id, position, size, source_kind } = stream; // Armed even without `prefer_dmabuf`: the DMA-BUF object is still offered in // second place, and xdg-desktop-portal-cosmic picks it there too (issue #1445). *shared_memory_fallback = fallback_fd.map(|fd| SharedMemoryFallback { fd, node_id, position, size, source_kind, }); let forward = sender.clone(); match shim::Session::start( fd, node_id, Box::new(move |event| { let _ = forward.send(Message::Stream(event)); }), frames.clone(), prefer_dmabuf, shared_memory_only, ) { Ok(started) => { *session = Some(started); *granted_kind = source_kind; *portal_stream = Some(StreamInfo { node_id, position, source_kind }); Ok(()) } Err(message) => { let _ = emitter.emit(&Event::Error { code: "pipewire-connect-failed".to_owned(), message, }); Err(()) } } } /// What survives from the portal reply after its fd has been handed to /// libpipewire — everything `stream-started` needs to report. struct StreamInfo { node_id: u32, position: Option<(i32, i32)>, source_kind: Option, } /// Separate portal remote kept only until a DMA-BUF-first stream starts. struct SharedMemoryFallback { fd: OwnedFd, node_id: u32, position: Option<(i32, i32)>, size: Option<(i32, i32)>, source_kind: Option, } /// Whether a crop-less window frame is held back rather than opening the encoder. /// See MAX_CROP_WAIT. fn holds_for_crop(expecting_crop: bool, waited: Duration) -> bool { expecting_crop && waited < MAX_CROP_WAIT } fn should_retry_shared_memory( state: &str, error: Option<&str>, capture_started: bool, fallback_available: bool, ) -> bool { state == "error" && !capture_started && fallback_available && error.is_some_and(|message| message.contains("alloc buffers")) } /// niri on the NVIDIA driver negotiates DMA-BUF, then marks every chunk /// corrupted with size 0 (issue #1503), so no frame ever reaches the encoder. /// Under the shim's five drop reports per stream, so the count is reached. const CORRUPTED_FRAMES_BEFORE_RETRY: u32 = 3; fn is_corrupted_dmabuf_frame(code: &str, detail: &str) -> bool { code == "frame-dropped" && detail.contains("(producer-marked-frame-corrupted)") && detail.contains("dataType=DmaBuf") } /// Drops the failed stream and reconnects with the portal's independent /// remote, offering the shm format alone; that attempt arms no further retry. #[allow(clippy::too_many_arguments)] fn retry_on_shared_memory( emitter: &mut Emitter, receiver: &Receiver, sender: &Sender, frames: &Option>, session: &mut Option, portal_stream: &mut Option, granted_kind: &mut Option, fallback: SharedMemoryFallback, shared_memory_fallback: &mut Option, reason: &str, ) -> Result<(), ()> { // Joined before reconnecting, so everything the old stream sent is queued. // Its events are dropped, or a late error would end the new stream; the // rest goes back on the channel. drop(session.take()); // A frame still in the mailbox, and the buffers queued for return, belong // to the stream just dropped: neither may reach the new one. if let Some(mailbox) = frames.as_ref() { let _ = mailbox.take(); let _ = mailbox.drain_requeue(); } for message in receiver.try_iter().collect::>() { if !matches!(message, Message::Stream(_)) { let _ = sender.send(message); } } *portal_stream = None; let _ = emitter.emit(&Event::Warning { code: "capture-retry-shared-memory".to_owned(), message: reason.to_owned(), }); let retry = portal::PortalStream { fd: fallback.fd, fallback_fd: None, node_id: fallback.node_id, position: fallback.position, size: fallback.size, source_kind: fallback.source_kind, }; begin_stream( emitter, sender, frames, session, portal_stream, granted_kind, retry, false, true, shared_memory_fallback, ) } /// A stream error ends the helper only before capture has started, or the app /// waits on a recording that never starts. "Started" is the first staged frame /// in video mode, and `streaming` in cursor-only mode, which has no `Capture`. fn stream_error_is_fatal( state: &str, video: bool, recording_started: bool, stream_was_live: bool, ) -> bool { let started = if video { recording_started } else { stream_was_live }; state == "error" && !started } fn run( emitter: &mut Emitter, receiver: mpsc::Receiver, sender: Sender, config: RunConfig, ) -> i32 { let constants = shim::constants(); // Held for the whole loop: dropping it stops and joins the PipeWire thread. let mut session: Option = None; let mut portal_stream: Option = None; let mut size: Option<(i32, i32)> = None; // What the portal granted, kept past the `StreamInfo` that is consumed at // Format: only a window stream is expected to carry a crop. let mut granted_kind: Option = None; // While a window's first usable crop rectangle is awaited: when the wait // began, and the latest crop-less frame, which starts the recording if the // wait runs out. See MAX_CROP_WAIT. let mut crop_wait_since: Option = None; let mut held_frame: Option = None; // A crop change is reported once, not once per frame. let mut crop_change_reported = false; // A `deferStart` grant held while the caller runs its countdown. let mut pending_portal: Option = None; // Retain the portal grant while the preferred DMA-BUF format is starting, // so a buffer-allocation failure can retry shared-memory-first once. let mut shared_memory_fallback: Option = None; // DMA-BUF frames the producer marked corrupted before any usable one. let mut corrupted_frames = 0; // `record` has been received. Latched, because it can arrive before the // picker has been answered. let mut armed = false; let mut cursor: Option = None; let mut known_assets: HashSet = HashSet::new(); let mut pending_asset: Option = None; // Wall-clock ms at which the pw_stream last reached `streaming` (mutter began // handing us frames), or `None` when it is not streaming. A press counts only // if its own read time is at or after this: before streaming the only thing on // screen is the portal picker, and the click that dismisses it (its "Share" // button) would otherwise ride out on the first sample as a phantom click at // t≈0. Comparing timestamps rather than a bare flag closes two gaps: the press // and the stream-state arrive on DIFFERENT channels, so a pre-stream press can // be dequeued after the flag flips (it is still dropped, its time is older); // and clearing it on disconnect stops clicks emitting against a stale cursor // after capture has stopped. let mut streaming_since: Option = None; let mut reported_cursor_meta = false; // Allocated up front so the PipeWire callback has somewhere to put frames // from the very first buffer; `None` in cursor-only mode, which is also what // tells `Session::start` not to map the buffers at all. let frames: Option> = config .output_path .as_ref() .map(|_| Arc::new(FrameMailbox::default())); // Offer dmabuf ahead of shm (issue #507) only for a video session, only when // the VAAPI import pipeline actually builds on this GPU, AND only when the // forced-encoder choice can consume it — `software`/`vulkan` cannot, and // offering dmabuf they can't import makes the first frame fail. The available() // probe constructs a VAAPI device and filtergraph once here, so a machine that // cannot import keeps the shm path with no per-recording cost. let prefer_dmabuf = frames.is_some() && crate::dmabuf_import::available() && encoder::forced_allows_dmabuf(config.forced_encoder); if frames.is_some() { let _ = emitter.emit(&Event::Debug { code: "dmabuf-import".to_owned(), data: json_map([("available", prefer_dmabuf.into())]), }); } let mut capture: Option = None; // Started before the portal picker so the streams are warm and the graph // has settled by the time the first video frame arrives. Everything they // record before that first frame is discarded — see `Capture::stage`. // `_audio_sessions` is bound, not dropped: dropping one stops its thread. let (_audio_sessions, audio_sources) = start_audio(emitter, &config.audio); // Buffered until the format is negotiated: the encoder cannot be opened // before the frame size is known, and `pause` can arrive first. let mut paused = false; // Backdated so the very first sample is not held back by the throttle. // `checked_sub` because a bare `Instant - Duration` panics on underflow, and // this runs milliseconds after process start. let mut last_emit = Instant::now() .checked_sub(config.sample_interval) .unwrap_or_else(Instant::now); let mut exit_code = 0; // Consecutive dmabuf import failures; reset on any staged frame. A run of them // means the import never works, which would otherwise record nothing — see // MAX_CONSECUTIVE_IMPORT_FAILURES. let mut consecutive_drops: u32 = 0; // When the capture clock last advanced. See the heartbeat at the top of the loop. let mut last_advance = Instant::now(); loop { // THE HEARTBEAT, CHECKED ON EVERY PASS, NOT ONLY ON A TIMEOUT. `advance` // holds the last picture forward and drains the audio rings, and it used // to run only on a new frame or when `recv_timeout` expired. Any other // message restarts that timeout, and cursor messages arrive at the // compositor's rate while the mouse moves: a static screen with a moving // mouse never timed out, so nothing drained and the 2 s rings dropped // their oldest audio (getopenscreen/openscreen#936). if heartbeat_due(last_advance, Instant::now(), config.tick) { last_advance = Instant::now(); if let Some(capture) = capture.as_mut() { if let Err(message) = capture.advance() { let _ = emitter.emit(&Event::Error { code: "encode-failed".to_owned(), message, }); exit_code = 1; break; } } } // A window still waiting for its crop when the wait runs out starts from // the frame it holds. Replayed through FrameReady so the encoder opens on // the one path; KWin may never send another frame to trigger it. if held_frame.is_some() && crop_wait_since.is_some_and(|since| since.elapsed() >= MAX_CROP_WAIT) { let _ = sender.send(Message::Stream(StreamEvent::FrameReady)); } // Return PipeWire buffers whose dmabuf imports completed last iteration // (or that were superseded on the capture thread). This MUST run on this // loop, not the PipeWire thread — `Session::requeue` takes the thread-loop // lock, which would deadlock from inside the loop. The one-tick delay is // harmless: the import has already copied the pixels, so the buffer is // free, and the pool has other buffers in flight meanwhile. if let (Some(session), Some(mailbox)) = (session.as_ref(), frames.as_ref()) { for handle in mailbox.drain_requeue() { session.requeue(handle); } } match receiver.recv_timeout(config.tick) { Ok(Message::Stop) => break, // Emit a click sample straight away at the current cursor position, // stamped with the press time the reader captured: immediate so it is // not backdated to the next throttled sample, and one sample per press // so a rapid double-click reads as two clicks rather than collapsing // into one. Counted only if the press happened at or after streaming // began (see `streaming_since`): a press while the picker is still up — // the click on its "Share" button — is older, so it is dropped even if // its message is delivered after the stream-state one. Ok(Message::PointerButton(press_ms)) => { if streaming_since.is_some_and(|since| press_ms >= since) { emit_sample(emitter, &cursor, content_rect(&capture, size), &mut pending_asset, Some(press_ms)); } } // A reader lost its device (typically an unplug). Report it — if it // was the only pointer, clicks stop until one is (re)connected, which // the input watcher will pick up. Ok(Message::PointerDeviceLost(reason)) => { let _ = emitter.emit(&Event::Warning { code: "click-capture-device-lost".to_owned(), message: format!( "a pointer device stopped delivering clicks ({reason}); if it was the \ only one, clicks are not captured until a device is reconnected" ), }); } Ok(Message::Pause) => { paused = true; if let Some(capture) = capture.as_mut() { capture.pause(); } } Ok(Message::Resume) => { paused = false; if let Some(capture) = capture.as_mut() { capture.resume(); } } Ok(Message::Stream(StreamEvent::FrameReady)) => { let Some(mailbox) = frames.as_ref() else { continue; }; // `take` can legitimately return None: several FrameReady // notifications can arrive for frames that superseded each other // in the mailbox before the loop got here. // A fresh frame supersedes the one held for a crop; with none, // this is the replay of the held frame once the wait ran out. let next = match mailbox.take() { Some(frame) => { if let Some(stale) = held_frame.take() { mailbox.recycle(stale.pixels); } Some(frame) } None => held_frame.take(), }; if let Some(frame) = next { // Opening the encoder is deferred to here because only a // frame carries the crop, and the crop — not the negotiated // format — is the size of a window. if capture.is_none() { if let Some(path) = config.output_path.as_ref() { // A WINDOW WITHOUT A CROP IS NOT YET TRUSTWORTHY. // mutter's rectangle intersection reports success // even when it produced an empty rect, and it records // one frame synchronously from `enable()` — before // the picked window is necessarily mapped. Committing // the encoder to that frame would pin a window // recording at monitor size for its whole duration, // which is the very bug being fixed, only intermittent. let expecting_crop = granted_kind == Some(portal::SourceKind::Window) && !frame.has_crop; let waited = crop_wait_since.get_or_insert_with(Instant::now).elapsed(); if holds_for_crop(expecting_crop, waited) { held_frame = Some(frame); continue; } if expecting_crop { let _ = emitter.emit(&Event::Warning { code: "window-crop-missing".to_owned(), message: format!( "a window was granted but no crop rectangle arrived in \ {} ms; recording the full {}x{} stream instead", waited.as_millis(), frame.width, frame.height ), }); } // H.264 chroma is subsampled 2x2, so odd dimensions // are a per-encoder lottery. A window's rect is its // own pixel size and is routinely odd. let width = frame.crop.width & !1; let height = frame.crop.height & !1; if width <= 0 || height <= 0 { let _ = emitter.emit(&Event::Error { code: "encoder-unavailable".to_owned(), message: format!( "the compositor reported a {}x{} content rectangle, which \ cannot be encoded", frame.crop.width, frame.crop.height ), }); exit_code = 1; break; } let _ = emitter.emit(&Event::Debug { code: "crop".to_owned(), data: json_map([ ("streamWidth", frame.width.into()), ("streamHeight", frame.height.into()), ("cropX", frame.crop.x.into()), ("cropY", frame.crop.y.into()), ("cropWidth", frame.crop.width.into()), ("cropHeight", frame.crop.height.into()), ("encodedWidth", width.into()), ("encodedHeight", height.into()), ("hasCrop", frame.has_crop.into()), ("cropWaitMs", (waited.as_millis() as u64).into()), ]), }); match Capture::start( path, width, height, config.fps, config.bitrate, config.forced_encoder, // Cloned, not taken: an unstarted capture is rebuilt // when the stream falls back to shared memory. audio_sources.clone(), frame.dmabuf.as_ref(), ) { Ok((started, selection)) => { let _ = emitter.emit(&Event::EncoderSelection { video: selection.backend.as_str().to_owned(), rejected: selection.rejected, }); capture = Some(started); if paused { // A `pause` that arrived while the portal // picker was still up applies to the // capture that picker was for. if let Some(capture) = capture.as_mut() { capture.pause(); } } } Err(message) => { let _ = emitter.emit(&Event::Error { code: "encoder-unavailable".to_owned(), message, }); exit_code = 1; break; } } } } let Some(capture) = capture.as_mut() else { // Cursor-only session: no encoder, nothing to stage. The // recycle below still has to run. shared_memory_fallback = None; mailbox.recycle(frame.pixels); continue; }; // A window that was resized mid-recording. The file keeps its // original dimensions — an MP4 track cannot change resolution // — so this is reported rather than silently reframed. if !crop_change_reported && capture.crop_diverged(&frame) { crop_change_reported = true; let _ = emitter.emit(&Event::Warning { code: "crop-changed".to_owned(), message: format!( "the recorded window changed size to {}x{} mid-recording; the \ file keeps the size it started at, so the framing may be cut \ off or padded from here on", frame.crop.width, frame.crop.height ), }); } let first = !capture.started(); let staged = capture.stage(&frame); let (width, height) = (frame.crop.width, frame.crop.height); mailbox.recycle(frame.pixels); match staged { // A real frame, or a pause-freeze — neither is an import // failure, so end any run of them. (A freeze holds the last // picture; `advance` is a no-op while paused.) Ok(capture::StageOutcome::Staged | capture::StageOutcome::Frozen) => { consecutive_drops = 0; } // Recoverable: one frame the GPU could not import. Warn so it // is answerable from the log, then carry on — `advance` holds // the previous frame — rather than ending the file. But a long // RUN of failures means the import path is broken and the file // would be empty, so past the threshold fail loudly with a way // out (the full auto-downgrade to shm remains a follow-up). Ok(capture::StageOutcome::Dropped(reason)) => { consecutive_drops += 1; if consecutive_drops >= MAX_CONSECUTIVE_IMPORT_FAILURES { let _ = emitter.emit(&Event::Error { code: "encode-failed".to_owned(), message: format!( "the GPU could not import {consecutive_drops} captured \ frames in a row ({reason}); the render node likely \ cannot map the compositor's buffers. Set \ OPENSCREEN_LINUX_RENDER_NODE to the correct /dev/dri \ node, or OPENSCREEN_LINUX_ENCODER=software for the CPU \ path." ), }); exit_code = 1; break; } let _ = emitter.emit(&Event::Warning { code: "frame-dropped".to_owned(), message: reason, }); } Err(message) => { let _ = emitter.emit(&Event::Error { code: "encode-failed".to_owned(), message, }); exit_code = 1; break; } } // Only once a frame has actually staged — a first frame that // dropped leaves capture unstarted, so this waits for a real one. if first && capture.started() { // A frame staged: the corrupted-frame retry is moot. Not // earlier, as a first frame can still fail its GPU import. shared_memory_fallback = None; let _ = emitter.emit(&Event::CaptureStarted { timestamp_ms: timestamp_ms(), path: config .output_path .as_ref() .map(|path| path.display().to_string()) .unwrap_or_default(), width, height, fps: config.fps, }); } } if let Some(capture) = capture.as_mut() { last_advance = Instant::now(); if let Err(message) = capture.advance() { let _ = emitter.emit(&Event::Error { code: "encode-failed".to_owned(), message, }); exit_code = 1; break; } } } Ok(Message::Record) => { // Arming is idempotent and may arrive before OR after the picker // is answered — the user can be slower than the countdown, or // faster. Both orderings have to converge on the same state, or a // slow picker produces a recording that never starts. armed = true; if let Some(stream) = pending_portal.take() { if let Err(()) = begin_stream( emitter, &sender, &frames, &mut session, &mut portal_stream, &mut granted_kind, stream, prefer_dmabuf, false, &mut shared_memory_fallback, ) { exit_code = 1; break; } } } Ok(Message::Portal(result)) => match *result { Ok(stream) => { // The portal's size is in the compositor's coordinate space // and can differ from the negotiated pixel size on a scaled // display. Logged rather than used: cursor positions arrive // in stream pixels, and `content_rect` places them from // there. let _ = emitter.emit(&Event::Debug { code: "portal-stream".to_owned(), data: json_map([ ("nodeId", stream.node_id.into()), ("logicalWidth", stream.size.map(|(w, _)| w).into()), ("logicalHeight", stream.size.map(|(_, h)| h).into()), ("positionX", stream.position.map(|(x, _)| x).into()), ("positionY", stream.position.map(|(_, y)| y).into()), ( "sourceKind", stream.source_kind.map(|kind| kind.as_str()).into(), ), ]), }); // PROTOCOL, NOT DIAGNOSTICS. The picker has been answered — // which is a different moment from "pixels are flowing", and // the only one at which an app can start a countdown without // it running before the user has been asked anything. let _ = emitter.emit(&Event::SourceSelected { timestamp_ms: timestamp_ms(), node_id: stream.node_id, source_kind: stream.source_kind.map(|kind| kind.as_str().to_owned()), position_x: stream.position.map(|(x, _)| x), position_y: stream.position.map(|(_, y)| y), }); granted_kind = stream.source_kind; if config.defer_start && !armed { // Hold the grant WITHOUT connecting a pw_stream. mutter // only enables its capture source on STREAMING, so an // unconnected session costs the compositor nothing and // produces no frames to throw away. Connecting it // inactive instead — OBS's pattern — would expose it to // WirePlumber's idle-node suspend, which tears down the // negotiated format after a few seconds and would lose it // during a countdown. pending_portal = Some(stream); } else if let Err(()) = begin_stream( emitter, &sender, &frames, &mut session, &mut portal_stream, &mut granted_kind, stream, prefer_dmabuf, false, &mut shared_memory_fallback, ) { exit_code = 1; break; } } Err(error) => { let _ = emitter.emit(&Event::Error { code: error.code().to_owned(), message: error.message(), }); exit_code = 1; break; } }, Ok(Message::Stream(StreamEvent::Format(format))) => { size = Some((format.width, format.height)); let _ = emitter.emit(&Event::Debug { code: "format".to_owned(), data: json_map([ ("videoFormatId", format.video_format.into()), ("framerateNum", format.framerate_num.into()), ("framerateDenom", format.framerate_denom.into()), ]), }); if let Some(stream) = portal_stream.take() { let _ = emitter.emit(&Event::StreamStarted { timestamp_ms: timestamp_ms(), node_id: stream.node_id, width: format.width, height: format.height, position_x: stream.position.map(|(x, _)| x), position_y: stream.position.map(|(_, y)| y), source_kind: stream .source_kind .map(|kind| kind.as_str().to_owned()), }); } // THE ENCODER IS NOT OPENED HERE ANY MORE, and that is the whole // window-capture fix. `format` is the size of the STREAM, which // for a window is the size of its monitor: a PipeWire stream // cannot be resized once negotiated but a window can, so mutter // pins the stream to the monitor and reports the window's live // rectangle as SPA_META_VideoCrop instead. Sizing the encoder // from `format` is what produced window recordings padded out to // screen size with black. The size now comes from the first frame // carrying a usable crop — see the FrameReady arm. if config.output_path.is_some() { if capture.is_some() { let _ = emitter.emit(&Event::Warning { code: "format-renegotiated".to_owned(), message: format!( "the compositor renegotiated to {}x{} mid-recording; the \ encoder keeps its original size and the file may be letterboxed \ or cropped from here on", format.width, format.height ), }); } } } Ok(Message::Stream(StreamEvent::BufferInfo { data_type, n_datas, has_cursor_meta, cursor_meta_size, metas, })) => { // What memory type does the compositor hand us, and which // metadata blocks survived negotiation? `metas` is the load- // bearing field: a missing Cursor next to a present Header means // our ParamMeta was sent and lost the size intersection, which is // otherwise indistinguishable from never having sent it. let _ = emitter.emit(&Event::Debug { code: "buffer-info".to_owned(), data: json_map([ ("dataType", data_type_name(&constants, data_type).into()), ("dataTypeId", data_type.into()), ("nDatas", n_datas.into()), ("hasCursorMeta", has_cursor_meta.into()), ("cursorMetaSize", cursor_meta_size.into()), ("metas", metas.clone().into()), ]), }); if !has_cursor_meta && config.cursor_mode.reports_cursor() { let _ = emitter.emit(&Event::Warning { code: "no-cursor-metadata".to_owned(), message: format!( "The negotiated buffers carry no SPA_META_Cursor, so no cursor \ samples will be produced. Metadata present: [{metas}]. A cursor \ block is dropped when the compositor's fixed SPA_PARAM_META_size \ falls outside the range this helper accepts." ), }); } } // Surfaced as a warning rather than logged, because these are the // conditions that leave a recording empty while every counter still // reads zero: a frame refused by the shim never reaches the mailbox, // so `frames-dropped` stays at 0 and the session stops "cleanly". // Without this the only honest symptom is a file with no frames in // it and nothing anywhere saying why. Ok(Message::Stream(StreamEvent::CaptureIssue { code, detail })) => { let corrupted = is_corrupted_dmabuf_frame(&code, &detail); let _ = emitter.emit(&Event::Warning { code, message: detail, }); if corrupted && shared_memory_fallback.is_some() { corrupted_frames += 1; if corrupted_frames >= CORRUPTED_FRAMES_BEFORE_RETRY { let fallback = shared_memory_fallback.take().expect("checked above"); // Nothing of the retired stream carries over: a frame // held for its crop, its crop wait, its streaming edge // (the click gate), and an encoder that one of its // frames opened but never started, which may be bound to // the DMA-BUF importer the shm stream cannot feed. if let Some(held) = held_frame.take() { if let Some(mailbox) = frames.as_ref() { mailbox.recycle(held.pixels); } } crop_wait_since = None; streaming_since = None; // Its audio rings outlive it through `audio_sources`. A // pause closed them; reopen them before it goes, or a // Resume that lands before the replacement opens has no // capture to resume and the new one records no audio. // The replacement re-pauses them if `paused` still holds. if let Some(mut retired) = capture.take() { retired.resume(); } if let Err(()) = retry_on_shared_memory( emitter, &receiver, &sender, &frames, &mut session, &mut portal_stream, &mut granted_kind, fallback, &mut shared_memory_fallback, "The compositor marks every DMA-BUF frame corrupted; retrying screen capture with shared-memory buffers.", ) { exit_code = 1; break; } } } } Ok(Message::Stream(StreamEvent::State { state, error })) => { // Every transition, not just the failures. Without this a run // that ends in "target not found" is ambiguous: there is no way // to tell a stream that never reached `streaming` from one that // streamed happily and then hit the error on teardown. let _ = emitter.emit(&Event::Debug { code: "stream-state".to_owned(), data: json_map([ ("state", state.clone().into()), ("error", error.clone().into()), ]), }); // Once frames are flowing, presses land on recorded content; the // picker (and the "Share" click that dismissed it) is behind us. // Stamped so a press read before this instant is dropped by time, // and cleared on disconnect so clicks stop emitting against a stale // cursor after capture ends. Only set on the FIRST streaming edge so // a transient renegotiation `paused`→`streaming` does not re-arm it. if state == "streaming" { if streaming_since.is_none() { streaming_since = Some(timestamp_ms()); } // No allocation retry is appropriate after the compositor // accepted the stream. A video session keeps the grant until // its first frame, for the corrupted-frame retry (#1503). if frames.is_none() { shared_memory_fallback = None; } } else if state == "unconnected" { streaming_since = None; } let recording_started = capture.as_ref().is_some_and(Capture::started); let stream_was_live = streaming_since.is_some(); if should_retry_shared_memory( &state, error.as_deref(), recording_started || stream_was_live, shared_memory_fallback.is_some(), ) { let fallback = shared_memory_fallback.take().expect("checked above"); if let Err(()) = retry_on_shared_memory( emitter, &receiver, &sender, &frames, &mut session, &mut portal_stream, &mut granted_kind, fallback, &mut shared_memory_fallback, "PipeWire buffer allocation failed; retrying screen capture with shared-memory buffers.", ) { exit_code = 1; break; } continue; } if let Some(error) = error { if stream_error_is_fatal( &state, frames.is_some(), recording_started, stream_was_live, ) { let _ = emitter.emit(&Event::Error { code: "pipewire-capture-failed".to_owned(), message: format!("OpenScreen could not start screen capture: {error}"), }); exit_code = 1; break; } let _ = emitter.emit(&Event::Warning { code: "stream-error".to_owned(), message: format!("PipeWire stream reported an error in state {state}: {error}"), }); } if state == "unconnected" && session.is_some() { let _ = emitter.emit(&Event::Warning { code: "stream-unconnected".to_owned(), message: "The PipeWire stream disconnected; no further cursor samples \ will arrive." .to_owned(), }); } } Ok(Message::Stream(StreamEvent::Cursor(event))) => { // Open question #2: does SPA_META_Cursor actually carry a bitmap? if !reported_cursor_meta { reported_cursor_meta = true; let _ = emitter.emit(&Event::Debug { code: "cursor-meta".to_owned(), data: json_map([ ("hasBitmap", event.bitmap.is_some().into()), ("cursorId", event.id.into()), ("hotspotX", event.hotspot_x.into()), ("hotspotY", event.hotspot_y.into()), ]), }); } let mut asset_is_new = false; if let Some(raw) = &event.bitmap { if emitter.debug_enabled() { let _ = emitter.emit(&Event::Debug { code: "cursor-bitmap".to_owned(), data: json_map([ ("formatId", raw.format.into()), ("width", raw.width.into()), ("height", raw.height.into()), ("stride", raw.stride.into()), ("bytes", raw.pixels.len().into()), ]), }); } match bitmap::encode(&constants, raw) { Ok(encoded) => { if known_assets.insert(encoded.id.clone()) { asset_is_new = true; pending_asset = Some(CursorAsset { id: encoded.id.clone(), image_data_url: encoded.image_data_url, width: encoded.width, height: encoded.height, hotspot_x: event.hotspot_x, hotspot_y: event.hotspot_y, }); } cursor = Some(CursorState { x: event.x, y: event.y, asset_id: Some(encoded.id), }); } Err(message) => { let _ = emitter.emit(&Event::Warning { code: "cursor-bitmap-unusable".to_owned(), message, }); cursor = Some(CursorState { x: event.x, y: event.y, asset_id: cursor.and_then(|state| state.asset_id), }); } } } else { cursor = Some(CursorState { x: event.x, y: event.y, asset_id: cursor.and_then(|state| state.asset_id), }); } // A new sprite ships immediately; positions respect the sample // interval so a 120fps compositor cannot flood stdout. if asset_is_new || last_emit.elapsed() >= config.sample_interval { emit_sample(emitter, &cursor, content_rect(&capture, size), &mut pending_asset, None); last_emit = Instant::now(); } } Err(RecvTimeoutError::Timeout) => { if cursor.is_some() && last_emit.elapsed() >= config.sample_interval { emit_sample(emitter, &cursor, content_rect(&capture, size), &mut pending_asset, None); last_emit = Instant::now(); } // No advance here: a timeout means a tick has passed, so the // heartbeat at the top of the next pass holds the picture forward. } Err(RecvTimeoutError::Disconnected) => break, } } // Before the file is closed: this joins the PipeWire thread, so no callback // can fire against a freed Sender or write into a mailbox that is about to // be dropped. drop(session); if let Some(capture) = capture { finish_capture(emitter, capture, frames.as_deref(), &mut exit_code); } exit_code } /// Writes the trailer and reports what the recording cost. /// /// Runs even when the loop broke on an error: a file whose trailer was never /// written loses its last fragment, and a partial recording is worth more to the /// user than none. fn finish_capture( emitter: &mut Emitter, capture: Capture, frames: Option<&FrameMailbox>, exit_code: &mut i32, ) { for (label, samples) in capture.dropped_audio() { let _ = emitter.emit(&Event::Warning { code: "audio-dropped".to_owned(), message: format!( "{samples} {label} sample(s) were discarded because the encoder could not \ keep up, and stand in the recording as that much silence. Unlike a dropped \ video frame, this is audible." ), }); } match capture.finish() { Ok(summary) => { let dropped = frames.map(FrameMailbox::dropped).unwrap_or(0); if dropped > 0 { let _ = emitter.emit(&Event::Warning { code: "frames-dropped".to_owned(), message: format!( "{dropped} captured frame(s) were replaced before the encoder could \ take them, so the recording holds an older picture across those \ moments. The machine could not keep up with the capture rate." ), }); } // The video timeline should equal real elapsed wall-clock time. With // wall-clock PTS it does by construction; a divergence beyond a frame // or two means frames are being stamped off the clock again (#511) — // surface it loudly rather than shipping a silently time-compressed // file that plays ahead of audio, webcam and the cursor overlay. const TIMELINE_SKEW_EPSILON_MS: i64 = 100; let skew = summary.duration_ms as i64 - summary.wall_clock_ms as i64; if skew.abs() > TIMELINE_SKEW_EPSILON_MS { let _ = emitter.emit(&Event::Warning { code: "timeline-divergence".to_owned(), message: format!( "the recorded video timeline is {} ms but the recording ran {} ms of \ wall-clock time (off by {} ms); the screen track may be out of sync \ with audio, webcam and the cursor overlay. See issue #511.", summary.duration_ms, summary.wall_clock_ms, skew.abs() ), }); } let _ = emitter.emit(&Event::CaptureStopped { timestamp_ms: timestamp_ms(), path: summary.path.display().to_string(), duration_ms: summary.duration_ms, frames: summary.frames, dropped, convert_ms: summary.stats.convert_ms(), upload_ms: summary.stats.upload_ms(), encode_ms: summary.stats.encode_ms(), }); } Err(message) => { let _ = emitter.emit(&Event::Error { code: "capture-finish-failed".to_owned(), message, }); *exit_code = 1; } } } /// The rectangle cursor positions are measured against. /// /// The encoder's content rectangle once a frame has been staged — that is the /// only rectangle the file shows. Before then, and for a cursor-only session /// that opens no encoder at all, the whole negotiated stream, which is what the /// consumer of a cursor-only recording is compositing over. /// Whether a tick has passed since the capture clock last advanced. fn heartbeat_due(last_advance: Instant, now: Instant, tick: Duration) -> bool { now.saturating_duration_since(last_advance) >= tick } fn content_rect(capture: &Option, size: Option<(i32, i32)>) -> Option { match capture { Some(capture) if capture.started() => Some(capture.content_rect()), _ => size.map(|(width, height)| shim::CropRect { x: 0, y: 0, width, height }), } } /// Emits one cursor sample, positioned inside `content`. /// /// `content` is the sub-rectangle of the stream the recording actually shows — /// [`Capture::content_rect`] once pixels are being encoded, the whole stream /// otherwise (a cursor-only session records no video of its own). The pointer /// arrives in STREAM pixels, so for a window stream it is measured from the /// corner of the monitor while the file starts at the corner of the window; /// subtracting the origin is what puts the two in the same space. The consumer /// normalises against the `width`/`height` reported here, so those must be the /// content's, not the stream's. /// /// `click` is `Some(press_ms)` for a left-button press — the sample is then /// tagged `"click"` and stamped with that press time — or `None` for an ordinary /// throttled position sample stamped now. Either way a pending new sprite rides /// out on it. fn emit_sample( emitter: &mut Emitter, cursor: &Option, content: Option, pending_asset: &mut Option, click: Option, ) { let (Some(state), Some(content)) = (cursor, content) else { return; }; let (x, y) = (state.x - content.x, state.y - content.y); // A pointer outside the recorded rectangle is REPORTED, not withheld: the // consumer clamps the position and carries `visible` so a renderer can hide // the sprite rather than pin it to an edge. let visible = x >= 0 && y >= 0 && x < content.width && y < content.height; // A click carries its own press time so the bounce lands when the button went // down, not up to a sample interval later; the position is the latest known, // which at the sample cadence has not moved enough to be wrong. let (timestamp, interaction_type) = match click { Some(press_ms) => (press_ms, Some("click".to_owned())), None => (timestamp_ms(), None), }; let _ = emitter.emit(&Event::CursorSample { timestamp_ms: timestamp, x, y, width: content.width, height: content.height, visible, asset_id: state.asset_id.clone(), asset: pending_asset.take(), interaction_type, }); } fn data_type_name(constants: &shim::Constants, data_type: u32) -> &'static str { if data_type == constants.data_mem_ptr { "MemPtr" } else if data_type == constants.data_mem_fd { "MemFd" } else if data_type == constants.data_dma_buf { "DmaBuf" } else { "unknown" } } fn json_map( entries: [(&str, serde_json::Value); N], ) -> serde_json::Map { entries .into_iter() .map(|(key, value)| (key.to_owned(), value)) .collect() } /// Resolves the microphone the user picked in the app to a PipeWire `node.name`. /// /// WHY A MATCH AND NOT A LOOKUP. The picker lists Chromium `MediaDeviceInfo`, /// whose `deviceId` is an opaque per-origin hash and whose `label` is a human /// string. Neither is a PipeWire node name, and `PW_KEY_TARGET_OBJECT` takes /// nothing else. On a PipeWire system Chromium's label IS the node's /// `node.description`, so matching on that is the bridge between the two /// namespaces — there is no id to look up. /// /// Returning `None` is a real answer, not a failure: the caller then passes no /// target and PipeWire links to the session default source. That is the old /// behaviour, and it is the right fallback for a label we cannot place. fn resolve_microphone_node(label: &str, sources: &[shim::AudioSourceInfo]) -> Option { let wanted = label.trim(); if wanted.is_empty() { return None; } // A caller that already knows the node name (a config file, a future // picker) should not be forced through fuzzy matching. if let Some(exact) = sources.iter().find(|s| s.name == wanted) { return Some(exact.name.clone()); } let folded = wanted.to_lowercase(); if let Some(exact) = sources.iter().find(|s| s.description.to_lowercase() == folded) { return Some(exact.name.clone()); } // Chromium decorates labels ("Digital Microphone (Family 17h/19h ...)"), // and some descriptions are longer than the label, so containment is // checked BOTH ways. Longest description first, so that when several nodes // match a short label the most specific one wins rather than whichever the // registry happened to announce first. let mut candidates: Vec<&shim::AudioSourceInfo> = sources .iter() .filter(|s| { let d = s.description.to_lowercase(); !d.is_empty() && (d.contains(&folded) || folded.contains(&d)) }) .collect(); candidates.sort_by_key(|s| std::cmp::Reverse(s.description.len())); candidates.first().map(|s| s.name.clone()) } #[cfg(test)] mod cursor_sample_tests { use super::*; fn sample_json(cursor: (i32, i32), content: Option) -> serde_json::Value { let mut buffer = Vec::new(); let mut emitter = Emitter::new(&mut buffer, false); emit_sample( &mut emitter, &Some(CursorState { x: cursor.0, y: cursor.1, asset_id: None }), content, &mut None, None, ); let line = String::from_utf8(buffer).expect("utf8"); serde_json::from_str(line.trim()).expect("json") } /// A full-screen capture crops nothing, so the pointer keeps the numbers the /// portal gave and is normalised against the whole stream. #[test] fn a_full_screen_capture_reports_stream_coordinates() { let value = sample_json( (960, 540), Some(shim::CropRect { x: 0, y: 0, width: 1920, height: 1080 }), ); assert_eq!(value["x"], 960); assert_eq!(value["y"], 540); assert_eq!(value["width"], 1920); assert_eq!(value["height"], 1080); assert_eq!(value["visible"], true); } /// THE WINDOW-CAPTURE BUG. mutter pins a window stream to the whole monitor /// and carves the window out through SPA_META_VideoCrop, so the pointer /// arrives measured from the monitor's corner while the file starts at the /// window's. Reporting the stream's size here normalised a monitor position /// against monitor dimensions and handed the compositor a fraction of the /// wrong rectangle: the cursor sat at the wrong place in every window /// recording, by the crop origin, at the wrong scale. #[test] fn a_window_capture_reports_coordinates_inside_the_window() { // A 1920x1080 monitor carrying a 640x480 window at (100, 50), pointer // one quarter into the window. let value = sample_json( (260, 170), Some(shim::CropRect { x: 100, y: 50, width: 640, height: 480 }), ); assert_eq!(value["x"], 160, "the crop origin has to come off the position"); assert_eq!(value["y"], 120); assert_eq!(value["width"], 640, "the consumer normalises against what it is told"); assert_eq!(value["height"], 480); assert_eq!(value["visible"], true); } /// Outside the window but still on the monitor. The old test was `x < width` /// against the STREAM, which called this visible — and since the consumer /// clamps to 0..1, it parked the sprite on the frame's edge for as long as /// the pointer was anywhere else on screen. #[test] fn a_pointer_outside_the_window_is_reported_invisible() { let outside = sample_json( (1500, 900), Some(shim::CropRect { x: 100, y: 50, width: 640, height: 480 }), ); assert_eq!(outside["visible"], false); // Above and to the left of the window, which goes negative rather than // past the far edge. let before = sample_json( (10, 10), Some(shim::CropRect { x: 100, y: 50, width: 640, height: 480 }), ); assert_eq!(before["visible"], false); } /// No content rectangle means the format has not been negotiated yet. There /// is nothing to measure against, so nothing is emitted — a sample stamped /// with a guess would be indistinguishable from a real one downstream. #[test] fn nothing_is_emitted_before_the_format_is_known() { let mut buffer = Vec::new(); let mut emitter = Emitter::new(&mut buffer, false); emit_sample( &mut emitter, &Some(CursorState { x: 10, y: 10, asset_id: None }), None, &mut None, None, ); assert!(buffer.is_empty(), "emitted {}", String::from_utf8_lossy(&buffer)); } /// #936: messages arriving faster than the tick must not hold the heartbeat /// off. The loop checks it on every pass, so what matters is that it comes /// due once a tick has passed, however many passes happened in between. #[test] fn the_heartbeat_comes_due_every_tick_whatever_else_arrives() { let tick = Duration::from_millis(16); let start = Instant::now(); let mut last_advance = start; let mut advances = 0; // A cursor message every 7 ms (144 Hz) for one second of loop passes. for pass in 0..=(1000 / 7) { let now = start + Duration::from_millis(pass * 7); if heartbeat_due(last_advance, now, tick) { last_advance = now; advances += 1; } } // Every 21 ms at worst (three 7 ms passes): about 47 a second, never zero. assert!(advances >= 1000 / 21, "{advances} advances in a second of cursor traffic"); assert!(!heartbeat_due(start, start + Duration::from_millis(15), tick)); } /// A cursor-only session opens no encoder, so it falls back to the whole /// stream; once pixels are flowing the encoder's rectangle wins. #[test] fn the_content_rect_prefers_the_encoder_once_it_has_started() { // No encoder yet (cursor-only, or before the first frame): the whole // negotiated stream, or nothing when the format is still unknown. assert_eq!( content_rect(&None, Some((1920, 1080))), Some(shim::CropRect { x: 0, y: 0, width: 1920, height: 1080 }) ); assert_eq!(content_rect(&None, None), None); // Once a window frame has been staged, the encoder's content rect — the // window carved out of the monitor-sized stream — wins over the stream // size. This is the branch the fix turns on, so it is the one that has // to be pinned. let output = std::env::temp_dir().join("openscreen-content-rect-encoder.mp4"); let (mut capture, _) = Capture::start( &output, 320, 240, 30, Some(1_000_000), Some(encoder::Backend::Software), Vec::new(), None, ) .expect("start"); let stride = 1920usize * 4; capture .stage(&shim::Frame { pixels: vec![0x30; stride * 1080], stride, width: 1920, height: 1080, video_format: shim::constants().video_format_bgrx, pts_ns: -1, crop: shim::CropRect { x: 100, y: 50, width: 320, height: 240 }, has_crop: true, dmabuf: None, }) .expect("stage"); assert_eq!( content_rect(&Some(capture), Some((1920, 1080))), Some(shim::CropRect { x: 100, y: 50, width: 320, height: 240 }), "a started encoder reports the window it read, not the monitor stream" ); let _ = std::fs::remove_file(&output); } } #[cfg(test)] mod microphone_resolution_tests { use super::*; fn sources() -> Vec { // The real graph of the machine this was debugged on. vec![ shim::AudioSourceInfo { name: "alsa_input.pci-0000_03_00.6.HiFi__hw_Generic_1__source".into(), description: "Family 17h/19h HD Audio Controller Headphones Stereo Microphone" .into(), }, shim::AudioSourceInfo { name: "alsa_input.pci-0000_03_00.6.HiFi__hw_acp6x__source".into(), description: "Family 17h/19h HD Audio Controller Digital Microphone".into(), }, ] } #[test] fn matches_the_description_chromium_reports_as_the_label() { // THE bug this exists for: the user picked the built-in microphone and // the recording captured the empty headphone jack, because no target was // passed and PipeWire fell back to the default source. let node = resolve_microphone_node( "Family 17h/19h HD Audio Controller Digital Microphone", &sources(), ); assert_eq!(node.as_deref(), Some("alsa_input.pci-0000_03_00.6.HiFi__hw_acp6x__source")); } #[test] fn matches_a_decorated_or_shortened_label() { // Chromium is not obliged to hand back the description verbatim. assert_eq!( resolve_microphone_node("Digital Microphone", &sources()).as_deref(), Some("alsa_input.pci-0000_03_00.6.HiFi__hw_acp6x__source") ); } #[test] fn a_node_name_passes_straight_through() { let name = "alsa_input.pci-0000_03_00.6.HiFi__hw_Generic_1__source"; assert_eq!(resolve_microphone_node(name, &sources()).as_deref(), Some(name)); } #[test] fn an_unknown_label_falls_back_to_the_default_source() { // `None` means "pass no target", which is the pre-fix behaviour. Better // a default microphone than no audio at all. assert_eq!(resolve_microphone_node("Blue Yeti", &sources()), None); assert_eq!(resolve_microphone_node(" ", &sources()), None); } #[test] fn prefers_the_most_specific_description_when_several_contain_the_label() { // "Microphone" is a substring of both. Picking the first announced would // make the result depend on registry ordering, which is not stable. let node = resolve_microphone_node("Microphone", &sources()); assert_eq!( node.as_deref(), Some("alsa_input.pci-0000_03_00.6.HiFi__hw_Generic_1__source"), "the longer description should win, deterministically" ); } } #[cfg(test)] mod capture_fallback_tests { use super::{ holds_for_crop, is_corrupted_dmabuf_frame, should_retry_shared_memory, stream_error_is_fatal, MAX_CROP_WAIT, }; use std::time::Duration; #[test] fn a_crop_is_awaited_for_a_bounded_time_not_a_frame_count() { // mutter: the crop follows the first frame within a few frames. assert!(holds_for_crop(true, Duration::ZERO)); // KWin: a still window sends one frame and never a crop, so the wait has // to end on time alone and start from that frame. assert!(!holds_for_crop(true, MAX_CROP_WAIT)); assert!(!holds_for_crop(false, Duration::ZERO)); } #[test] fn a_stream_error_is_fatal_only_before_capture_starts() { // Video: started means a staged frame. assert!(stream_error_is_fatal("error", true, false, true)); assert!(!stream_error_is_fatal("error", true, true, true)); // Cursor-only: started means the stream reached `streaming`. assert!(stream_error_is_fatal("error", false, false, false)); assert!(!stream_error_is_fatal("error", false, false, true)); assert!(!stream_error_is_fatal("paused", true, false, false)); } #[test] fn retries_only_for_pre_recording_pipewire_buffer_allocation_errors() { assert!(should_retry_shared_memory( "error", Some("error alloc buffers: Invalid argument"), false, true, )); } #[test] fn does_not_retry_after_capture_has_started() { assert!(!should_retry_shared_memory( "error", Some("error alloc buffers: Invalid argument"), true, true, )); } #[test] fn does_not_retry_without_a_retained_portal_grant() { assert!(!should_retry_shared_memory( "error", Some("error alloc buffers: Invalid argument"), false, false, )); } #[test] fn a_corrupted_dmabuf_frame_counts_toward_the_shared_memory_retry() { // The shim's report for niri on NVIDIA (issue #1503). let corrupted = |data_type: &str| { format!( "the compositor's buffer failed validation (producer-marked-frame-corrupted), so this frame was dropped: dataType={data_type} stride=10240 width=2560 height=1080 chunkOffset=0 chunkSize=0 available=11059200" ) }; assert!(is_corrupted_dmabuf_frame("frame-dropped", &corrupted("DmaBuf"))); // Shared memory already is the fallback. assert!(!is_corrupted_dmabuf_frame("frame-dropped", &corrupted("MemFd"))); assert!(!is_corrupted_dmabuf_frame( "frame-dropped", "the compositor's buffer failed validation (stride-shorter-than-row), so this frame was dropped: dataType=DmaBuf", )); assert!(!is_corrupted_dmabuf_frame("dmabuf-import-failed", &corrupted("DmaBuf"))); } #[test] fn does_not_retry_unrelated_pipewire_errors_or_non_error_states() { assert!(!should_retry_shared_memory( "error", Some("target not found"), false, true, )); assert!(!should_retry_shared_memory( "paused", Some("error alloc buffers: Invalid argument"), false, true, )); } }