//! Everything behind the window: the GPU, the sources, the program clock that //! draws the canvas and mixes the audio, and, while on air ([`air`]), the //! broadcast it encodes into: the program, at the canvas size and at half of //! it ([`crate::ladder`]), and each published scene's beneath it, at its own //! size and half. The relays serve those broadcasts, and recordings read the //! program or a scene, on air or not. Each local source's own track stays in //! a broadcast of the studio's own, which recordings read too, as they read //! each broadcast source from the relays. //! //! Threads: //! - **program** draws preview, program, and each published scene once per //! canvas frame and hands each to each rendition's encoder its gate lets //! encode, scaled to the rendition and stamped with the program clock. //! - **audio** advances the audio bus, and with it each audio scene's mix, on //! the same clock. //! - **audio-record** encodes the audio scene recordings hold into the //! recording broadcast's own audio while a recording reads it. //! - While on air, the program audio, each published scene's, and each //! guest's return feed, a mix-minus, encode on the tokio runtime. //! - **encode**: one per program rendition, which waits for each program copy //! and encodes it. A new canvas size or frame rate starts new renditions. //! - Recordings write their files on the tokio runtime. //! - **vcam**: one per virtual camera on, the program's or a published //! scene's, which waits for each copy the camera takes and hands it to the //! camera's backend. //! - **source-***: one per running source, since a capture stream may not //! leave the thread that opened it. //! - **autosave** writes the collection and its history to their files once //! edits to them settle, and takes or holds an outside edit to the scene file. //! - The UI runs on the main thread and edits [`State`] under its lock. use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, mpsc}; use std::time::Duration; use anyhow::Context; use bytes::Bytes; use kio::clock::Instant; use moq_net::bandwidth; use moq_scene::publish::{Encoding, Reclaim, Sessions}; use moq_scene::source::{self, Driver, Sources, Status}; use moq_scene::{Canvas, Collection, Compositor, Draft, Edit, History, Scaler}; use moq_video::render::export::{Exporter, Pending, Route}; use tokio::sync::oneshot::error::TryRecvError; use crate::autosave::{self, Autosave}; use crate::ladder::{self, Gate, Joins, Step}; use crate::{health, mix, record}; pub mod air; pub use air::{Aired, Destination, OnAir}; /// The GPU everything shares: sources upload to it, the compositor draws on /// it, the window presents from it, and the encoder reads the program from it. #[derive(Clone)] pub struct Gpu { pub instance: wgpu::Instance, pub adapter: wgpu::Adapter, pub device: wgpu::Device, pub queue: wgpu::Queue, } impl Gpu { /// Open the best adapter among `backends`. pub async fn new(backends: wgpu::Backends) -> anyhow::Result { let instance = wgpu::Instance::new(wgpu::InstanceDescriptor { backends, ..wgpu::InstanceDescriptor::new_without_display_handle() }); let adapter = instance .request_adapter(&wgpu::RequestAdapterOptions { power_preference: wgpu::PowerPreference::HighPerformance, ..Default::default() }) .await .context("no GPU adapter")?; // Lets a hardware-decoded or captured frame reach the compositor, and the // program reach the encoder, without a copy where the driver allows. let descriptor = wgpu::DeviceDescriptor { label: Some("moq-studio"), ..Default::default() }; let (device, queue) = moq_video::render::request_device(&adapter, &descriptor) .await .context("cannot open the GPU")?; Ok(Self { instance, adapter, device, queue, }) } } /// What the UI edits and the program clock draws. pub struct State { /// Changed only through [`commit`](Self::commit) and the history, so every /// edit can be undone. collection: Collection, history: History, /// History entries the last load dropped, since they no longer applied to /// the scene file. pub dropped: usize, /// How many times the collection was reloaded from its files, so the UI /// can let go of what a reload may have removed. pub reloads: u64, /// How many times an edit the UI did not make, such as a guest appearing /// on their own, inserted or replaced a layer of a scene, and the count /// each scene last did so at, so the UI can let go of a layer there whose /// index it may have given to another. See [`shifted`](Self::shifted). pub shifts: (u64, BTreeMap), /// How many times the preview was taken to program, by a cut or a /// transition, so the UI can mark each however it was fired. pub takes: u64, pub sources: Sources, pub compositor: Compositor, /// Each virtual camera on: the program's under `None`, and each published /// scene's under its name. cameras: BTreeMap, crate::vcam::Camera>, /// Edited since the sources were last synced. stale: bool, /// For the autosave, which writes the collection once edits settle. edits: autosave::Edits, /// What became of each scene the edits, undos, and redos since the window /// last took them renamed or removed, one map per edit as /// [`Collection::renames`] reads it, so what the window follows a scene by /// name with, such as a hotkey, follows it too. renamed: Vec>>, /// Sources a view shows gone dark for so long, as no capture can be in /// a view's headless build. #[cfg(test)] pub dark: BTreeMap, /// Sources a view shows stalled for so long, as no camera or broadcast /// can be in a view's headless build. #[cfg(test)] pub stalled: BTreeMap, } /// Why a layer shows no new picture, once that counts: what its badge says. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Lost { /// A camera, display, or window all dark this long. Dark(Duration), /// A camera or broadcast without a new picture this long, so its last /// one repeats. Stalled(Duration), } impl Lost { /// How long it has gone without a picture. pub fn since(&self) -> Duration { match self { Self::Dark(since) | Self::Stalled(since) => *since, } } } impl State { /// What stalled of the source `name`, a copy's what it copies, live at /// `now`, once that counts; see [`source::Feed::stalled`]. pub fn stalled(&self, name: &str, now: Instant) -> Option { let capture = self.collection.capture(name); #[cfg(test)] if let Some(stalled) = self.stalled.get(capture) { return Some(source::Stall::Picture(*stalled)); } self.sources.get(capture)?.stalled(now) } /// Why `name`, a copy showing what it copies, shows no new picture at /// `now`, once that counts: a picture stalled, else gone dark. pub fn lost(&self, name: &str, now: Instant) -> Option { if let Some(source::Stall::Picture(since)) = self.stalled(name, now) { return Some(Lost::Stalled(since)); } self.dark(name, now).map(Lost::Dark) } /// How long the camera, display, or window `name` shows, a copy showing /// what it copies, has been all dark while live at `now`, once that is /// long enough to count; see [`Compositor::dark`]. pub fn dark(&self, name: &str, now: Instant) -> Option { let capture = self.collection.capture(name); #[cfg(test)] if let Some(dark) = self.dark.get(capture) { return Some(*dark); } self.compositor.dark(capture, now) } /// How the program's virtual camera, or the published scene `scene`'s, /// is doing, or `None` while it is off. pub fn camera(&self, scene: Option<&str>) -> Option { self.cameras .get(&scene.map(str::to_string)) .map(crate::vcam::Camera::status) } /// How each virtual camera on is doing. pub fn cameras(&self) -> impl Iterator { self.cameras.values().map(crate::vcam::Camera::status) } pub fn collection(&self) -> &Collection { &self.collection } pub fn history(&self) -> &History { &self.history } /// Apply `draft` and record it in the history; the sources follow it on /// the next frame, and the files shortly after. `gesture` names the /// interaction that made it, so the steps of one drag undo together. pub fn commit(&mut self, draft: Draft, gesture: Option) -> anyhow::Result<()> { let edit = draft.finish()?; let mut collection = self.collection.clone(); collection.apply(&edit)?; self.adopt(collection, std::slice::from_ref(&edit))?; self.history.record(edit, gesture); self.edited(); Ok(()) } /// Note that an edit the UI did not make inserted or replaced a layer of /// `scene`. pub fn shifted(&mut self, scene: &str) { let (count, scenes) = &mut self.shifts; *count += 1; scenes.insert(scene.to_string(), *count); } /// Apply `draft`, a change the user did not make, such as a guest joining, /// without recording it: it is not undone, and drops nothing that could be /// redone. See [`History::apply_outside`] for what it may change. pub fn apply_outside(&mut self, draft: Draft) -> anyhow::Result<()> { let edit = draft.finish()?; self.history.apply_outside(&mut self.collection, &edit)?; self.edited(); Ok(()) } /// Undo the newest edit, returning its label, or `None` with none left. pub fn undo(&mut self) -> anyhow::Result> { let Some(position) = self.history.position().checked_sub(1) else { return Ok(None); }; let label = self.label(position); self.seek(position)?; Ok(label) } /// Redo the oldest undone edit, returning its label, or `None` with none left. pub fn redo(&mut self) -> anyhow::Result> { let position = self.history.position(); let Some(label) = self.label(position) else { return Ok(None); }; self.seek(position + 1)?; Ok(Some(label)) } fn label(&self, index: usize) -> Option { self.history.entries().nth(index).map(|edit| edit.label.clone()) } /// Undo or redo to just after the history's `position`th entry. pub fn seek(&mut self, position: usize) -> anyhow::Result<()> { let mut history = self.history.clone(); let mut collection = self.collection.clone(); history.seek(&mut collection, position)?; // What the seek did, in order: the entries it redid, or the inverse of // those it undid, newest first. let from = self.history.position(); let entries: Vec<&Edit> = self.history.entries().collect(); let made: Vec = if position < from { entries[position..from] .iter() .rev() .map(|edit| edit.inverse()) .collect() } else { entries[from..position].iter().map(|&edit| edit.clone()).collect() }; let moved = self.adopt(collection, &made)?; match position < from { true => self.undone(&moved), false => self.hold(&moved), } self.history = history; self.edited(); Ok(()) } /// Note each layer keeping clear in `moved`, as [`moved`] lists them, as /// moved by hand: a redo or a reload put it elsewhere, so it stays where /// the scene file has it until the content under it changes. fn hold(&mut self, moved: &[Moved]) { for moved in moved { self.compositor .redone(&moved.scene, moved.index, &moved.source, moved.rect); } } /// Note each layer keeping clear in `moved`, as [`moved`] lists them, as /// put back by an undo: one back where it was saved before a move by hand /// held it, or before keeping clear was switched off, goes back to where /// it was then, such as the slot it had glided to, so the undo puts back what the streamer saw; any other /// holds, as [`hold`](Self::hold) says. fn undone(&mut self, moved: &[Moved]) { let now = kio::clock::now(); for moved in moved { self.compositor .undone(&moved.scene, moved.index, &moved.source, moved.rect, now); } } /// Make `collection`, which the edits `made` led to from the current one, /// the one drawn, once the compositor fits its scenes' sizes. The scenes /// in preview and on program follow the renames and removals `made` holds, /// each edit read against the collection it was made to, and any /// `collection` lacks is dropped. Edits that remove a scene on air, leave /// one drawing the program, or make one another size than the program, /// are refused, changing nothing. Returns the layers keeping clear it /// saved elsewhere, as [`moved`] finds them. fn adopt(&mut self, collection: Collection, made: &[Edit]) -> anyhow::Result> { let mut before = self.collection.clone(); let mut renames = Vec::with_capacity(made.len()); for edit in made { renames.push(before.renames(edit)); before.apply(edit)?; } keep_on_air(self.compositor.on_air(), &renames, &collection)?; let moved = moved(&self.collection, &collection, &renames); self.compositor.fit(&collection)?; for renames in &renames { self.compositor.rename(renames); } self.compositor.retain(&collection); self.collection = collection; // A removed scene may have been the one cued. self.preview_cued(); self.renamed .extend(renames.into_iter().filter(|renames| !renames.is_empty())); Ok(moved) } /// The scene renames and removals since the last call, oldest first. pub fn take_renames(&mut self) -> Vec>> { std::mem::take(&mut self.renamed) } /// What the renames not yet taken call `scene` now, or `None` if one /// removed it. pub fn follow(&self, scene: &str) -> Option { let mut scene = scene.to_string(); for renames in &self.renamed { match renames.get(&scene) { Some(Some(to)) => scene = to.clone(), Some(None) => return None, None => {} } } Some(scene) } /// The sources whose pictures program draws, through clones: those the /// scenes on air, including one leaving in a transition, and the top layer /// [`place`](Self::placed), and the clip a stinger under way plays with /// each mask its source's filters read. fn on_program(&self) -> BTreeSet { let collection = &self.collection; let stinger = self.compositor.stinger().into_iter().flat_map(|clip| { let filters = collection.video.get(clip).into_iter().flat_map(|video| &video.filters); std::iter::once(clip).chain(masks(filters).map(String::as_str)) }); let mut shown = self.placed(self.compositor.on_air().map(Some).chain([None])); shown.extend(stinger.map(|source| collection.capture(source).to_string())); shown } /// Each published or recorded scene drawn beside program, not on air, and /// the sources whose pictures it [places](Self::placed). fn beside(&self) -> Vec<(&str, BTreeSet)> { let on_air: Vec<&str> = self.compositor.on_air().collect(); let scenes = self.compositor.live().filter(|scene| !on_air.contains(scene)); scenes.map(|scene| (scene, self.placed([Some(scene)]))).collect() } /// The sources whose pictures program or a scene [beside](Self::beside) /// it shows. fn shown(&self) -> BTreeSet { let mut shown = self.on_program(); shown.extend(self.beside().into_iter().flat_map(|(_, placed)| placed)); shown } /// The sources whose pictures `scenes` place, `None` being the top layer, /// through clones: each visible item's, of the scenes they draw, and each /// mask's it reads. fn placed<'a>(&self, scenes: impl IntoIterator>) -> BTreeSet { let collection = &self.collection; let items = scenes.into_iter().flat_map(|scene| self.drawn(scene)); items .flat_map(|item| std::iter::once(&item.source).chain(masks(collection.filters(item)))) .map(|source| collection.capture(source).to_string()) .collect() } /// The visible items the scene called `scene`, or the top layer for /// `None`, draws where viewers see it, through the scenes they draw: in /// each picture the show goes out in while it is on air, or for the top /// layer, and in the landscape one alone for a scene drawn beside program. fn drawn(&self, scene: Option<&str>) -> Vec<&moq_scene::Item> { let on_air = scene.is_none_or(|scene| self.compositor.on_air().any(|on| on == scene)); let layer = moq_scene::Layer { scene: scene.map(str::to_string), }; let formats = moq_scene::Format::ALL.into_iter(); let formats = formats.filter(|format| on_air || *format == moq_scene::Format::Landscape); formats .flat_map(|format| self.collection.drawn([layer.clone()], format)) .collect() } /// Put the preview on program at once; see [`Compositor::cut`]. pub fn cut(&mut self) -> anyhow::Result<()> { self.compositor.cut(&self.collection, &self.sources)?; self.takes += 1; self.preview_cued(); Ok(()) } /// Take the preview to program by `transition`, starting at `now`; see /// [`Compositor::transition`]. pub fn transition(&mut self, transition: moq_scene::Transition, now: Instant) -> anyhow::Result<()> { self.compositor .transition(&self.collection, transition, &self.sources, now)?; self.takes += 1; self.preview_cued(); Ok(()) } /// Take the scene called `scene` on program by the show's transition, as /// its pad or hotkey does, following any rename the window has yet to move /// a key through; the name it went on by. The scene on program stays as it /// is, leaving nothing [cued](Self::cued). The preview follows, so nothing /// is cued after a take either. /// /// Refused, changing nothing, unless the scene can go on program, as /// [`Collection::takes`] says. pub fn take(&mut self, scene: &str, now: Instant) -> anyhow::Result { let scene = self.named(scene)?; if self.compositor.program_scene() == Some(scene.as_str()) { self.uncue(); return Ok(scene); } self.collection.takes(&scene)?; let previewed = self.compositor.preview_scene().map(str::to_string); self.compositor.set_preview(Some(scene.clone())); let transition = self.collection.transition.clone(); if let Err(err) = self.transition(transition, now) { self.compositor.set_preview(previewed); return Err(err); } Ok(scene) } /// The scene cued for Studio mode's TAKE and AUTO, or open for editing off /// air: the one in preview, unless it is the one on program. pub fn cued(&self) -> Option<&str> { let preview = self.compositor.preview_scene()?; (self.compositor.program_scene() != Some(preview)).then_some(preview) } /// Cue the scene called `scene`, as its pad or hotkey does in Studio mode, /// following renames as [`take`](Self::take) does; the name it was cued /// by. Its keyframes and overlays start over, as the audience will see /// them, unless it was cued already. Cueing the scene on program leaves /// nothing cued. /// /// Refused, changing nothing, unless the scene can go on program. pub fn cue(&mut self, scene: &str) -> anyhow::Result { let scene = self.named(scene)?; self.collection.takes(&scene)?; self.stage(&scene); Ok(scene) } /// Open the scene called `scene` for editing off air: cue it as /// [`cue`](Self::cue) does, even one that cannot go on program. pub fn open_off_air(&mut self, scene: &str) -> anyhow::Result { let scene = self.named(scene)?; self.stage(&scene); Ok(scene) } /// Why viewers see the scene called `scene`, which is not on program, if /// they do, so an edit to it off air reaches them: it is leaving program /// in a transition, published or recorded on its own, or drawn through a /// scene source by a scene they see or by an always-on-top layer. pub fn seen(&self, scene: &str) -> Option { let collection = &self.collection; let program = self.compositor.program_scene(); if self.compositor.on_air().any(|on| on == scene) && program != Some(scene) { return Some("it is leaving program in a transition".into()); } if collection.scene(scene).is_some_and(|found| found.published) { return Some("it is published as a broadcast of its own".into()); } if self.compositor.live().any(|drawn| drawn == scene) { return Some("it is recorded".into()); } let published = collection.scenes.iter().filter(|found| found.published); let roots = self.compositor.live().chain(published.map(|found| found.name.as_str())); let roots = roots.map(Some).chain([None]); for root in roots { let items = self.drawn(root); if items .iter() .any(|item| collection.nested(&item.source) == Some(moq_scene::Nested::Scene(scene))) { return Some(match root { Some(root) => format!("`{root}` shows it"), None => "an always-on-top layer shows it".into(), }); } } None } /// Leave nothing cued, back to editing the scene on program. pub fn uncue(&mut self) { if self.cued().is_some() { let program = self.compositor.program_scene().map(str::to_string); self.compositor.set_preview(program); } self.preview_cued(); } /// Put the cued scene on program: at once without a `transition`, or by /// it, starting at its instant. The name it went on by. /// /// Refused, changing nothing, while nothing is cued or the cued scene /// cannot go on program. pub fn take_cued(&mut self, transition: Option<(moq_scene::Transition, Instant)>) -> anyhow::Result { let scene = self.cued().context("nothing is cued")?.to_string(); match transition { None => self.cut()?, Some((transition, now)) => self.transition(transition, now)?, } Ok(scene) } /// Put the scene called `scene` in preview, unless it is there already, /// where its keyframes play on. fn stage(&mut self, scene: &str) { if self.compositor.preview_scene() != Some(scene) { self.compositor.set_preview(Some(scene.to_string())); } self.preview_cued(); } /// Draw the preview while a scene is cued, which the window shows, and /// not otherwise. Called on every change to what is cued, so a new cue is /// drawn from the next render and never shows the one before, and a take /// from a hotkey while the window is hidden stops drawing it. fn preview_cued(&mut self) { self.compositor.set_previewing(self.cued().is_some()); } /// The scene called `scene`, following any rename the window has yet to /// move a key through, if the collection has it. fn named(&self, scene: &str) -> anyhow::Result { self.follow(scene) .filter(|scene| self.collection.scene(scene).is_some()) .with_context(|| format!("no scene is called `{scene}`")) } /// The panic key: mute every microphone and every guest of `rooms`, hide /// every item of the top layer, in one edit undo takes back, and cut at /// once to the scene the show marks safe, if it marks one. Says what it /// did, or fails saying what it did before it failed. pub fn panic(&mut self, rooms: &[moq_net::Path]) -> anyhow::Result { let mut done = Vec::new(); let (draft, muted, hidden) = silence(&self.collection, rooms)?; let named = |names: &[String]| { names .iter() .map(|name| format!("`{name}`")) .collect::>() .join(", ") }; match muted.is_empty() { true => done.push("no microphone or guest was on".to_string()), false => done.push(format!("muted {}", named(&muted))), } match hidden.is_empty() { true => done.push("no always-on-top layer was shown".to_string()), false => done.push(format!("hid {} from the always-on-top layers", named(&hidden))), } if !muted.is_empty() || !hidden.is_empty() { self.commit(draft, None)?; } match self.collection.safe().map(str::to_string) { None => done.push("no scene is marked safe to cut to".into()), // A transition bringing it in still shows what it leaves, so that // cuts too. Some(safe) if self.compositor.program_scene() == Some(safe.as_str()) && !self.compositor.transitioning() => { done.push(format!("`{safe}` is on program")); } Some(safe) => { let previewed = self.compositor.preview_scene().map(str::to_string); self.compositor.set_preview(Some(safe.clone())); if let Err(err) = self.cut() { self.compositor.set_preview(previewed); anyhow::bail!("{}, but cannot cut to `{safe}`: {err:#}", done.join(", ")); } done.push(format!("cut to `{safe}`")); } } Ok(done.join(", and ")) } /// Restart the source called `name`, as an edit would, such as to reload a /// web page. pub fn restart(&mut self, name: &str) { self.sources.restart(name); self.stale = true; } /// Restart the display or window called `name`, asking the system picker /// afresh rather than reopening what it was granted. pub fn pick_again(&mut self, name: &str) -> anyhow::Result<()> { self.sources.pick_again(name)?; self.stale = true; Ok(()) } /// After any change to the collection or its history, which the autosave /// writes once they settle. fn edited(&mut self) { self.stale = true; self.edits.record(); } } /// The edit panic makes: it mutes every microphone `collection` has, and /// every source playing a guest of one of `rooms`, that is not muted, and hides every /// visible item of the top layer, whatever it shows. Also the sources it /// mutes, and the sources it hides, each once. Other broadcasts and media stay /// on. The edit is empty when there is nothing to do. fn silence( collection: &Collection, rooms: &[moq_net::Path], ) -> Result<(Draft, Vec, Vec), moq_scene::Error> { let voice = |source: &moq_scene::Source| { matches!(source, moq_scene::Source::Microphone { .. }) || rooms.iter().any(|room| crate::guests::plays_guest(room, source)) }; let muted: Vec = collection .sources .iter() .filter(|(_, source)| voice(source)) .map(|(name, _)| name) .filter(|name| !collection.audio.get(*name).is_some_and(|audio| audio.muted)) .cloned() .collect(); let mut draft = collection.draft("Panic: mute every microphone and guest, hide the always-on-top layers"); for name in &muted { let audio = collection.audio.get(name).cloned().unwrap_or_default(); draft.set_audio(name, Some(moq_scene::Audio { muted: true, ..audio }))?; } let mut hidden: Vec = Vec::new(); let top = moq_scene::Layer::top(); for (index, item) in collection.top.iter().enumerate().filter(|(_, item)| item.visible) { draft.set_item( &top, index, moq_scene::Item { visible: false, ..item.clone() }, )?; if !hidden.contains(&item.source) { hidden.push(item.source.clone()); } } Ok((draft, muted, hidden)) } /// The sources the masks among `filters` read: a mask's picture shapes what /// the program draws too. fn masks<'a>(filters: impl Iterator) -> impl Iterator { filters.filter_map(|filter| match filter { moq_scene::Filter::Mask { source, .. } => Some(source), _ => None, }) } impl autosave::Document for State { /// Both under one lock, so the history describes the collection beside it. fn text(&self) -> anyhow::Result { autosave::Text::new(&self.collection, &self.history) } /// Swap in a collection and history reloaded from their files. fn replace(&mut self, loaded: autosave::Loaded) -> anyhow::Result<()> { let autosave::Loaded { collection, history, dropped, } = loaded; // A scene in preview or on program that the file no longer has, such as // one renamed by hand, leaves it until another is picked. let moved = self.adopt(collection, &[])?; self.hold(&moved); self.history = history; self.dropped = dropped; self.reloads += 1; self.stale = true; Ok(()) } } /// A scene item keeping clear that an edit saved elsewhere, as [`moved`] /// finds it. #[derive(Clone, Debug, PartialEq)] struct Moved { scene: String, index: usize, source: String, /// Where it is saved now. rect: moq_scene::Rect, } /// Each scene item keeping clear in `after` that `before` saved elsewhere or /// had not keeping clear: the same scene's item at the same index showing /// the same source, at another x or y or switched back on, or one switched /// back on that `before` showed elsewhere in the scene, as an undo of a /// swap into a slot puts it. A scene is the one `renames`, each edit's /// between them, oldest first, called it in `before`. Its keeper is still /// under that place, as the compositor follows items only as it renders. A /// change of order alone moves nothing. fn moved(before: &Collection, after: &Collection, renames: &[BTreeMap>]) -> Vec { let mut moved = Vec::new(); for scene in &after.scenes { let Some(was) = was_named(renames, &scene.name).and_then(|name| before.scene(&name)) else { continue; }; if reordered(&was.items, &scene.items) { continue; } for (index, item) in scene.items.iter().enumerate() { let Some(old) = was.items.get(index) else { continue }; let switched_on = |old: &moq_scene::Item| old.source == item.source && old.keep_clear.is_none(); if item.keep_clear.is_some() && (switched_on(old) || (old.source == item.source && (old.rect.x, old.rect.y) != (item.rect.x, item.rect.y)) || (old.source != item.source && was.items.iter().any(switched_on))) { moved.push(Moved { scene: scene.name.clone(), index, source: item.source.clone(), rect: item.rect, }); } } } moved } /// What the scene called `name` after `renames`, each edit's, oldest first, /// was called before them, or `None` if one of them added it. fn was_named(renames: &[BTreeMap>], name: &str) -> Option { let mut name = name.to_string(); for renames in renames.iter().rev() { if let Some((from, _)) = renames.iter().find(|(_, to)| to.as_deref() == Some(name.as_str())) { name = from.clone(); } else if renames.contains_key(&name) { return None; } } Some(name) } /// Whether `after` holds each of `before`'s items once, in any order. fn reordered(before: &[moq_scene::Item], after: &[moq_scene::Item]) -> bool { let mut left: Vec<&moq_scene::Item> = before.iter().collect(); before.len() == after.len() && after.iter().all(|item| { let Some(at) = left.iter().position(|old| *old == item) else { return false; }; left.swap_remove(at); true }) } /// Refuse edits, each read as its [`Collection::renames`], that remove a /// scene `on_air` program draws, so a slip cannot put black on air, or that /// leave one in `collection` unable to go on program, as /// [`Collection::refuses`] says: drawing the program into itself, another /// size than the program, or of its own size while the show goes out in /// portrait too, with no portrait picture. A reload that leaves one out /// still drops it, having nothing to show instead. fn keep_on_air<'a>( on_air: impl IntoIterator, renames: &'a [BTreeMap>], collection: &Collection, ) -> anyhow::Result<()> { for mut scene in on_air { for renames in renames { match renames.get(scene) { Some(Some(to)) => scene = to, Some(None) => anyhow::bail!("`{scene}` is on air; cut to another scene before removing it"), None => {} } } // A reload that leaves it out drops it, having nothing to show. if collection.scene(scene).is_none() { continue; } if let Some(why) = collection.refuses(scene) { anyhow::bail!("`{scene}` is on air, so cut to another scene first: {why}"); } } Ok(()) } /// How the output is doing, for the stream panel. #[derive(Default)] pub struct Stats { /// Program frames drawn. pub rendered: AtomicU64, /// The program video. pub video: Arc, /// The recording's own video, of the program or the scene recordings /// hold. pub recording: Arc, /// Each published scene's video, by the scene's name, while it is /// published, and the portrait picture's, as [`moq_scene::PORTRAIT`], /// while the show goes out in both. published: Mutex>>, /// The video encoders open: the program's, each published scene's, the /// recording's, and the sources'. pub sessions: Sessions, /// Each published track's share of the connection's bandwidth, by name: a /// published scene's prefixed with its own. grants: Mutex>, /// Why the audio stopped, if it did. audio_error: Mutex>, /// Why the recording's audio stopped, if it did, which leaves the published /// audio running. recording_audio_error: Mutex>, } /// How the program's video, a published scene's, or the recording's is /// doing. #[derive(Default)] pub struct VideoStats { /// What each of its video encoders costs: a published video's full size /// and half size, or, third, the recording's. pub renditions: [Load; 3], /// Whether the encoder reads the program on the GPU, rather than each frame /// being read back to the CPU. pub zero_copy: AtomicBool, /// Whether anyone subscribes to any of the video's renditions: a viewer /// through the relay, or a recording. pub watched: AtomicBool, /// Whether the program is handed to the canvas size's encoder now: not /// while on demand and unwatched, nor while a new rendition opens, nor /// while the grant cannot fund it. pub encoding: AtomicBool, /// How many sizes the video has now: the canvas size, and the half size /// unless the canvas is too small or its encoder refused it. pub sizes: std::sync::atomic::AtomicUsize, /// The encoder backend, once the current rendition opened one. encoder: Mutex, /// Why the video stopped, if it did, until a new rendition starts. error: Mutex>, /// Why a published scene's own audio stopped, if it did. audio_error: Mutex>, /// Which video this is, so a scene published again under its name is /// told from the one before, whose counters it does not continue. generation: Generation, } /// A number no other [`VideoStats`] has. struct Generation(u64); impl Default for Generation { fn default() -> Self { static NEXT: AtomicU64 = AtomicU64::new(0); Self(NEXT.fetch_add(1, Ordering::Relaxed)) } } /// What one of the program's renditions costs. #[derive(Default)] pub struct Load { /// Program frames handed to its encoder. pub copied: AtomicU64, /// Microseconds the last one waited on the GPU, and on a readback converted /// pixels. pub copy_us: AtomicU64, /// Program frames read back to the CPU for its encoder. pub read_back: AtomicU64, /// Program frames not encoded because its encoder still held every copy. pub dropped: AtomicU64, /// Program frames encoded. pub encoded: AtomicU64, /// Microseconds the last encode took. pub encode_us: AtomicU64, /// Why this encoder stopped, or could not start, while the canvas size /// carries on, until the next canvas. The canvas size's own failure stops /// the video, which [`VideoStats::error`] reports. error: Mutex>, } impl Load { /// Why this encoder stopped, if it did; see [`Load::error`]. pub fn error(&self) -> Option { self.error.lock().unwrap().clone() } fn fail(&self, err: impl std::fmt::Display) { tracing::warn!(%err, "a program rendition stopped"); *self.error.lock().unwrap() = Some(err.to_string()); } } /// The encoder backend the current rendition opened, if it has. #[derive(Default)] struct EncoderName { /// Counts the renditions, so one a resize retired while its encoder was /// still opening cannot report that encoder afterwards. rendition: u64, name: Option, } impl VideoStats { /// The encoder backend, once the current rendition opened one. pub fn encoder(&self) -> Option { self.encoder.lock().unwrap().name.clone() } /// The number of the current rendition, which its encode thread reports /// its encoder under. fn rendition(&self) -> u64 { self.encoder.lock().unwrap().rendition } /// Retire the current rendition, so its encoder is no longer reported, even /// one that opens afterwards. fn retire_encoder(&self) { let mut encoder = self.encoder.lock().unwrap(); encoder.rendition += 1; encoder.name = None; } /// Report `name` as the encoder, unless `rendition` was retired. fn opened_encoder(&self, rendition: u64, name: &str) { let mut encoder = self.encoder.lock().unwrap(); if encoder.rendition == rendition { encoder.name = Some(name.to_string()); } } /// Why the video stopped, if it did. pub fn error(&self) -> Option { self.error.lock().unwrap().clone() } fn fail(&self, err: impl std::fmt::Display) { tracing::error!(%err, "video stopped"); *self.error.lock().unwrap() = Some(format!("video: {err}")); } /// Why a published scene's own audio stopped, if it did. pub fn audio_error(&self) -> Option { self.audio_error.lock().unwrap().clone() } fn fail_audio(&self, err: impl std::fmt::Display) { *self.audio_error.lock().unwrap() = Some(format!("audio: {err}")); } } impl Stats { /// Each published scene's video, by the scene's name. pub fn published(&self) -> BTreeMap> { self.published.lock().unwrap().clone() } /// Each published track's share of the connection's bandwidth now, by name: /// `None` while the connection has no estimate. pub fn grants(&self) -> BTreeMap> { let grants = self.grants.lock().unwrap(); grants .iter() .map(|(name, grant)| (name.clone(), grant.peek())) .collect() } /// Why the program's output stopped, if it did. pub fn error(&self) -> Option { let video = self.video.error(); let audio = self.audio_error.lock().unwrap().clone(); match (video, audio) { (Some(video), Some(audio)) => Some(format!("{video}; {audio}")), (video, audio) => video.or(audio), } } /// The recording's audio encoder failed, which stops what recordings hold but /// not the published program. fn fail_recording_audio(&self, err: impl std::fmt::Display) { tracing::error!(%err, "the recording's audio stopped"); *self.recording_audio_error.lock().unwrap() = Some(format!("{err}")); } /// Why the recording's audio stopped, if it did. pub fn recording_audio_error(&self) -> Option { self.recording_audio_error.lock().unwrap().clone() } fn fail_audio(&self, err: impl std::fmt::Display) { tracing::error!(%err, "audio stopped"); *self.audio_error.lock().unwrap() = Some(format!("audio: {err}")); } } /// A relay the program is published to. #[derive(Clone)] pub struct Relay { pub client: moq_tokio::Client, pub url: url::Url, } /// What the program's broadcasts share for as long as the engine runs, on air /// or not: the clock, the local sources' tracks, and the recording's own /// broadcast. pub struct Program { /// What every catalog's timestamps are on: the recording's, the program's /// each time it goes live, and each published scene's. clock: moq_mux::Clock, /// Where the program clock started on [`clock`](Self::clock). epoch: Duration, /// Where the local sources keep their own tracks, on the same clock. sources: moq_scene::publish::Publish, /// [`Renditions::reclaim`]. reclaim: Option, /// The sources published independently, whose tracks the broadcast on air /// carries beside the program's. published: tokio::sync::watch::Sender>, /// What a recording reads the program from, on the same clock. recording: moq_net::origin::Route, /// The recording broadcast's catalog, held for as long as the engine runs, /// so a recording still reading as the studio stops reads it to its end. recorded: moq_scene::publish::Catalog, /// The origin the studio's own broadcasts are read through, which no relay /// connection carries. local: moq_net::origin::Producer, } impl Program { /// Finish the recording's catalog, once every track it lists has ended, so /// a recording reading it reaches its end rather than a cut. fn finish(&self) { // Fails only once the catalog failed, which ended its tracks already. let _ = self.recorded.clone().finish(); } /// What a replay of the local source `name` copies: the video or sound of /// its own track, up to now, or `None` while it publishes neither. pub fn source(&self, name: &str) -> Option { let mut catalog = self.sources.catalog(); let listed = catalog.ext.studio.sources.remove(name)?; let video = listed.video.renditions.into_iter().next(); let audio = listed.audio.renditions.into_iter().next(); if video.is_none() && audio.is_none() { return None; } Some(moq_scene::replay::Target { broadcast: self.sources.consume(), video, audio, end: self.clock.now(), }) } /// Encode the local sources' tracks on program, whether or not anything /// reads them, so a replay of one has its past: the pictures `shown` in a /// scene on air or a published or recorded scene drawn, and the sound the /// audio scene on program plays. A media file heard and not shown encodes /// only its sound. fn on_program(&self, state: &State, shown: BTreeSet) { let program = state.collection.program_audio().unwrap_or_default(); let heard = state .sources .iter() .filter(|(name, _)| state.sources.plays(program, name)) .map(|(name, _)| name.to_string()) .collect(); let published = &state.collection.published; self.published.send_if_modified(|current| { let changed = current != published; current.clone_from(published); changed }); // Published, a source encodes as though on program. let mut shown = shown; let mut heard: BTreeSet = heard; shown.extend(published.iter().cloned()); heard.extend(published.iter().cloned()); self.sources .set_on_program(moq_scene::publish::OnProgram { shown, heard }); } } /// The running studio. pub struct Engine { /// For opening the virtual camera, which reads the program back from it. gpu: Gpu, pub state: Mutex, pub autosave: Autosave, pub stats: Arc, /// How the stream is delivered, graded every [`health::INTERVAL`]. pub health: Mutex, /// What grades it: the stream on air's delivery, or only the studio's /// own work while off air. sampler: Mutex, /// The audio scene the program audio carries, as the collection picks it. program_audio: mix::Pick, /// The audio scene recordings hold, as the collection picks it. recorded_audio: mix::Pick, /// Plays an audio scene to the speakers while the creator listens. monitor: Mutex>, /// The stream, while on air, going live, or ending. air: Mutex, /// The last stream to go off air. aired: Mutex>, /// Whether the program is on air, for the window and for quitting, which /// waits out a stream going live or ending. on_air: tokio::sync::watch::Sender, /// What the program clock takes from going live and gives back on ending. handoff: Mutex, /// The program's broadcast on air, which guests' return feeds show. on_air_program: tokio::sync::watch::Sender>, /// Records the program, and each source too if asked. pub recorder: record::Recorder, /// The room guests join, while the show has one, and what follows it. room: Mutex>, /// The path of every room followed this run, so the panic key still mutes /// a guest added from a room left since. rooms: Mutex>, /// Whether a guest who joins takes the next tile of the scene on program, /// as the show's settings say. guests_appear: std::sync::atomic::AtomicBool, /// Each guest's return feed, once the studio's session in the room dials. feeds: Mutex>, /// What the program's broadcasts share, on air or not. pub program: Program, /// Where broadcast sources, guests, and viewers' reports arrive from the /// relays the studio [watches](Self::watch), open even while none is. subscribe: moq_net::origin::Producer, /// A session to each relay set for the show, which only subscribes, so /// broadcast sources and guests are seen off air too. watching: Mutex>, /// Set by [`stop`](Self::stop), which ends the program clock and the tasks /// that run only for as long as the engine does. stopped: tokio::sync::watch::Sender, /// The program clock's and the audio's threads, which [`stop`](Self::stop) /// waits on. threads: Mutex>>, /// Each source thread's end, which [`close`](Self::close) waits on. running: Mutex>>, } /// The audio the program publishes. pub const AUDIO_BITRATE: moq_net::bandwidth::Rate = moq_net::bandwidth::Rate::from_kbps(128); /// How often the audio thread advances the bus. Short enough that a block is never /// close to the bus's buffering, long enough to cost nothing. pub(crate) const AUDIO_PERIOD: Duration = Duration::from_millis(10); /// The longest [`Engine::close`] waits for the sources to stop and the relays /// to drain, so a source or relay that never answers cannot hold up quitting. /// Each relay session bounds its own drain at about a second. const CLOSE: Duration = Duration::from_secs(3); /// Everything starting the engine takes besides the scene file, which guided /// setup holds back until the creator has settled on a collection. #[derive(Clone)] pub struct Launch { pub gpu: Gpu, /// The directory recordings are saved in. pub recordings: PathBuf, /// Where the video encoders count the encode sessions they hold. pub sessions: Sessions, } /// The room the engine follows, until dropped: the guests who join it, their /// renewals, and who has a return feed. struct Joined { room: Arc, tasks: Vec>, } impl Drop for Joined { fn drop(&mut self) { for task in &self.tasks { task.abort(); } } } impl Engine { /// Open the `loaded` collection and history on `launch.gpu`, start the /// program clock, and save both to `autosave`'s files as they change, /// starting at once. Recordings go into `launch.recordings`. Guests join /// once the engine [joins](Self::join) a room. Nothing is published until /// the studio [goes live](Self::go_live). /// /// Call inside the tokio runtime that serves the relay connections. pub async fn start(launch: Launch, autosave: Autosave, loaded: autosave::Loaded) -> anyhow::Result> { let Launch { gpu, recordings, sessions, } = launch; let (bus, driver) = moq_audio::bus::Bus::new(Default::default())?; let subscribe = moq_tokio::origin::spawn(); let sources = subscribe.consume(); let autosave::Loaded { collection, history, dropped, } = loaded; let canvas = collection.canvas; let stats = Arc::new(Stats { sessions, ..Default::default() }); // The program clock's start, from which the audio clock advances the // bus too, so the sources place their sound on it. let start = Instant::now(); let (program, publish) = publish(&gpu, &stats, canvas, start).await?; // Nothing in either local broadcast refers to another. let local = |broadcast: moq_net::broadcast::Consumer| moq_mux::Source::new(program.local.consume(), broadcast); let inputs = record::Inputs { program: local(program.recording.broadcast().consume()), sources: local(program.sources.consume()), origin: sources.clone(), }; let recorder = record::Recorder::new(inputs, &recordings, autosave.path())?; // Absolute, so a copied image names the same file in another collection. let dir = autosave.path().parent().filter(|dir| !dir.as_os_str().is_empty()); let dir = std::path::absolute(dir.unwrap_or(Path::new(".")))?; let (mixer, mixed) = source::Mixer::new(driver, start); let sources = Sources::new(source::Context { bus: bus.clone(), mixed, origin: sources, dir, output: moq_video::render::output(&gpu.device), publish: Some(publish.sources.clone()), browser: browser(&gpu)?, grants: grants(autosave.path())?, }); let mut compositor = Compositor::new(&gpu.device, &gpu.queue, collection.canvas)?; compositor.fit(&collection)?; // The first scene that can go on program starts there, and in preview. let first = collection .scenes .iter() .find(|scene| collection.takes(&scene.name).is_ok()); compositor.set_preview(first.map(|scene| scene.name.clone())); compositor.cut(&collection, &sources)?; // Nothing is cued, so nothing shows the preview. compositor.set_previewing(false); let epoch = program.epoch; let engine = Arc::new(Self { gpu, state: Mutex::new(State { collection, history, dropped, reloads: 0, shifts: (0, BTreeMap::new()), takes: 0, sources, compositor, cameras: BTreeMap::new(), stale: true, edits: autosave.edits(), renamed: Vec::new(), #[cfg(test)] dark: BTreeMap::new(), #[cfg(test)] stalled: BTreeMap::new(), }), autosave, stats, health: Mutex::default(), sampler: Mutex::new(health::Sampler::new(None)), program_audio: mix::Pick::default(), recorded_audio: mix::Pick::default(), monitor: Mutex::default(), air: Mutex::new(air::Slot::Off), aired: Mutex::default(), on_air: tokio::sync::watch::Sender::new(OnAir::Off), handoff: Mutex::default(), on_air_program: tokio::sync::watch::Sender::new(None), recorder, room: Mutex::default(), rooms: Mutex::default(), guests_appear: Default::default(), feeds: Mutex::default(), program, subscribe, watching: Mutex::default(), stopped: tokio::sync::watch::Sender::new(false), threads: Mutex::default(), running: Mutex::default(), }); let Publish { recording, recorded_audio, sources: _, } = publish; // Probed now, so a scene a recording cannot encode says so before // anyone records it, and opened before the program clock starts the // sources. It opens its session only once recorded. let (_, recorded) = recorded(engine.state.lock().unwrap().collection()); let recording = Line::open(recording, recorded).await; // Each worker holds the engine, so one that cannot start stops those // already running, which would otherwise keep this engine and its // publication alive beside a retry's. let starting = Starting(Some(engine.clone())); // The audio clock first, so the bus keeps up with the program clock by // the time the program starts the sources, whose sound otherwise waits // for it. let mix = engine.clone(); let config = bus.config().clone(); let rate = config.sample_rate; let thread = clocked("audio".into(), async move || mix.run_audio(start, config, mixer).await)?; engine.threads.lock().unwrap().push(thread); let program = engine.clone(); let thread = clocked("program".into(), async move || { program.run_program(start, epoch, recording).await })?; engine.threads.lock().unwrap().push(thread); // On its own thread, so an encoder that falls behind skips its own // audio rather than holding up the bus. let encode = engine.clone(); let follow = engine.recorded_audio.follow(); let thread = clocked("audio-record".into(), async move || { encode.encode_audio(epoch, rate, follow, recorded_audio).await })?; engine.threads.lock().unwrap().push(thread); let save = engine.clone(); let mut stopped = engine.stopped.subscribe(); let thread = clocked("autosave".into(), async move || { tokio::select! { () = save.autosave.run(&save.state) => {} // Its last write is the one quitting makes. _ = stopped.wait_for(|stopped| *stopped) => {} } })?; engine.threads.lock().unwrap().push(thread); starting.started(); engine.until_stopped(engine.clone().grade()); Ok(engine) } /// The room guests join, while the engine follows one. pub fn room(&self) -> Option> { let joined = self.room.lock().unwrap(); joined.as_ref().map(|joined| joined.room.clone()) } /// The path of every room followed this run, the one followed now /// included, and of every room [remembered](Self::remember_room). pub fn rooms(&self) -> Vec { self.rooms.lock().unwrap().clone() } /// Whether a guest who joins takes the next tile of the scene on program /// on their own, rather than waiting under Add. pub fn guests_appear(&self) -> bool { self.guests_appear.load(std::sync::atomic::Ordering::Relaxed) } /// Set whether guests appear on their own, as the show's settings say. pub fn set_guests_appear(&self, appear: bool) { self.guests_appear.store(appear, std::sync::atomic::Ordering::Relaxed); } /// Keep `room` among the [rooms](Self::rooms) Panic mutes the guests of, /// as one the settings named, even if it could not be joined: a guest of /// it already in a scene may still play through the relays followed. pub fn remember_room(&self, room: &moq_net::Path) { let mut rooms = self.rooms.lock().unwrap(); if !rooms.contains(room) { rooms.push(room.clone()); } } /// Follow `joining`'s room in place of the one followed now, if any: /// add each guest who joins it as a source, renew their tokens through its /// session, and give each a return feed. `None` leaves the room. Guests /// already added stay sources, and a guest of the room left keeps their /// link until it expires, unrenewed. /// /// Call inside the tokio runtime that serves the relay connections. pub fn join(self: &Arc, joining: Option) -> anyhow::Result<()> { // Held throughout, as `close` takes it once `stop` has run, so a join // racing a close either ends before it, and is left, or finds the // engine stopped. let mut joined = self.room.lock().unwrap(); // The room left stops renewing and feeding before the next starts, so // the two never publish at once. *joined = None; *self.feeds.lock().unwrap() = None; let Some(crate::guests::Joining { room, renewal }) = joining else { return Ok(()); }; anyhow::ensure!(!*self.stopped.borrow(), "the show is closing"); // Every relay feeds `subscribe`, so a guest either one routes is seen. let announced = self .subscribe .consume() .scope(room.path(), &moq_auth::Pattern::all().into()) .context("cannot follow the room")?; if let Some(origin) = room.origin() { let program = self.on_air_program.subscribe(); let clock = self.program.clock.clone(); let feeds = crate::feed::Feeds::new(room.clone(), origin.clone(), program, clock, self.program.epoch)?; *self.feeds.lock().unwrap() = Some(feeds); } // Each ends with the engine too, so none outlives a show closed. let mut tasks = vec![self.until_stopped(crate::guests::run(self.clone(), room.clone(), announced))]; if let Some(renewal) = renewal { tasks.push(self.until_stopped(renewal.run())); } // Who has a return feed follows who is renewed, synced with the audio, // including any change made since the feeds started. let mut changes = room.changes(); changes.mark_changed(); let engine = Arc::downgrade(self); tasks.push(self.until_stopped(async move { while changes.changed().await.is_ok() { let Some(engine) = engine.upgrade() else { return }; engine.state.lock().unwrap().stale = true; } })); self.remember_room(room.path()); *joined = Some(Joined { room, tasks }); Ok(()) } /// Stop the program clock, then wait for each rendition's encoder, of /// every picture, to publish the copies on their way to it and what it /// still holds, and to end its video track, so quitting loses none of the /// program's last pictures; a published scene's catalog then finishes too. /// The audio ends its tracks too, and then the recording's catalog /// finishes, so a recording reading the program reaches its end. Then stop /// every source, which finishes its own track on its thread, and the /// browser sources' helper, waiting for it to exit, so their profile is /// saved and free for the next launch. The autosave and the health grading /// end too, so a show closed without quitting leaves nothing running. A /// stream on air ends with [`close`](Self::close). pub fn stop(&self) { self.stopped.send_replace(true); let threads = std::mem::take(&mut *self.threads.lock().unwrap()); for thread in threads { let name = thread.thread().name().unwrap_or_default().to_string(); if thread.join().is_err() { tracing::error!(name, "the thread panicked"); } } self.program.finish(); // Taken out first, so the wait holds no lock the window draws with. let browser = { let mut state = self.state.lock().unwrap(); state.sources.stop(); state.sources.context().browser.clone() }; if let Some(browser) = browser { browser.stop(); } } /// Once [`stop`](Self::stop) has ended the tracks of every picture, end /// the stream on air as [`end_stream`](Self::end_stream) does, closing /// each relay session once the relay has their tails, so a viewer or a /// recording on the relay receives them rather than a cut; one going live /// or ending is waited out first. Meanwhile wait for each source's thread /// to end, having finished its own track, and then finish the sources' /// local catalog. Bounded at [`CLOSE`]. The studio's session in the guests' /// room ends too. pub async fn close(&self) { // Its session and followers hold the engine until they end. *self.room.lock().unwrap() = None; *self.feeds.lock().unwrap() = None; let running = std::mem::take(&mut *self.running.lock().unwrap()); let sources = async { for ended in running { // Fails once the thread ends, which is all it says. let _ = ended.await; } // Every source has left the local catalog, so it lists nothing more. // Fails only once it failed, which ended its tracks already. let _ = self.program.sources.finish(); }; let relays = async { let mut on_air = self.on_air.subscribe(); // Fails only once the engine is gone, which holds the sender. let _ = on_air .wait_for(|on_air| !matches!(on_air, OnAir::Starting | OnAir::Ending)) .await; if let Err(err) = self.end_stream().await { tracing::debug!(error = format!("{err:#}"), "nothing on air to end"); } self.unwatch().await; }; // A source's track is in the studio's own local broadcast, which no // relay session carries, so neither waits on the other. let closing = async { tokio::join!(sources, relays) }; if kio::clock::timeout(CLOSE, closing).await.is_err() { tracing::warn!(timeout = ?CLOSE, "quitting before every source stopped and every relay drained"); } } /// Run `task` on the runtime until [`stop`](Self::stop), for work that /// belongs to this engine and would otherwise outlive a show closed. fn until_stopped(&self, task: impl Future + Send + 'static) -> tokio::task::JoinHandle<()> { let mut stopped = self.stopped.subscribe(); tokio::spawn(async move { tokio::select! { () = task => {} _ = stopped.wait_for(|stopped| *stopped) => {} } }) } /// Write the collection and its history to their files unless they /// already hold them, or the scene file changed outside the studio. pub fn save(&self) -> anyhow::Result<()> { self.autosave.save(&self.state) } /// Replace the collection and its history with what the files hold now, /// dropping any unsaved edit. pub fn reload(&self) -> anyhow::Result<()> { self.autosave.reload(&self.state) } /// Write the collection and its history over what the files hold now. pub fn keep(&self) -> anyhow::Result<()> { self.autosave.keep(&self.state) } /// Play an audio scene to an output on `runtime`, from the next frame, or /// stop with `None`. A monitored audio scene that leaves the collection /// hands over to the one the program carries, and another output takes /// over from the old one. /// /// Takes the `state` already locked, so an edit renaming the audio scene /// and the pick that follows it land before the next frame's sync. pub fn listen( &self, state: &mut State, runtime: &tokio::runtime::Handle, listen: Option>, ) -> anyhow::Result<()> { let mut monitor = self.monitor.lock().unwrap(); match (listen, monitor.as_mut()) { (None, _) => *monitor = None, (Some(listen), Some(monitor)) if monitor.failed().is_none() => { monitor.set_name(listen.scene); monitor.set_speaker(listen.speaker); } // Listening again after a failure starts over. (Some(listen), _) => *monitor = Some(mix::Monitor::start(runtime, listen.scene, listen.speaker)?), } // Synced on the next frame, which picks the audio scene by its name. state.stale = true; Ok(()) } /// The audio scene played to the speakers, and why it stopped if it did. pub fn listening(&self) -> Option<(String, Option)> { let monitor = self.monitor.lock().unwrap(); monitor .as_ref() .map(|monitor| (monitor.name().to_string(), monitor.failed())) } /// Point the program audio, the recorded audio, the monitor, and each /// guest's return feed at the audio scenes `state` now has. fn pick_audio(&self, state: &State) { let collection = &state.collection; let program = collection.program_audio().unwrap_or_default(); if let Some(mix) = state.sources.mix(program) { self.program_audio.set(mix.clone()); } if let Some(mix) = collection.recorded_audio().and_then(|scene| state.sources.mix(scene)) { self.recorded_audio.set(mix.clone()); } // From the audio scene on program, which is what each guest follows. if let Some(feeds) = self.feeds.lock().unwrap().as_mut() { feeds.sync(collection, &state.sources); } if let Some(monitor) = self.monitor.lock().unwrap().as_mut() { if collection.audio_scene(monitor.name()).is_none() { monitor.set_name(program); } // What viewers hear, published sources included. if let Some(mix) = state.sources.listened(monitor.name()) { monitor.play(mix.clone()); } } } /// Grade the stream's health every interval, for the Health panel. async fn grade(self: Arc) { let mut interval = kio::clock::interval(health::INTERVAL); interval.set_missed_tick_behavior(kio::clock::MissedTickBehavior::Skip); loop { let now = interval.tick().await; let local = self.local(now); let report = self.sampler.lock().unwrap().sample(now, local); *self.health.lock().unwrap() = report; } } /// What only the studio knows about its stream at `now`. fn local(&self, now: Instant) -> health::Local { let stats = &self.stats; // A stalled picture matters where a live scene draws it: a scene on air, // including one leaving in a transition, the top layer over them, or a // published or recorded scene drawn. // Stalled sound matters only if an audio scene viewers hear plays it: // the one on program or one a published scene carries, not one only // listened to. A source that stopped matters either way. let (stalled, dark, stopped, uploaded) = { let state = self.state.lock().unwrap(); let on_program = state.on_program(); let beside = state.beside(); let shown = |name: &str| health::Live { program: on_program.contains(name), scenes: beside .iter() .filter(|(_, placed)| placed.contains(name)) .map(|(scene, _)| scene.to_string()) .collect(), }; let collection = &state.collection; let program_audio = collection.program_audio(); let published: Vec<(&str, Option<&str>)> = collection .scenes .iter() .filter(|scene| scene.published) .map(|scene| (scene.name.as_str(), collection.published_audio(&scene.name))) .collect(); let plays = |audio: Option<&str>, name: &str| audio.is_some_and(|audio| state.sources.plays(audio, name)); let heard = |name: &str| health::Live { program: plays(program_audio, name), scenes: published .iter() .filter(|(_, audio)| plays(*audio, name)) .map(|(scene, _)| scene.to_string()) .collect(), }; let stalled = state .sources .iter() .filter_map(|(name, _)| { let stall = state.stalled(name, now)?; let live = match stall { source::Stall::Picture(_) => shown(name), source::Stall::Sound(_) => heard(name), }; live.any().then(|| (name.to_string(), stall, live)) }) .collect(); let dark = state .sources .iter() .filter_map(|(name, _)| { // A stalled picture is reported as stalled, as its row says, never also as dark. let Some(Lost::Dark(dark)) = state.lost(name, now) else { return None; }; let live = shown(name); live.any().then(|| (name.to_string(), dark, live)) }) .collect(); // A stinger's clip rests ended between stingers, unless a live scene // places it. let stinger = state.collection.transition.stinger(); let placed = state.placed(state.compositor.live().map(Some).chain([None])); let stopped = state .sources .iter() .filter_map(|(name, feed)| { let reach = health::Reach { shown: shown(name), heard: heard(name), }; // A dark or stalled source is reported so, as its layer's row says. if !reach.any() || state.lost(name, now).is_some() { return None; } // A picture that fails to draw fails the source as the Sources panel shows it. let status = match state.compositor.error(name) { Some(err) => Status::Failed(err.to_string()), None => feed.status(), }; let stopped = match status { Status::Ended => stinger != Some(name) || placed.contains(name), Status::Failed(_) => true, _ => false, }; stopped.then(|| (name.to_string(), status, reach)) }) .collect(); let uploaded = state .sources .iter() .filter_map(|(name, feed)| match feed.page()?.paint()? { moq_scene::browser::Paint::Cpu(why) => { let live = shown(name); live.any().then(|| (name.to_string(), why, live)) } moq_scene::browser::Paint::Gpu => None, }) .collect(); (stalled, dark, stopped, uploaded) }; let video = |video: &VideoStats, failed: Option| { let full = &video.renditions[Role::Canvas as usize]; health::Video { dropped: full.dropped.load(Ordering::Relaxed), encoded: full.encoded.load(Ordering::Relaxed), encoding: video.encoding.load(Ordering::Relaxed), watched: video.watched.load(Ordering::Relaxed), encoder: video.encoder(), failed, generation: video.generation.0, } }; // The program's output stops with the audio too. let program = (None, video(&stats.video, stats.error())); let published = stats.published().into_iter(); let published = published.map(|(name, stats)| { let failed = stats.error().or_else(|| stats.audio_error()); (Some(name), video(&stats, failed)) }); health::Local { rendered: stats.rendered.load(Ordering::Relaxed), broadcasts: std::iter::once(program).chain(published).collect(), grants: stats.grants(), stalled, dark, stopped, uploaded, } } /// Turn the program's virtual camera, or the published scene `scene`'s, /// on or off. Turning it on fails, saying why, when the scene is not /// published or this machine has no way to publish a camera. A scene's /// camera turns off once the scene is no longer published under its name. pub fn set_camera(&self, scene: Option<&str>, on: bool) -> anyhow::Result<()> { let canvas = { let state = self.state.lock().unwrap(); match scene { None => state.compositor.canvas(), Some(scene) => published(&state.collection) .into_iter() .find(|(name, _)| name == scene) .map(|(_, canvas)| canvas) .with_context(|| format!("scene `{scene}` is not published"))?, } }; // Opened and closed outside the lock, so the program clock does not // wait on the camera's backend. let key = scene.map(str::to_string); let previous = match on { true => { let camera = crate::vcam::Camera::open(&self.gpu.device, &self.gpu.queue, scene, canvas)?; self.state.lock().unwrap().cameras.insert(key, camera) } false => self.state.lock().unwrap().cameras.remove(&key), }; drop(previous); Ok(()) } /// Draw the program and each published scene once per tick of the program /// clock, and hand each to its renditions' encoders, stamped `epoch` past /// `start`, while their gates let them encode. On air, that is the full /// size every frame unless on demand, otherwise only while anyone watches /// it, and the half size only while anyone watches it; off air, the /// program and the published scenes encode nothing. The program, or the /// scene recordings hold, goes to `recording`, the recording's own /// encoder, which encodes only while recording. /// /// A published or recorded scene is drawn only while one of its encoders /// or virtual camera takes the frame, so one nobody reads costs no /// drawing. Drawing outlives the encoders: a failed output stops copying /// its picture, and the scenes keep moving. Each virtual camera on takes /// every frame of what it shows while an app reads it, and one whose /// scene is no longer published turns off. /// /// Runs until the studio [stops](Self::stop), and returns once every /// rendition's encode thread has finished its track. async fn run_program(&self, start: Instant, epoch: Duration, mut recording: Line) { let mut framerate = self.state.lock().unwrap().compositor.canvas().framerate; let mut clock = program_clock(start, framerate); // The encoders of lines since ended, waited on as the studio stops so // each finishes its track. let mut retired: Vec = Vec::new(); // The encoders of the stream on air. let mut air: Option = None; loop { let tick = clock.tick().await; // The cameras of scenes no longer published, closed once the lock // is let go, so the UI does not wait on their backends. let mut closed: Vec = Vec::new(); if *self.stopped.borrow() { // A stream handed over since is ended too, and one asked to end // finds nothing left to wait for. let (start, _end) = self.handoff.lock().unwrap().close(); // Without a sender, each encoder publishes what it was sent and // still holds, then finishes its track. retired.push(recording.end(None)); let mut compositions = Vec::new(); for lines in air.take().into_iter().chain(start) { let ending = lines.end(); retired.extend(ending.ended); compositions.extend(ending.composition); } for ended in retired { ended.join(); } // Once the video it composes has ended. for mut composition in compositions { composition.finish(); } return; } retired.retain(|ended| !ended.is_finished()); // Ended before another starts, so the two never hold encode sessions // at once. let (mut starting, end) = self.handoff.lock().unwrap().take(); if let Some(reply) = end { // The stream on air, or one handed over since that never drew a // frame, such as one ended at once at a low frame rate. let mut ending = air .take() .or_else(|| starting.take()) .map(air::Lines::end) .unwrap_or_default(); // The scenes unpublished on air too, so each finishes its // track before the stream's sessions close. ending.ended.append(&mut retired); if let Err(ending) = reply.send(ending) { retired.extend(ending.ended); } self.stats.published.lock().unwrap().clear(); } if let Some(starting) = starting { air = Some(starting); } // Where this frame lands on the media clock. let at = epoch + tick.saturating_duration_since(start); let drawn = { let mut guard = self.state.lock().unwrap(); let state = &mut *guard; let synced = std::mem::take(&mut state.stale); if synced { let drivers = state.sources.sync(&state.collection); self.recorder.set_sources(&state.collection); if !drivers.is_empty() { let mut running = self.running.lock().unwrap(); // Forgets the threads that have ended. running.retain_mut(|ended| ended.try_recv() == Err(TryRecvError::Empty)); running.extend(drivers.into_iter().map(spawn)); } self.pick_audio(state); } // On air, each published scene's line starts once it is // published, in a broadcast of its own, and ends once it no // longer is, as does the portrait picture's while the show // goes out in portrait. let published = published(&state.collection); let portrait = state.collection.portrait_canvas(); let extras: Vec<(Extra, Canvas)> = published .iter() .map(|(name, canvas)| (Extra::Scene(name.clone()), *canvas)) .chain(portrait.map(|canvas| (Extra::Portrait, canvas))) .collect(); if let Some(air) = &mut air { // One whose name now carries the other kind, such as a scene // named as the portrait picture before it went out in both, // ends too, so its viewers see it end. let gone: Vec = air .lines .keys() .filter(|line| !extras.iter().any(|(extra, _)| extra == *line)) .cloned() .collect(); for extra in gone { if let Some(Some((line, broadcast))) = air.lines.remove(&extra) { retired.push(line.end(Some(broadcast))); } self.stats.published.lock().unwrap().remove(extra.name()); } for (extra, canvas) in &extras { let name = extra.name(); let started = !air.lines.contains_key(extra); let extras = &air.extras; let line = air.lines.entry(extra.clone()).or_insert_with(|| { let video = Arc::new(VideoStats::default()); self.stats .published .lock() .unwrap() .insert(name.to_string(), video.clone()); extras .start(name, *canvas, video.clone()) .inspect_err(|err| video.fail(format!("{err:#}"))) .ok() }); // Its audio follows the audio scene it carries from now on: // the portrait picture's, the program's, with the sources // published independently in it, as it carries no // composition. let mix = match extra { Extra::Portrait => state .collection .program_audio() .and_then(|audio| state.sources.listened(audio)), Extra::Scene(name) => state .collection .published_audio(name) .and_then(|audio| state.sources.mix(audio)), }; if (synced || started) && let Some((_, broadcast)) = line && let Some(mix) = mix { broadcast.audio.set(mix.clone()); } } } closed.extend( state .cameras .extract_if(.., |scene, _| { scene .as_ref() .is_some_and(|scene| !published.iter().any(|(name, _)| name == scene)) }) .map(|(_, camera)| camera), ); let (recorded, recorded_canvas) = recorded(&state.collection); let cameras = &state.cameras; let lines = air.as_ref().map(|air| &air.lines); let wants = |name: &str| { let extra = Extra::Scene(name.to_string()); matches!(lines.and_then(|lines| lines.get(&extra)), Some(Some((line, _))) if line.wants()) || cameras .get(&Some(name.to_string())) .is_some_and(crate::vcam::Camera::wanted) }; let mut outputs: BTreeMap<&str, bool> = published.iter().map(|(name, _)| (name.as_str(), wants(name))).collect(); if let Some(scene) = &recorded { *outputs.entry(scene).or_default() |= recording.wants(); } state.compositor.set_outputs(outputs); // Before drawing, so a media file that restarts going on program // is cleared in this frame rather than showing where it ended. A // scene beside program starting to draw restarts none: its // sound would start over in the program mix too. state.sources.set_on_program(&state.on_program()); let shown = state.shown(); if let Err(err) = state.compositor.render(&state.collection, &state.sources, tick) { tracing::error!(%err, "cannot draw the canvas"); continue; } self.program.on_program(state, shown); if let Some(air) = &mut air { air.composition.compose(state, at, tick); } let compositor = &state.compositor; for (scene, camera) in &mut state.cameras { match scene { // An app reading it cannot place the layers itself. None => camera.offer(Some(compositor.layered()), compositor.canvas()), Some(scene) => { let Some((_, canvas)) = published.iter().find(|(name, _)| name == scene) else { continue; }; let picture = compositor.output(scene).and_then(moq_scene::Output::texture); camera.offer(picture, *canvas); } } } let program = (compositor.program().clone(), compositor.canvas()); let texture = |scene: &str| compositor.output(scene).and_then(moq_scene::Output::texture).cloned(); let portrait = compositor.portrait().map(|(texture, canvas)| (texture.clone(), canvas)); Drawn { published: published .into_iter() .map(|(name, canvas)| { let texture = texture(&name); (Extra::Scene(name), texture, canvas) }) .chain(portrait.map(|(texture, canvas)| (Extra::Portrait, Some(texture), canvas))) .collect(), recorded: match recorded { Some(scene) => (texture(&scene), recorded_canvas), None => (Some(program.0.clone()), program.1), }, program, } }; drop(closed); self.stats.rendered.fetch_add(1, Ordering::Relaxed); let timestamp = moq_net::Timestamp::try_from(at).ok(); // Every scene shares the program's frame rate. let (program, canvas) = drawn.program; if canvas.framerate != framerate { framerate = canvas.framerate; clock = program_clock(tick.saturating_add(period(framerate)), framerate); } let (texture, recorded) = drawn.recorded; recording.feed(texture.as_ref(), recorded, timestamp); if let Some(air) = &mut air { air.main.feed(Some(&program), canvas, timestamp); for (extra, texture, canvas) in drawn.published { if let Some(Some((line, _))) = air.lines.get_mut(&extra) { line.feed(texture.as_ref(), canvas, timestamp); } } } } } /// Advance the bus as the program clock advances, from `start`. /// /// Runs without a relay too, so the meters move and sources never back up, /// until the studio [stops](Self::stop), which ends every mix, so each /// output reads its mix to the end. async fn run_audio(&self, start: Instant, config: moq_audio::bus::Config, mut mixer: source::Mixer) { let rate = u128::from(config.sample_rate); let mut clock = kio::clock::interval_at(start, AUDIO_PERIOD); clock.set_missed_tick_behavior(kio::clock::MissedTickBehavior::Skip); let mut written: u128 = 0; loop { let tick = clock.tick().await; if *self.stopped.borrow() { drop(mixer); self.program_audio.close(); self.recorded_audio.close(); return; } // Sample-exact against the program clock, so audio and video stay // aligned however the ticks jitter. let due = tick.saturating_duration_since(start).as_nanos() * rate / 1_000_000_000; let frames = (due - written) as usize; if frames == 0 { continue; } mixer.advance(frames); written = due; } } /// Encode the mix `follow` reads into the recording's own `audio`, stamped /// `epoch` past the bus's start at `rate`, until the bus stops, and finish /// the track. /// /// It encodes only while a recording reads it, and marks the gap as the /// last one stops, so the next starts on the program clock rather than /// where the samples left off. Its failing leaves the published audio /// running, so it is the recording's to report. async fn encode_audio( &self, epoch: Duration, rate: u32, mut follow: mix::Follow, mut audio: moq_audio::encode::Producer, ) { let fail = |err: moq_audio::Error| self.stats.fail_recording_audio(err); let rate = u128::from(rate); let mut next = None; let mut wanted = false; while let Some(block) = follow.read().await { let was = std::mem::replace(&mut wanted, audio.demand().wanted().is_some()); if !wanted { if was && let Err(err) = audio.discontinuity() { fail(err); audio.abort(moq_net::StreamError::Internal.into()); return; } next = None; continue; } // Audio this encoder fell too far behind to read leaves a gap in // the track rather than pulling what follows early. if next.is_some_and(|next| next != block.position) { audio.reanchor(); } next = Some(block.position + block.frames() as u64); let position = u128::from(block.position); let micros = epoch.as_micros() + position * 1_000_000 / rate; let Ok(timestamp) = moq_net::Timestamp::from_micros(micros as u64) else { continue; }; let data: Vec = block.data.iter().flat_map(|sample| sample.to_le_bytes()).collect(); if let Err(err) = audio.write(&moq_audio::Frame::new(Bytes::from(data), timestamp)) { fail(err); // Readers see the audio fail rather than go away unfinished. audio.abort(moq_net::StreamError::Internal.into()); return; } } if let Err(err) = audio.finish() { fail(err); } } } /// An engine whose workers are starting. Unless every one /// [started](Self::started), dropping it stops those that did, waiting for /// them, which takes about a frame and the encoders' tails. struct Starting(Option>); impl Starting { fn started(mut self) { self.0 = None; } } impl Drop for Starting { fn drop(&mut self) { if let Some(engine) = self.0.take() { engine.stop(); } } } /// What starting the engine sets up for the program's broadcasts beside /// [`Program`], handed to the threads that encode it. struct Publish { /// Where the recording's own video goes, of the program or the scene it /// holds. recording: Renditions, /// The recording broadcast's own audio encoder. recorded_audio: moq_audio::encode::Producer, /// Where each local source keeps its own track. sources: moq_scene::publish::Publish, } /// Set up what the program's broadcasts share for as long as the engine runs: /// the clock, with the program clock's `start` on it, the local sources' /// tracks, and the recording's own broadcast, which reads the program at /// `canvas` until a recording asks for another. async fn publish(gpu: &Gpu, stats: &Arc, canvas: Canvas, start: Instant) -> anyhow::Result<(Program, Publish)> { // What a recording reads the program from: its own full-size encoder, of // the program or the scene it holds, at full quality whatever the grant, // and the mix it holds, in a broadcast no relay sees: announced on an // origin of its own, which no relay connection carries, so a recording // reads it as it would the published program. let local = moq_tokio::origin::spawn(); let recording = local .route("recording", moq_net::origin::RouteOptions::hidden()) .context("cannot create the recording broadcast")?; recording .announce() .context("cannot announce the recording broadcast")?; let mut recorder = recording.broadcast().clone(); let config = moq_mux::catalog::Config::default().with_catalog(Default::default()); let recorded = moq_scene::publish::Catalog::new(&mut recorder, config)?; // Every other catalog shares the recording's clock, so a recording, the // program on air, and a source's track agree on when a frame was drawn. let clock = recorded.clock(); // Where the program clock starts on the catalog's, so the wall time the // catalog advertises for a frame is when the program drew it. let epoch = Duration::try_from(clock.capture(start)?)?; // Probed now, so a machine that cannot encode the canvas says so before // going live, which tries it again. Sources take the program's tier: // hardware when the program found it, so a session the GPU refuses fails // that source loudly instead of falling to software. When the program could // not use hardware, such as NVENC beside an integrated GPU it cannot read, a // source may still reach it from the CPU. let kind = match Probe::open(gpu, canvas, false).await { Ok(probe) if !probe.hardware() => moq_video::encode::Kind::Auto, Ok(_) => moq_video::encode::Kind::Hardware, Err(err) => { stats.video.fail(format!("{err:#}")); moq_video::encode::Kind::Hardware } }; let capped = matches!(kind, moq_video::encode::Kind::Hardware) || stats.sessions.is_capped(); let reclaim = capped.then(|| stats.sessions.clone()); let sources = moq_scene::publish::Publish::new(clock.clone(), kind, stats.sessions.clone())?; let recorded_video = Renditions { gpu: gpu.clone(), broadcast: recorder.clone(), timebase: recorded.timebase(), record: true, reclaim: reclaim.clone(), bandwidth: moq_net::bandwidth::Allocator::unlimited(), on_demand: true, live: false, stats: stats.clone(), video: stats.recording.clone(), grants: String::new(), }; let (input, options) = audio_options(); let recorded_audio = moq_audio::encode::Producer::new(&mut recorder, recorded.clone(), input, &options)?; let (published, _) = tokio::sync::watch::channel(BTreeSet::new()); Ok(( Program { clock, epoch, sources: sources.clone(), reclaim, published, recording, recorded, local, }, Publish { recording: recorded_video, recorded_audio, sources, }, )) } /// What the program audio, the recording's, and each published scene's encode: /// the bus's mix, as Opus at [`AUDIO_BITRATE`] on a track called `audio`. fn audio_options() -> (moq_audio::encode::Input, moq_audio::encode::Options) { let bus = moq_audio::bus::Config::default(); let mut input = moq_audio::encode::Input::new(bus.sample_rate, bus.layout); input.format = moq_audio::Format::F32; let mut options = moq_audio::encode::Options::default(); options.track = Some("audio".into()); options.config = moq_audio::encode::Config::from_input(moq_audio::encode::Codec::Opus, &input); options.config.bitrate = Some(AUDIO_BITRATE); (input, options) } /// What one of the program's video encoders is for, and its place in /// [`Stats::renditions`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum Role { /// The canvas size, published. Canvas = 0, /// Half of it, published. Half = 1, /// The canvas size again for a recording, in the studio's own recording /// broadcast, at full quality whatever the bandwidth grant. Record = 2, } impl Role { /// Whether viewers on a relay see it. fn published(self) -> bool { self != Self::Record } } /// Where the program's or a published scene's video is published, a canvas /// at a time, each rendition its own track, since a track never changes what /// it carries; or where the recording's own video goes, of the program or the /// scene it holds. #[derive(Clone)] struct Renditions { gpu: Gpu, /// A published broadcast, or the recording's: one the studio keeps to /// itself, holding the recording's video and its own encoding of the mix /// it holds, listed in a catalog of its own on the program's clock. broadcast: moq_net::broadcast::Producer, /// Shared by every rendition, so each lands on the catalog's clock the same /// way and a switch keeps the video in step with the audio. timebase: moq_mux::catalog::Timebase, /// Whether this is the recording's: its one rendition the full size, at /// full quality whatever the grant, rather than the published full and /// half sizes. record: bool, /// Where the program reclaims the sources' replay encoders' sessions when /// the GPU refuses it the hardware encoder it found at launch, or `None` /// when it found none and the sessions are not /// [capped](Sessions::capped), so the sources' encoders hold none either. reclaim: Option, /// The connection's bandwidth, which each published rendition follows its /// share of. bandwidth: bandwidth::Allocator, /// Whether the canvas size encodes only while someone watches. on_demand: bool, /// Whether the program goes out on a relay, where viewers can watch it. live: bool, stats: Arc, /// How this video is doing. video: Arc, /// What each track's grant is listed under in [`Stats::grants`], before /// its name: empty for the program, and a published scene's name and a /// slash. grants: String, } impl Renditions { /// Probe for `canvas`'s renditions and [`start`](Self::start) them, waiting /// for the full size's encoder to open, so a canvas the GPU cannot encode /// fails here, even after reclaiming the sources' replay encoders. async fn open(&self, canvas: Canvas) -> anyhow::Result { let probe = Probe::new(self, canvas).await?; let (ladder, opened) = self.start(probe)?; opened.await.context("the encode thread stopped")??; Ok(ladder) } /// List each of `probe`'s renditions in its catalog under a new track, and /// start their encode threads. The receiver reports once the canvas size's /// encoder opens; the half size opens its own once watched, and the /// recording's once recorded. fn start(&self, probe: Probe) -> anyhow::Result<(Ladder, tokio::sync::oneshot::Receiver>)> { // Before the encode threads start, which report their own failures. *self.video.error.lock().unwrap() = None; for (role, load) in [Role::Half, Role::Record] .into_iter() .map(|role| (role, &self.video.renditions[role as usize])) { let refused = probe.refused.iter().find(|(refused, _)| *refused == role); *load.error.lock().unwrap() = refused.map(|(_, reason)| reason.clone()); if let Some((_, reason)) = refused { tracing::warn!(?role, %reason, "the encoder refused a rendition"); } } // The published sizes, largest first, each funded over the next. let ceilings: Vec<_> = probe .rungs .iter() .filter(|probed| probed.role.published()) .map(|probed| ladder::ceiling(probed.config.size(), probed.config.framerate)) .collect(); let (opened, open) = tokio::sync::oneshot::channel(); let mut opened = Some(opened); let mut reclaim = probe.reclaim; let number = self.video.rendition(); let mut rungs = Vec::with_capacity(probe.rungs.len()); for probed in probe.rungs { let Probed { role, exporter, config, video, } = probed; let (broadcast, timebase) = (&self.broadcast, &self.timebase); let reserved = timebase.reserve().with_interval(config.effective_join()); let suffix = moq_scene::publish::suffix(&video.codec)?; let rank = role as usize; let priority = match role { Role::Record => hang::catalog::PRIORITY.video, _ => ladder::priority(rank, ceilings.len()), }; let info = reserved.track_info(priority).with_refresh(ladder::REFRESH); let track = broadcast.unique_track(suffix, info)?; let producer = moq_video::encode::Producer::new(track.clone(), reserved, video)?; let (gate, grant) = match role { // Reads nothing from the connection, so it always encodes at its // ceiling, and only while recorded. Role::Record => { let gate = Gate::new( &bandwidth::Allocator::unlimited(), producer.demand(), ladder::ceiling(config.size(), config.framerate), None, false, ); (gate, None) } _ => { let gate = Gate::new( &self.bandwidth, producer.demand(), ceilings[rank], ceilings.get(rank + 1).copied(), role == Role::Canvas && !self.on_demand, ); let grant = format!("{}{}", self.grants, track.name()); self.stats.grants.lock().unwrap().insert(grant.clone(), gate.grant()); (gate, Some(grant)) } }; // One more than the exporter's pool, for a skip behind a full pool. let (frames, pending) = mpsc::sync_channel(Exporter::DEPTH + 1); // Live, the canvas size holds its session from the start, so a // source past the GPU's session cap is the one that fails. Without // a relay nobody can watch it, so it opens nothing. let opened = match role { Role::Canvas if self.live => opened.take(), _ => None, }; let job = Job { sessions: self.stats.sessions.clone(), reclaim_from: self.reclaim.clone(), // Kept from the probe until the canvas size opens, so no source // takes back a session meanwhile. reclaim: opened.is_some().then(|| reclaim.take()).flatten(), video: self.video.clone(), role, rendition: (role == Role::Canvas).then_some(number), zero_copy: exporter.route() == Route::Gpu, producer, track, config, // The canvas size holds its session while it skips frames. hold: role == Role::Canvas, }; let thread = std::thread::Builder::new() .name("encode".into()) .spawn(move || encode(job, opened, pending))?; let scaler = (role == Role::Half).then(|| Scaler::new(&self.gpu.device, &self.gpu.queue, exporter.size())); rungs.push(Rung { role, gate, grant, scaler, exporter, frames, skipped: None, thread, }); } if let Some(opened) = opened { let _ = opened.send(Ok(())); } Ok((Ladder { rungs }, open)) } } /// Encoders proven to take this GPU's export of a canvas for each of its /// renditions, before they are listed. struct Probe { /// The canvas size first. rungs: Vec, /// Why the encoder refused a rendition the program does without. refused: Vec<(Role, String)>, /// The sources' replay encoders' sessions, reclaimed for these encoders. reclaim: Option, } /// One rendition of a [`Probe`]. struct Probed { role: Role, exporter: Exporter, config: moq_video::encode::Config, video: hang::catalog::VideoConfig, } impl Probe { /// [`open`](Self::open) `renditions` of `canvas`. When the GPU refuses /// them the hardware encoder found at launch, as one at its session cap /// does, the program comes first: this closes the sources' replay /// encoders, which replay a source's recent past, and probes again in /// their sessions, holding them until the full size's encoder opens. async fn new(renditions: &Renditions, canvas: Canvas) -> anyhow::Result { let probe = Self::open(&renditions.gpu, canvas, renditions.record).await; let Some(sessions) = renditions.reclaim.as_ref().filter(|sessions| sessions.spare() > 0) else { return probe; }; if probe.as_ref().is_ok_and(Self::hardware) { return probe; } match &probe { Ok(_) => tracing::warn!("closing the sources' replay encoders for the program's hardware encoder"), Err(err) => tracing::warn!(%err, "closing the sources' replay encoders for the program's encoder"), } drop(probe); let reclaim = sessions.reclaim(); reclaim.closed().await; let mut probe = Self::open(&renditions.gpu, canvas, renditions.record).await?; probe.reclaim = Some(reclaim); Ok(probe) } /// Whether the canvas size took a hardware encoder. fn hardware(&self) -> bool { !matches!(&self.rungs[0].config.kind, moq_video::encode::Kind::Named(name) if name == "openh264") } /// Open a throwaway encoder of each of `canvas`'s renditions on what `gpu` /// exports, the published sizes or, if `record`, the recording's alone, /// which also settles whether the encoder reads the program on the GPU. /// The first picks the encoder, which the other uses too. async fn open(gpu: &Gpu, canvas: Canvas, record: bool) -> anyhow::Result { let framerate = canvas.framerate(); let size = moq_video::Size::new(canvas.width, canvas.height); let roles: Vec<(Role, moq_video::Size)> = match record { true => vec![(Role::Record, size)], false => [Role::Canvas, Role::Half] .into_iter() .zip(ladder::sizes(size)) .collect(), }; let mut kind = moq_video::encode::Kind::Auto; let mut rungs = Vec::new(); let mut refused = Vec::new(); for (role, size) in roles { let mut config = moq_video::encode::Config::new(size.width, size.height, framerate); config.bitrate = Some(ladder::ceiling(size, framerate)); config.kind = kind.clone(); let probed = Exporter::new(&gpu.device, &gpu.queue, &config) .await .with_context(|| format!("cannot encode a {}x{} canvas", size.width, size.height)); let (exporter, config, video) = match probed { Ok(probed) => probed, Err(err) if role != Role::Half => return Err(err), // The program goes on without it. Err(err) => { refused.push((role, format!("{err:#}"))); continue; } }; kind = config.kind.clone(); rungs.push(Probed { role, exporter, config, video, }); } Ok(Self { rungs, refused, reclaim: None, }) } /// [`new`](Self::new) on a thread of its own, off the program clock, once /// the encode threads `ending` have closed their sessions. fn spawn( renditions: Renditions, canvas: Canvas, ending: Vec>, ) -> std::io::Result>> { std::thread::Builder::new().name("probe".into()).spawn(move || { for thread in ending { // One that panicked has closed its session all the same. let _ = thread.join(); } let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; runtime.block_on(Self::new(&renditions, canvas)) }) } } /// One canvas's program video as the program clock feeds it: the renditions /// of the canvas it last drew, the probe opening the next, and the encoders /// ending. struct Line { renditions: Renditions, ladder: Option, /// The canvas `ladder` is for, or will be once probed. canvas: Canvas, /// One probe at a time, with the canvas it is for, since a GPU caps its /// encode sessions. probing: Option<(Canvas, std::thread::JoinHandle>)>, /// The encode threads of renditions ended or failed since, whose sessions /// the next probe waits to be closed. ending: Vec>, /// Whether the canvas drawn has its renditions, or failed to open them, so /// a failure is not retried until the canvas changes. settled: bool, } /// What a [`Line`] leaves running once ended, to wait on. struct Ended { threads: Vec>, /// Holds the encode threads it was waiting on. probe: Option>>, /// A published scene's broadcast, dropped only after the threads, so its /// catalog finishes after the video it lists. _broadcast: Option, } impl Ended { fn is_finished(&self) -> bool { self.threads.iter().all(std::thread::JoinHandle::is_finished) && self.probe.as_ref().is_none_or(std::thread::JoinHandle::is_finished) } fn join(self) { for thread in self.threads { if thread.join().is_err() { tracing::error!("an encode thread panicked"); } } if let Some(probe) = self.probe { let _ = probe.join(); } } } impl Line { /// `renditions` of `canvas`, opened at once, as at startup. One whose /// encoder cannot open draws on without video, saying why. async fn open(renditions: Renditions, canvas: Canvas) -> Self { let ladder = renditions.open(canvas).await; let ladder = ladder.inspect_err(|err| renditions.video.fail(format!("{err:#}"))); Self::new(renditions, canvas, ladder.ok(), true) } /// Whether any rendition would take the next frame: one its grant funds /// that encodes always, or that anyone reads. A probe done opens the renditions on the next /// feed, so their first frame is drawn then rather than one frame late. /// Without renditions open or opening, none would. fn wants(&self) -> bool { let opening = self.probing.as_ref().is_some_and(|(_, probe)| probe.is_finished()); let rungs = self.ladder.iter().flat_map(|ladder| &ladder.rungs); opening || rungs.into_iter().any(|rung| rung.gate.wanted()) } /// `renditions` of `canvas`: `ladder` when already open, or `None` to probe /// for them on the first feed unless `settled`, as a canvas that failed /// to open at startup is. fn new(renditions: Renditions, canvas: Canvas, ladder: Option, settled: bool) -> Self { if let Some(ladder) = &ladder { ladder.adopted(&renditions.video); } Self { renditions, ladder, canvas, probing: None, ending: Vec::new(), settled, } } /// Hand `program`, drawn at `drawn`, to each rendition its gate lets /// encode, stamped `timestamp`; with no `program`, as for a scene not /// drawn since none [wanted](Self::wants) it, each skips the frame. A new /// canvas size or frame rate ends the renditions and probes for others, /// since a track never changes what it carries. fn feed(&mut self, program: Option<&wgpu::Texture>, drawn: Canvas, timestamp: Option) { let renditions = &self.renditions; let video = &renditions.video; if drawn != self.canvas { self.canvas = drawn; self.settled = false; // Ended before the next open, so the two never hold encode sessions // at once. self.ending .extend(self.ladder.take().into_iter().flat_map(|ladder| ladder.end(renditions))); video.retire_encoder(); } // Probed beside the program clock, so drawing carries on. if let Some((probed, probe)) = self.probing.take_if(|(_, probe)| probe.is_finished()) { let probe = probe .join() .unwrap_or_else(|_| Err(anyhow::anyhow!("the encoder probe panicked"))); // One for a canvas since left is dropped unlisted. if probed == self.canvas { self.settled = true; match probe.and_then(|probe| renditions.start(probe)) { Ok((started, _)) => { started.adopted(video); self.ladder = Some(started); } Err(err) => video.fail(format!("{err:#}")), } } } if self.probing.is_none() && !self.settled { let ending = std::mem::take(&mut self.ending); match Probe::spawn(renditions.clone(), self.canvas, ending) { Ok(probe) => self.probing = Some((self.canvas, probe)), Err(err) => { self.settled = true; video.fail(format!("cannot start an encoder probe: {err}")); } } } let Some(current) = &mut self.ladder else { video.watched.store(false, Ordering::Relaxed); video.encoding.store(false, Ordering::Relaxed); return; }; let watched = current .rungs .iter() .any(|rung| rung.role.published() && rung.gate.watched()); video.watched.store(watched, Ordering::Relaxed); let Some(timestamp) = timestamp else { return }; let mut encoding = false; let mut failed = Vec::new(); for (index, rung) in current.rungs.iter_mut().enumerate() { let role = rung.role; match rung.feed(&video.renditions[role as usize], program, timestamp) { Some(step) => encoding |= role == Role::Canvas && matches!(step, Step::Encode(_)), None => failed.push((index, role)), } } video.encoding.store(encoding, Ordering::Relaxed); // An encoder stops only having reported why. Without the canvas size // the video stops; without another rendition it carries on. if failed.iter().any(|(_, role)| *role == Role::Canvas) { self.ending .extend(self.ladder.take().into_iter().flat_map(|ladder| ladder.end(renditions))); } else { for (index, _) in failed.into_iter().rev() { self.ending.push(current.remove(index, renditions)); } } } /// Stop handing the renditions frames, so each encode thread publishes /// what it holds and finishes its track, then ends a published scene's /// `broadcast`. fn end(mut self, broadcast: Option) -> Ended { let mut threads = std::mem::take(&mut self.ending); threads.extend( self.ladder .take() .into_iter() .flat_map(|ladder| ladder.end(&self.renditions)), ); self.renditions.video.watched.store(false, Ordering::Relaxed); self.renditions.video.encoding.store(false, Ordering::Relaxed); Ended { threads, probe: self.probing.map(|(_, probe)| probe), _broadcast: broadcast, } } } /// Each published scene's line and broadcast, and the portrait picture's, or /// `None` once it failed to start, which is not retried until it starts /// again. type Lines = BTreeMap>; /// A picture going out as a broadcast of its own, beneath the program's. #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] enum Extra { /// A published scene, by name. Scene(String), /// The portrait picture, while the show goes out in both. Portrait, } impl Extra { /// Where it goes out, beneath the program's broadcast. fn name(&self) -> &str { match self { Self::Scene(name) => name, Self::Portrait => moq_scene::PORTRAIT, } } } /// What one tick drew, for the encoders. struct Drawn { /// The program, at its canvas. program: (wgpu::Texture, Canvas), /// Each published scene, unless not drawn, at its canvas, and the /// portrait picture, while the show goes out in portrait. published: Vec<(Extra, Option, Canvas)>, /// What recordings hold, unless not drawn, at its canvas. recorded: (Option, Canvas), } /// Each published scene in `collection`, with its canvas. fn published(collection: &Collection) -> Vec<(String, Canvas)> { let published = collection.scenes.iter().filter(|scene| scene.published); published .map(|scene| (scene.name.clone(), scene.canvas(collection.canvas))) .collect() } /// What recordings hold in `collection`: the scene it records, or `None` for /// the program, with its canvas. fn recorded(collection: &Collection) -> (Option, Canvas) { let scene = collection.recorded(); let canvas = collection.canvas(scene).unwrap_or(collection.canvas); (scene.map(str::to_string), canvas) } /// A published scene's broadcast, whose catalog finishes and which closes /// when dropped. struct Broadcast { publication: moq_net::origin::Route, /// Lists the scene's video and audio. catalog: moq_scene::publish::Catalog, /// The audio scene its audio carries, as the collection picks it. audio: mix::Pick, /// Dropped to finish its audio track. _audio_stop: tokio::sync::oneshot::Sender<()>, /// Its audio's claim on the connection's bandwidth, which it does not /// follow, as the program's. _audio_grant: bandwidth::Reservation, /// Its grading, while live, which stops with it. _graded: Option, } impl Drop for Broadcast { fn drop(&mut self) { // Fails only once the catalog failed, which ended its track already. let _ = self.catalog.finish(); self.publication.broadcast().close(); } } /// What starts each published scene's broadcast: beneath the program's path, /// on its clock, sharing its connection's bandwidth. struct Extras { origin: moq_net::origin::Producer, path: moq_net::Path, clock: moq_mux::Clock, /// Where the program audio's timestamps start on that clock, which each /// scene's audio is stamped against too. epoch: Duration, /// Where each scene's audio encodes: never the program thread that /// starts it, so encoding never holds up a frame. runtime: tokio::runtime::Handle, gpu: Gpu, bandwidth: bandwidth::Allocator, on_demand: bool, live: bool, stats: Arc, /// [`Renditions::reclaim`]. reclaim: Option, /// Where each published scene's broadcast is graded, while live. graded: Option, } impl Extras { /// Publish the scene `name`, drawn at `canvas`, and the [`Line`] that /// encodes its video, reporting in `video`. Its encoders are probed for on /// the line's first feed, off the program clock, for a scene published /// while the studio runs. fn start(&self, name: &str, canvas: Canvas, video: Arc) -> anyhow::Result<(Line, Broadcast)> { let (renditions, broadcast) = self.publish(name, video)?; Ok((Line::new(renditions, canvas, None, false), broadcast)) } /// Publish the scene `name` at its own path beneath the program's /// broadcast, with audio of its own, which encodes the audio scene it is /// [picked](Broadcast::audio) on the runtime, and where its video goes, /// reporting in `video`. Its own, rather than the program's by /// reference, so picking another audio scene never changes the track its /// viewers play. fn publish(&self, name: &str, video: Arc) -> anyhow::Result<(Renditions, Broadcast)> { let path = moq_net::Path::new(name) .and_then(|name| self.path.join(&name)) .with_context(|| format!("cannot publish scene `{name}`"))?; if self.live { let (_, echo) = health::echo(&self.path)?; anyhow::ensure!( path != echo, "scene `{name}` would publish where viewers report back; rename it to publish it" ); } anyhow::ensure!( name != moq_scene::composition::PATH, "scene `{name}` would publish where the composition is; rename it to publish it" ); let publication = self .origin .route(&path, moq_net::origin::RouteOptions::hidden()) .with_context(|| format!("cannot create the broadcast of scene `{name}`"))?; let mut broadcast = publication.broadcast().clone(); let config = moq_mux::catalog::Config::default() .with_catalog(Default::default()) .with_clock(self.clock.clone()); let mut catalog = moq_scene::publish::Catalog::new(&mut broadcast, config)?; let (input, options) = audio_options(); let rate = input.sample_rate; let mut audio = moq_audio::encode::Producer::new(&mut broadcast, catalog.clone(), input, &options)?; let audio_grant = self.bandwidth.reserve(&audio.demand(), AUDIO_BITRATE); let pick = mix::Pick::default(); let mut follow = pick.follow(); let (audio_stop, stopped) = tokio::sync::oneshot::channel::<()>(); let epoch = self.epoch; let failed = video.clone(); let scene = name.to_string(); self.runtime.spawn(async move { let encoded = tokio::select! { encoded = mix::encode(&mut follow, &mut audio, epoch, rate) => encoded, // Dropped with the broadcast. _ = stopped => Ok(()), }; match encoded { Ok(()) => { // Fails only once the broadcast closed, which ended the track. let _ = audio.finish(); } Err(err) => { tracing::error!(%scene, error = format!("{err:#}"), "a published scene's audio stopped"); failed.fail_audio(format!("{err:#}")); // Readers see the audio fail rather than go away unfinished. audio.abort(moq_net::StreamError::Internal.into()); } } }); // Its viewers report back beneath it, as the program's do. if self.graded.is_some() { health::solicit(&mut catalog, &path)?; } publication .announce() .with_context(|| format!("cannot announce the broadcast of scene `{name}`"))?; let graded = match &self.graded { Some(scenes) => { let observer = hang::health::Observer::new(path.clone(), Some(publication.instance())); Some(scenes.follow(name, observer, catalog.clone())?) } None => None, }; let renditions = Renditions { gpu: self.gpu.clone(), broadcast, timebase: catalog.timebase(), record: false, reclaim: self.reclaim.clone(), bandwidth: self.bandwidth.clone(), on_demand: self.on_demand, live: self.live, stats: self.stats.clone(), video, grants: format!("{name}/"), }; let broadcast = Broadcast { publication, catalog, audio: pick, _audio_stop: audio_stop, _audio_grant: audio_grant, _graded: graded, }; Ok((renditions, broadcast)) } } /// The program video of one canvas: each of its renditions, the canvas size /// first. struct Ladder { rungs: Vec, } impl Ladder { /// Report these renditions as the canvas's from now on. fn adopted(&self, video: &VideoStats) { let zero_copy = self.rungs[0].exporter.route() == Route::Gpu; video.zero_copy.store(zero_copy, Ordering::Relaxed); self.count(video); } /// Report how many sizes the canvas publishes. fn count(&self, video: &VideoStats) { let sizes = self.rungs.iter().filter(|rung| rung.role.published()).count(); video.sizes.store(sizes, Ordering::Relaxed); } /// Stop handing the renditions frames, so each encode thread encodes those /// it holds, finishes its track, and returns. fn end(self, renditions: &Renditions) -> Vec> { renditions.video.sizes.store(0, Ordering::Relaxed); let mut grants = renditions.stats.grants.lock().unwrap(); self.rungs .into_iter() .map(|rung| { if let Some(grant) = &rung.grant { grants.remove(grant); } rung.thread }) .collect() } /// Stop handing rendition `index` frames, as [`end`](Self::end) does. /// The canvas size left alone no longer disables below the half size, /// since no viewer has a smaller one to move to. fn remove(&mut self, index: usize, renditions: &Renditions) -> std::thread::JoinHandle<()> { let rung = self.rungs.remove(index); if let Some(grant) = &rung.grant { renditions.stats.grants.lock().unwrap().remove(grant); } self.count(&renditions.video); if rung.role == Role::Half { for canvas in self.rungs.iter_mut().filter(|rung| rung.role == Role::Canvas) { canvas.gate.alone(); } } rung.thread } } /// One rendition of the program video, as the program clock feeds it: its /// gate, the exporter it copies into, and the encode thread behind it. struct Rung { role: Role, gate: Gate, /// Its grant's name in [`Stats::grants`]: a published rendition's alone. grant: Option, /// Draws the program at the rendition's size; `None` at the canvas size. scaler: Option, exporter: Exporter, frames: mpsc::SyncSender