use fframes::error::FFramesError; use fframes::{ AudioTimelineFrames, Duration, Markers, ResolvedAudioMap, ResolvedRenderingTimeline, ResolvedScenesTimeline, Scenes, ScenesWithAudio, StaticMediaProvider, TimeBase, TimelineMarker, Video, }; use futures::future::join_all; use std::collections::HashMap; use wasm_bindgen::{JsValue, prelude::wasm_bindgen}; #[wasm_bindgen(module = "@fframes/editor")] extern "C" { #[wasm_bindgen(catch)] async fn load_audio_wasm_callback(path: &str) -> Result; } #[wasm_bindgen] pub struct AudioData { pub sample_rate: u32, name: &'static str, fltp_data: js_sys::Float32Array, } impl AudioData { pub fn new(audio: &fframes::AudioData, name: &'static str) -> Self { let preloaded_data = match audio { fframes::AudioData::Preloaded(data) => data, fframes::AudioData::Lazy => panic!("Lazy audio is not supported in wasm"), }; // The editor plays one channel: mix stereo files down instead of dropping the right. let fltp_data = match &preloaded_data.right { Some(right) => { let mixed: Vec = preloaded_data .samples .iter() .zip(right.iter()) .map(|(left, right)| (left + right) * 0.5) .collect(); js_sys::Float32Array::from(mixed.as_slice()) } None => js_sys::Float32Array::from(preloaded_data.samples.as_ref()), }; Self { name, fltp_data, sample_rate: preloaded_data.sample_rate, } } } #[wasm_bindgen] impl AudioData { /// An expensive cloning of the whole audio buffer, should be called once per /// the app lifetime. #[wasm_bindgen(getter)] pub fn fltp_data(&self) -> js_sys::Float32Array { self.fltp_data.clone() } #[wasm_bindgen(getter)] pub fn name(&self) -> String { self.name.to_string() } } /// Durations (in frames) of every media file the video and scene timelines /// are resolved from. /// /// Audio comes from static media or is decoded through the browser; video /// durations come from the metadata the editor registered for the file /// (`resolve_video_duration`, in seconds), falling back to the audio track /// when the metadata is not registered yet. async fn resolve_used_media_durations<'a, 'media: 'a, TStaticMedia: StaticMediaProvider<'media>>( duration: &'a Duration<'a>, scene_audios: &'a ScenesWithAudio<'media>, global_audio_map: &'a fframes::AudioMap<'a>, markers: &'a Markers<'a>, static_media: &'media TStaticMedia, resolve_video_duration: impl Fn(&str) -> Option, ) -> fframes::error::Result> { // Durations in seconds, the timeline converts them to frames or samples. let mut duration_map: HashMap = HashMap::new(); let mut all_used_files = global_audio_map .used_audio_files::>() .unwrap_or_default(); if let Some(duration_audios) = duration.used_audio_files() { all_used_files.extend(duration_audios.into_iter()); } if let Some(duration_audios) = scene_audios.used_audio_files() { all_used_files.extend(duration_audios.into_iter()); } all_used_files.extend(markers.used_audio_files()); // The editor registers video metadata while `prepare` is already // running, so a video that is not registered yet is measured through // its audio track instead (like any audio file). let mut video_durations: HashMap = HashMap::new(); for file in duration .used_video_files() .into_iter() .chain(scene_audios.used_video_files()) .chain(Some(markers.used_video_files())) .flatten() { match resolve_video_duration(file) { Some(seconds) => { video_durations.insert(file.to_string(), seconds as f64); } None => all_used_files.push(file), } } if all_used_files.is_empty() { return Ok(video_durations); } all_used_files.sort_unstable(); all_used_files.dedup(); let all_used_files = all_used_files .into_iter() .filter(|file| { let static_audio = static_media.resolve_audio(file); // if audio is not static than we need to fetch and load it through the browser xhr static_audio .map(|audio| { duration_map.insert(file.to_string(), audio.duration_in_seconds() as f64); }) .is_none() }) .collect::>(); if all_used_files.is_empty() { duration_map.extend(video_durations); return Ok(duration_map); } join_all( all_used_files .iter() .map(|file| load_audio_wasm_callback(file)), ) .await .into_iter() .zip(all_used_files.into_iter()) .try_for_each(|(js_value, file)| match js_value { Ok(val) => val .as_f64() .map(|seconds| { duration_map.insert(file.to_string(), seconds); }) .ok_or_else(|| FFramesError::CanNotProcessAudioDuration(file.to_string())), Err(_) => Err(FFramesError::CanNotProcessAudioDuration(file.to_string())), })?; // A video file's own duration wins over its audio track, as in the renderer. duration_map.extend(video_durations); Ok(duration_map) } pub async fn prepare_video_with_audio< 'a, TVideo: Video, TStaticMedia: StaticMediaProvider<'static>, >( video: &TVideo, tb: &TimeBase, static_media: &'static TStaticMedia, scenes: &Scenes<'static>, resolve_video_duration: impl Fn(&str) -> Option, ) -> Result< ( usize, Option>, Option>, Vec, ), FFramesError, > { let video_duration = video.duration(); let audio_map = video.audio(); let markers = video.markers(); let scene_audios = ScenesWithAudio::new(scenes); let audio_durations = resolve_used_media_durations( &video_duration, &scene_audios, &audio_map, &markers, static_media, resolve_video_duration, ) .await .unwrap(); let resolve_audio_duration_in_frames = |val: &str| { audio_durations .get(val) .copied() .ok_or_else(|| FFramesError::CanNotProcessAudioDuration(val.to_string())) }; let ResolvedRenderingTimeline { duration_in_frames: duration, scenes, audio_map, markers, } = fframes::resolve_timeline::( &video_duration, &scene_audios, tb, &audio_map, &markers, resolve_audio_duration_in_frames, )?; Ok((duration, scenes, audio_map, markers)) }