use fframes::{FFramesContext, FFramesMode, Frame, StaticMediaProvider, Video, VideoSize}; use std::{cell::RefCell, marker::PhantomPinned, pin::Pin, sync::Mutex}; use wasm_bindgen::prelude::*; use crate::{video_metadata, wasm_audio, wasm_font_source}; #[wasm_bindgen] extern "C" { #[wasm_bindgen(catch)] fn __fframes_get_video_frame( filename: &str, pts: f64, preview: bool, ) -> Result; } fn resolve_video_frame(filename: &str, pts: i64, preview: bool) -> Option { // A JS exception must not unwind through the editor's `RefCell` borrows. let result = __fframes_get_video_frame(filename, pts as f64, preview).ok()?; if result.is_null() || result.is_undefined() { return None; } let url = js_sys::Reflect::get(&result, &"url".into()) .ok()? .as_string()?; let width = js_sys::Reflect::get(&result, &"width".into()) .ok()? .as_f64()? as u32; let height = js_sys::Reflect::get(&result, &"height".into()) .ok()? .as_f64()? as u32; let preview_url = js_sys::Reflect::get(&result, &"posterUrl".into()) .ok() .and_then(|v| v.as_string()); Some(fframes::VideoFrameData { url, preview_url, width, height, }) } struct VideoCtx { video: T, raw_scenes: fframes::Scenes<'static>, _marker: PhantomPinned, } impl VideoCtx { fn new(video: T) -> Pin> { let inner = VideoCtx { video, raw_scenes: fframes::Scenes::empty(), _marker: PhantomPinned, }; let mut pinned = Box::pin(inner); // SAFETY: We're creating a self-reference, but we ensure the struct // cannot be moved by pinning it and using PhantomPinned unsafe { let self_ptr: &T = &pinned.as_ref().video; let scenes = self_ptr.define_scenes(); let scenes = std::mem::transmute::, fframes::Scenes<'static>>(scenes); Pin::get_unchecked_mut(Pin::as_mut(&mut pinned)).raw_scenes = scenes; } pinned } } pub struct WasmEditor + 'static> { video_ctx: RefCell>>>, static_media: &'static TMedia, duration_in_frames: Mutex, break_lines_cache: fframes::TextCache, fonts: Mutex, scenes: Mutex>>, markers: Mutex>, time_base: Mutex>, media_provider: Mutex>, video_decoders: fframes::VideoDecodersWorker, } impl + 'static> WasmEditor { pub fn new(video: TVideo, static_media: &'static TMedia) -> Self { fframes::set_video_frame_resolver(resolve_video_frame); #[cfg(not(feature = "lottie"))] let media_provider = fframes::DynamicMediaProvider::default(); // `render_frame` reads media from this provider. The static animations go into it too. #[cfg(feature = "lottie")] let media_provider = static_media.get_all_lottie_data().into_iter().fold( fframes::DynamicMediaProvider::default(), |provider, (lottie, name)| provider.with_lottie(name, lottie.clone()), ); Self { video_ctx: RefCell::new(VideoCtx::new(video)), static_media, duration_in_frames: Mutex::new(0), break_lines_cache: fframes::TextCache::new(20).unwrap(), fonts: Mutex::new(wasm_font_source::WasmFontSource::new()), scenes: Mutex::new(None), markers: Mutex::new(Vec::new()), time_base: Mutex::new(None), media_provider: Mutex::new(media_provider), video_decoders: fframes::VideoDecodersWorker::default(), } } pub async fn replace(&self, video: TVideo) -> Result { let fps = { self.time_base.lock().unwrap().as_ref().map(|tb| tb.fps) }; self.video_ctx.replace(VideoCtx::new(video)); self.prepare(fps).await } pub async fn prepare( &self, fps: Option, ) -> Result { let tb = fframes::TimeBase { fps: fps.unwrap_or(TVideo::FPS), sample_rate: 44100, }; self.time_base.lock().unwrap().replace(tb); let video_ctx = self.video_ctx.borrow(); let (duration, scenes, audio_map, markers) = wasm_audio::prepare_video_with_audio( &video_ctx.video, &tb, self.static_media, &video_ctx.raw_scenes, |file| { self.media_provider .lock() .unwrap() .videos .get(file) .and_then(|video| video.metadata.as_ref()) .map(|metadata| metadata.duration) }, ) .await .map_err(|err| JsValue::from_str(&err.to_string()))?; let mut duration_mutex_ref = self.duration_in_frames.lock().unwrap(); *duration_mutex_ref = duration; let video_metadata = video_metadata::VideoMetadata::new::( duration as i32, tb.fps, audio_map, scenes.as_ref(), markers.clone(), ); if let Some(scenes_data) = scenes { self.scenes.lock().unwrap().replace(scenes_data.clone()); } *self.markers.lock().unwrap() = markers; Ok(video_metadata) } pub fn add_audio_source(&self, file: String, sample_rate: i32, input: &[f32]) { let audio_data = fframes::media::PreloadedAudioData { sample_rate: sample_rate as u32, // Can't guarantee the lifetime of the input slice samples: std::borrow::Cow::Owned(input.to_vec()), right: None, }; let mut media_provider = self.media_provider.lock().unwrap(); media_provider .audio .insert(file.clone(), fframes::AudioData::Preloaded(audio_data)); } pub fn add_subtitles_source(&self, file: String, content: String) -> usize { use fframes::media::FFramesSubtitles; let parsed_subtitle = fframes::media::Subtitles::parse(&content).unwrap(); let phrases_count = (&parsed_subtitle).cues_count(); self.media_provider .lock() .unwrap() .subtitles .insert(file, parsed_subtitle); phrases_count } /// Parses a Lottie file of the dynamic media folder and returns its length in seconds. /// Returns `None` for an invalid file, or when the `lottie` feature is off. pub fn add_lottie_source(&self, file: String, content: String) -> Option { #[cfg(feature = "lottie")] { let lottie = fframes::Lottie::parse(content) .map_err(|error| fframes::log!("{file} is not a Lottie file: {error}")) .ok()?; let seconds = lottie.seconds(); self.media_provider .lock() .unwrap() .lottie .insert(file, lottie); Some(seconds) } #[cfg(not(feature = "lottie"))] { let _ = content; fframes::log!( "{file}: enable the `lottie` feature of fframes-editor-controller to load Lottie files" ); None } } pub fn add_image_source( &self, file: String, url: String, width: u32, height: u32, base64_data: Option, ) { let mut media_provider = self.media_provider.lock().unwrap(); let base64_data = fframes::media::OwnedSharedString::new_owned( base64_data.unwrap_or_else(|| url.clone()), ); media_provider.images.insert( file.clone(), fframes::media::ImageData::new_from_web_source( base64_data, url, file, fframes::media::ImageMetadata { width: width as u32, height: height as u32, }, ), ); } pub fn add_video_source_placeholder( &self, file: String, url: String, width: u32, height: u32, duration: f32, ) { let mut media_provider = self.media_provider.lock().unwrap(); let metadata = fframes::media::GeneralVideoFileMetadata { width: width as u32, height: height as u32, duration, fps: 30.0, // hardcoding for user convenience }; media_provider.videos.insert( file.clone(), fframes::media::VideoMedia { metadata: Some(metadata), path: std::path::PathBuf::from(url), }, ); } pub fn render_frame(&self, frame: i64) -> String { use std::ops::Deref; let time_base = self .time_base .lock() .unwrap() .expect("TimeBase must be set up before rendering."); self.video_ctx .borrow() .video .render_frame( Frame::__internal_make_for_renderer( frame as usize, frame as usize, time_base.fps, Some(self.break_lines_cache.clone()), self.video_decoders.clone(), ), &FFramesContext { duration_in_frames: *self.duration_in_frames.lock().unwrap(), current_video_size: VideoSize { width: TVideo::WIDTH, height: TVideo::HEIGHT, }, mode: FFramesMode::Editor, time_base, font_source: Some(self.fonts.lock().unwrap().deref()), scenes: self.scenes.lock().unwrap().as_ref(), markers: self.markers.lock().unwrap().as_slice(), media_source: Some(self.media_provider.lock().unwrap().deref()), abort_signal: None, }, ) .to_string() } pub fn render_preview_frame(&self, frame: i64) -> String { use std::ops::Deref; let time_base = self .time_base .lock() .unwrap() .expect("TimeBase must be set up before rendering."); #[cfg(target_arch = "wasm32")] fframes::media::IS_PREVIEW_RENDERING.with(|force_base64| { force_base64.store(true, std::sync::atomic::Ordering::Relaxed); }); let result = self .video_ctx .borrow() .video .render_frame( Frame::__internal_make_for_renderer( frame as usize, frame as usize, time_base.fps as usize, Some(self.break_lines_cache.clone()), self.video_decoders.clone(), ), &FFramesContext { time_base, current_video_size: VideoSize { width: TVideo::WIDTH, height: TVideo::HEIGHT, }, duration_in_frames: *self.duration_in_frames.lock().unwrap(), mode: FFramesMode::EditorTimelinePreview, scenes: self.scenes.lock().unwrap().as_ref(), markers: self.markers.lock().unwrap().as_slice(), media_source: Some(self.media_provider.lock().unwrap().deref()), font_source: Some(self.fonts.lock().unwrap().deref()), abort_signal: None, }, ) .to_string(); fframes::media::IS_PREVIEW_RENDERING.with(|force_base64| { force_base64.store(false, std::sync::atomic::Ordering::Relaxed); }); result } pub fn ingest_font(&self, slice: &[u8]) -> JsValue { let mut fonts = self.fonts.lock().unwrap(); let face_info = fonts.insert_font(slice.to_vec().into(), None); serde_wasm_bindgen::to_value(&face_info).unwrap() } pub fn populate_static_fonts_db_with_static_fonts(&self) { let mut font_wasm_db = self.fonts.lock().unwrap(); for (font_data, filename) in self.static_media.get_all_font_data() { font_wasm_db.insert_font(std::borrow::Cow::Borrowed(font_data), Some(filename)); } } pub fn get_static_font_data_by_index( &self, index: usize, ) -> Option { let fonts = self.fonts.lock().unwrap(); fonts.static_fonts.get(index).cloned() } pub fn get_static_audio_data_by_index(&self, index: usize) -> Option { let audios = self.static_media.get_all_audio_data(); audios .get(index) .map(|(static_audio, filename)| wasm_audio::AudioData::new(*static_audio, filename)) } }