From 266337415d24a59a810aac59cd368b6475423b9d Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 21 Dec 2025 02:04:36 +0000 Subject: [PATCH 01/37] Improve video frame handling and YUV conversion performance --- crates/recording/src/studio_recording.rs | 12 +- crates/rendering/src/decoder/avassetreader.rs | 418 ++++++++++-------- crates/rendering/src/decoder/ffmpeg.rs | 14 + crates/rendering/src/decoder/mod.rs | 98 +++- crates/rendering/src/layers/camera.rs | 46 +- crates/rendering/src/layers/display.rs | 2 +- crates/rendering/src/lib.rs | 2 + crates/rendering/src/yuv_converter.rs | 184 ++++++-- 8 files changed, 532 insertions(+), 244 deletions(-) diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 325ee7509e..f93ab0c87a 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -13,8 +13,8 @@ use crate::{ #[cfg(target_os = "macos")] use crate::output_pipeline::{ - AVFoundationCameraMuxer, AVFoundationCameraMuxerConfig, FragmentedAVFoundationCameraMuxer, - FragmentedAVFoundationCameraMuxerConfig, + AVFoundationCameraMuxer, AVFoundationCameraMuxerConfig, MacOSFragmentedM4SCameraMuxer, + MacOSFragmentedM4SCameraMuxerConfig, }; #[cfg(windows)] @@ -843,11 +843,13 @@ async fn create_segment_pipeline( let (display, crop) = target_to_display_and_crop(&base_inputs.capture_target).context("target_display_crop")?; + let max_fps = if fragmented { 60 } else { 120 }; + let screen_config = ScreenCaptureConfig::::init( display, crop, !custom_cursor_capture, - 120, + max_fps, start_time.system_time(), base_inputs.capture_system_audio, #[cfg(windows)] @@ -887,8 +889,8 @@ async fn create_segment_pipeline( OutputPipeline::builder(fragments_dir) .with_video::(camera_feed) .with_timestamps(start_time) - .build::( - FragmentedAVFoundationCameraMuxerConfig::default(), + .build::( + MacOSFragmentedM4SCameraMuxerConfig::default(), ) .instrument(error_span!("camera-out")) .await diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index 408490097b..f32846a9fb 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -20,48 +20,47 @@ use crate::{DecodedFrame, PixelFormat}; use super::frame_converter::{copy_bgra_to_rgba, copy_rgba_plane}; use super::{DecoderInitResult, DecoderType, FRAME_CACHE_SIZE, VideoDecoderMessage, pts_to_frame}; +#[derive(Clone)] +struct FrameData { + data: Arc>, + y_stride: u32, + uv_stride: u32, +} + #[derive(Clone)] struct ProcessedFrame { _number: u32, - data: Arc>, width: u32, height: u32, format: PixelFormat, - y_stride: u32, - uv_stride: u32, - image_buf: Option>, + frame_data: FrameData, } impl ProcessedFrame { fn to_decoded_frame(&self) -> DecodedFrame { + let FrameData { + data, + y_stride, + uv_stride, + } = &self.frame_data; + match self.format { - PixelFormat::Rgba => DecodedFrame::new((*self.data).clone(), self.width, self.height), - PixelFormat::Nv12 => { - if let Some(image_buf) = &self.image_buf { - DecodedFrame::new_nv12_with_iosurface( - (*self.data).clone(), - self.width, - self.height, - self.y_stride, - self.uv_stride, - image_buf.retained(), - ) - } else { - DecodedFrame::new_nv12( - (*self.data).clone(), - self.width, - self.height, - self.y_stride, - self.uv_stride, - ) - } + PixelFormat::Rgba => { + DecodedFrame::new_with_arc(Arc::clone(data), self.width, self.height) } - PixelFormat::Yuv420p => DecodedFrame::new_yuv420p( - (*self.data).clone(), + PixelFormat::Nv12 => DecodedFrame::new_nv12_with_arc( + Arc::clone(data), self.width, self.height, - self.y_stride, - self.uv_stride, + *y_stride, + *uv_stride, + ), + PixelFormat::Yuv420p => DecodedFrame::new_yuv420p_with_arc( + Arc::clone(data), + self.width, + self.height, + *y_stride, + *uv_stride, ), } } @@ -205,22 +204,18 @@ impl CachedFrame { fn new(processor: &ImageBufProcessor, mut image_buf: R, number: u32) -> Self { let width = image_buf.width() as u32; let height = image_buf.height() as u32; - let (data, format, y_stride, uv_stride) = processor.extract_raw(&mut image_buf); - let retain_iosurface = format == PixelFormat::Nv12 && image_buf.io_surf().is_some(); + let (data, format, y_stride, uv_stride) = processor.extract_raw(&mut image_buf); let frame = ProcessedFrame { _number: number, - data: Arc::new(data), width, height, format, - y_stride, - uv_stride, - image_buf: if retain_iosurface { - Some(image_buf) - } else { - None + frame_data: FrameData { + data: Arc::new(data), + y_stride, + uv_stride, }, }; Self(frame) @@ -291,191 +286,238 @@ impl AVAssetReaderDecoder { #[allow(unused)] let mut last_active_frame = None::; let last_sent_frame = Rc::new(RefCell::new(None::)); + let first_ever_frame = Rc::new(RefCell::new(None::)); + let mut backward_stale_count: u32 = 0; let mut frames = this.inner.frames(); let processor = ImageBufProcessor::new(); + struct PendingRequest { + frame: u32, + sender: oneshot::Sender, + } + while let Ok(r) = rx.recv() { + let mut pending_requests: Vec = Vec::with_capacity(8); + match r { VideoDecoderMessage::GetFrame(requested_time, sender) => { - if sender.is_closed() { - continue; + let frame = (requested_time * fps as f32).floor() as u32; + if !sender.is_closed() { + pending_requests.push(PendingRequest { frame, sender }); } + } + } - let requested_frame = (requested_time * fps as f32).floor() as u32; - - const BACKWARD_SEEK_TOLERANCE: u32 = 120; - let cache_frame_min_early = cache.keys().next().copied(); - let cache_frame_max_early = cache.keys().next_back().copied(); - - if let (Some(c_min), Some(_c_max)) = - (cache_frame_min_early, cache_frame_max_early) - { - let is_backward_within_tolerance = requested_frame < c_min - && requested_frame + BACKWARD_SEEK_TOLERANCE >= c_min; - if is_backward_within_tolerance - && let Some(closest_frame) = cache.get(&c_min) - { - let data = closest_frame.data().clone(); - if sender.send(data.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } - *last_sent_frame.borrow_mut() = Some(data); - continue; + while let Ok(msg) = rx.try_recv() { + match msg { + VideoDecoderMessage::GetFrame(requested_time, sender) => { + let frame = (requested_time * fps as f32).floor() as u32; + if !sender.is_closed() { + pending_requests.push(PendingRequest { frame, sender }); } } + } + } - let mut sender = if let Some(cached) = cache.get(&requested_frame) { - let data = cached.data().clone(); - if sender.send(data.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } - *last_sent_frame.borrow_mut() = Some(data); - continue; - } else { - let last_sent_frame = last_sent_frame.clone(); - Some(move |data: ProcessedFrame| { - *last_sent_frame.borrow_mut() = Some(data.clone()); - if sender.send(data.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } - }) - }; + pending_requests.sort_by_key(|r| r.frame); - let cache_min = requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); - let cache_max = requested_frame + FRAME_CACHE_SIZE as u32 / 2; + let mut i = 0; + while i < pending_requests.len() { + let request = &pending_requests[i]; + if let Some(cached) = cache.get(&request.frame) { + let data = cached.data().clone(); + let req = pending_requests.remove(i); + if req.sender.send(data.to_decoded_frame()).is_err() { + debug!("frame receiver dropped before send"); + } + *last_sent_frame.borrow_mut() = Some(data); + } else { + i += 1; + } + } - let cache_frame_min = cache.keys().next().copied(); - let cache_frame_max = cache.keys().next_back().copied(); + if pending_requests.is_empty() { + continue; + } - let needs_reset = - if let (Some(c_min), Some(c_max)) = (cache_frame_min, cache_frame_max) { - let is_backward_seek_beyond_tolerance = - requested_frame + BACKWARD_SEEK_TOLERANCE < c_min; - let is_forward_seek_beyond_cache = - requested_frame > c_max + FRAME_CACHE_SIZE as u32 / 4; - is_backward_seek_beyond_tolerance || is_forward_seek_beyond_cache - } else { - true - }; - - if needs_reset { - this.reset(requested_time); - frames = this.inner.frames(); - *last_sent_frame.borrow_mut() = None; - cache.retain(|&f, _| f >= cache_min && f <= cache_max); + let min_requested_frame = pending_requests.iter().map(|r| r.frame).min().unwrap(); + let max_requested_frame = pending_requests.iter().map(|r| r.frame).max().unwrap(); + let requested_frame = min_requested_frame; + let requested_time = requested_frame as f32 / fps as f32; + + const BACKWARD_SEEK_TOLERANCE: u32 = 120; + const MAX_STALE_FRAMES: u32 = 3; + let cache_frame_min_early = cache.keys().next().copied(); + let cache_frame_max_early = cache.keys().next_back().copied(); + + if let (Some(c_min), Some(_c_max)) = (cache_frame_min_early, cache_frame_max_early) { + let is_backward_within_tolerance = + requested_frame < c_min && requested_frame + BACKWARD_SEEK_TOLERANCE >= c_min; + if is_backward_within_tolerance + && backward_stale_count < MAX_STALE_FRAMES + && let Some(closest_frame) = cache.get(&c_min) + { + backward_stale_count += 1; + let data = closest_frame.data().clone(); + *last_sent_frame.borrow_mut() = Some(data.clone()); + for req in pending_requests.drain(..) { + if req.sender.send(data.to_decoded_frame()).is_err() { + debug!("frame receiver dropped before send"); + } } + continue; + } + } - last_active_frame = Some(requested_frame); + let cache_min = min_requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); + let cache_max = max_requested_frame + FRAME_CACHE_SIZE as u32 / 2; + + let cache_frame_min = cache.keys().next().copied(); + let cache_frame_max = cache.keys().next_back().copied(); + + let needs_reset = if let (Some(c_min), Some(c_max)) = (cache_frame_min, cache_frame_max) + { + let is_backward_seek_beyond_tolerance = + requested_frame + BACKWARD_SEEK_TOLERANCE < c_min; + let is_forward_seek_beyond_cache = + requested_frame > c_max + FRAME_CACHE_SIZE as u32 / 4; + let stale_limit_exceeded = + backward_stale_count >= MAX_STALE_FRAMES && requested_frame < c_min; + is_backward_seek_beyond_tolerance + || is_forward_seek_beyond_cache + || stale_limit_exceeded + } else { + true + }; + + if needs_reset { + this.reset(requested_time); + frames = this.inner.frames(); + *last_sent_frame.borrow_mut() = None; + backward_stale_count = 0; + cache.retain(|&f, _| f >= cache_min && f <= cache_max); + } - let mut exit = false; + last_active_frame = Some(requested_frame); - for frame in &mut frames { - let Ok(frame) = frame.map_err(|e| format!("read frame / {e}")) else { - continue; - }; + let mut exit = false; - let current_frame = pts_to_frame( - frame.pts().value, - Rational::new(1, frame.pts().scale), - fps, - ); + for frame in &mut frames { + let Ok(frame) = frame.map_err(|e| format!("read frame / {e}")) else { + continue; + }; - let Some(frame) = frame.image_buf() else { - continue; - }; + let current_frame = + pts_to_frame(frame.pts().value, Rational::new(1, frame.pts().scale), fps); - let cache_frame = - CachedFrame::new(&processor, frame.retained(), current_frame); + let Some(frame) = frame.image_buf() else { + continue; + }; - this.is_done = false; + let cache_frame = CachedFrame::new(&processor, frame.retained(), current_frame); - if let Some(most_recent_prev_frame) = - cache.iter().rev().find(|v| *v.0 < requested_frame) - && let Some(sender) = sender.take() - { - (sender)(most_recent_prev_frame.1.data().clone()); - } + if first_ever_frame.borrow().is_none() { + *first_ever_frame.borrow_mut() = Some(cache_frame.data().clone()); + } - let exceeds_cache_bounds = current_frame > cache_max; - let too_small_for_cache_bounds = current_frame < cache_min; - - if !too_small_for_cache_bounds { - if cache.len() >= FRAME_CACHE_SIZE { - if let Some(last_active_frame) = &last_active_frame { - let frame = if requested_frame > *last_active_frame { - *cache.keys().next().unwrap() - } else if requested_frame < *last_active_frame { - *cache.keys().next_back().unwrap() - } else { - let min = *cache.keys().min().unwrap(); - let max = *cache.keys().max().unwrap(); - - if current_frame > max { min } else { max } - }; - - cache.remove(&frame); - } else { - cache.clear() - } - } + this.is_done = false; + + let exceeds_cache_bounds = current_frame > cache_max; + let too_small_for_cache_bounds = current_frame < cache_min; + + if !too_small_for_cache_bounds { + if cache.len() >= FRAME_CACHE_SIZE { + if let Some(last_active) = &last_active_frame { + let frame_to_remove = if requested_frame > *last_active { + *cache.keys().next().unwrap() + } else if requested_frame < *last_active { + *cache.keys().next_back().unwrap() + } else { + let min = *cache.keys().min().unwrap(); + let max = *cache.keys().max().unwrap(); + if current_frame > max { min } else { max } + }; + cache.remove(&frame_to_remove); + } else { + cache.clear() + } + } - cache.insert(current_frame, cache_frame.clone()); + cache.insert(current_frame, cache_frame.clone()); + backward_stale_count = 0; - if current_frame == requested_frame - && let Some(sender) = sender.take() - { - (sender)(cache_frame.data().clone()); - break; + let mut remaining_requests = Vec::with_capacity(pending_requests.len()); + for req in pending_requests.drain(..) { + if req.frame == current_frame { + let data = cache_frame.data().clone(); + *last_sent_frame.borrow_mut() = Some(data.clone()); + if req.sender.send(data.to_decoded_frame()).is_err() { + debug!("frame receiver dropped before send"); } - } - - if current_frame > requested_frame && sender.is_some() { - // not inlining this is important so that last_sent_frame is dropped before the sender is invoked - let last_sent_frame = last_sent_frame.borrow().clone(); - - if let Some((sender, last_sent_frame)) = - last_sent_frame.and_then(|l| Some((sender.take()?, l))) - { - // info!( - // "sending previous frame {} for {requested_frame}", - // last_sent_frame.0 - // ); - - (sender)(last_sent_frame); - } else if let Some(sender) = sender.take() { - (sender)(cache_frame.data().clone()); + } else if req.frame < current_frame { + let prev_frame_data = last_sent_frame.borrow().clone(); + if let Some(data) = prev_frame_data { + if req.sender.send(data.to_decoded_frame()).is_err() { + debug!("frame receiver dropped before send"); + } + } else { + let data = cache_frame.data().clone(); + if req.sender.send(data.to_decoded_frame()).is_err() { + debug!("frame receiver dropped before send"); + } } + } else { + remaining_requests.push(req); } + } + pending_requests = remaining_requests; + } - exit = exit || exceeds_cache_bounds; + *last_sent_frame.borrow_mut() = Some(cache_frame.data().clone()); - if exit { - break; - } - } + exit = exit || exceeds_cache_bounds; - this.is_done = true; + if pending_requests.is_empty() || exit { + break; + } + } - let last_sent_frame = last_sent_frame.borrow().clone(); - if let Some(sender) = sender.take() { - if let Some(last_sent_frame) = last_sent_frame { - (sender)(last_sent_frame); - } else { - let black_frame_data = - vec![0u8; (video_width * video_height * 4) as usize]; - let black_frame = ProcessedFrame { - _number: requested_frame, - data: Arc::new(black_frame_data), - width: video_width, - height: video_height, - format: PixelFormat::Rgba, - y_stride: video_width * 4, - uv_stride: 0, - image_buf: None, - }; - (sender)(black_frame); - } + this.is_done = true; + + for req in pending_requests.drain(..) { + let prev_data = last_sent_frame.borrow().clone(); + if let Some(data) = prev_data { + if req.sender.send(data.to_decoded_frame()).is_err() { + debug!("frame receiver dropped before send"); + } + } else if let Some(first_frame) = first_ever_frame.borrow().clone() { + debug!( + "Returning first decoded frame as fallback for request {}", + req.frame + ); + if req.sender.send(first_frame.to_decoded_frame()).is_err() { + debug!("frame receiver dropped before send"); + } + } else { + debug!( + "No frames available for request {}, returning black frame", + req.frame + ); + let black_frame_data = vec![0u8; (video_width * video_height * 4) as usize]; + let black_frame = ProcessedFrame { + _number: req.frame, + width: video_width, + height: video_height, + format: PixelFormat::Rgba, + frame_data: FrameData { + data: Arc::new(black_frame_data), + y_stride: video_width * 4, + uv_stride: 0, + }, + }; + if req.sender.send(black_frame.to_decoded_frame()).is_err() { + debug!("frame receiver dropped before send"); } } } diff --git a/crates/rendering/src/decoder/ffmpeg.rs b/crates/rendering/src/decoder/ffmpeg.rs index 961e4cec28..b49ee06110 100644 --- a/crates/rendering/src/decoder/ffmpeg.rs +++ b/crates/rendering/src/decoder/ffmpeg.rs @@ -187,6 +187,7 @@ impl FfmpegDecoder { let mut last_active_frame = None::; let last_sent_frame = Rc::new(RefCell::new(None::)); + let first_ever_frame = Rc::new(RefCell::new(None::)); let mut frames = this.frames(); let mut converter = FrameConverter::new(); @@ -275,6 +276,14 @@ impl FfmpegDecoder { number: current_frame, }; + if first_ever_frame.borrow().is_none() { + let processed = cache_frame.process(&mut converter); + *first_ever_frame.borrow_mut() = Some(processed); + cache_frame = CachedFrame::Processed( + first_ever_frame.borrow().as_ref().unwrap().clone(), + ); + } + // Handles frame skips. // We use the cache instead of last_sent_frame as newer non-matching frames could have been decoded. if let Some(most_recent_prev_frame) = @@ -357,6 +366,11 @@ impl FfmpegDecoder { if let Some(sender) = sender.take() { if let Some(last_sent_frame) = last_sent_frame { (sender)(last_sent_frame); + } else if let Some(first_frame) = first_ever_frame.borrow().clone() { + debug!( + "Returning first decoded frame as fallback for request {requested_frame}" + ); + (sender)(first_frame); } else { debug!( "No frames available for request {requested_frame}, sending black frame" diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs index bad0ddbaf1..5a6e31db9b 100644 --- a/crates/rendering/src/decoder/mod.rs +++ b/crates/rendering/src/decoder/mod.rs @@ -207,6 +207,21 @@ impl DecodedFrame { } } + pub fn new_with_arc(data: Arc>, width: u32, height: u32) -> Self { + Self { + data, + width, + height, + format: PixelFormat::Rgba, + y_stride: width * 4, + uv_stride: 0, + #[cfg(target_os = "macos")] + iosurface_backing: None, + #[cfg(target_os = "windows")] + d3d11_texture_backing: None, + } + } + pub fn new_nv12(data: Vec, width: u32, height: u32, y_stride: u32, uv_stride: u32) -> Self { Self { data: Arc::new(data), @@ -222,6 +237,27 @@ impl DecodedFrame { } } + pub fn new_nv12_with_arc( + data: Arc>, + width: u32, + height: u32, + y_stride: u32, + uv_stride: u32, + ) -> Self { + Self { + data, + width, + height, + format: PixelFormat::Nv12, + y_stride, + uv_stride, + #[cfg(target_os = "macos")] + iosurface_backing: None, + #[cfg(target_os = "windows")] + d3d11_texture_backing: None, + } + } + pub fn new_yuv420p( data: Vec, width: u32, @@ -243,6 +279,27 @@ impl DecodedFrame { } } + pub fn new_yuv420p_with_arc( + data: Arc>, + width: u32, + height: u32, + y_stride: u32, + uv_stride: u32, + ) -> Self { + Self { + data, + width, + height, + format: PixelFormat::Yuv420p, + y_stride, + uv_stride, + #[cfg(target_os = "macos")] + iosurface_backing: None, + #[cfg(target_os = "windows")] + d3d11_texture_backing: None, + } + } + #[cfg(target_os = "macos")] pub fn new_nv12_with_iosurface( data: Vec, @@ -263,6 +320,45 @@ impl DecodedFrame { } } + #[cfg(target_os = "macos")] + pub fn new_nv12_with_iosurface_arc( + data: Arc>, + width: u32, + height: u32, + y_stride: u32, + uv_stride: u32, + image_buf: R, + ) -> Self { + Self { + data, + width, + height, + format: PixelFormat::Nv12, + y_stride, + uv_stride, + iosurface_backing: Some(Arc::new(SendableImageBuf::new(image_buf))), + } + } + + #[cfg(target_os = "macos")] + pub fn new_nv12_zero_copy( + width: u32, + height: u32, + y_stride: u32, + uv_stride: u32, + image_buf: R, + ) -> Self { + Self { + data: Arc::new(Vec::new()), + width, + height, + format: PixelFormat::Nv12, + y_stride, + uv_stride, + iosurface_backing: Some(Arc::new(SendableImageBuf::new(image_buf))), + } + } + #[cfg(target_os = "macos")] #[allow(clippy::redundant_closure)] pub fn iosurface_backing(&self) -> Option<&cv::ImageBuf> { @@ -451,7 +547,7 @@ pub fn pts_to_frame(pts: i64, time_base: Rational, fps: u32) -> u32 { .round() as u32 } -pub const FRAME_CACHE_SIZE: usize = 750; +pub const FRAME_CACHE_SIZE: usize = 60; #[derive(Clone)] pub struct AsyncVideoDecoderHandle { diff --git a/crates/rendering/src/layers/camera.rs b/crates/rendering/src/layers/camera.rs index 17754734e0..7a26d266be 100644 --- a/crates/rendering/src/layers/camera.rs +++ b/crates/rendering/src/layers/camera.rs @@ -14,7 +14,7 @@ pub struct CameraLayer { bind_groups: [Option; 2], pipeline: CompositeVideoFramePipeline, hidden: bool, - last_frame_ptr: usize, + last_recording_time: Option, yuv_converter: YuvToRgbaConverter, } @@ -50,7 +50,7 @@ impl CameraLayer { bind_groups: [bind_group_0, bind_group_1], pipeline, hidden: false, - last_frame_ptr: 0, + last_recording_time: None, yuv_converter, } } @@ -59,19 +59,22 @@ impl CameraLayer { &mut self, device: &wgpu::Device, queue: &wgpu::Queue, - data: Option<(CompositeVideoFrameUniforms, XY, &DecodedFrame)>, + data: Option<(CompositeVideoFrameUniforms, XY, &DecodedFrame, f32)>, ) { self.hidden = data.is_none(); - let Some((uniforms, frame_size, camera_frame)) = data else { + let Some((uniforms, frame_size, camera_frame, recording_time)) = data else { return; }; let frame_data = camera_frame.data(); - let frame_ptr = frame_data.as_ptr() as usize; let format = camera_frame.format(); - if frame_ptr != self.last_frame_ptr { + let is_same_frame = self + .last_recording_time + .is_some_and(|last| (last - recording_time).abs() < 0.001); + + if !is_same_frame { let next_texture = 1 - self.current_texture; if self.frame_textures[next_texture].width() != frame_size.x @@ -118,6 +121,35 @@ impl CameraLayer { ); } PixelFormat::Nv12 => { + #[cfg(target_os = "macos")] + let iosurface_result = camera_frame.iosurface_backing().map(|image_buf| { + self.yuv_converter + .convert_nv12_from_iosurface(device, queue, image_buf) + }); + + #[cfg(target_os = "macos")] + if let Some(Ok(_)) = iosurface_result { + self.copy_from_yuv_output(device, queue, next_texture, frame_size); + } else if let (Some(y_data), Some(uv_data)) = + (camera_frame.y_plane(), camera_frame.uv_plane()) + && self + .yuv_converter + .convert_nv12( + device, + queue, + y_data, + uv_data, + frame_size.x, + frame_size.y, + camera_frame.y_stride(), + camera_frame.uv_stride(), + ) + .is_ok() + { + self.copy_from_yuv_output(device, queue, next_texture, frame_size); + } + + #[cfg(not(target_os = "macos"))] if let (Some(y_data), Some(uv_data)) = (camera_frame.y_plane(), camera_frame.uv_plane()) && self @@ -162,7 +194,7 @@ impl CameraLayer { } } - self.last_frame_ptr = frame_ptr; + self.last_recording_time = Some(recording_time); self.current_texture = next_texture; } diff --git a/crates/rendering/src/layers/display.rs b/crates/rendering/src/layers/display.rs index 78e3dd9fce..90f9cc34da 100644 --- a/crates/rendering/src/layers/display.rs +++ b/crates/rendering/src/layers/display.rs @@ -91,7 +91,7 @@ impl DisplayLayer { let skipped = self .last_recording_time - .is_some_and(|last| (last - current_recording_time).abs() < f32::EPSILON); + .is_some_and(|last| (last - current_recording_time).abs() < 0.001); if !skipped { let next_texture = 1 - self.current_texture; diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 4da4bc794f..0f5a8594bc 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -1699,6 +1699,7 @@ impl RendererLayers { uniforms.camera?, constants.options.camera_size?, segment_frames.camera_frame.as_ref()?, + segment_frames.recording_time, )) })(), ); @@ -1711,6 +1712,7 @@ impl RendererLayers { uniforms.camera_only?, constants.options.camera_size?, segment_frames.camera_frame.as_ref()?, + segment_frames.recording_time, )) })(), ); diff --git a/crates/rendering/src/yuv_converter.rs b/crates/rendering/src/yuv_converter.rs index 76126c7cc9..36d3749022 100644 --- a/crates/rendering/src/yuv_converter.rs +++ b/crates/rendering/src/yuv_converter.rs @@ -133,6 +133,120 @@ fn validate_dimensions( Ok((new_width, new_height, true)) } +struct BindGroupCache { + nv12_bind_groups: [Option; 2], + yuv420p_bind_groups: [Option; 2], + cached_width: u32, + cached_height: u32, +} + +impl BindGroupCache { + fn new() -> Self { + Self { + nv12_bind_groups: [None, None], + yuv420p_bind_groups: [None, None], + cached_width: 0, + cached_height: 0, + } + } + + fn invalidate(&mut self) { + self.nv12_bind_groups = [None, None]; + self.yuv420p_bind_groups = [None, None]; + self.cached_width = 0; + self.cached_height = 0; + } + + fn get_or_create_nv12( + &mut self, + device: &wgpu::Device, + layout: &wgpu::BindGroupLayout, + y_view: &wgpu::TextureView, + uv_view: &wgpu::TextureView, + output_view: &wgpu::TextureView, + output_index: usize, + width: u32, + height: u32, + ) -> &wgpu::BindGroup { + if self.cached_width != width || self.cached_height != height { + self.invalidate(); + self.cached_width = width; + self.cached_height = height; + } + + if self.nv12_bind_groups[output_index].is_none() { + self.nv12_bind_groups[output_index] = + Some(device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("NV12 Converter Bind Group (Cached)"), + layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(y_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(uv_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::TextureView(output_view), + }, + ], + })); + } + + self.nv12_bind_groups[output_index].as_ref().unwrap() + } + + fn get_or_create_yuv420p( + &mut self, + device: &wgpu::Device, + layout: &wgpu::BindGroupLayout, + y_view: &wgpu::TextureView, + u_view: &wgpu::TextureView, + v_view: &wgpu::TextureView, + output_view: &wgpu::TextureView, + output_index: usize, + width: u32, + height: u32, + ) -> &wgpu::BindGroup { + if self.cached_width != width || self.cached_height != height { + self.invalidate(); + self.cached_width = width; + self.cached_height = height; + } + + if self.yuv420p_bind_groups[output_index].is_none() { + self.yuv420p_bind_groups[output_index] = + Some(device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("YUV420P Converter Bind Group (Cached)"), + layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(y_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(u_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::TextureView(v_view), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::TextureView(output_view), + }, + ], + })); + } + + self.yuv420p_bind_groups[output_index].as_ref().unwrap() + } +} + pub struct YuvToRgbaConverter { nv12_pipeline: wgpu::ComputePipeline, yuv420p_pipeline: wgpu::ComputePipeline, @@ -152,6 +266,7 @@ pub struct YuvToRgbaConverter { allocated_width: u32, allocated_height: u32, gpu_max_texture_size: u32, + bind_group_cache: BindGroupCache, #[cfg(target_os = "macos")] iosurface_cache: Option, #[cfg(target_os = "windows")] @@ -331,6 +446,7 @@ impl YuvToRgbaConverter { allocated_width: initial_width, allocated_height: initial_height, gpu_max_texture_size, + bind_group_cache: BindGroupCache::new(), #[cfg(target_os = "macos")] iosurface_cache: IOSurfaceTextureCache::new(), #[cfg(target_os = "windows")] @@ -507,6 +623,7 @@ impl YuvToRgbaConverter { self.output_views = output_views; self.allocated_width = new_width; self.allocated_height = new_height; + self.bind_group_cache.invalidate(); } pub fn gpu_max_texture_size(&self) -> u32 { @@ -574,24 +691,17 @@ impl YuvToRgbaConverter { }, ); - let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("NV12 Converter Bind Group"), - layout: &self.nv12_bind_group_layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&self.y_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::TextureView(&self.uv_view), - }, - wgpu::BindGroupEntry { - binding: 2, - resource: wgpu::BindingResource::TextureView(self.current_output_view()), - }, - ], - }); + let output_index = self.current_output; + let bind_group = self.bind_group_cache.get_or_create_nv12( + device, + &self.nv12_bind_group_layout, + &self.y_view, + &self.uv_view, + &self.output_views[output_index], + output_index, + self.allocated_width, + self.allocated_height, + ); let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("NV12 Conversion Encoder"), @@ -603,7 +713,7 @@ impl YuvToRgbaConverter { ..Default::default() }); compute_pass.set_pipeline(&self.nv12_pipeline); - compute_pass.set_bind_group(0, &bind_group, &[]); + compute_pass.set_bind_group(0, bind_group, &[]); compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1); } @@ -740,28 +850,18 @@ impl YuvToRgbaConverter { "V", )?; - let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("YUV420P Converter Bind Group"), - layout: &self.yuv420p_bind_group_layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&self.y_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::TextureView(&self.u_view), - }, - wgpu::BindGroupEntry { - binding: 2, - resource: wgpu::BindingResource::TextureView(&self.v_view), - }, - wgpu::BindGroupEntry { - binding: 3, - resource: wgpu::BindingResource::TextureView(self.current_output_view()), - }, - ], - }); + let output_index = self.current_output; + let bind_group = self.bind_group_cache.get_or_create_yuv420p( + device, + &self.yuv420p_bind_group_layout, + &self.y_view, + &self.u_view, + &self.v_view, + &self.output_views[output_index], + output_index, + self.allocated_width, + self.allocated_height, + ); let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("YUV420P Conversion Encoder"), @@ -773,7 +873,7 @@ impl YuvToRgbaConverter { ..Default::default() }); compute_pass.set_pipeline(&self.yuv420p_pipeline); - compute_pass.set_bind_group(0, &bind_group, &[]); + compute_pass.set_bind_group(0, bind_group, &[]); compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1); } From 636f5b34551dc67d259fa05e34f1fbfcc29ee5e4 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 21 Dec 2025 02:04:51 +0000 Subject: [PATCH 02/37] Improve audio-video sync and adjust playback buffer sizes --- crates/editor/src/audio.rs | 4 ++++ crates/editor/src/playback.rs | 36 +++++++++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/crates/editor/src/audio.rs b/crates/editor/src/audio.rs index 0e1032bb75..215018ff09 100644 --- a/crates/editor/src/audio.rs +++ b/crates/editor/src/audio.rs @@ -277,6 +277,10 @@ impl AudioPlaybackBuffer { self.frame_buffer.set_playhead(playhead, project); } + pub fn current_playhead(&self) -> f64 { + self.frame_buffer.elapsed_samples_to_playhead() + } + pub fn buffer_reaching_limit(&self) -> bool { self.resampled_buffer.vacant_len() <= 2 * (Self::PROCESSING_SAMPLES_COUNT as usize) * self.resampler.output.channels diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index f02469dcca..9d333ec77f 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -30,11 +30,11 @@ use crate::{ segments::get_audio_segments, }; -const PREFETCH_BUFFER_SIZE: usize = 180; -const PARALLEL_DECODE_TASKS: usize = 20; -const MAX_PREFETCH_AHEAD: u32 = 240; -const PREFETCH_BEHIND: u32 = 60; -const FRAME_CACHE_SIZE: usize = 150; +const PREFETCH_BUFFER_SIZE: usize = 30; +const PARALLEL_DECODE_TASKS: usize = 8; +const MAX_PREFETCH_AHEAD: u32 = 90; +const PREFETCH_BEHIND: u32 = 15; +const FRAME_CACHE_SIZE: usize = 30; #[derive(Debug)] pub enum PlaybackStartError { @@ -333,12 +333,16 @@ impl Playback { f64::MAX }; + let (audio_playhead_tx, audio_playhead_rx) = + watch::channel(self.start_frame_number as f64 / fps as f64); + AudioPlayback { segments: get_audio_segments(&self.segment_medias), stop_rx: stop_rx.clone(), start_frame_number: self.start_frame_number, project: self.project.clone(), fps, + playhead_rx: audio_playhead_rx, } .spawn(); @@ -627,6 +631,7 @@ impl Playback { frame_number = frame_number.saturating_add(1); let _ = playback_position_tx.send(frame_number); + let _ = audio_playhead_tx.send(frame_number as f64 / fps_f64); let expected_frame = self.start_frame_number + (start.elapsed().as_secs_f64() * fps_f64).floor() as u32; @@ -646,6 +651,7 @@ impl Playback { prefetch_buffer.retain(|p| p.frame_number >= frame_number); let _ = frame_request_tx.send(frame_number); let _ = playback_position_tx.send(frame_number); + let _ = audio_playhead_tx.send(frame_number as f64 / fps_f64); } } } @@ -676,6 +682,7 @@ struct AudioPlayback { start_frame_number: u32, project: watch::Receiver, fps: u32, + playhead_rx: watch::Receiver, } impl AudioPlayback { @@ -761,7 +768,7 @@ impl AudioPlayback { project, segments, fps, - .. + playhead_rx, } = self; let mut base_output_info = AudioInfo::from_stream_config(&supported_config); @@ -915,6 +922,9 @@ impl AudioPlayback { let project_for_stream = project.clone(); let headroom_for_stream = headroom_samples; + let mut playhead_rx_for_stream = playhead_rx.clone(); + let mut last_video_playhead = playhead; + const SYNC_THRESHOLD_SECS: f64 = 0.15; let stream_result = device.build_output_stream( &config, @@ -923,6 +933,20 @@ impl AudioPlayback { let project = project_for_stream.borrow(); + if playhead_rx_for_stream.has_changed().unwrap_or(false) { + let video_playhead = *playhead_rx_for_stream.borrow_and_update(); + let audio_playhead = audio_renderer.current_playhead(); + let drift = (video_playhead - audio_playhead).abs(); + + if drift > SYNC_THRESHOLD_SECS + || (video_playhead - last_video_playhead).abs() > SYNC_THRESHOLD_SECS + { + audio_renderer + .set_playhead(video_playhead + initial_compensation_secs, &project); + } + last_video_playhead = video_playhead; + } + let playback_samples = buffer.len(); let min_headroom = headroom_for_stream.max(playback_samples * 2); audio_renderer.fill(buffer, &project, min_headroom); From b7011d8206c9d613518bc0fda726115479a77286 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 21 Dec 2025 02:05:13 +0000 Subject: [PATCH 03/37] Remove fragmented_mp4, add segmented_stream encoder --- crates/enc-ffmpeg/src/lib.rs | 3 - crates/enc-ffmpeg/src/mux/fragmented_mp4.rs | 170 ----- crates/enc-ffmpeg/src/mux/mod.rs | 2 +- crates/enc-ffmpeg/src/mux/segmented_audio.rs | 6 +- crates/enc-ffmpeg/src/mux/segmented_stream.rs | 603 ++++++++++++++++++ crates/enc-ffmpeg/src/remux.rs | 143 +++++ crates/enc-ffmpeg/src/video/h264.rs | 89 ++- 7 files changed, 829 insertions(+), 187 deletions(-) delete mode 100644 crates/enc-ffmpeg/src/mux/fragmented_mp4.rs create mode 100644 crates/enc-ffmpeg/src/mux/segmented_stream.rs diff --git a/crates/enc-ffmpeg/src/lib.rs b/crates/enc-ffmpeg/src/lib.rs index d57097346c..2dd64f560d 100644 --- a/crates/enc-ffmpeg/src/lib.rs +++ b/crates/enc-ffmpeg/src/lib.rs @@ -13,6 +13,3 @@ pub mod remux; pub mod segmented_audio { pub use crate::mux::segmented_audio::*; } -pub mod fragmented_mp4 { - pub use crate::mux::fragmented_mp4::*; -} diff --git a/crates/enc-ffmpeg/src/mux/fragmented_mp4.rs b/crates/enc-ffmpeg/src/mux/fragmented_mp4.rs deleted file mode 100644 index 20f1b034ef..0000000000 --- a/crates/enc-ffmpeg/src/mux/fragmented_mp4.rs +++ /dev/null @@ -1,170 +0,0 @@ -use cap_media_info::RawVideoFormat; -use ffmpeg::{format, frame}; -use std::{path::PathBuf, time::Duration}; -use tracing::*; - -use crate::{ - audio::AudioEncoder, - h264, - video::h264::{H264Encoder, H264EncoderError}, -}; - -pub struct FragmentedMP4File { - output: format::context::Output, - video: H264Encoder, - audio: Option>, - is_finished: bool, - has_frames: bool, -} - -#[derive(thiserror::Error, Debug)] -pub enum InitError { - #[error("{0:?}")] - Ffmpeg(ffmpeg::Error), - #[error("Video/{0}")] - VideoInit(H264EncoderError), - #[error("Audio/{0}")] - AudioInit(Box), - #[error("Failed to create output directory: {0}")] - CreateDirectory(std::io::Error), -} - -#[derive(thiserror::Error, Debug)] -pub enum FinishError { - #[error("Already finished")] - AlreadyFinished, - #[error("{0}")] - WriteTrailerFailed(ffmpeg::Error), -} - -pub struct FinishResult { - pub video_finish: Result<(), ffmpeg::Error>, - pub audio_finish: Result<(), ffmpeg::Error>, -} - -impl FragmentedMP4File { - pub fn init( - mut output_path: PathBuf, - video: impl FnOnce(&mut format::context::Output) -> Result, - audio: impl FnOnce( - &mut format::context::Output, - ) - -> Option, Box>>, - ) -> Result { - output_path.set_extension("mp4"); - - if let Some(parent) = output_path.parent() { - std::fs::create_dir_all(parent).map_err(InitError::CreateDirectory)?; - } - - let mut output = format::output_as(&output_path, "mp4").map_err(InitError::Ffmpeg)?; - - unsafe { - let opts = output.as_mut_ptr(); - let key = std::ffi::CString::new("movflags").unwrap(); - let value = - std::ffi::CString::new("frag_keyframe+empty_moov+default_base_moof").unwrap(); - ffmpeg::ffi::av_opt_set((*opts).priv_data, key.as_ptr(), value.as_ptr(), 0); - } - - trace!("Preparing encoders for fragmented mp4 file"); - - let video = video(&mut output).map_err(InitError::VideoInit)?; - let audio = audio(&mut output) - .transpose() - .map_err(InitError::AudioInit)?; - - info!("Prepared encoders for fragmented mp4 file"); - - output.write_header().map_err(InitError::Ffmpeg)?; - - Ok(Self { - output, - video, - audio, - is_finished: false, - has_frames: false, - }) - } - - pub fn video_format() -> RawVideoFormat { - RawVideoFormat::Yuv420p - } - - pub fn queue_video_frame( - &mut self, - frame: frame::Video, - timestamp: Duration, - ) -> Result<(), h264::QueueFrameError> { - if self.is_finished { - return Ok(()); - } - - self.has_frames = true; - self.video.queue_frame(frame, timestamp, &mut self.output) - } - - pub fn queue_audio_frame(&mut self, frame: frame::Audio) { - if self.is_finished { - return; - } - - let Some(audio) = &mut self.audio else { - return; - }; - - self.has_frames = true; - audio.send_frame(frame, &mut self.output); - } - - pub fn finish(&mut self) -> Result { - if self.is_finished { - return Err(FinishError::AlreadyFinished); - } - - self.is_finished = true; - - tracing::info!("FragmentedMP4File: Finishing encoding"); - - let video_finish = self.video.flush(&mut self.output).inspect_err(|e| { - error!("Failed to finish video encoder: {e:#}"); - }); - - let audio_finish = self - .audio - .as_mut() - .map(|enc| { - tracing::info!("FragmentedMP4File: Flushing audio encoder"); - enc.flush(&mut self.output).inspect_err(|e| { - error!("Failed to finish audio encoder: {e:#}"); - }) - }) - .unwrap_or(Ok(())); - - tracing::info!("FragmentedMP4File: Writing trailer"); - self.output - .write_trailer() - .map_err(FinishError::WriteTrailerFailed)?; - - Ok(FinishResult { - video_finish, - audio_finish, - }) - } - - pub fn video(&self) -> &H264Encoder { - &self.video - } - - pub fn video_mut(&mut self) -> &mut H264Encoder { - &mut self.video - } -} - -impl Drop for FragmentedMP4File { - fn drop(&mut self) { - if let Err(e) = self.finish() { - error!("Failed to finish FragmentedMP4File in Drop: {e}"); - } - } -} diff --git a/crates/enc-ffmpeg/src/mux/mod.rs b/crates/enc-ffmpeg/src/mux/mod.rs index 8ed8362d1b..28b1fcf0db 100644 --- a/crates/enc-ffmpeg/src/mux/mod.rs +++ b/crates/enc-ffmpeg/src/mux/mod.rs @@ -1,5 +1,5 @@ pub mod fragmented_audio; -pub mod fragmented_mp4; pub mod mp4; pub mod ogg; pub mod segmented_audio; +pub mod segmented_stream; diff --git a/crates/enc-ffmpeg/src/mux/segmented_audio.rs b/crates/enc-ffmpeg/src/mux/segmented_audio.rs index 1f54d3e743..e24a8ff058 100644 --- a/crates/enc-ffmpeg/src/mux/segmented_audio.rs +++ b/crates/enc-ffmpeg/src/mux/segmented_audio.rs @@ -367,6 +367,10 @@ impl SegmentedAudioEncoder { pub fn finish_with_timestamp(&mut self, timestamp: Duration) -> Result<(), FinishError> { let segment_path = self.current_segment_path(); let segment_start = self.segment_start_time; + let effective_end_timestamp = self + .last_frame_timestamp + .map(|last| last.max(timestamp)) + .unwrap_or(timestamp); if let Some(mut encoder) = self.current_encoder.take() { if encoder.has_frames { @@ -383,7 +387,7 @@ impl SegmentedAudioEncoder { sync_file(&segment_path); if let Some(start) = segment_start { - let final_duration = timestamp.saturating_sub(start); + let final_duration = effective_end_timestamp.saturating_sub(start); let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); self.completed_segments.push(SegmentInfo { diff --git a/crates/enc-ffmpeg/src/mux/segmented_stream.rs b/crates/enc-ffmpeg/src/mux/segmented_stream.rs new file mode 100644 index 0000000000..8dab6f9aea --- /dev/null +++ b/crates/enc-ffmpeg/src/mux/segmented_stream.rs @@ -0,0 +1,603 @@ +use cap_media_info::VideoInfo; +use ffmpeg::{format, frame}; +use serde::Serialize; +use std::{ + ffi::CString, + io::Write, + path::{Path, PathBuf}, + time::Duration, +}; + +use crate::video::h264::{H264Encoder, H264EncoderBuilder, H264EncoderError, H264Preset}; + +const INIT_SEGMENT_NAME: &str = "init.mp4"; + +fn atomic_write_json(path: &Path, data: &T) -> std::io::Result<()> { + let temp_path = path.with_extension("json.tmp"); + let json = serde_json::to_string_pretty(data) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + + let mut file = std::fs::File::create(&temp_path)?; + file.write_all(json.as_bytes())?; + file.sync_all()?; + + std::fs::rename(&temp_path, path)?; + + if let Some(parent) = path.parent() + && let Ok(dir) = std::fs::File::open(parent) + && let Err(e) = dir.sync_all() + { + tracing::warn!( + "Directory fsync failed after rename for {}: {e}", + parent.display() + ); + } + + Ok(()) +} + +fn sync_file(path: &Path) { + if let Ok(file) = std::fs::File::open(path) { + if let Err(e) = file.sync_all() { + tracing::warn!("File fsync failed for {}: {e}", path.display()); + } + } +} + +pub struct SegmentedVideoEncoder { + base_path: PathBuf, + + encoder: H264Encoder, + output: format::context::Output, + + current_index: u32, + segment_duration: Duration, + segment_start_time: Option, + last_frame_timestamp: Option, + frames_in_segment: u32, + + completed_segments: Vec, +} + +#[derive(Debug, Clone)] +pub struct VideoSegmentInfo { + pub path: PathBuf, + pub index: u32, + pub duration: Duration, + pub file_size: Option, +} + +#[derive(Serialize)] +struct SegmentEntry { + path: String, + index: u32, + duration: f64, + is_complete: bool, + #[serde(skip_serializing_if = "Option::is_none")] + file_size: Option, +} + +const MANIFEST_VERSION: u32 = 4; + +#[derive(Serialize)] +struct Manifest { + version: u32, + #[serde(rename = "type")] + manifest_type: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + init_segment: Option, + segments: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + total_duration: Option, + is_complete: bool, +} + +#[derive(thiserror::Error, Debug)] +pub enum InitError { + #[error("FFmpeg: {0}")] + FFmpeg(#[from] ffmpeg::Error), + #[error("Encoder: {0}")] + Encoder(#[from] H264EncoderError), + #[error("IO: {0}")] + Io(#[from] std::io::Error), +} + +#[derive(thiserror::Error, Debug)] +pub enum QueueFrameError { + #[error("FFmpeg: {0}")] + FFmpeg(#[from] ffmpeg::Error), + #[error("Init: {0}")] + Init(#[from] InitError), + #[error("Encode: {0}")] + Encode(#[from] crate::video::h264::QueueFrameError), +} + +#[derive(thiserror::Error, Debug)] +pub enum FinishError { + #[error("FFmpeg: {0}")] + FFmpeg(#[from] ffmpeg::Error), +} + +pub struct SegmentedVideoEncoderConfig { + pub segment_duration: Duration, + pub preset: H264Preset, + pub bpp: f32, + pub output_size: Option<(u32, u32)>, +} + +impl Default for SegmentedVideoEncoderConfig { + fn default() -> Self { + Self { + segment_duration: Duration::from_secs(3), + preset: H264Preset::Ultrafast, + bpp: H264EncoderBuilder::QUALITY_BPP, + output_size: None, + } + } +} + +impl SegmentedVideoEncoder { + pub fn init( + base_path: PathBuf, + video_config: VideoInfo, + config: SegmentedVideoEncoderConfig, + ) -> Result { + std::fs::create_dir_all(&base_path)?; + + let manifest_path = base_path.join("dash_manifest.mpd"); + + let mut output = format::output_as(&manifest_path, "dash")?; + + unsafe { + let opts = output.as_mut_ptr(); + + let set_opt = |key: &str, value: &str| { + let k = CString::new(key).unwrap(); + let v = CString::new(value).unwrap(); + ffmpeg::ffi::av_opt_set((*opts).priv_data, k.as_ptr(), v.as_ptr(), 0); + }; + + set_opt("init_seg_name", INIT_SEGMENT_NAME); + set_opt("media_seg_name", "segment_$Number%03d$.m4s"); + set_opt( + "seg_duration", + &config.segment_duration.as_secs().to_string(), + ); + set_opt("use_timeline", "0"); + set_opt("use_template", "1"); + set_opt("single_file", "0"); + } + + let mut builder = H264EncoderBuilder::new(video_config) + .with_preset(config.preset) + .with_bpp(config.bpp); + + if let Some((width, height)) = config.output_size { + builder = builder.with_output_size(width, height)?; + } + + let encoder = builder.build(&mut output)?; + + output.write_header()?; + + tracing::info!( + path = %base_path.display(), + segment_duration_secs = config.segment_duration.as_secs(), + "Initialized segmented video encoder with FFmpeg DASH muxer (init.mp4 + m4s segments)" + ); + + let instance = Self { + base_path, + encoder, + output, + current_index: 1, + segment_duration: config.segment_duration, + segment_start_time: None, + last_frame_timestamp: None, + frames_in_segment: 0, + completed_segments: Vec::new(), + }; + + instance.write_in_progress_manifest(); + + Ok(instance) + } + + pub fn queue_frame( + &mut self, + frame: frame::Video, + timestamp: Duration, + ) -> Result<(), QueueFrameError> { + if self.segment_start_time.is_none() { + self.segment_start_time = Some(timestamp); + } + + self.last_frame_timestamp = Some(timestamp); + + let prev_segment_index = self.detect_current_segment_index(); + + self.encoder + .queue_frame(frame, timestamp, &mut self.output)?; + self.frames_in_segment += 1; + + let new_segment_index = self.detect_current_segment_index(); + + if new_segment_index > prev_segment_index { + self.on_segment_completed(prev_segment_index, timestamp)?; + } + + Ok(()) + } + + fn detect_current_segment_index(&self) -> u32 { + let next_segment_path = self + .base_path + .join(format!("segment_{:03}.m4s", self.current_index + 1)); + if next_segment_path.exists() { + self.current_index + 1 + } else { + self.current_index + } + } + + fn on_segment_completed( + &mut self, + completed_index: u32, + timestamp: Duration, + ) -> Result<(), QueueFrameError> { + let segment_path = self + .base_path + .join(format!("segment_{completed_index:03}.m4s")); + + if segment_path.exists() { + sync_file(&segment_path); + + let segment_start = self.segment_start_time.unwrap_or(Duration::ZERO); + let segment_duration = timestamp.saturating_sub(segment_start); + + let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); + + tracing::debug!( + segment_index = completed_index, + duration_secs = segment_duration.as_secs_f64(), + file_size = ?file_size, + frames = self.frames_in_segment, + "Segment completed" + ); + + self.completed_segments.push(VideoSegmentInfo { + path: segment_path, + index: completed_index, + duration: segment_duration, + file_size, + }); + + self.current_index = completed_index + 1; + self.segment_start_time = Some(timestamp); + self.frames_in_segment = 0; + + self.write_manifest(); + self.write_in_progress_manifest(); + } + + Ok(()) + } + + fn current_segment_path(&self) -> PathBuf { + self.base_path + .join(format!("segment_{:03}.m4s", self.current_index)) + } + + fn write_manifest(&self) { + let manifest = Manifest { + version: MANIFEST_VERSION, + manifest_type: "m4s_segments", + init_segment: Some(INIT_SEGMENT_NAME.to_string()), + segments: self + .completed_segments + .iter() + .map(|s| SegmentEntry { + path: s + .path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned(), + index: s.index, + duration: s.duration.as_secs_f64(), + is_complete: true, + file_size: s.file_size, + }) + .collect(), + total_duration: None, + is_complete: false, + }; + + let manifest_path = self.base_path.join("manifest.json"); + if let Err(e) = atomic_write_json(&manifest_path, &manifest) { + tracing::warn!( + "Failed to write manifest to {}: {e}", + manifest_path.display() + ); + } + } + + fn write_in_progress_manifest(&self) { + let mut segments: Vec = self + .completed_segments + .iter() + .map(|s| SegmentEntry { + path: s + .path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned(), + index: s.index, + duration: s.duration.as_secs_f64(), + is_complete: true, + file_size: s.file_size, + }) + .collect(); + + segments.push(SegmentEntry { + path: self + .current_segment_path() + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned(), + index: self.current_index, + duration: 0.0, + is_complete: false, + file_size: None, + }); + + let manifest = Manifest { + version: MANIFEST_VERSION, + manifest_type: "m4s_segments", + init_segment: Some(INIT_SEGMENT_NAME.to_string()), + segments, + total_duration: None, + is_complete: false, + }; + + let manifest_path = self.base_path.join("manifest.json"); + if let Err(e) = atomic_write_json(&manifest_path, &manifest) { + tracing::warn!( + "Failed to write in-progress manifest to {}: {e}", + manifest_path.display() + ); + } + } + + pub fn finish(&mut self) -> Result<(), FinishError> { + let segment_start = self.segment_start_time; + let last_timestamp = self.last_frame_timestamp; + let frames_before_flush = self.frames_in_segment; + + if let Err(e) = self.encoder.flush(&mut self.output) { + tracing::warn!("Video encoder flush warning: {e}"); + } + + if let Err(e) = self.output.write_trailer() { + tracing::warn!("Video write_trailer warning: {e}"); + } + + self.finalize_pending_tmp_files(); + + let end_timestamp = + last_timestamp.unwrap_or_else(|| segment_start.unwrap_or(Duration::ZERO)); + self.collect_orphaned_segments(segment_start, end_timestamp, frames_before_flush); + + self.finalize_manifest(); + + Ok(()) + } + + pub fn finish_with_timestamp(&mut self, timestamp: Duration) -> Result<(), FinishError> { + let segment_start = self.segment_start_time; + let frames_before_flush = self.frames_in_segment; + + if let Err(e) = self.encoder.flush(&mut self.output) { + tracing::warn!("Video encoder flush warning: {e}"); + } + + if let Err(e) = self.output.write_trailer() { + tracing::warn!("Video write_trailer warning: {e}"); + } + + self.finalize_pending_tmp_files(); + + let effective_end_timestamp = self + .last_frame_timestamp + .map(|last| last.max(timestamp)) + .unwrap_or(timestamp); + + self.collect_orphaned_segments(segment_start, effective_end_timestamp, frames_before_flush); + + self.finalize_manifest(); + + Ok(()) + } + + fn finalize_pending_tmp_files(&self) { + let Ok(entries) = std::fs::read_dir(&self.base_path) else { + return; + }; + + for entry in entries.flatten() { + let path = entry.path(); + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + if name.starts_with("segment_") && name.ends_with(".m4s.tmp") { + let final_name = name.trim_end_matches(".tmp"); + let final_path = self.base_path.join(final_name); + + if let Ok(metadata) = std::fs::metadata(&path) { + if metadata.len() > 0 { + if let Err(e) = std::fs::rename(&path, &final_path) { + tracing::warn!( + "Failed to rename tmp segment {} to {}: {}", + path.display(), + final_path.display(), + e + ); + } else { + tracing::debug!( + "Finalized pending segment: {} ({} bytes)", + final_path.display(), + metadata.len() + ); + sync_file(&final_path); + } + } + } + } + } + } + } + + fn collect_orphaned_segments( + &mut self, + segment_start: Option, + end_timestamp: Duration, + frames_before_flush: u32, + ) { + let completed_indices: std::collections::HashSet = + self.completed_segments.iter().map(|s| s.index).collect(); + + let Ok(entries) = std::fs::read_dir(&self.base_path) else { + return; + }; + + let mut orphaned: Vec<(u32, PathBuf)> = Vec::new(); + + for entry in entries.flatten() { + let path = entry.path(); + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + if name.starts_with("segment_") && name.ends_with(".m4s") && !name.contains(".tmp") + { + if let Some(index_str) = name + .strip_prefix("segment_") + .and_then(|s| s.strip_suffix(".m4s")) + { + if let Ok(index) = index_str.parse::() { + if !completed_indices.contains(&index) { + orphaned.push((index, path)); + } + } + } + } + } + } + + orphaned.sort_by_key(|(idx, _)| *idx); + + for (index, segment_path) in orphaned { + if let Ok(metadata) = std::fs::metadata(&segment_path) { + let file_size = metadata.len(); + + if file_size < 100 { + tracing::debug!( + "Skipping tiny orphaned segment {} ({} bytes)", + segment_path.display(), + file_size + ); + continue; + } + + sync_file(&segment_path); + + let duration = if index == self.current_index && frames_before_flush > 0 { + if let Some(start) = segment_start { + end_timestamp.saturating_sub(start) + } else { + self.segment_duration + } + } else { + self.segment_duration + }; + + tracing::info!( + "Recovered orphaned segment {} with {} bytes, estimated duration {:?}", + segment_path.display(), + file_size, + duration + ); + + self.completed_segments.push(VideoSegmentInfo { + path: segment_path, + index, + duration, + file_size: Some(file_size), + }); + } + } + + self.completed_segments.sort_by_key(|s| s.index); + } + + fn finalize_manifest(&self) { + let total_duration: Duration = self.completed_segments.iter().map(|s| s.duration).sum(); + + let manifest = Manifest { + version: MANIFEST_VERSION, + manifest_type: "m4s_segments", + init_segment: Some(INIT_SEGMENT_NAME.to_string()), + segments: self + .completed_segments + .iter() + .map(|s| SegmentEntry { + path: s + .path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned(), + index: s.index, + duration: s.duration.as_secs_f64(), + is_complete: true, + file_size: s.file_size, + }) + .collect(), + total_duration: Some(total_duration.as_secs_f64()), + is_complete: true, + }; + + let manifest_path = self.base_path.join("manifest.json"); + if let Err(e) = atomic_write_json(&manifest_path, &manifest) { + tracing::warn!( + "Failed to write final manifest to {}: {e}", + manifest_path.display() + ); + } + } + + pub fn completed_segments(&self) -> &[VideoSegmentInfo] { + &self.completed_segments + } + + pub fn current_encoder(&self) -> Option<&H264Encoder> { + Some(&self.encoder) + } + + pub fn current_encoder_mut(&mut self) -> Option<&mut H264Encoder> { + Some(&mut self.encoder) + } + + pub fn base_path(&self) -> &Path { + &self.base_path + } + + pub fn segment_duration(&self) -> Duration { + self.segment_duration + } + + pub fn current_index(&self) -> u32 { + self.current_index + } + + pub fn init_segment_path(&self) -> PathBuf { + self.base_path.join(INIT_SEGMENT_NAME) + } +} diff --git a/crates/enc-ffmpeg/src/remux.rs b/crates/enc-ffmpeg/src/remux.rs index cf858002e2..5c658fff9d 100644 --- a/crates/enc-ffmpeg/src/remux.rs +++ b/crates/enc-ffmpeg/src/remux.rs @@ -364,3 +364,146 @@ pub fn get_video_fps(path: &Path) -> Option { } Some((rate.numerator() as f64 / rate.denominator() as f64).round() as u32) } + +pub fn probe_m4s_can_decode_with_init( + init_path: &Path, + segment_path: &Path, +) -> Result { + let temp_path = segment_path.with_extension("probe_temp.mp4"); + + let init_data = std::fs::read(init_path) + .map_err(|e| format!("Failed to read init segment {}: {e}", init_path.display()))?; + let segment_data = std::fs::read(segment_path) + .map_err(|e| format!("Failed to read segment {}: {e}", segment_path.display()))?; + + { + let mut temp_file = std::fs::File::create(&temp_path) + .map_err(|e| format!("Failed to create temp file: {e}"))?; + temp_file + .write_all(&init_data) + .map_err(|e| format!("Failed to write init data: {e}"))?; + temp_file + .write_all(&segment_data) + .map_err(|e| format!("Failed to write segment data: {e}"))?; + temp_file + .sync_all() + .map_err(|e| format!("Failed to sync temp file: {e}"))?; + } + + let result = probe_video_can_decode(&temp_path); + + let _ = std::fs::remove_file(&temp_path); + + result +} + +pub fn concatenate_m4s_segments_with_init( + init_path: &Path, + segments: &[PathBuf], + output: &Path, +) -> Result<(), RemuxError> { + if segments.is_empty() { + return Err(RemuxError::NoFragments); + } + + if !init_path.exists() { + return Err(RemuxError::FragmentNotFound(init_path.to_path_buf())); + } + + for segment in segments { + if !segment.exists() { + return Err(RemuxError::FragmentNotFound(segment.clone())); + } + } + + let combined_path = output.with_extension("combined_fmp4.mp4"); + + { + let init_data = std::fs::read(init_path)?; + let mut combined_file = std::fs::File::create(&combined_path)?; + combined_file.write_all(&init_data)?; + + for segment in segments { + let segment_data = std::fs::read(segment)?; + combined_file.write_all(&segment_data)?; + } + combined_file.sync_all()?; + } + + let result = remux_to_regular_mp4(&combined_path, output); + + let _ = std::fs::remove_file(&combined_path); + + result +} + +fn remux_to_regular_mp4(input_path: &Path, output_path: &Path) -> Result<(), RemuxError> { + let mut ictx = avformat::input(input_path)?; + let mut octx = avformat::output(output_path)?; + + let mut stream_mapping: Vec> = Vec::new(); + let mut output_stream_index = 0usize; + + for input_stream in ictx.streams() { + let codec_params = input_stream.parameters(); + let medium = codec_params.medium(); + + if medium == ffmpeg::media::Type::Video || medium == ffmpeg::media::Type::Audio { + stream_mapping.push(Some(output_stream_index)); + output_stream_index += 1; + + let mut output_stream = octx.add_stream(None)?; + output_stream.set_parameters(codec_params); + unsafe { + (*output_stream.as_mut_ptr()).time_base = (*input_stream.as_ptr()).time_base; + } + } else { + stream_mapping.push(None); + } + } + + octx.write_header()?; + + let mut last_dts: Vec = vec![i64::MIN; output_stream_index]; + let mut dts_offset: Vec = vec![0; output_stream_index]; + + for (input_stream, packet) in ictx.packets() { + let input_stream_index = input_stream.index(); + + if let Some(Some(output_index)) = stream_mapping.get(input_stream_index) { + let output_index = *output_index; + let mut packet = packet; + let input_time_base = input_stream.time_base(); + let output_time_base = octx.stream(output_index).unwrap().time_base(); + + packet.rescale_ts(input_time_base, output_time_base); + + let current_dts = packet.dts().unwrap_or(0); + + if last_dts[output_index] != i64::MIN && current_dts <= last_dts[output_index] { + dts_offset[output_index] = last_dts[output_index] - current_dts + 1; + } + + let adjusted_dts = current_dts + dts_offset[output_index]; + let adjusted_pts = packet.pts().map(|pts| pts + dts_offset[output_index]); + + unsafe { + (*packet.as_mut_ptr()).dts = adjusted_dts; + if let Some(pts) = adjusted_pts { + (*packet.as_mut_ptr()).pts = pts; + } + } + + last_dts[output_index] = adjusted_dts; + + packet.set_stream(output_index); + packet.set_position(-1); + + packet.write_interleaved(&mut octx)?; + } + } + + octx.write_trailer()?; + + Ok(()) +} diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs index fa440c6f93..d8e76718ca 100644 --- a/crates/enc-ffmpeg/src/video/h264.rs +++ b/crates/enc-ffmpeg/src/video/h264.rs @@ -120,7 +120,23 @@ impl H264EncoderBuilder { self.external_conversion, ) { Ok(encoder) => { - debug!("Using encoder {}", codec_name); + let is_hardware = matches!( + codec_name.as_str(), + "h264_videotoolbox" | "h264_nvenc" | "h264_qsv" | "h264_amf" | "h264_mf" + ); + if is_hardware { + debug!( + encoder = %codec_name, + "Selected hardware H264 encoder" + ); + } else { + error!( + encoder = %codec_name, + input_width = input_config.width, + input_height = input_config.height, + "WARNING: Using SOFTWARE H264 encoder (high CPU usage expected)" + ); + } return Ok(encoder); } Err(err) => { @@ -156,16 +172,18 @@ impl H264EncoderBuilder { input_config.pixel_format } else { needs_pixel_conversion = true; - let format = ffmpeg::format::Pixel::NV12; - if !external_conversion { - debug!( - "Converting from {:?} to {:?} for H264 encoding", - input_config.pixel_format, format - ); - } - format + ffmpeg::format::Pixel::NV12 }; + debug!( + encoder = %codec.name(), + input_format = ?input_config.pixel_format, + output_format = ?output_format, + needs_pixel_conversion = needs_pixel_conversion, + external_conversion = external_conversion, + "Encoder pixel format configuration" + ); + if is_420(output_format) && (!output_width.is_multiple_of(2) || !output_height.is_multiple_of(2)) { @@ -187,8 +205,10 @@ impl H264EncoderBuilder { let converter = if external_conversion { debug!( - "External conversion enabled, skipping internal converter. Expected input: {:?} {}x{}", - output_format, output_width, output_height + output_format = ?output_format, + output_width = output_width, + output_height = output_height, + "External conversion enabled, skipping internal converter" ); None } else if needs_pixel_conversion || needs_scaling { @@ -207,7 +227,18 @@ impl H264EncoderBuilder { output_height, flags, ) { - Ok(context) => Some(context), + Ok(context) => { + debug!( + encoder = %codec.name(), + src_format = ?input_config.pixel_format, + src_size = %format!("{}x{}", input_config.width, input_config.height), + dst_format = ?output_format, + dst_size = %format!("{}x{}", output_width, output_height), + needs_scaling = needs_scaling, + "Created SOFTWARE scaler for pixel format conversion (CPU-intensive)" + ); + Some(context) + } Err(e) => { if needs_pixel_conversion { error!( @@ -223,6 +254,10 @@ impl H264EncoderBuilder { } } } else { + debug!( + encoder = %codec.name(), + "No pixel format conversion needed (zero-copy path)" + ); None }; @@ -351,6 +386,36 @@ impl H264Encoder { Ok(()) } + pub fn queue_frame_reusable( + &mut self, + frame: &mut frame::Video, + converted_frame: &mut Option, + timestamp: Duration, + output: &mut format::context::Output, + ) -> Result<(), QueueFrameError> { + self.base.update_pts(frame, timestamp, &mut self.encoder); + + let frame_to_send = if let Some(converter) = &mut self.converter { + let pts = frame.pts(); + let converted = converted_frame.get_or_insert_with(|| { + frame::Video::new(self.output_format, self.output_width, self.output_height) + }); + converter + .run(frame, converted) + .map_err(QueueFrameError::Converter)?; + converted.set_pts(pts); + converted as &frame::Video + } else { + frame as &frame::Video + }; + + self.base + .send_frame(frame_to_send, output, &mut self.encoder) + .map_err(QueueFrameError::Encode)?; + + Ok(()) + } + pub fn queue_preconverted_frame( &mut self, mut frame: frame::Video, From a1224363136b5d92fcc8b5dfd4254390403fa612 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 21 Dec 2025 02:05:32 +0000 Subject: [PATCH 04/37] Remove segmented MP4 encoder and enhance MP4 encoder settings --- crates/enc-avfoundation/src/lib.rs | 2 - crates/enc-avfoundation/src/mp4.rs | 28 +- crates/enc-avfoundation/src/segmented.rs | 429 ----------------------- 3 files changed, 26 insertions(+), 433 deletions(-) delete mode 100644 crates/enc-avfoundation/src/segmented.rs diff --git a/crates/enc-avfoundation/src/lib.rs b/crates/enc-avfoundation/src/lib.rs index 8683aafff5..1db0f16db5 100644 --- a/crates/enc-avfoundation/src/lib.rs +++ b/crates/enc-avfoundation/src/lib.rs @@ -1,7 +1,5 @@ #![cfg(target_os = "macos")] mod mp4; -mod segmented; pub use mp4::*; -pub use segmented::*; diff --git a/crates/enc-avfoundation/src/mp4.rs b/crates/enc-avfoundation/src/mp4.rs index e0be1560cc..2498624be0 100644 --- a/crates/enc-avfoundation/src/mp4.rs +++ b/crates/enc-avfoundation/src/mp4.rs @@ -79,6 +79,15 @@ impl MP4Encoder { audio_config: Option, output_height: Option, ) -> Result { + info!( + width = video_config.width, + height = video_config.height, + pixel_format = ?video_config.pixel_format, + frame_rate = ?video_config.frame_rate, + output_height = ?output_height, + has_audio = audio_config.is_some(), + "Initializing AVFoundation MP4 encoder (VideoToolbox hardware encoding)" + ); debug!("{video_config:#?}"); debug!("{audio_config:#?}"); @@ -122,11 +131,23 @@ impl MP4Encoder { debug!("recording bitrate: {bitrate}"); + let keyframe_interval = (fps * 2.0) as i32; + output_settings.insert( av::video_settings_keys::compression_props(), ns::Dictionary::with_keys_values( - &[unsafe { AVVideoAverageBitRateKey }], - &[ns::Number::with_f32(bitrate).as_id_ref()], + &[ + unsafe { AVVideoAverageBitRateKey }, + unsafe { AVVideoAllowFrameReorderingKey }, + unsafe { AVVideoExpectedSourceFrameRateKey }, + unsafe { AVVideoMaxKeyFrameIntervalKey }, + ], + &[ + ns::Number::with_f32(bitrate).as_id_ref(), + ns::Number::with_bool(false).as_id_ref(), + ns::Number::with_f32(fps).as_id_ref(), + ns::Number::with_i32(keyframe_interval).as_id_ref(), + ], ) .as_id_ref(), ); @@ -524,6 +545,9 @@ impl Drop for MP4Encoder { #[link(name = "AVFoundation", kind = "framework")] unsafe extern "C" { static AVVideoAverageBitRateKey: &'static ns::String; + static AVVideoAllowFrameReorderingKey: &'static ns::String; + static AVVideoExpectedSourceFrameRateKey: &'static ns::String; + static AVVideoMaxKeyFrameIntervalKey: &'static ns::String; static AVVideoTransferFunctionKey: &'static ns::String; static AVVideoColorPrimariesKey: &'static ns::String; static AVVideoYCbCrMatrixKey: &'static ns::String; diff --git a/crates/enc-avfoundation/src/segmented.rs b/crates/enc-avfoundation/src/segmented.rs deleted file mode 100644 index 1c0cff38af..0000000000 --- a/crates/enc-avfoundation/src/segmented.rs +++ /dev/null @@ -1,429 +0,0 @@ -use crate::{FinishError, InitError, MP4Encoder, QueueFrameError, wait_for_writer_finished}; -use cap_media_info::{AudioInfo, VideoInfo}; -use cidre::arc; -use ffmpeg::frame; -use serde::Serialize; -use std::{ - io::Write, - path::{Path, PathBuf}, - time::Duration, -}; -use tracing::warn; - -fn atomic_write_json(path: &Path, data: &T) -> std::io::Result<()> { - let temp_path = path.with_extension("json.tmp"); - let json = serde_json::to_string_pretty(data) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - - let mut file = std::fs::File::create(&temp_path)?; - file.write_all(json.as_bytes())?; - file.sync_all()?; - - std::fs::rename(&temp_path, path)?; - - if let Some(parent) = path.parent() - && let Ok(dir) = std::fs::File::open(parent) - { - let _ = dir.sync_all(); - } - - Ok(()) -} - -fn sync_file(path: &Path) { - if let Ok(file) = std::fs::File::open(path) { - let _ = file.sync_all(); - } -} - -pub struct SegmentedMP4Encoder { - base_path: PathBuf, - video_config: VideoInfo, - audio_config: Option, - output_height: Option, - - current_encoder: Option, - current_index: u32, - segment_duration: Duration, - segment_start_time: Option, - - completed_segments: Vec, -} - -#[derive(Debug, Clone)] -pub struct SegmentInfo { - pub path: PathBuf, - pub index: u32, - pub duration: Duration, - pub file_size: Option, - pub is_failed: bool, -} - -#[derive(Serialize)] -struct FragmentEntry { - path: String, - index: u32, - duration: f64, - is_complete: bool, - #[serde(skip_serializing_if = "Option::is_none")] - file_size: Option, - #[serde(skip_serializing_if = "std::ops::Not::not")] - is_failed: bool, -} - -const MANIFEST_VERSION: u32 = 2; - -#[derive(Serialize)] -struct Manifest { - version: u32, - fragments: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - total_duration: Option, - is_complete: bool, -} - -impl SegmentedMP4Encoder { - pub fn init( - base_path: PathBuf, - video_config: VideoInfo, - audio_config: Option, - output_height: Option, - segment_duration: Duration, - ) -> Result { - std::fs::create_dir_all(&base_path).map_err(|_| InitError::NoSettingsAssistant)?; - - let segment_path = base_path.join("fragment_000.mp4"); - let encoder = MP4Encoder::init(segment_path, video_config, audio_config, output_height)?; - - let instance = Self { - base_path, - video_config, - audio_config, - output_height, - current_encoder: Some(encoder), - current_index: 0, - segment_duration, - segment_start_time: None, - completed_segments: Vec::new(), - }; - - instance.write_in_progress_manifest(); - - Ok(instance) - } - - pub fn queue_video_frame( - &mut self, - frame: arc::R, - timestamp: Duration, - ) -> Result<(), QueueFrameError> { - if self.segment_start_time.is_none() { - self.segment_start_time = Some(timestamp); - } - - let segment_elapsed = - timestamp.saturating_sub(self.segment_start_time.unwrap_or(Duration::ZERO)); - - if segment_elapsed >= self.segment_duration { - self.rotate_segment(timestamp)?; - } - - if let Some(encoder) = &mut self.current_encoder { - encoder.queue_video_frame(frame, timestamp) - } else { - Err(QueueFrameError::NoEncoder) - } - } - - pub fn queue_audio_frame( - &mut self, - frame: &frame::Audio, - timestamp: Duration, - ) -> Result<(), QueueFrameError> { - if let Some(encoder) = &mut self.current_encoder { - encoder.queue_audio_frame(frame, timestamp) - } else { - Err(QueueFrameError::NoEncoder) - } - } - - fn rotate_segment(&mut self, timestamp: Duration) -> Result<(), QueueFrameError> { - let segment_start = self.segment_start_time.unwrap_or(Duration::ZERO); - let segment_duration = timestamp.saturating_sub(segment_start); - let completed_segment_path = self.current_segment_path(); - let current_index = self.current_index; - - if let Some(mut encoder) = self.current_encoder.take() { - let finish_failed = match encoder.finish_nowait(Some(timestamp)) { - Ok(writer) => { - let path_for_sync = completed_segment_path.clone(); - std::thread::spawn(move || { - if let Err(e) = wait_for_writer_finished(&writer) { - warn!( - "Background writer finalization failed for segment {current_index}: {e}" - ); - } - sync_file(&path_for_sync); - }); - false - } - Err(e) => { - tracing::error!( - "Failed to finish encoder during rotation for segment {}: {e}", - current_index - ); - true - } - }; - - let file_size = std::fs::metadata(&completed_segment_path) - .ok() - .map(|m| m.len()); - - self.completed_segments.push(SegmentInfo { - path: completed_segment_path, - index: current_index, - duration: segment_duration, - file_size, - is_failed: finish_failed, - }); - - self.write_manifest(); - - if finish_failed { - tracing::warn!( - "Segment {} marked as failed in manifest, continuing with new segment", - current_index - ); - } - } - - self.current_index += 1; - self.segment_start_time = Some(timestamp); - - let new_path = self.current_segment_path(); - self.current_encoder = Some( - MP4Encoder::init( - new_path, - self.video_config, - self.audio_config, - self.output_height, - ) - .map_err(|e| { - tracing::error!( - "Failed to create new encoder for segment {}: {e}", - self.current_index - ); - QueueFrameError::Failed - })?, - ); - - self.write_in_progress_manifest(); - - Ok(()) - } - - fn current_segment_path(&self) -> PathBuf { - self.base_path - .join(format!("fragment_{:03}.mp4", self.current_index)) - } - - fn write_manifest(&self) { - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: !s.is_failed, - file_size: s.file_size, - is_failed: s.is_failed, - }) - .collect(), - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = atomic_write_json(&manifest_path, &manifest) { - tracing::warn!( - "Failed to write manifest to {}: {e}", - manifest_path.display() - ); - } - } - - fn write_in_progress_manifest(&self) { - let mut fragments: Vec = self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: !s.is_failed, - file_size: s.file_size, - is_failed: s.is_failed, - }) - .collect(); - - fragments.push(FragmentEntry { - path: self - .current_segment_path() - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: self.current_index, - duration: 0.0, - is_complete: false, - file_size: None, - is_failed: false, - }); - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments, - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = atomic_write_json(&manifest_path, &manifest) { - tracing::warn!( - "Failed to write in-progress manifest to {}: {e}", - manifest_path.display() - ); - } - } - - pub fn pause(&mut self) { - if let Some(encoder) = &mut self.current_encoder { - encoder.pause(); - } - } - - pub fn resume(&mut self) { - if let Some(encoder) = &mut self.current_encoder { - encoder.resume(); - } - } - - pub fn finish(&mut self, timestamp: Option) -> Result<(), FinishError> { - let segment_path = self.current_segment_path(); - let segment_start = self.segment_start_time; - let current_index = self.current_index; - - if let Some(mut encoder) = self.current_encoder.take() { - match encoder.finish_nowait(timestamp) { - Ok(writer) => { - let path_for_sync = segment_path.clone(); - std::thread::spawn(move || { - if let Err(e) = wait_for_writer_finished(&writer) { - warn!( - "Background writer finalization failed for segment {current_index}: {e}" - ); - } - sync_file(&path_for_sync); - }); - - if let Some(start) = segment_start { - let final_duration = timestamp.unwrap_or(start).saturating_sub(start); - let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); - - self.completed_segments.push(SegmentInfo { - path: segment_path, - index: current_index, - duration: final_duration, - file_size, - is_failed: false, - }); - } - } - Err(e) => { - tracing::error!("Failed to finish final segment {current_index}: {e}"); - - if let Some(start) = segment_start { - let final_duration = timestamp.unwrap_or(start).saturating_sub(start); - let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); - - self.completed_segments.push(SegmentInfo { - path: segment_path, - index: current_index, - duration: final_duration, - file_size, - is_failed: true, - }); - } - } - } - } - - self.finalize_manifest(); - - Ok(()) - } - - fn finalize_manifest(&self) { - let total_duration: Duration = self.completed_segments.iter().map(|s| s.duration).sum(); - let has_failed_segments = self.completed_segments.iter().any(|s| s.is_failed); - - if has_failed_segments { - tracing::warn!( - "Recording completed with {} failed segment(s)", - self.completed_segments - .iter() - .filter(|s| s.is_failed) - .count() - ); - } - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: !s.is_failed, - file_size: s.file_size, - is_failed: s.is_failed, - }) - .collect(), - total_duration: Some(total_duration.as_secs_f64()), - is_complete: true, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = atomic_write_json(&manifest_path, &manifest) { - tracing::warn!( - "Failed to write final manifest to {}: {e}", - manifest_path.display() - ); - } - } - - pub fn completed_segments(&self) -> &[SegmentInfo] { - &self.completed_segments - } -} From 84cba3cf7676b513ad4b7e261dcc66f3d610d7ea Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 21 Dec 2025 02:05:49 +0000 Subject: [PATCH 05/37] Replace segmented muxers with M4S muxer on macOS --- crates/recording/src/capture_pipeline.rs | 4 +- crates/recording/src/cursor.rs | 111 +-- crates/recording/src/output_pipeline/core.rs | 22 +- .../src/output_pipeline/fragmented.rs | 294 ------ .../output_pipeline/macos_fragmented_m4s.rs | 891 ++++++++++++++++++ .../output_pipeline/macos_segmented_ffmpeg.rs | 746 --------------- crates/recording/src/output_pipeline/mod.rs | 8 +- crates/recording/src/recovery.rs | 274 ++++-- .../src/sources/screen_capture/macos.rs | 93 +- 9 files changed, 1241 insertions(+), 1202 deletions(-) delete mode 100644 crates/recording/src/output_pipeline/fragmented.rs create mode 100644 crates/recording/src/output_pipeline/macos_fragmented_m4s.rs delete mode 100644 crates/recording/src/output_pipeline/macos_segmented_ffmpeg.rs diff --git a/crates/recording/src/capture_pipeline.rs b/crates/recording/src/capture_pipeline.rs index 1caa3afe8e..d5575bc846 100644 --- a/crates/recording/src/capture_pipeline.rs +++ b/crates/recording/src/capture_pipeline.rs @@ -6,7 +6,7 @@ use crate::{ }; #[cfg(target_os = "macos")] -use crate::output_pipeline::{MacOSSegmentedMuxer, MacOSSegmentedMuxerConfig}; +use crate::output_pipeline::{MacOSFragmentedM4SMuxer, MacOSFragmentedM4SMuxerConfig}; use anyhow::anyhow; use cap_timestamp::Timestamps; use std::{path::PathBuf, sync::Arc}; @@ -86,7 +86,7 @@ impl MakeCapturePipeline for screen_capture::CMSampleBufferCapture { OutputPipeline::builder(fragments_dir) .with_video::(screen_capture) .with_timestamps(start_time) - .build::(MacOSSegmentedMuxerConfig::default()) + .build::(MacOSFragmentedM4SMuxerConfig::default()) .await } else { OutputPipeline::builder(output_path.clone()) diff --git a/crates/recording/src/cursor.rs b/crates/recording/src/cursor.rs index a3701dfeaa..405c97bf59 100644 --- a/crates/recording/src/cursor.rs +++ b/crates/recording/src/cursor.rs @@ -100,9 +100,10 @@ pub fn spawn_cursor_recorder( let mut last_flush = Instant::now(); let flush_interval = Duration::from_secs(CURSOR_FLUSH_INTERVAL_SECS); + let mut last_cursor_id = "default".to_string(); loop { - let sleep = tokio::time::sleep(Duration::from_millis(10)); + let sleep = tokio::time::sleep(Duration::from_millis(16)); let Either::Right(_) = futures::future::select(pin!(stop_token_child.cancelled()), pin!(sleep)).await else { @@ -112,68 +113,70 @@ pub fn spawn_cursor_recorder( let elapsed = start_time.instant().elapsed().as_secs_f64() * 1000.0; let mouse_state = device_state.get_mouse(); - let cursor_data = get_cursor_data(); - let cursor_id = if let Some(data) = cursor_data { - let mut hasher = DefaultHasher::default(); - data.image.hash(&mut hasher); - let id = hasher.finish(); + let position = cap_cursor_capture::RawCursorPosition::get(); + let position_changed = position != last_position; - if let Some(existing_id) = response.cursors.get(&id) { - existing_id.id.to_string() - } else { - let cursor_id = response.next_cursor_id.to_string(); - let file_name = format!("cursor_{cursor_id}.png"); - let cursor_path = cursors_dir.join(&file_name); - - if let Ok(image) = image::load_from_memory(&data.image) { - let rgba_image = image.into_rgba8(); - - if let Err(e) = rgba_image.save(&cursor_path) { - error!("Failed to save cursor image: {}", e); - } else { - info!("Saved cursor {cursor_id} image to: {:?}", file_name); - response.cursors.insert( - id, - Cursor { - file_name, - id: response.next_cursor_id, - hotspot: data.hotspot, - shape: data.shape, - }, - ); - response.next_cursor_id += 1; + let cursor_id = if position_changed { + last_position = position; + if let Some(data) = get_cursor_data() { + let mut hasher = DefaultHasher::default(); + data.image.hash(&mut hasher); + let id = hasher.finish(); + + let cursor_id = if let Some(existing_id) = response.cursors.get(&id) { + existing_id.id.to_string() + } else { + let cursor_id = response.next_cursor_id.to_string(); + let file_name = format!("cursor_{cursor_id}.png"); + let cursor_path = cursors_dir.join(&file_name); + + if let Ok(image) = image::load_from_memory(&data.image) { + let rgba_image = image.into_rgba8(); + + if let Err(e) = rgba_image.save(&cursor_path) { + error!("Failed to save cursor image: {}", e); + } else { + info!("Saved cursor {cursor_id} image to: {:?}", file_name); + response.cursors.insert( + id, + Cursor { + file_name, + id: response.next_cursor_id, + hotspot: data.hotspot, + shape: data.shape, + }, + ); + response.next_cursor_id += 1; + } } - } + cursor_id + }; + last_cursor_id = cursor_id.clone(); cursor_id + } else { + last_cursor_id.clone() } } else { - "default".to_string() + last_cursor_id.clone() }; - let position = cap_cursor_capture::RawCursorPosition::get(); - - let position = (position != last_position).then(|| { - last_position = position; - + if position_changed { let cropped_norm_pos = position - .relative_to_display(display)? - .normalize()? - .with_crop(crop_bounds); - - Some((cropped_norm_pos.x(), cropped_norm_pos.y())) - }); - - if let Some((x, y)) = position.flatten() { - let mouse_event = CursorMoveEvent { - active_modifiers: vec![], - cursor_id: cursor_id.clone(), - time_ms: elapsed, - x, - y, - }; - - response.moves.push(mouse_event); + .relative_to_display(display) + .and_then(|p| p.normalize()) + .map(|p| p.with_crop(crop_bounds)); + + if let Some(pos) = cropped_norm_pos { + let mouse_event = CursorMoveEvent { + active_modifiers: vec![], + cursor_id: cursor_id.clone(), + time_ms: elapsed, + x: pos.x(), + y: pos.y(), + }; + response.moves.push(mouse_event); + } } for (num, &pressed) in mouse_state.button_pressed.iter().enumerate() { diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rs index 44148ec89c..0946b8b1b0 100644 --- a/crates/recording/src/output_pipeline/core.rs +++ b/crates/recording/src/output_pipeline/core.rs @@ -455,7 +455,6 @@ fn spawn_video_encoder, TVideo: V let mut first_tx = Some(first_tx); let mut frame_count = 0u64; - let res = stop_token .run_until_cancelled(async { while let Some(frame) = video_rx.next().await { @@ -487,9 +486,20 @@ fn spawn_video_encoder, TVideo: V if was_cancelled { info!("mux-video cancelled, draining remaining frames from channel"); + let drain_start = std::time::Instant::now(); + let drain_timeout = Duration::from_secs(2); + let max_drain_frames = 30u64; let mut drained = 0u64; + let mut skipped = 0u64; + while let Some(frame) = video_rx.next().await { frame_count += 1; + + if drain_start.elapsed() > drain_timeout || drained >= max_drain_frames { + skipped += 1; + continue; + } + drained += 1; let timestamp = frame.timestamp(); @@ -506,14 +516,16 @@ fn spawn_video_encoder, TVideo: V Ok(()) => {} Err(e) => { warn!("Error processing drained frame: {e}"); - break; + skipped += 1; } } } - if drained > 0 { + if drained > 0 || skipped > 0 { info!( - "mux-video drained {} additional frames after cancellation", - drained + "mux-video drain complete: {} frames processed, {} skipped in {:?}", + drained, + skipped, + drain_start.elapsed() ); } } diff --git a/crates/recording/src/output_pipeline/fragmented.rs b/crates/recording/src/output_pipeline/fragmented.rs deleted file mode 100644 index a5dd3d933a..0000000000 --- a/crates/recording/src/output_pipeline/fragmented.rs +++ /dev/null @@ -1,294 +0,0 @@ -#[cfg(target_os = "macos")] -use crate::{ - VideoFrame, - output_pipeline::{AudioFrame, AudioMuxer, Muxer, TaskPool, VideoMuxer}, - sources::screen_capture, -}; - -#[cfg(target_os = "macos")] -use anyhow::anyhow; -#[cfg(target_os = "macos")] -use cap_enc_avfoundation::SegmentedMP4Encoder; -#[cfg(target_os = "macos")] -use cap_media_info::{AudioInfo, VideoInfo}; -#[cfg(target_os = "macos")] -use cap_timestamp::Timestamp; -#[cfg(target_os = "macos")] -use std::{ - path::PathBuf, - sync::{Arc, atomic::AtomicBool}, - time::Duration, -}; - -#[cfg(target_os = "macos")] -pub struct FragmentedAVFoundationMp4Muxer { - inner: SegmentedMP4Encoder, - pause_flag: Arc, -} - -#[cfg(target_os = "macos")] -pub struct FragmentedAVFoundationMp4MuxerConfig { - pub output_height: Option, - pub segment_duration: Duration, -} - -#[cfg(target_os = "macos")] -impl Default for FragmentedAVFoundationMp4MuxerConfig { - fn default() -> Self { - Self { - output_height: None, - segment_duration: Duration::from_secs(3), - } - } -} - -#[cfg(target_os = "macos")] -impl FragmentedAVFoundationMp4Muxer { - const MAX_QUEUE_RETRIES: u32 = 1500; -} - -#[cfg(target_os = "macos")] -#[derive(Clone)] -pub struct FragmentedNativeCameraFrame { - pub sample_buf: cidre::arc::R, - pub timestamp: Timestamp, -} - -#[cfg(target_os = "macos")] -unsafe impl Send for FragmentedNativeCameraFrame {} -#[cfg(target_os = "macos")] -unsafe impl Sync for FragmentedNativeCameraFrame {} - -#[cfg(target_os = "macos")] -impl VideoFrame for FragmentedNativeCameraFrame { - fn timestamp(&self) -> Timestamp { - self.timestamp - } -} - -#[cfg(target_os = "macos")] -impl Muxer for FragmentedAVFoundationMp4Muxer { - type Config = FragmentedAVFoundationMp4MuxerConfig; - - async fn setup( - config: Self::Config, - output_path: PathBuf, - video_config: Option, - audio_config: Option, - pause_flag: Arc, - _tasks: &mut TaskPool, - ) -> anyhow::Result { - let video_config = - video_config.ok_or_else(|| anyhow!("Invariant: No video source provided"))?; - - Ok(Self { - inner: SegmentedMP4Encoder::init( - output_path, - video_config, - audio_config, - config.output_height, - config.segment_duration, - ) - .map_err(|e| anyhow!("{e}"))?, - pause_flag, - }) - } - - fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - Ok(self.inner.finish(Some(timestamp)).map(Ok)?) - } -} - -#[cfg(target_os = "macos")] -impl VideoMuxer for FragmentedAVFoundationMp4Muxer { - type VideoFrame = screen_capture::VideoFrame; - - fn send_video_frame( - &mut self, - frame: Self::VideoFrame, - timestamp: Duration, - ) -> anyhow::Result<()> { - if self.pause_flag.load(std::sync::atomic::Ordering::Relaxed) { - self.inner.pause(); - } else { - self.inner.resume(); - } - - let mut retry_count = 0; - loop { - match self - .inner - .queue_video_frame(frame.sample_buf.clone(), timestamp) - { - Ok(()) => break, - Err(cap_enc_avfoundation::QueueFrameError::NotReadyForMore) => { - retry_count += 1; - if retry_count >= Self::MAX_QUEUE_RETRIES { - return Err(anyhow!( - "send_video_frame/timeout after {} retries", - Self::MAX_QUEUE_RETRIES - )); - } - std::thread::sleep(Duration::from_millis(2)); - continue; - } - Err(e) => return Err(anyhow!("send_video_frame/{e}")), - } - } - - Ok(()) - } -} - -#[cfg(target_os = "macos")] -impl AudioMuxer for FragmentedAVFoundationMp4Muxer { - fn send_audio_frame(&mut self, frame: AudioFrame, timestamp: Duration) -> anyhow::Result<()> { - let mut retry_count = 0; - loop { - match self.inner.queue_audio_frame(&frame.inner, timestamp) { - Ok(()) => break, - Err(cap_enc_avfoundation::QueueFrameError::NotReadyForMore) => { - retry_count += 1; - if retry_count >= Self::MAX_QUEUE_RETRIES { - return Err(anyhow!( - "send_audio_frame/retries_exceeded after {} retries", - Self::MAX_QUEUE_RETRIES - )); - } - std::thread::sleep(Duration::from_millis(2)); - continue; - } - Err(e) => return Err(anyhow!("send_audio_frame/{e}")), - } - } - - Ok(()) - } -} - -#[cfg(target_os = "macos")] -pub struct FragmentedAVFoundationCameraMuxer { - inner: SegmentedMP4Encoder, - pause_flag: Arc, -} - -#[cfg(target_os = "macos")] -pub struct FragmentedAVFoundationCameraMuxerConfig { - pub output_height: Option, - pub segment_duration: Duration, -} - -#[cfg(target_os = "macos")] -impl Default for FragmentedAVFoundationCameraMuxerConfig { - fn default() -> Self { - Self { - output_height: None, - segment_duration: Duration::from_secs(3), - } - } -} - -#[cfg(target_os = "macos")] -impl FragmentedAVFoundationCameraMuxer { - const MAX_QUEUE_RETRIES: u32 = 1500; -} - -#[cfg(target_os = "macos")] -impl Muxer for FragmentedAVFoundationCameraMuxer { - type Config = FragmentedAVFoundationCameraMuxerConfig; - - async fn setup( - config: Self::Config, - output_path: PathBuf, - video_config: Option, - audio_config: Option, - pause_flag: Arc, - _tasks: &mut TaskPool, - ) -> anyhow::Result { - let video_config = - video_config.ok_or_else(|| anyhow!("Invariant: No video source provided"))?; - - Ok(Self { - inner: SegmentedMP4Encoder::init( - output_path, - video_config, - audio_config, - config.output_height, - config.segment_duration, - ) - .map_err(|e| anyhow!("{e}"))?, - pause_flag, - }) - } - - fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - Ok(self.inner.finish(Some(timestamp)).map(Ok)?) - } -} - -#[cfg(target_os = "macos")] -impl VideoMuxer for FragmentedAVFoundationCameraMuxer { - type VideoFrame = crate::output_pipeline::NativeCameraFrame; - - fn send_video_frame( - &mut self, - frame: Self::VideoFrame, - timestamp: Duration, - ) -> anyhow::Result<()> { - if self.pause_flag.load(std::sync::atomic::Ordering::Relaxed) { - self.inner.pause(); - } else { - self.inner.resume(); - } - - let mut retry_count = 0; - loop { - match self - .inner - .queue_video_frame(frame.sample_buf.clone(), timestamp) - { - Ok(()) => break, - Err(cap_enc_avfoundation::QueueFrameError::NotReadyForMore) => { - retry_count += 1; - if retry_count >= Self::MAX_QUEUE_RETRIES { - return Err(anyhow!( - "send_video_frame/timeout after {} retries", - Self::MAX_QUEUE_RETRIES - )); - } - std::thread::sleep(Duration::from_millis(2)); - continue; - } - Err(e) => return Err(anyhow!("send_video_frame/{e}")), - } - } - - Ok(()) - } -} - -#[cfg(target_os = "macos")] -impl AudioMuxer for FragmentedAVFoundationCameraMuxer { - fn send_audio_frame(&mut self, frame: AudioFrame, timestamp: Duration) -> anyhow::Result<()> { - let mut retry_count = 0; - loop { - match self.inner.queue_audio_frame(&frame.inner, timestamp) { - Ok(()) => break, - Err(cap_enc_avfoundation::QueueFrameError::NotReadyForMore) => { - retry_count += 1; - if retry_count >= Self::MAX_QUEUE_RETRIES { - return Err(anyhow!( - "send_audio_frame/retries_exceeded after {} retries", - Self::MAX_QUEUE_RETRIES - )); - } - std::thread::sleep(Duration::from_millis(2)); - continue; - } - Err(e) => return Err(anyhow!("send_audio_frame/{e}")), - } - } - - Ok(()) - } -} diff --git a/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs b/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs new file mode 100644 index 0000000000..484171588c --- /dev/null +++ b/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs @@ -0,0 +1,891 @@ +use crate::{ + AudioFrame, AudioMuxer, Muxer, TaskPool, VideoMuxer, output_pipeline::NativeCameraFrame, + screen_capture, +}; +use anyhow::{Context, anyhow}; +use cap_enc_ffmpeg::h264::{H264EncoderBuilder, H264Preset}; +use cap_enc_ffmpeg::segmented_stream::{SegmentedVideoEncoder, SegmentedVideoEncoderConfig}; +use cap_media_info::{AudioInfo, VideoInfo}; +use std::{ + path::PathBuf, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, + mpsc::{SyncSender, sync_channel}, + }, + thread::JoinHandle, + time::Duration, +}; +use tracing::*; + +fn get_muxer_buffer_size() -> usize { + std::env::var("CAP_MUXER_BUFFER_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(3) +} + +struct PauseTracker { + flag: Arc, + paused_at: Option, + offset: Duration, +} + +impl PauseTracker { + fn new(flag: Arc) -> Self { + Self { + flag, + paused_at: None, + offset: Duration::ZERO, + } + } + + fn adjust(&mut self, timestamp: Duration) -> anyhow::Result> { + if self.flag.load(Ordering::Relaxed) { + if self.paused_at.is_none() { + self.paused_at = Some(timestamp); + } + return Ok(None); + } + + if let Some(start) = self.paused_at.take() { + let delta = timestamp.checked_sub(start).ok_or_else(|| { + anyhow!( + "Frame timestamp went backward during unpause (resume={start:?}, current={timestamp:?})" + ) + })?; + + self.offset = self.offset.checked_add(delta).ok_or_else(|| { + anyhow!( + "Pause offset overflow (offset={:?}, delta={delta:?})", + self.offset + ) + })?; + } + + let adjusted = timestamp.checked_sub(self.offset).ok_or_else(|| { + anyhow!( + "Adjusted timestamp underflow (timestamp={timestamp:?}, offset={:?})", + self.offset + ) + })?; + + Ok(Some(adjusted)) + } +} + +struct FrameDropTracker { + drops_in_window: u32, + frames_in_window: u32, + total_drops: u64, + total_frames: u64, + last_check: std::time::Instant, +} + +impl FrameDropTracker { + fn new() -> Self { + Self { + drops_in_window: 0, + frames_in_window: 0, + total_drops: 0, + total_frames: 0, + last_check: std::time::Instant::now(), + } + } + + fn record_frame(&mut self) { + self.frames_in_window += 1; + self.total_frames += 1; + self.check_drop_rate(); + } + + fn record_drop(&mut self) { + self.drops_in_window += 1; + self.total_drops += 1; + self.check_drop_rate(); + } + + fn check_drop_rate(&mut self) { + if self.last_check.elapsed() >= Duration::from_secs(5) { + let total_in_window = self.frames_in_window + self.drops_in_window; + if total_in_window > 0 { + let drop_rate = 100.0 * self.drops_in_window as f64 / total_in_window as f64; + if drop_rate > 5.0 { + warn!( + frames = self.frames_in_window, + drops = self.drops_in_window, + drop_rate_pct = format!("{:.1}%", drop_rate), + total_frames = self.total_frames, + total_drops = self.total_drops, + "M4S muxer frame drop rate exceeds 5% threshold" + ); + } else if self.drops_in_window > 0 { + debug!( + frames = self.frames_in_window, + drops = self.drops_in_window, + drop_rate_pct = format!("{:.1}%", drop_rate), + "M4S muxer frame stats" + ); + } + } + self.drops_in_window = 0; + self.frames_in_window = 0; + self.last_check = std::time::Instant::now(); + } + } +} + +struct EncoderState { + video_tx: SyncSender, Duration)>>, + encoder: Arc>, + encoder_handle: Option>>, +} + +pub struct MacOSFragmentedM4SMuxer { + base_path: PathBuf, + video_config: VideoInfo, + segment_duration: Duration, + state: Option, + pause: PauseTracker, + frame_drops: FrameDropTracker, + started: bool, +} + +pub struct MacOSFragmentedM4SMuxerConfig { + pub segment_duration: Duration, + pub preset: H264Preset, + pub output_size: Option<(u32, u32)>, +} + +impl Default for MacOSFragmentedM4SMuxerConfig { + fn default() -> Self { + Self { + segment_duration: Duration::from_secs(3), + preset: H264Preset::Ultrafast, + output_size: None, + } + } +} + +impl Muxer for MacOSFragmentedM4SMuxer { + type Config = MacOSFragmentedM4SMuxerConfig; + + async fn setup( + config: Self::Config, + output_path: PathBuf, + video_config: Option, + _audio_config: Option, + pause_flag: Arc, + _tasks: &mut TaskPool, + ) -> anyhow::Result + where + Self: Sized, + { + let video_config = + video_config.ok_or_else(|| anyhow!("invariant: video config expected"))?; + + std::fs::create_dir_all(&output_path) + .with_context(|| format!("Failed to create segments directory: {output_path:?}"))?; + + Ok(Self { + base_path: output_path, + video_config, + segment_duration: config.segment_duration, + state: None, + pause: PauseTracker::new(pause_flag), + frame_drops: FrameDropTracker::new(), + started: false, + }) + } + + fn stop(&mut self) { + if let Some(state) = &self.state + && let Err(e) = state.video_tx.send(None) + { + trace!("M4S encoder channel already closed during stop: {e}"); + } + } + + fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { + if let Some(mut state) = self.state.take() { + if let Err(e) = state.video_tx.send(None) { + trace!("M4S encoder channel already closed during finish: {e}"); + } + + if let Some(handle) = state.encoder_handle.take() { + let timeout = Duration::from_secs(5); + let start = std::time::Instant::now(); + loop { + if handle.is_finished() { + if let Err(panic_payload) = handle.join() { + warn!( + "M4S encoder thread panicked during finish: {:?}", + panic_payload + ); + } + break; + } + if start.elapsed() > timeout { + warn!( + "M4S encoder thread did not finish within {:?}, abandoning", + timeout + ); + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + } + + if let Ok(mut encoder) = state.encoder.lock() { + if let Err(e) = encoder.finish_with_timestamp(timestamp) { + warn!("Failed to finish segmented encoder: {e}"); + } + } + } + + Ok(Ok(())) + } +} + +impl MacOSFragmentedM4SMuxer { + fn start_encoder(&mut self) -> anyhow::Result<()> { + let buffer_size = get_muxer_buffer_size(); + debug!( + buffer_size = buffer_size, + "M4S muxer encoder channel buffer size" + ); + + let (video_tx, video_rx) = + sync_channel::, Duration)>>(buffer_size); + let (ready_tx, ready_rx) = sync_channel::>(1); + + let encoder_config = SegmentedVideoEncoderConfig { + segment_duration: self.segment_duration, + preset: H264Preset::Ultrafast, + bpp: H264EncoderBuilder::QUALITY_BPP, + output_size: None, + }; + + let encoder = + SegmentedVideoEncoder::init(self.base_path.clone(), self.video_config, encoder_config)?; + let encoder = Arc::new(Mutex::new(encoder)); + let encoder_clone = encoder.clone(); + let video_config = self.video_config; + + let encoder_handle = std::thread::Builder::new() + .name("m4s-segment-encoder".to_string()) + .spawn(move || { + let pixel_format = match video_config.pixel_format { + cap_media_info::Pixel::NV12 => ffmpeg::format::Pixel::NV12, + cap_media_info::Pixel::BGRA => ffmpeg::format::Pixel::BGRA, + cap_media_info::Pixel::UYVY422 => ffmpeg::format::Pixel::UYVY422, + _ => ffmpeg::format::Pixel::NV12, + }; + + let mut frame_pool = + FramePool::new(pixel_format, video_config.width, video_config.height); + + if ready_tx.send(Ok(())).is_err() { + return Err(anyhow!("Failed to send ready signal - receiver dropped")); + } + + let mut slow_convert_count = 0u32; + let mut slow_encode_count = 0u32; + let mut total_frames = 0u64; + const SLOW_THRESHOLD_MS: u128 = 5; + + while let Ok(Some((sample_buf, timestamp))) = video_rx.recv() { + let convert_start = std::time::Instant::now(); + let frame = frame_pool.get_frame(); + let fill_result = fill_frame_from_sample_buf(&sample_buf, frame); + let convert_elapsed_ms = convert_start.elapsed().as_millis(); + + if convert_elapsed_ms > SLOW_THRESHOLD_MS { + slow_convert_count += 1; + if slow_convert_count <= 5 || slow_convert_count % 100 == 0 { + debug!( + elapsed_ms = convert_elapsed_ms, + count = slow_convert_count, + "fill_frame_from_sample_buf exceeded {}ms threshold", + SLOW_THRESHOLD_MS + ); + } + } + + match fill_result { + Ok(()) => { + let encode_start = std::time::Instant::now(); + let owned_frame = frame_pool.take_frame(); + + if let Ok(mut encoder) = encoder_clone.lock() { + if let Err(e) = encoder.queue_frame(owned_frame, timestamp) { + warn!("Failed to encode frame: {e}"); + } + } + + let encode_elapsed_ms = encode_start.elapsed().as_millis(); + + if encode_elapsed_ms > SLOW_THRESHOLD_MS { + slow_encode_count += 1; + if slow_encode_count <= 5 || slow_encode_count % 100 == 0 { + debug!( + elapsed_ms = encode_elapsed_ms, + count = slow_encode_count, + "encoder.queue_frame exceeded {}ms threshold", + SLOW_THRESHOLD_MS + ); + } + } + } + Err(e) => { + warn!("Failed to convert frame: {e:?}"); + } + } + + total_frames += 1; + } + + if total_frames > 0 { + debug!( + total_frames = total_frames, + slow_converts = slow_convert_count, + slow_encodes = slow_encode_count, + slow_convert_pct = format!( + "{:.1}%", + 100.0 * slow_convert_count as f64 / total_frames as f64 + ), + slow_encode_pct = format!( + "{:.1}%", + 100.0 * slow_encode_count as f64 / total_frames as f64 + ), + "M4S encoder timing summary (using SegmentedVideoEncoder)" + ); + } + + Ok(()) + })?; + + ready_rx + .recv() + .map_err(|_| anyhow!("M4S encoder thread ended unexpectedly"))??; + + self.state = Some(EncoderState { + video_tx, + encoder, + encoder_handle: Some(encoder_handle), + }); + + self.started = true; + + info!( + path = %self.base_path.display(), + "Started M4S fragmented video encoder" + ); + + Ok(()) + } +} + +impl VideoMuxer for MacOSFragmentedM4SMuxer { + type VideoFrame = screen_capture::VideoFrame; + + fn send_video_frame( + &mut self, + frame: Self::VideoFrame, + timestamp: Duration, + ) -> anyhow::Result<()> { + let Some(adjusted_timestamp) = self.pause.adjust(timestamp)? else { + return Ok(()); + }; + + if !self.started { + self.start_encoder()?; + } + + if let Some(state) = &self.state { + match state + .video_tx + .try_send(Some((frame.sample_buf, adjusted_timestamp))) + { + Ok(()) => { + self.frame_drops.record_frame(); + } + Err(e) => match e { + std::sync::mpsc::TrySendError::Full(_) => { + self.frame_drops.record_drop(); + } + std::sync::mpsc::TrySendError::Disconnected(_) => { + trace!("M4S encoder channel disconnected"); + } + }, + } + } + + Ok(()) + } +} + +impl AudioMuxer for MacOSFragmentedM4SMuxer { + fn send_audio_frame(&mut self, _frame: AudioFrame, _timestamp: Duration) -> anyhow::Result<()> { + Ok(()) + } +} + +fn copy_plane_data( + src: &[u8], + dest: &mut [u8], + height: usize, + row_width: usize, + src_stride: usize, + dest_stride: usize, +) { + if src_stride == row_width && dest_stride == row_width { + let total_bytes = height * row_width; + dest[..total_bytes].copy_from_slice(&src[..total_bytes]); + } else if src_stride == dest_stride { + let total_bytes = height * src_stride; + dest[..total_bytes].copy_from_slice(&src[..total_bytes]); + } else { + for y in 0..height { + let src_row = &src[y * src_stride..y * src_stride + row_width]; + let dest_row = &mut dest[y * dest_stride..y * dest_stride + row_width]; + dest_row.copy_from_slice(src_row); + } + } +} + +struct FramePool { + frame: Option, + pixel_format: ffmpeg::format::Pixel, + width: u32, + height: u32, +} + +impl FramePool { + fn new(pixel_format: ffmpeg::format::Pixel, width: u32, height: u32) -> Self { + Self { + frame: Some(ffmpeg::frame::Video::new(pixel_format, width, height)), + pixel_format, + width, + height, + } + } + + fn get_frame(&mut self) -> &mut ffmpeg::frame::Video { + if self.frame.is_none() { + self.frame = Some(ffmpeg::frame::Video::new( + self.pixel_format, + self.width, + self.height, + )); + } + self.frame.as_mut().unwrap() + } + + fn take_frame(&mut self) -> ffmpeg::frame::Video { + self.frame.take().unwrap_or_else(|| { + ffmpeg::frame::Video::new(self.pixel_format, self.width, self.height) + }) + } +} + +fn fill_frame_from_sample_buf( + sample_buf: &cidre::cm::SampleBuf, + frame: &mut ffmpeg::frame::Video, +) -> Result<(), SampleBufConversionError> { + use cidre::cv::{self, pixel_buffer::LockFlags}; + + let Some(image_buf_ref) = sample_buf.image_buf() else { + return Err(SampleBufConversionError::NoImageBuffer); + }; + let mut image_buf = image_buf_ref.retained(); + + let width = image_buf.width(); + let height = image_buf.height(); + let pixel_format = image_buf.pixel_format(); + let plane0_stride = image_buf.plane_bytes_per_row(0); + let plane1_stride = image_buf.plane_bytes_per_row(1); + + let bytes_lock = BaseAddrLockGuard::lock(image_buf.as_mut(), LockFlags::READ_ONLY) + .map_err(SampleBufConversionError::BaseAddrLock)?; + + match pixel_format { + cv::PixelFormat::_420V => { + let dest_stride0 = frame.stride(0); + let dest_stride1 = frame.stride(1); + + copy_plane_data( + bytes_lock.plane_data(0), + frame.data_mut(0), + height, + width, + plane0_stride, + dest_stride0, + ); + + copy_plane_data( + bytes_lock.plane_data(1), + frame.data_mut(1), + height / 2, + width, + plane1_stride, + dest_stride1, + ); + } + cv::PixelFormat::_32_BGRA => { + let row_width = width * 4; + let dest_stride = frame.stride(0); + copy_plane_data( + bytes_lock.plane_data(0), + frame.data_mut(0), + height, + row_width, + plane0_stride, + dest_stride, + ); + } + cv::PixelFormat::_2VUY => { + let row_width = width * 2; + let dest_stride = frame.stride(0); + copy_plane_data( + bytes_lock.plane_data(0), + frame.data_mut(0), + height, + row_width, + plane0_stride, + dest_stride, + ); + } + format => return Err(SampleBufConversionError::UnsupportedFormat(format)), + } + + Ok(()) +} + +#[derive(Debug)] +#[allow(dead_code)] +enum SampleBufConversionError { + UnsupportedFormat(cidre::cv::PixelFormat), + BaseAddrLock(cidre::os::Error), + NoImageBuffer, +} + +struct BaseAddrLockGuard<'a>( + &'a mut cidre::cv::ImageBuf, + cidre::cv::pixel_buffer::LockFlags, +); + +impl<'a> BaseAddrLockGuard<'a> { + fn lock( + image_buf: &'a mut cidre::cv::ImageBuf, + flags: cidre::cv::pixel_buffer::LockFlags, + ) -> cidre::os::Result { + unsafe { image_buf.lock_base_addr(flags) }.result()?; + Ok(Self(image_buf, flags)) + } + + fn plane_data(&self, index: usize) -> &[u8] { + let base_addr = self.0.plane_base_address(index); + let plane_size = self.0.plane_bytes_per_row(index); + unsafe { std::slice::from_raw_parts(base_addr, plane_size * self.0.plane_height(index)) } + } +} + +impl Drop for BaseAddrLockGuard<'_> { + fn drop(&mut self) { + let _ = unsafe { self.0.unlock_lock_base_addr(self.1) }; + } +} + +pub struct MacOSFragmentedM4SCameraMuxer { + base_path: PathBuf, + video_config: VideoInfo, + segment_duration: Duration, + state: Option, + pause: PauseTracker, + frame_drops: FrameDropTracker, + started: bool, +} + +pub struct MacOSFragmentedM4SCameraMuxerConfig { + pub segment_duration: Duration, + pub preset: H264Preset, + pub output_size: Option<(u32, u32)>, +} + +impl Default for MacOSFragmentedM4SCameraMuxerConfig { + fn default() -> Self { + Self { + segment_duration: Duration::from_secs(3), + preset: H264Preset::Ultrafast, + output_size: None, + } + } +} + +impl Muxer for MacOSFragmentedM4SCameraMuxer { + type Config = MacOSFragmentedM4SCameraMuxerConfig; + + async fn setup( + config: Self::Config, + output_path: PathBuf, + video_config: Option, + _audio_config: Option, + pause_flag: Arc, + _tasks: &mut TaskPool, + ) -> anyhow::Result + where + Self: Sized, + { + let video_config = + video_config.ok_or_else(|| anyhow!("invariant: video config expected for camera"))?; + + std::fs::create_dir_all(&output_path).with_context(|| { + format!("Failed to create camera segments directory: {output_path:?}") + })?; + + Ok(Self { + base_path: output_path, + video_config, + segment_duration: config.segment_duration, + state: None, + pause: PauseTracker::new(pause_flag), + frame_drops: FrameDropTracker::new(), + started: false, + }) + } + + fn stop(&mut self) { + if let Some(state) = &self.state + && let Err(e) = state.video_tx.send(None) + { + trace!("M4S camera encoder channel already closed during stop: {e}"); + } + } + + fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { + if let Some(mut state) = self.state.take() { + if let Err(e) = state.video_tx.send(None) { + trace!("M4S camera encoder channel already closed during finish: {e}"); + } + + if let Some(handle) = state.encoder_handle.take() { + let timeout = Duration::from_secs(5); + let start = std::time::Instant::now(); + loop { + if handle.is_finished() { + if let Err(panic_payload) = handle.join() { + warn!( + "M4S camera encoder thread panicked during finish: {:?}", + panic_payload + ); + } + break; + } + if start.elapsed() > timeout { + warn!( + "M4S camera encoder thread did not finish within {:?}, abandoning", + timeout + ); + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + } + + if let Ok(mut encoder) = state.encoder.lock() { + if let Err(e) = encoder.finish_with_timestamp(timestamp) { + warn!("Failed to finish camera segmented encoder: {e}"); + } + } + } + + Ok(Ok(())) + } +} + +impl MacOSFragmentedM4SCameraMuxer { + fn start_encoder(&mut self) -> anyhow::Result<()> { + let buffer_size = get_muxer_buffer_size(); + debug!( + buffer_size = buffer_size, + "M4S camera muxer encoder channel buffer size" + ); + + let (video_tx, video_rx) = + sync_channel::, Duration)>>(buffer_size); + let (ready_tx, ready_rx) = sync_channel::>(1); + + let encoder_config = SegmentedVideoEncoderConfig { + segment_duration: self.segment_duration, + preset: H264Preset::Ultrafast, + bpp: H264EncoderBuilder::QUALITY_BPP, + output_size: None, + }; + + let encoder = + SegmentedVideoEncoder::init(self.base_path.clone(), self.video_config, encoder_config)?; + let encoder = Arc::new(Mutex::new(encoder)); + let encoder_clone = encoder.clone(); + let video_config = self.video_config; + + let encoder_handle = std::thread::Builder::new() + .name("m4s-camera-segment-encoder".to_string()) + .spawn(move || { + let pixel_format = match video_config.pixel_format { + cap_media_info::Pixel::NV12 => ffmpeg::format::Pixel::NV12, + cap_media_info::Pixel::BGRA => ffmpeg::format::Pixel::BGRA, + cap_media_info::Pixel::UYVY422 => ffmpeg::format::Pixel::UYVY422, + _ => ffmpeg::format::Pixel::NV12, + }; + + let mut frame_pool = + FramePool::new(pixel_format, video_config.width, video_config.height); + + if ready_tx.send(Ok(())).is_err() { + return Err(anyhow!( + "Failed to send ready signal - camera receiver dropped" + )); + } + + let mut slow_convert_count = 0u32; + let mut slow_encode_count = 0u32; + let mut total_frames = 0u64; + const SLOW_THRESHOLD_MS: u128 = 5; + + while let Ok(Some((sample_buf, timestamp))) = video_rx.recv() { + let convert_start = std::time::Instant::now(); + let frame = frame_pool.get_frame(); + let fill_result = fill_frame_from_sample_buf(&sample_buf, frame); + let convert_elapsed_ms = convert_start.elapsed().as_millis(); + + if convert_elapsed_ms > SLOW_THRESHOLD_MS { + slow_convert_count += 1; + if slow_convert_count <= 5 || slow_convert_count % 100 == 0 { + debug!( + elapsed_ms = convert_elapsed_ms, + count = slow_convert_count, + "Camera fill_frame_from_sample_buf exceeded {}ms threshold", + SLOW_THRESHOLD_MS + ); + } + } + + match fill_result { + Ok(()) => { + let encode_start = std::time::Instant::now(); + let owned_frame = frame_pool.take_frame(); + + if let Ok(mut encoder) = encoder_clone.lock() { + if let Err(e) = encoder.queue_frame(owned_frame, timestamp) { + warn!("Failed to encode camera frame: {e}"); + } + } + + let encode_elapsed_ms = encode_start.elapsed().as_millis(); + + if encode_elapsed_ms > SLOW_THRESHOLD_MS { + slow_encode_count += 1; + if slow_encode_count <= 5 || slow_encode_count % 100 == 0 { + debug!( + elapsed_ms = encode_elapsed_ms, + count = slow_encode_count, + "Camera encoder.queue_frame exceeded {}ms threshold", + SLOW_THRESHOLD_MS + ); + } + } + } + Err(e) => { + warn!("Failed to convert camera frame: {e:?}"); + } + } + + total_frames += 1; + } + + if total_frames > 0 { + debug!( + total_frames = total_frames, + slow_converts = slow_convert_count, + slow_encodes = slow_encode_count, + slow_convert_pct = format!( + "{:.1}%", + 100.0 * slow_convert_count as f64 / total_frames as f64 + ), + slow_encode_pct = format!( + "{:.1}%", + 100.0 * slow_encode_count as f64 / total_frames as f64 + ), + "M4S camera encoder timing summary" + ); + } + + Ok(()) + })?; + + ready_rx + .recv() + .map_err(|_| anyhow!("M4S camera encoder thread ended unexpectedly"))??; + + self.state = Some(EncoderState { + video_tx, + encoder, + encoder_handle: Some(encoder_handle), + }); + + self.started = true; + + info!( + path = %self.base_path.display(), + "Started M4S fragmented camera encoder" + ); + + Ok(()) + } +} + +impl VideoMuxer for MacOSFragmentedM4SCameraMuxer { + type VideoFrame = NativeCameraFrame; + + fn send_video_frame( + &mut self, + frame: Self::VideoFrame, + timestamp: Duration, + ) -> anyhow::Result<()> { + let Some(adjusted_timestamp) = self.pause.adjust(timestamp)? else { + return Ok(()); + }; + + if !self.started { + self.start_encoder()?; + } + + if let Some(state) = &self.state { + match state + .video_tx + .try_send(Some((frame.sample_buf, adjusted_timestamp))) + { + Ok(()) => { + self.frame_drops.record_frame(); + } + Err(e) => match e { + std::sync::mpsc::TrySendError::Full(_) => { + self.frame_drops.record_drop(); + } + std::sync::mpsc::TrySendError::Disconnected(_) => { + trace!("M4S camera encoder channel disconnected"); + } + }, + } + } + + Ok(()) + } +} + +impl AudioMuxer for MacOSFragmentedM4SCameraMuxer { + fn send_audio_frame(&mut self, _frame: AudioFrame, _timestamp: Duration) -> anyhow::Result<()> { + Ok(()) + } +} diff --git a/crates/recording/src/output_pipeline/macos_segmented_ffmpeg.rs b/crates/recording/src/output_pipeline/macos_segmented_ffmpeg.rs deleted file mode 100644 index ad128644d9..0000000000 --- a/crates/recording/src/output_pipeline/macos_segmented_ffmpeg.rs +++ /dev/null @@ -1,746 +0,0 @@ -use crate::{AudioFrame, AudioMuxer, Muxer, TaskPool, VideoMuxer, fragmentation, screen_capture}; -use anyhow::{Context, anyhow}; -use cap_media_info::{AudioInfo, VideoInfo}; -use serde::Serialize; -use std::{ - path::PathBuf, - sync::{ - Arc, Mutex, - atomic::{AtomicBool, Ordering}, - mpsc::{SyncSender, sync_channel}, - }, - thread::JoinHandle, - time::Duration, -}; -use tracing::*; - -#[derive(Debug, Clone)] -pub struct SegmentInfo { - pub path: PathBuf, - pub index: u32, - pub duration: Duration, - pub file_size: Option, -} - -#[derive(Serialize)] -struct FragmentEntry { - path: String, - index: u32, - duration: f64, - is_complete: bool, - #[serde(skip_serializing_if = "Option::is_none")] - file_size: Option, -} - -const MANIFEST_VERSION: u32 = 2; - -#[derive(Serialize)] -struct Manifest { - version: u32, - fragments: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - total_duration: Option, - is_complete: bool, -} - -struct SegmentState { - video_tx: SyncSender, Duration)>>, - output: Arc>, - encoder_handle: Option>>, -} - -struct PauseTracker { - flag: Arc, - paused_at: Option, - offset: Duration, -} - -struct FrameDropTracker { - count: u32, - last_warning: std::time::Instant, -} - -impl FrameDropTracker { - fn new() -> Self { - Self { - count: 0, - last_warning: std::time::Instant::now(), - } - } - - fn record_drop(&mut self) { - self.count += 1; - if self.count >= 30 && self.last_warning.elapsed() > Duration::from_secs(5) { - warn!( - "Dropped {} screen frames due to encoder backpressure", - self.count - ); - self.count = 0; - self.last_warning = std::time::Instant::now(); - } - } - - fn reset(&mut self) { - if self.count > 0 { - trace!("Frame drop count at segment boundary: {}", self.count); - } - self.count = 0; - } -} - -impl PauseTracker { - fn new(flag: Arc) -> Self { - Self { - flag, - paused_at: None, - offset: Duration::ZERO, - } - } - - fn adjust(&mut self, timestamp: Duration) -> anyhow::Result> { - if self.flag.load(Ordering::Relaxed) { - if self.paused_at.is_none() { - self.paused_at = Some(timestamp); - } - return Ok(None); - } - - if let Some(start) = self.paused_at.take() { - let delta = timestamp.checked_sub(start).ok_or_else(|| { - anyhow!( - "Frame timestamp went backward during unpause (resume={start:?}, current={timestamp:?})" - ) - })?; - - self.offset = self.offset.checked_add(delta).ok_or_else(|| { - anyhow!( - "Pause offset overflow (offset={:?}, delta={delta:?})", - self.offset - ) - })?; - } - - let adjusted = timestamp.checked_sub(self.offset).ok_or_else(|| { - anyhow!( - "Adjusted timestamp underflow (timestamp={timestamp:?}, offset={:?})", - self.offset - ) - })?; - - Ok(Some(adjusted)) - } -} - -pub struct MacOSSegmentedMuxer { - base_path: PathBuf, - segment_duration: Duration, - current_index: u32, - segment_start_time: Option, - completed_segments: Vec, - pending_segments: Arc>>, - - current_state: Option, - - video_config: VideoInfo, - - pause: PauseTracker, - frame_drops: FrameDropTracker, -} - -pub struct MacOSSegmentedMuxerConfig { - pub segment_duration: Duration, -} - -impl Default for MacOSSegmentedMuxerConfig { - fn default() -> Self { - Self { - segment_duration: Duration::from_secs(3), - } - } -} - -impl Muxer for MacOSSegmentedMuxer { - type Config = MacOSSegmentedMuxerConfig; - - async fn setup( - config: Self::Config, - output_path: PathBuf, - video_config: Option, - _audio_config: Option, - pause_flag: Arc, - _tasks: &mut TaskPool, - ) -> anyhow::Result - where - Self: Sized, - { - let video_config = - video_config.ok_or_else(|| anyhow!("invariant: video config expected"))?; - - std::fs::create_dir_all(&output_path) - .with_context(|| format!("Failed to create segments directory: {output_path:?}"))?; - - Ok(Self { - base_path: output_path, - segment_duration: config.segment_duration, - current_index: 0, - segment_start_time: None, - completed_segments: Vec::new(), - pending_segments: Arc::new(Mutex::new(Vec::new())), - current_state: None, - video_config, - pause: PauseTracker::new(pause_flag), - frame_drops: FrameDropTracker::new(), - }) - } - - fn stop(&mut self) { - if let Some(state) = &self.current_state - && let Err(e) = state.video_tx.send(None) - { - trace!("Screen encoder channel already closed during stop: {e}"); - } - } - - fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - self.collect_pending_segments(); - - let segment_path = self.current_segment_path(); - let segment_start = self.segment_start_time; - let current_index = self.current_index; - - if let Some(mut state) = self.current_state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("Screen encoder channel already closed during finish: {e}"); - } - - if let Some(handle) = state.encoder_handle.take() { - let timeout = Duration::from_secs(5); - let start = std::time::Instant::now(); - loop { - if handle.is_finished() { - if let Err(panic_payload) = handle.join() { - warn!( - "Screen encoder thread panicked during finish: {:?}", - panic_payload - ); - } - break; - } - if start.elapsed() > timeout { - warn!( - "Screen encoder thread did not finish within {:?}, abandoning", - timeout - ); - break; - } - std::thread::sleep(Duration::from_millis(50)); - } - } - - if let Ok(mut output) = state.output.lock() - && let Err(e) = output.write_trailer() - { - warn!("Failed to write trailer for segment {current_index}: {e}"); - } - - fragmentation::sync_file(&segment_path); - - if let Some(start) = segment_start { - let final_duration = timestamp.saturating_sub(start); - let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); - - self.completed_segments.push(SegmentInfo { - path: segment_path, - index: current_index, - duration: final_duration, - file_size, - }); - } - } - - self.finalize_manifest(); - - Ok(Ok(())) - } -} - -impl MacOSSegmentedMuxer { - fn current_segment_path(&self) -> PathBuf { - self.base_path - .join(format!("fragment_{:03}.mp4", self.current_index)) - } - - fn write_manifest(&self) { - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(), - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write manifest to {}: {e}", - manifest_path.display() - ); - } - } - - fn finalize_manifest(&self) { - let total_duration: Duration = self.completed_segments.iter().map(|s| s.duration).sum(); - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(), - total_duration: Some(total_duration.as_secs_f64()), - is_complete: true, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write final manifest to {}: {e}", - manifest_path.display() - ); - } - } - - fn collect_pending_segments(&mut self) { - if let Ok(mut pending) = self.pending_segments.lock() { - for segment in pending.drain(..) { - self.completed_segments.push(segment); - } - self.completed_segments.sort_by_key(|s| s.index); - } - } - - fn create_segment(&mut self) -> anyhow::Result<()> { - let segment_path = self.current_segment_path(); - - let (video_tx, video_rx) = - sync_channel::, Duration)>>(8); - let (ready_tx, ready_rx) = sync_channel::>(1); - let output = ffmpeg::format::output(&segment_path)?; - let output = Arc::new(Mutex::new(output)); - - let video_config = self.video_config; - let output_clone = output.clone(); - - let encoder_handle = std::thread::Builder::new() - .name(format!("segment-encoder-{}", self.current_index)) - .spawn(move || { - let encoder = (|| { - let mut output_guard = match output_clone.lock() { - Ok(guard) => guard, - Err(poisoned) => { - return Err(anyhow!( - "MacOSSegmentedEncoder: failed to lock output mutex: {}", - poisoned - )); - } - }; - - cap_enc_ffmpeg::h264::H264Encoder::builder(video_config) - .build(&mut output_guard) - .map_err(|e| anyhow!("MacOSSegmentedEncoder/{e}")) - })(); - - let mut encoder = match encoder { - Ok(encoder) => { - if ready_tx.send(Ok(())).is_err() { - error!("Failed to send ready signal - receiver dropped"); - return Ok(()); - } - encoder - } - Err(e) => { - error!("Encoder setup failed: {:#}", e); - if let Err(send_err) = ready_tx.send(Err(anyhow!("{e}"))) { - error!("failed to send ready_tx error: {send_err}"); - } - return Err(anyhow!("{e}")); - } - }; - - let mut first_timestamp: Option = None; - - while let Ok(Some((sample_buf, timestamp))) = video_rx.recv() { - let Ok(mut output) = output_clone.lock() else { - continue; - }; - - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; - - let frame = sample_buf_to_ffmpeg_frame(&sample_buf); - - match frame { - Ok(frame) => { - if let Err(e) = encoder.queue_frame(frame, relative, &mut output) { - warn!("Failed to encode frame: {e}"); - } - } - Err(e) => { - warn!("Failed to convert frame: {e:?}"); - } - } - } - - if let Ok(mut output) = output_clone.lock() - && let Err(e) = encoder.flush(&mut output) - { - warn!("Failed to flush encoder: {e}"); - } - - drop(encoder); - - Ok(()) - })?; - - ready_rx - .recv() - .map_err(|_| anyhow!("Encoder thread ended unexpectedly"))??; - - output - .lock() - .map_err(|_| anyhow!("output mutex poisoned when writing header"))? - .write_header()?; - - self.current_state = Some(SegmentState { - video_tx, - output, - encoder_handle: Some(encoder_handle), - }); - - Ok(()) - } - - fn rotate_segment(&mut self, timestamp: Duration) -> anyhow::Result<()> { - self.collect_pending_segments(); - - let segment_start = self.segment_start_time.unwrap_or(Duration::ZERO); - let segment_duration = timestamp.saturating_sub(segment_start); - let completed_segment_path = self.current_segment_path(); - let current_index = self.current_index; - - if let Some(mut state) = self.current_state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("Screen encoder channel already closed during rotation: {e}"); - } - - let output = state.output.clone(); - let encoder_handle = state.encoder_handle.take(); - let path_for_sync = completed_segment_path.clone(); - let pending_segments = self.pending_segments.clone(); - - std::thread::spawn(move || { - if let Some(handle) = encoder_handle { - let timeout = Duration::from_secs(5); - let start = std::time::Instant::now(); - loop { - if handle.is_finished() { - if let Err(panic_payload) = handle.join() { - warn!( - "Screen encoder thread panicked during rotation: {:?}", - panic_payload - ); - } - break; - } - if start.elapsed() > timeout { - warn!( - "Screen encoder thread did not finish within {:?} during rotation, abandoning", - timeout - ); - break; - } - std::thread::sleep(Duration::from_millis(50)); - } - } - - if let Ok(mut output) = output.lock() - && let Err(e) = output.write_trailer() - { - warn!("Failed to write trailer for segment {current_index}: {e}"); - } - - fragmentation::sync_file(&path_for_sync); - - let file_size = std::fs::metadata(&path_for_sync).ok().map(|m| m.len()); - - if let Ok(mut pending) = pending_segments.lock() { - pending.push(SegmentInfo { - path: path_for_sync, - index: current_index, - duration: segment_duration, - file_size, - }); - } - }); - - self.write_manifest(); - } - - self.frame_drops.reset(); - self.current_index += 1; - self.segment_start_time = Some(timestamp); - - self.create_segment()?; - self.write_in_progress_manifest(); - - info!( - "Rotated to segment {} at {:?}", - self.current_index, timestamp - ); - - Ok(()) - } - - fn write_in_progress_manifest(&self) { - let mut fragments: Vec = self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(); - - fragments.push(FragmentEntry { - path: self - .current_segment_path() - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: self.current_index, - duration: 0.0, - is_complete: false, - file_size: None, - }); - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments, - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write in-progress manifest to {}: {e}", - manifest_path.display() - ); - } - } -} - -impl VideoMuxer for MacOSSegmentedMuxer { - type VideoFrame = screen_capture::VideoFrame; - - fn send_video_frame( - &mut self, - frame: Self::VideoFrame, - timestamp: Duration, - ) -> anyhow::Result<()> { - let Some(adjusted_timestamp) = self.pause.adjust(timestamp)? else { - return Ok(()); - }; - - if self.current_state.is_none() { - self.segment_start_time = Some(adjusted_timestamp); - self.create_segment()?; - self.write_in_progress_manifest(); - } - - if self.segment_start_time.is_none() { - self.segment_start_time = Some(adjusted_timestamp); - } - - let segment_elapsed = - adjusted_timestamp.saturating_sub(self.segment_start_time.unwrap_or(Duration::ZERO)); - - if segment_elapsed >= self.segment_duration { - self.rotate_segment(adjusted_timestamp)?; - } - - if let Some(state) = &self.current_state - && let Err(e) = state - .video_tx - .try_send(Some((frame.sample_buf, adjusted_timestamp))) - { - match e { - std::sync::mpsc::TrySendError::Full(_) => { - self.frame_drops.record_drop(); - } - std::sync::mpsc::TrySendError::Disconnected(_) => { - trace!("Screen encoder channel disconnected"); - } - } - } - - Ok(()) - } -} - -impl AudioMuxer for MacOSSegmentedMuxer { - fn send_audio_frame(&mut self, _frame: AudioFrame, _timestamp: Duration) -> anyhow::Result<()> { - Ok(()) - } -} - -fn sample_buf_to_ffmpeg_frame( - sample_buf: &cidre::cm::SampleBuf, -) -> Result { - use cidre::cv::{self, pixel_buffer::LockFlags}; - - let Some(image_buf_ref) = sample_buf.image_buf() else { - return Err(SampleBufConversionError::NoImageBuffer); - }; - let mut image_buf = image_buf_ref.retained(); - - let width = image_buf.width(); - let height = image_buf.height(); - let pixel_format = image_buf.pixel_format(); - let plane0_stride = image_buf.plane_bytes_per_row(0); - let plane1_stride = image_buf.plane_bytes_per_row(1); - - let bytes_lock = BaseAddrLockGuard::lock(image_buf.as_mut(), LockFlags::READ_ONLY) - .map_err(SampleBufConversionError::BaseAddrLock)?; - - Ok(match pixel_format { - cv::PixelFormat::_420V => { - let mut ff_frame = - ffmpeg::frame::Video::new(ffmpeg::format::Pixel::NV12, width as u32, height as u32); - - let src_stride = plane0_stride; - let dest_stride = ff_frame.stride(0); - - let src_bytes = bytes_lock.plane_data(0); - let dest_bytes = &mut ff_frame.data_mut(0); - - for y in 0..height { - let row_width = width; - let src_row = &src_bytes[y * src_stride..y * src_stride + row_width]; - let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width]; - - dest_row.copy_from_slice(src_row); - } - - let src_stride = plane1_stride; - let dest_stride = ff_frame.stride(1); - - let src_bytes = bytes_lock.plane_data(1); - let dest_bytes = &mut ff_frame.data_mut(1); - - for y in 0..height / 2 { - let row_width = width; - let src_row = &src_bytes[y * src_stride..y * src_stride + row_width]; - let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width]; - - dest_row.copy_from_slice(src_row); - } - - ff_frame - } - cv::PixelFormat::_32_BGRA => { - let mut ff_frame = - ffmpeg::frame::Video::new(ffmpeg::format::Pixel::BGRA, width as u32, height as u32); - - let src_stride = plane0_stride; - let dest_stride = ff_frame.stride(0); - - let src_bytes = bytes_lock.plane_data(0); - let dest_bytes = &mut ff_frame.data_mut(0); - - for y in 0..height { - let row_width = width * 4; - let src_row = &src_bytes[y * src_stride..y * src_stride + row_width]; - let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width]; - - dest_row.copy_from_slice(src_row); - } - - ff_frame - } - format => return Err(SampleBufConversionError::UnsupportedFormat(format)), - }) -} - -#[derive(Debug)] -pub enum SampleBufConversionError { - UnsupportedFormat(cidre::cv::PixelFormat), - BaseAddrLock(cidre::os::Error), - NoImageBuffer, -} - -struct BaseAddrLockGuard<'a>( - &'a mut cidre::cv::ImageBuf, - cidre::cv::pixel_buffer::LockFlags, -); - -impl<'a> BaseAddrLockGuard<'a> { - fn lock( - image_buf: &'a mut cidre::cv::ImageBuf, - flags: cidre::cv::pixel_buffer::LockFlags, - ) -> cidre::os::Result { - unsafe { image_buf.lock_base_addr(flags) }.result()?; - Ok(Self(image_buf, flags)) - } - - fn plane_data(&self, index: usize) -> &[u8] { - let base_addr = self.0.plane_base_address(index); - let plane_size = self.0.plane_bytes_per_row(index); - unsafe { std::slice::from_raw_parts(base_addr, plane_size * self.0.plane_height(index)) } - } -} - -impl Drop for BaseAddrLockGuard<'_> { - fn drop(&mut self) { - let _ = unsafe { self.0.unlock_lock_base_addr(self.1) }; - } -} diff --git a/crates/recording/src/output_pipeline/mod.rs b/crates/recording/src/output_pipeline/mod.rs index bf1df859ef..2f2bf6d2f1 100644 --- a/crates/recording/src/output_pipeline/mod.rs +++ b/crates/recording/src/output_pipeline/mod.rs @@ -2,17 +2,13 @@ mod async_camera; mod core; pub mod ffmpeg; #[cfg(target_os = "macos")] -mod fragmented; -#[cfg(target_os = "macos")] -mod macos_segmented_ffmpeg; +mod macos_fragmented_m4s; pub use async_camera::*; pub use core::*; pub use ffmpeg::*; #[cfg(target_os = "macos")] -pub use fragmented::*; -#[cfg(target_os = "macos")] -pub use macos_segmented_ffmpeg::*; +pub use macos_fragmented_m4s::*; #[cfg(target_os = "macos")] mod macos; diff --git a/crates/recording/src/recovery.rs b/crates/recording/src/recovery.rs index 4490ac1c53..96f24578bb 100644 --- a/crates/recording/src/recovery.rs +++ b/crates/recording/src/recovery.rs @@ -4,8 +4,9 @@ use std::{ }; use cap_enc_ffmpeg::remux::{ - concatenate_audio_to_ogg, concatenate_video_fragments, get_media_duration, get_video_fps, - probe_media_valid, probe_video_can_decode, + concatenate_audio_to_ogg, concatenate_m4s_segments_with_init, concatenate_video_fragments, + get_media_duration, get_video_fps, probe_m4s_can_decode_with_init, probe_media_valid, + probe_video_can_decode, }; use cap_project::{ AudioMeta, Cursors, MultipleSegment, MultipleSegments, ProjectConfiguration, RecordingMeta, @@ -27,7 +28,9 @@ pub struct IncompleteRecording { pub struct RecoverableSegment { pub index: u32, pub display_fragments: Vec, + pub display_init_segment: Option, pub camera_fragments: Option>, + pub camera_init_segment: Option, pub mic_fragments: Option>, pub system_audio_fragments: Option>, pub cursor_path: Option, @@ -39,6 +42,12 @@ pub struct RecoveredRecording { pub meta: StudioRecordingMeta, } +#[derive(Debug, Clone)] +struct FragmentsInfo { + fragments: Vec, + init_segment: Option, +} + #[derive(Debug, thiserror::Error)] pub enum RecoveryError { #[error("IO error: {0}")] @@ -133,13 +142,16 @@ impl RecoveryManager { let segment_path = segment_entry.path(); let display_dir = segment_path.join("display"); - let mut display_fragments = Self::find_complete_fragments(&display_dir); + let display_info = Self::find_complete_fragments_with_init(&display_dir); + let mut display_fragments = display_info.fragments; + let mut display_init_segment = display_info.init_segment; if display_fragments.is_empty() && let Some(display_mp4) = Self::probe_single_file(&segment_path.join("display.mp4")) { display_fragments = vec![display_mp4]; + display_init_segment = None; } if display_fragments.is_empty() { @@ -151,12 +163,15 @@ impl RecoveryManager { } let camera_dir = segment_path.join("camera"); - let camera_fragments = { - let frags = Self::find_complete_fragments(&camera_dir); - if frags.is_empty() { - Self::probe_single_file(&segment_path.join("camera.mp4")).map(|p| vec![p]) + let (camera_fragments, camera_init_segment) = { + let camera_info = Self::find_complete_fragments_with_init(&camera_dir); + if camera_info.fragments.is_empty() { + ( + Self::probe_single_file(&segment_path.join("camera.mp4")).map(|p| vec![p]), + None, + ) } else { - Some(frags) + (Some(camera_info.fragments), camera_info.init_segment) } }; @@ -173,7 +188,9 @@ impl RecoveryManager { recoverable_segments.push(RecoverableSegment { index: index as u32, display_fragments, + display_init_segment, camera_fragments, + camera_init_segment, mic_fragments, system_audio_fragments, cursor_path, @@ -201,6 +218,10 @@ impl RecoveryManager { } fn find_complete_fragments(dir: &Path) -> Vec { + Self::find_complete_fragments_with_init(dir).fragments + } + + fn find_complete_fragments_with_init(dir: &Path) -> FragmentsInfo { use crate::fragmentation::CURRENT_MANIFEST_VERSION; let manifest_path = dir.join("manifest.json"); @@ -208,82 +229,141 @@ impl RecoveryManager { if manifest_path.exists() && let Ok(content) = std::fs::read_to_string(&manifest_path) && let Ok(manifest) = serde_json::from_str::(&content) - && let Some(fragments) = manifest.get("fragments").and_then(|f| f.as_array()) { let manifest_version = manifest .get("version") .and_then(|v| v.as_u64()) .unwrap_or(1) as u32; - if manifest_version > CURRENT_MANIFEST_VERSION { + + let manifest_type = manifest + .get("type") + .and_then(|t| t.as_str()) + .unwrap_or("fragments"); + + let max_supported_version = if manifest_type == "m4s_segments" { + 4 + } else { + CURRENT_MANIFEST_VERSION + }; + + if manifest_version > max_supported_version { warn!( - "Manifest version {} is newer than supported {}", - manifest_version, CURRENT_MANIFEST_VERSION + "Manifest version {} is newer than supported {} for type {}", + manifest_version, max_supported_version, manifest_type ); } - let expected_file_size = |f: &serde_json::Value| -> Option { - f.get("file_size").and_then(|s| s.as_u64()) + let init_segment = manifest + .get("init_segment") + .and_then(|i| i.as_str()) + .map(|name| dir.join(name)) + .filter(|p| p.exists()); + + let entries = if manifest_type == "m4s_segments" { + manifest.get("segments").and_then(|s| s.as_array()) + } else { + manifest.get("fragments").and_then(|f| f.as_array()) }; - let result: Vec = fragments - .iter() - .filter(|f| { - f.get("is_complete") - .and_then(|c| c.as_bool()) - .unwrap_or(false) - }) - .filter_map(|f| { - let path_str = f.get("path").and_then(|p| p.as_str())?; - let path = dir.join(path_str); - if !path.exists() { - return None; - } + if let Some(entries) = entries { + let expected_file_size = |f: &serde_json::Value| -> Option { + f.get("file_size").and_then(|s| s.as_u64()) + }; + + let result: Vec = entries + .iter() + .filter(|f| { + f.get("is_complete") + .and_then(|c| c.as_bool()) + .unwrap_or(false) + }) + .filter_map(|f| { + let path_str = f.get("path").and_then(|p| p.as_str())?; + let path = dir.join(path_str); + if !path.exists() { + return None; + } - if let Some(expected_size) = expected_file_size(f) - && let Ok(metadata) = std::fs::metadata(&path) - && metadata.len() != expected_size - { - warn!( - "Fragment {} size mismatch: expected {}, got {}", - path.display(), - expected_size, - metadata.len() - ); - return None; - } + if let Some(expected_size) = expected_file_size(f) + && let Ok(metadata) = std::fs::metadata(&path) + && metadata.len() != expected_size + { + warn!( + "Fragment {} size mismatch: expected {}, got {}", + path.display(), + expected_size, + metadata.len() + ); + return None; + } - if Self::is_video_file(&path) { - match probe_video_can_decode(&path) { - Ok(true) => Some(path), - Ok(false) => { - warn!("Fragment {} has no decodable frames", path.display()); - None - } - Err(e) => { - warn!("Fragment {} validation failed: {}", path.display(), e); - None + if Self::is_video_file(&path) { + if let Some(init_path) = &init_segment { + match probe_m4s_can_decode_with_init(init_path, &path) { + Ok(true) => Some(path), + Ok(false) => { + warn!( + "M4S segment {} has no decodable frames (with init)", + path.display() + ); + None + } + Err(e) => { + warn!( + "M4S segment {} validation failed: {}", + path.display(), + e + ); + None + } + } + } else { + match probe_video_can_decode(&path) { + Ok(true) => Some(path), + Ok(false) => { + warn!( + "Fragment {} has no decodable frames", + path.display() + ); + None + } + Err(e) => { + warn!( + "Fragment {} validation failed: {}", + path.display(), + e + ); + None + } + } } + } else if probe_media_valid(&path) { + Some(path) + } else { + warn!("Fragment {} is not valid media", path.display()); + None } - } else if probe_media_valid(&path) { - Some(path) - } else { - warn!("Fragment {} is not valid media", path.display()); - None - } - }) - .collect(); - - if !result.is_empty() { - return result; + }) + .collect(); + + if !result.is_empty() { + return FragmentsInfo { + fragments: result, + init_segment, + }; + } } } - Self::probe_fragments_in_dir(dir) + FragmentsInfo { + fragments: Self::probe_fragments_in_dir(dir), + init_segment: None, + } } fn is_video_file(path: &Path) -> bool { path.extension() - .map(|e| e.eq_ignore_ascii_case("mp4")) + .map(|e| e.eq_ignore_ascii_case("mp4") || e.eq_ignore_ascii_case("m4s")) .unwrap_or(false) } @@ -301,7 +381,7 @@ impl RecoveryManager { .and_then(|e| e.to_str()) .map(|e| e.to_lowercase()); match ext.as_deref() { - Some("mp4") => match probe_video_can_decode(p) { + Some("mp4") | Some("m4s") => match probe_video_can_decode(p) { Ok(true) => true, Ok(false) => { debug!("Skipping {} - no decodable frames", p.display()); @@ -398,7 +478,7 @@ impl RecoveryManager { .join(format!("segment-{}", segment.index)); let display_output = segment_dir.join("display.mp4"); - if segment.display_fragments.len() == 1 { + if segment.display_fragments.len() == 1 && segment.display_init_segment.is_none() { let source = &segment.display_fragments[0]; if source != &display_output { info!("Moving single display fragment to {:?}", display_output); @@ -410,20 +490,39 @@ impl RecoveryManager { debug!("Failed to clean up display dir {:?}: {e}", display_dir); } } - } else if segment.display_fragments.len() > 1 { - info!( - "Concatenating {} display fragments to {:?}", - segment.display_fragments.len(), - display_output - ); - concatenate_video_fragments(&segment.display_fragments, &display_output) + } else if !segment.display_fragments.is_empty() { + if let Some(init_path) = &segment.display_init_segment { + info!( + "Concatenating {} M4S display segments with init to {:?}", + segment.display_fragments.len(), + display_output + ); + concatenate_m4s_segments_with_init( + init_path, + &segment.display_fragments, + &display_output, + ) .map_err(RecoveryError::VideoConcat)?; + } else { + info!( + "Concatenating {} display fragments to {:?}", + segment.display_fragments.len(), + display_output + ); + concatenate_video_fragments(&segment.display_fragments, &display_output) + .map_err(RecoveryError::VideoConcat)?; + } for fragment in &segment.display_fragments { if let Err(e) = std::fs::remove_file(fragment) { debug!("Failed to remove display fragment {:?}: {e}", fragment); } } + if let Some(init_path) = &segment.display_init_segment { + if let Err(e) = std::fs::remove_file(init_path) { + debug!("Failed to remove display init segment {:?}: {e}", init_path); + } + } let display_dir = segment_dir.join("display"); if display_dir.exists() && let Err(e) = std::fs::remove_dir_all(&display_dir) @@ -434,7 +533,7 @@ impl RecoveryManager { if let Some(camera_frags) = &segment.camera_fragments { let camera_output = segment_dir.join("camera.mp4"); - if camera_frags.len() == 1 { + if camera_frags.len() == 1 && segment.camera_init_segment.is_none() { let source = &camera_frags[0]; if source != &camera_output { info!("Moving single camera fragment to {:?}", camera_output); @@ -446,20 +545,35 @@ impl RecoveryManager { debug!("Failed to clean up camera dir {:?}: {e}", camera_dir); } } - } else if camera_frags.len() > 1 { - info!( - "Concatenating {} camera fragments to {:?}", - camera_frags.len(), - camera_output - ); - concatenate_video_fragments(camera_frags, &camera_output) - .map_err(RecoveryError::VideoConcat)?; + } else if !camera_frags.is_empty() { + if let Some(init_path) = &segment.camera_init_segment { + info!( + "Concatenating {} M4S camera segments with init to {:?}", + camera_frags.len(), + camera_output + ); + concatenate_m4s_segments_with_init(init_path, camera_frags, &camera_output) + .map_err(RecoveryError::VideoConcat)?; + } else { + info!( + "Concatenating {} camera fragments to {:?}", + camera_frags.len(), + camera_output + ); + concatenate_video_fragments(camera_frags, &camera_output) + .map_err(RecoveryError::VideoConcat)?; + } for fragment in camera_frags { if let Err(e) = std::fs::remove_file(fragment) { debug!("Failed to remove camera fragment {:?}: {e}", fragment); } } + if let Some(init_path) = &segment.camera_init_segment { + if let Err(e) = std::fs::remove_file(init_path) { + debug!("Failed to remove camera init segment {:?}: {e}", init_path); + } + } let camera_dir = segment_dir.join("camera"); if camera_dir.exists() && let Err(e) = std::fs::remove_dir_all(&camera_dir) diff --git a/crates/recording/src/sources/screen_capture/macos.rs b/crates/recording/src/sources/screen_capture/macos.rs index 51916b5e25..a5322af97e 100644 --- a/crates/recording/src/sources/screen_capture/macos.rs +++ b/crates/recording/src/sources/screen_capture/macos.rs @@ -13,7 +13,7 @@ use futures::{FutureExt as _, channel::mpsc, future::BoxFuture}; use std::{ sync::{ Arc, - atomic::{self, AtomicBool, AtomicU32}, + atomic::{self, AtomicBool, AtomicU32, AtomicU64}, }, time::Duration, }; @@ -24,6 +24,20 @@ use tokio_util::{ }; use tracing::{debug, warn}; +fn get_screen_buffer_size() -> usize { + std::env::var("CAP_SCREEN_BUFFER_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(4) +} + +fn get_max_queue_depth() -> isize { + std::env::var("CAP_MAX_QUEUE_DEPTH") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(4) +} + #[derive(Debug)] pub struct CMSampleBufferCapture; @@ -31,7 +45,7 @@ impl ScreenCaptureFormat for CMSampleBufferCapture { type VideoFormat = cidre::arc::R; fn pixel_format() -> ffmpeg::format::Pixel { - ffmpeg::format::Pixel::BGRA + ffmpeg::format::Pixel::NV12 } fn audio_info() -> AudioInfo { @@ -68,8 +82,10 @@ impl ScreenCaptureConfig { &self, ) -> anyhow::Result<(VideoSourceConfig, Option)> { let (error_tx, error_rx) = broadcast::channel(1); - // Increased from 4 to 12 to provide more buffer tolerance for frame processing delays - let (video_tx, video_rx) = flume::bounded(12); + let buffer_size = get_screen_buffer_size(); + debug!(buffer_size = buffer_size, "Screen capture buffer size"); + let (video_tx, video_rx) = flume::bounded(buffer_size); + let drop_counter: Arc = Arc::new(AtomicU64::new(0)); let (mut audio_tx, audio_rx) = if self.system_audio { let (tx, rx) = mpsc::channel(32); (Some(tx), Some(rx)) @@ -129,8 +145,14 @@ impl ScreenCaptureConfig { debug!("size: {:?}", size); - let queue_depth = ((self.config.fps as f32 / 30.0 * 5.0).ceil() as isize).clamp(3, 8); - debug!("Using queue depth: {}", queue_depth); + let max_queue_depth = get_max_queue_depth(); + let queue_depth = + ((self.config.fps as f32 / 30.0 * 5.0).ceil() as isize).clamp(3, max_queue_depth); + debug!( + queue_depth = queue_depth, + max_queue_depth = max_queue_depth, + "Screen capture queue depth" + ); let mut settings = scap_screencapturekit::StreamCfgBuilder::default() .with_width(size.width() as usize) @@ -141,7 +163,7 @@ impl ScreenCaptureConfig { .with_queue_depth(queue_depth) .build(); - settings.set_pixel_format(cv::PixelFormat::_32_BGRA); + settings.set_pixel_format(cv::PixelFormat::_420V); settings.set_color_space_name(cg::color_space::names::srgb()); if let Some(crop_bounds) = self.config.crop_bounds { @@ -163,6 +185,7 @@ impl ScreenCaptureConfig { let builder = scap_screencapturekit::Capturer::builder(content_filter, settings) .with_output_sample_buf_cb({ let video_frame_count = video_frame_counter.clone(); + let drop_counter = drop_counter.clone(); move |frame| { let sample_buffer = frame.sample_buf(); @@ -185,10 +208,15 @@ impl ScreenCaptureConfig { video_frame_count.fetch_add(1, atomic::Ordering::Relaxed); - let _ = video_tx.try_send(VideoFrame { - sample_buf: sample_buffer.retained(), - timestamp, - }); + if video_tx + .try_send(VideoFrame { + sample_buf: sample_buffer.retained(), + timestamp, + }) + .is_err() + { + drop_counter.fetch_add(1, atomic::Ordering::Relaxed); + } } scap_screencapturekit::Frame::Audio(_) => { use ffmpeg::ChannelLayout; @@ -243,6 +271,7 @@ impl ScreenCaptureConfig { capturer: capturer.clone(), error_rx: error_rx.resubscribe(), video_frame_counter: video_frame_counter.clone(), + drop_counter, cancel_token: cancel_token.clone(), drop_guard: cancel_token.drop_guard(), }, @@ -341,12 +370,14 @@ pub struct VideoSourceConfig { cancel_token: CancellationToken, drop_guard: DropGuard, video_frame_counter: Arc, + drop_counter: Arc, } pub struct VideoSource { inner: ChannelVideoSource, capturer: Capturer, cancel_token: CancellationToken, video_frame_counter: Arc, + drop_counter: Arc, _drop_guard: DropGuard, } @@ -369,6 +400,7 @@ impl output_pipeline::VideoSource for VideoSource { cancel_token, drop_guard, video_frame_counter, + drop_counter, } = config; let monitor_capturer = capturer.clone(); @@ -415,6 +447,7 @@ impl output_pipeline::VideoSource for VideoSource { cancel_token, _drop_guard: drop_guard, video_frame_counter, + drop_counter, }) } @@ -424,13 +457,43 @@ impl output_pipeline::VideoSource for VideoSource { tokio::spawn({ let video_frame_count = self.video_frame_counter.clone(); + let drop_counter = self.drop_counter.clone(); async move { + let mut prev_frames = 0u32; + let mut prev_drops = 0u64; loop { tokio::time::sleep(Duration::from_secs(5)).await; - debug!( - "Captured {} frames", - video_frame_count.load(atomic::Ordering::Relaxed) - ); + let current_frames = video_frame_count.load(atomic::Ordering::Relaxed); + let current_drops = drop_counter.load(atomic::Ordering::Relaxed); + + let frame_delta = current_frames.saturating_sub(prev_frames); + let drop_delta = current_drops.saturating_sub(prev_drops); + + if frame_delta > 0 { + let drop_rate = 100.0 * drop_delta as f64 + / (frame_delta as f64 + drop_delta as f64); + if drop_rate > 5.0 { + warn!( + frames = frame_delta, + drops = drop_delta, + drop_rate_pct = format!("{:.1}%", drop_rate), + total_frames = current_frames, + total_drops = current_drops, + "Screen capture frame drop rate exceeds 5% threshold" + ); + } else { + debug!( + frames = frame_delta, + drops = drop_delta, + drop_rate_pct = format!("{:.1}%", drop_rate), + total_frames = current_frames, + "Screen capture stats" + ); + } + } + + prev_frames = current_frames; + prev_drops = current_drops; } } .with_cancellation_token_owned(self.cancel_token.clone()) From b33d74cad09ce3313b61dddaa5ce5b3b88f4fcfa Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 21 Dec 2025 02:06:02 +0000 Subject: [PATCH 06/37] Add async finalization for fragmented recordings --- Cargo.lock | 13 ++ apps/desktop/src-tauri/src/lib.rs | 66 +++++++- apps/desktop/src-tauri/src/recording.rs | 197 ++++++++++++++++++++---- crates/recording/Cargo.toml | 4 + 4 files changed, 247 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4aa78c2b5f..90a9a5dd1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1546,6 +1546,8 @@ dependencies = [ "indexmap 2.11.4", "inquire", "kameo", + "libc", + "libproc", "objc", "objc2-app-kit", "relative-path", @@ -4870,6 +4872,17 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "libproc" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78a09b56be5adbcad5aa1197371688dc6bb249a26da3bca2011ee2fb987ebfb" +dependencies = [ + "bindgen 0.70.1", + "errno", + "libc", +] + [[package]] name = "libredox" version = "0.1.10" diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index e914955174..6bdb84b4e6 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -90,7 +90,7 @@ use tauri_plugin_notification::{NotificationExt, PermissionState}; use tauri_plugin_opener::OpenerExt; use tauri_plugin_shell::ShellExt; use tauri_specta::Event; -use tokio::sync::{RwLock, oneshot}; +use tokio::sync::{RwLock, oneshot, watch}; use tracing::*; use upload::{create_or_get_video, upload_image, upload_video}; use web_api::AuthedApiError; @@ -106,6 +106,34 @@ use crate::{ }; use crate::{recording::start_recording, upload::build_video_meta}; +#[derive(Default)] +pub struct FinalizingRecordings { + recordings: std::sync::Mutex< + std::collections::HashMap, watch::Receiver)>, + >, +} + +impl FinalizingRecordings { + pub fn start_finalizing(&self, path: PathBuf) -> watch::Receiver { + let mut recordings = self.recordings.lock().unwrap(); + let (tx, rx) = watch::channel(false); + recordings.insert(path, (tx, rx.clone())); + rx + } + + pub fn finish_finalizing(&self, path: &Path) { + let mut recordings = self.recordings.lock().unwrap(); + if let Some((tx, _)) = recordings.remove(path) { + tx.send(true).ok(); + } + } + + pub fn is_finalizing(&self, path: &Path) -> Option> { + let recordings = self.recordings.lock().unwrap(); + recordings.get(path).map(|(_, rx)| rx.clone()) + } +} + #[allow(clippy::large_enum_variant)] pub enum RecordingState { None, @@ -2597,6 +2625,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { app.manage(http_client::HttpClient::default()); app.manage(http_client::RetryableHttpClient::default()); app.manage(PendingScreenshots::default()); + app.manage(FinalizingRecordings::default()); gpu_context::prewarm_gpu(); @@ -3142,6 +3171,8 @@ async fn create_editor_instance_impl( ) -> Result, String> { let app = app.clone(); + wait_for_recording_ready(&app, &path).await?; + let instance = { let app = app.clone(); EditorInstance::new( @@ -3170,6 +3201,39 @@ async fn create_editor_instance_impl( Ok(instance) } +async fn wait_for_recording_ready(app: &AppHandle, path: &Path) -> Result<(), String> { + let finalizing_state = app.state::(); + + if let Some(mut rx) = finalizing_state.is_finalizing(path) { + info!("Recording is being finalized, waiting for completion..."); + rx.wait_for(|&ready| ready) + .await + .map_err(|_| "Finalization was cancelled".to_string())?; + info!("Recording finalization completed"); + return Ok(()); + } + + let meta = match RecordingMeta::load_for_project(path) { + Ok(meta) => meta, + Err(e) => { + return Err(format!("Failed to load recording meta: {e}")); + } + }; + + if let Some(studio_meta) = meta.studio_meta() { + if recording::needs_fragment_remux(path, studio_meta) { + info!("Recording needs remux (crash recovery), starting remux..."); + let path = path.to_path_buf(); + tokio::task::spawn_blocking(move || recording::remux_fragmented_recording(&path)) + .await + .map_err(|e| format!("Remux task panicked: {e}"))??; + info!("Crash recovery remux completed"); + } + } + + Ok(()) +} + fn recordings_path(app: &AppHandle) -> PathBuf { let path = app.path().app_data_dir().unwrap().join("recordings"); std::fs::create_dir_all(&path).unwrap_or_default(); diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index a796749932..4265cbcdf1 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -52,8 +52,8 @@ use crate::camera::{CameraPreviewManager, CameraPreviewShape}; use crate::general_settings; use crate::web_api::AuthedApiError; use crate::{ - App, CurrentRecordingChanged, MutableState, NewStudioRecordingAdded, RecordingState, - RecordingStopped, VideoUploadInfo, + App, CurrentRecordingChanged, FinalizingRecordings, MutableState, NewStudioRecordingAdded, + RecordingState, RecordingStopped, VideoUploadInfo, api::PresignedS3PutRequestMethod, audio::AppSounds, auth::AuthStore, @@ -1379,22 +1379,75 @@ async fn handle_recording_finish( .map_err(|e| format!("Failed to save recording meta: {e}"))?; } - let updated_studio_meta = if needs_fragment_remux(&recording_dir, &recording.meta) { - info!("Recording has fragments that need remuxing"); - if let Err(e) = remux_fragmented_recording(&recording_dir) { - error!("Failed to remux fragmented recording: {e}"); - return Err(format!("Failed to remux fragmented recording: {e}")); + let needs_remux = needs_fragment_remux(&recording_dir, &recording.meta); + + if needs_remux { + info!("Recording has fragments that need remuxing - opening editor immediately"); + + let finalizing_state = app.state::(); + finalizing_state.start_finalizing(recording_dir.clone()); + + let post_behaviour = GeneralSettingsStore::get(app) + .ok() + .flatten() + .map(|v| v.post_studio_recording_behaviour) + .unwrap_or(PostStudioRecordingBehaviour::OpenEditor); + + match post_behaviour { + PostStudioRecordingBehaviour::OpenEditor => { + let _ = ShowCapWindow::Editor { + project_path: recording_dir.clone(), + } + .show(app) + .await; + } + PostStudioRecordingBehaviour::ShowOverlay => { + let _ = ShowCapWindow::RecordingsOverlay.show(app).await; + + let app_clone = AppHandle::clone(app); + let recording_dir_clone = recording_dir.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(1000)).await; + let _ = NewStudioRecordingAdded { + path: recording_dir_clone, + } + .emit(&app_clone); + }); + } } - let updated_meta = RecordingMeta::load_for_project(&recording_dir) - .map_err(|e| format!("Failed to reload recording meta: {e}"))?; - updated_meta - .studio_meta() - .ok_or_else(|| "Expected studio meta after remux".to_string())? - .clone() - } else { - recording.meta.clone() - }; + AppSounds::StopRecording.play(); + + let app = app.clone(); + let recording_dir_for_finalize = recording_dir.clone(); + let screenshots_dir = screenshots_dir.clone(); + let default_preset = PresetsStore::get_default_preset(&app) + .ok() + .flatten() + .map(|p| p.config); + + tokio::spawn(async move { + let result = finalize_studio_recording( + &app, + recording_dir_for_finalize.clone(), + screenshots_dir, + recording, + default_preset, + ) + .await; + + if let Err(e) = result { + error!("Failed to finalize recording: {e}"); + } + + app.state::() + .finish_finalizing(&recording_dir_for_finalize); + }); + + return Ok(()); + } + + let updated_studio_meta = recording.meta.clone(); let display_output_path = match &updated_studio_meta { StudioRecordingMeta::SingleSegment { segment } => { @@ -1588,6 +1641,72 @@ async fn handle_recording_finish( Ok(()) } +async fn finalize_studio_recording( + app: &AppHandle, + recording_dir: PathBuf, + screenshots_dir: PathBuf, + recording: cap_recording::studio_recording::CompletedRecording, + default_preset: Option, +) -> Result<(), String> { + info!("Starting background finalization for recording"); + + let recording_dir_for_remux = recording_dir.clone(); + let remux_result = + tokio::task::spawn_blocking(move || remux_fragmented_recording(&recording_dir_for_remux)) + .await + .map_err(|e| format!("Remux task panicked: {e}"))?; + + if let Err(e) = remux_result { + error!("Failed to remux fragmented recording: {e}"); + return Err(format!("Failed to remux fragmented recording: {e}")); + } + + let updated_meta = RecordingMeta::load_for_project(&recording_dir) + .map_err(|e| format!("Failed to reload recording meta: {e}"))?; + let updated_studio_meta = updated_meta + .studio_meta() + .ok_or_else(|| "Expected studio meta after remux".to_string())? + .clone(); + + let display_output_path = match &updated_studio_meta { + StudioRecordingMeta::SingleSegment { segment } => { + segment.display.path.to_path(&recording_dir) + } + StudioRecordingMeta::MultipleSegments { inner, .. } => { + inner.segments[0].display.path.to_path(&recording_dir) + } + }; + + let display_screenshot = screenshots_dir.join("display.jpg"); + tokio::spawn(create_screenshot( + display_output_path, + display_screenshot, + None, + )); + + let recordings = ProjectRecordingsMeta::new(&recording_dir, &updated_studio_meta) + .map_err(|e| format!("Failed to create project recordings meta: {e}"))?; + + let config = project_config_from_recording( + app, + &cap_recording::studio_recording::CompletedRecording { + project_path: recording.project_path, + meta: updated_studio_meta, + cursor_data: recording.cursor_data, + }, + &recordings, + default_preset, + ); + + config + .write(&recording_dir) + .map_err(|e| format!("Failed to write project config: {e}"))?; + + info!("Background finalization completed for recording"); + + Ok(()) +} + /// Core logic for generating zoom segments based on mouse click events. /// This is an experimental feature that automatically creates zoom effects /// around user interactions to highlight important moments. @@ -1873,7 +1992,7 @@ fn project_config_from_recording( config } -fn needs_fragment_remux(recording_dir: &Path, meta: &StudioRecordingMeta) -> bool { +pub fn needs_fragment_remux(recording_dir: &Path, meta: &StudioRecordingMeta) -> bool { let StudioRecordingMeta::MultipleSegments { inner, .. } = meta else { return false; }; @@ -1888,7 +2007,7 @@ fn needs_fragment_remux(recording_dir: &Path, meta: &StudioRecordingMeta) -> boo false } -fn remux_fragmented_recording(recording_dir: &Path) -> Result<(), String> { +pub fn remux_fragmented_recording(recording_dir: &Path) -> Result<(), String> { let meta = RecordingMeta::load_for_project(recording_dir) .map_err(|e| format!("Failed to load recording meta: {e}"))?; @@ -1924,10 +2043,12 @@ fn analyze_recording_for_remux( for (index, segment) in inner.segments.iter().enumerate() { let display_path = segment.display.path.to_path(project_path); - let display_fragments = if display_path.is_dir() { - find_fragments_in_dir(&display_path) + let (display_fragments, display_init_segment) = if display_path.is_dir() { + let frags = find_fragments_in_dir(&display_path); + let init = display_path.join("init.mp4"); + (frags, if init.exists() { Some(init) } else { None }) } else if display_path.exists() { - vec![display_path] + (vec![display_path], None) } else { continue; }; @@ -1936,17 +2057,27 @@ fn analyze_recording_for_remux( continue; } - let camera_fragments = segment.camera.as_ref().and_then(|cam| { - let cam_path = cam.path.to_path(project_path); - if cam_path.is_dir() { - let frags = find_fragments_in_dir(&cam_path); - if frags.is_empty() { None } else { Some(frags) } - } else if cam_path.exists() { - Some(vec![cam_path]) - } else { - None - } - }); + let (camera_fragments, camera_init_segment) = segment + .camera + .as_ref() + .map(|cam| { + let cam_path = cam.path.to_path(project_path); + if cam_path.is_dir() { + let frags = find_fragments_in_dir(&cam_path); + let init = cam_path.join("init.mp4"); + let init_seg = if init.exists() { Some(init) } else { None }; + if frags.is_empty() { + (None, None) + } else { + (Some(frags), init_seg) + } + } else if cam_path.exists() { + (Some(vec![cam_path]), None) + } else { + (None, None) + } + }) + .unwrap_or((None, None)); let cursor_path = segment .cursor @@ -1981,7 +2112,9 @@ fn analyze_recording_for_remux( recoverable_segments.push(RecoverableSegment { index: index as u32, display_fragments, + display_init_segment, camera_fragments, + camera_init_segment, mic_fragments, system_audio_fragments, cursor_path, diff --git a/crates/recording/Cargo.toml b/crates/recording/Cargo.toml index 36aceb0a2e..409c19cb0a 100644 --- a/crates/recording/Cargo.toml +++ b/crates/recording/Cargo.toml @@ -62,6 +62,7 @@ objc2-app-kit = "0.3.1" core-graphics = "0.24.0" core-foundation = "0.10.0" foreign-types-shared = "0.3" +libc = "0.2" scap-screencapturekit = { path = "../scap-screencapturekit" } cap-enc-avfoundation = { path = "../enc-avfoundation" } @@ -87,3 +88,6 @@ scap-cpal = { path = "../scap-cpal" } [dev-dependencies] tempfile = "3.20.0" tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } + +[target.'cfg(target_os = "macos")'.dev-dependencies] +libproc = "0.14" From 04066dd02bf0aea44d911bd7c364d21c4156509c Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 21 Dec 2025 02:06:11 +0000 Subject: [PATCH 07/37] Refactor macOS memory usage reporting to use libproc --- .../examples/memory-leak-detector.rs | 138 +++--------------- 1 file changed, 24 insertions(+), 114 deletions(-) diff --git a/crates/recording/examples/memory-leak-detector.rs b/crates/recording/examples/memory-leak-detector.rs index 0e77765732..efb5c70045 100644 --- a/crates/recording/examples/memory-leak-detector.rs +++ b/crates/recording/examples/memory-leak-detector.rs @@ -18,106 +18,19 @@ const DEFAULT_DURATION_SECS: u64 = 120; #[cfg(target_os = "macos")] fn get_memory_usage() -> Option { - use std::process::Command; + use libproc::libproc::pid_rusage::{RUsageInfoV4, pidrusage}; - let pid = std::process::id(); - - let ps_output = Command::new("ps") - .args(["-o", "rss=,vsz=", "-p", &pid.to_string()]) - .output() - .ok()?; - - let stdout = String::from_utf8_lossy(&ps_output.stdout); - let parts: Vec<&str> = stdout.split_whitespace().collect(); - - let (rss_mb, vsz_mb) = if parts.len() >= 2 { - let rss_kb: u64 = parts[0].parse().ok()?; - let vsz_kb: u64 = parts[1].parse().ok()?; - (rss_kb as f64 / 1024.0, vsz_kb as f64 / 1024.0) - } else { - return None; - }; - - let (footprint_mb, dirty_mb) = Command::new("footprint") - .arg(pid.to_string()) - .output() - .ok() - .filter(|o| o.status.success()) - .and_then(|output| { - let stdout = String::from_utf8_lossy(&output.stdout); - parse_footprint_values(&stdout) - }) - .unwrap_or((None, None)); + let pid = std::process::id() as i32; + let rusage: RUsageInfoV4 = pidrusage(pid).ok()?; Some(MemoryStats { - resident_mb: rss_mb, - virtual_mb: vsz_mb, - footprint_mb, - dirty_mb, + resident_mb: rusage.ri_resident_size as f64 / 1024.0 / 1024.0, + footprint_mb: Some(rusage.ri_phys_footprint as f64 / 1024.0 / 1024.0), + dirty_mb: None, compressed_mb: None, }) } -#[cfg(target_os = "macos")] -fn parse_footprint_values(output: &str) -> Option<(Option, Option)> { - let mut footprint_kb: Option = None; - let mut dirty_kb: Option = None; - - for line in output.lines() { - if line.contains("phys_footprint:") { - let parts: Vec<&str> = line.split_whitespace().collect(); - if parts.len() >= 2 { - footprint_kb = parse_size_kb(parts[1]); - } - } else if line.contains("TOTAL") && dirty_kb.is_none() { - let parts: Vec<&str> = line.split_whitespace().collect(); - if !parts.is_empty() { - dirty_kb = parse_size_kb(parts[0]); - } - } - } - - Some(( - footprint_kb.map(|v| v / 1024.0), - dirty_kb.map(|v| v / 1024.0), - )) -} - -#[cfg(target_os = "macos")] -fn parse_size_kb(s: &str) -> Option { - let s = s.trim(); - if s.ends_with("KB") || s.ends_with("kb") { - s.trim_end_matches("KB") - .trim_end_matches("kb") - .trim() - .parse() - .ok() - } else if s.ends_with("MB") || s.ends_with("mb") { - s.trim_end_matches("MB") - .trim_end_matches("mb") - .trim() - .parse::() - .ok() - .map(|v| v * 1024.0) - } else if s.ends_with("GB") || s.ends_with("gb") { - s.trim_end_matches("GB") - .trim_end_matches("gb") - .trim() - .parse::() - .ok() - .map(|v| v * 1024.0 * 1024.0) - } else if s.ends_with('B') || s.ends_with('b') { - s.trim_end_matches('B') - .trim_end_matches('b') - .trim() - .parse::() - .ok() - .map(|v| v / 1024.0) - } else { - s.parse().ok() - } -} - #[cfg(not(target_os = "macos"))] fn get_memory_usage() -> Option { None @@ -126,7 +39,6 @@ fn get_memory_usage() -> Option { #[derive(Debug, Clone, Copy)] struct MemoryStats { resident_mb: f64, - virtual_mb: f64, footprint_mb: Option, #[allow(dead_code)] dirty_mb: Option, @@ -136,11 +48,11 @@ struct MemoryStats { impl MemoryStats { fn primary_metric(&self) -> f64 { - self.resident_mb + self.footprint_mb.unwrap_or(self.resident_mb) } fn metric_name() -> &'static str { - "RSS" + "Footprint" } } @@ -170,10 +82,10 @@ impl MemoryTracker { if let Some(baseline) = self.baseline { println!( - "Baseline: {:.1} MB {} (Footprint: {:.1} MB)", + "Baseline: {:.1} MB {} (RSS: {:.1} MB)", baseline.primary_metric(), MemoryStats::metric_name(), - baseline.footprint_mb.unwrap_or(0.0) + baseline.resident_mb ); } @@ -195,10 +107,10 @@ impl MemoryTracker { println!("\nMemory Timeline:"); println!( - "{:>8} {:>12} {:>12} {:>12} {:>12}", - "Time(s)", "RSS(MB)", "Delta", "Footprint", "VSZ(MB)" + "{:>8} {:>14} {:>10} {:>12}", + "Time(s)", "Footprint(MB)", "Delta", "RSS(MB)" ); - println!("{:-<70}", ""); + println!("{:-<50}", ""); let mut prev_memory = first.1.primary_metric(); for (time, stats) in &self.samples { @@ -210,20 +122,19 @@ impl MemoryTracker { "~0".to_string() }; println!( - "{:>8.1} {:>12.1} {:>12} {:>12.1} {:>12.1}", + "{:>8.1} {:>14.1} {:>10} {:>12.1}", time.as_secs_f64(), current, delta_str, - stats.footprint_mb.unwrap_or(0.0), - stats.virtual_mb + stats.resident_mb ); prev_memory = current; } println!("\n=== Summary ==="); println!("Duration: {duration_secs:.1}s"); - println!("Start RSS: {:.1} MB", first.1.primary_metric()); - println!("End RSS: {:.1} MB", last.1.primary_metric()); + println!("Start Footprint: {:.1} MB", first.1.primary_metric()); + println!("End Footprint: {:.1} MB", last.1.primary_metric()); println!("Total growth: {memory_growth:.1} MB"); println!( "Growth rate: {:.2} MB/s ({:.1} MB/10s)", @@ -251,7 +162,7 @@ impl MemoryTracker { } println!( - "\nPeak RSS: {:.1} MB", + "\nPeak Footprint: {:.1} MB", self.samples .iter() .map(|(_, s)| s.primary_metric()) @@ -344,7 +255,7 @@ async fn run_memory_test( ) .await?; - let sample_interval = Duration::from_secs(5); + let sample_interval = Duration::from_secs(1); let mut next_sample = start + sample_interval; while start.elapsed() < Duration::from_secs(duration_secs) { @@ -355,11 +266,10 @@ async fn run_memory_test( let current = get_memory_usage(); if let Some(stats) = current { println!( - "[{:>5.1}s] RSS: {:.1} MB, Footprint: {:.1} MB, VSZ: {:.1} MB", + "[{:>5.1}s] Footprint: {:.1} MB, RSS: {:.1} MB", start.elapsed().as_secs_f64(), - stats.resident_mb, stats.footprint_mb.unwrap_or(0.0), - stats.virtual_mb + stats.resident_mb ); } next_sample = Instant::now() + sample_interval; @@ -407,7 +317,7 @@ async fn run_camera_only_test(duration_secs: u64) -> Result<(), Box Result<(), Box5.1}s] RSS: {:.1} MB, Footprint: {:.1} MB, Frames: {}, Queue: {}", + "[{:>5.1}s] Footprint: {:.1} MB, RSS: {:.1} MB, Frames: {}, Queue: {}", start.elapsed().as_secs_f64(), - stats.resident_mb, stats.footprint_mb.unwrap_or(0.0), + stats.resident_mb, frame_count, queue_len ); From 3bfdbbcf0e852df7d9e7c105f999e194d62f3a57 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 00:39:09 +0000 Subject: [PATCH 08/37] Remove backward stale frame handling in AVAssetReaderDecoder --- crates/rendering/src/decoder/avassetreader.rs | 73 ++----------------- 1 file changed, 6 insertions(+), 67 deletions(-) diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index f32846a9fb..61e0efcb69 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -287,7 +287,6 @@ impl AVAssetReaderDecoder { let mut last_active_frame = None::; let last_sent_frame = Rc::new(RefCell::new(None::)); let first_ever_frame = Rc::new(RefCell::new(None::)); - let mut backward_stale_count: u32 = 0; let mut frames = this.inner.frames(); let processor = ImageBufProcessor::new(); @@ -347,28 +346,6 @@ impl AVAssetReaderDecoder { let requested_time = requested_frame as f32 / fps as f32; const BACKWARD_SEEK_TOLERANCE: u32 = 120; - const MAX_STALE_FRAMES: u32 = 3; - let cache_frame_min_early = cache.keys().next().copied(); - let cache_frame_max_early = cache.keys().next_back().copied(); - - if let (Some(c_min), Some(_c_max)) = (cache_frame_min_early, cache_frame_max_early) { - let is_backward_within_tolerance = - requested_frame < c_min && requested_frame + BACKWARD_SEEK_TOLERANCE >= c_min; - if is_backward_within_tolerance - && backward_stale_count < MAX_STALE_FRAMES - && let Some(closest_frame) = cache.get(&c_min) - { - backward_stale_count += 1; - let data = closest_frame.data().clone(); - *last_sent_frame.borrow_mut() = Some(data.clone()); - for req in pending_requests.drain(..) { - if req.sender.send(data.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } - } - continue; - } - } let cache_min = min_requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); let cache_max = max_requested_frame + FRAME_CACHE_SIZE as u32 / 2; @@ -382,11 +359,7 @@ impl AVAssetReaderDecoder { requested_frame + BACKWARD_SEEK_TOLERANCE < c_min; let is_forward_seek_beyond_cache = requested_frame > c_max + FRAME_CACHE_SIZE as u32 / 4; - let stale_limit_exceeded = - backward_stale_count >= MAX_STALE_FRAMES && requested_frame < c_min; - is_backward_seek_beyond_tolerance - || is_forward_seek_beyond_cache - || stale_limit_exceeded + is_backward_seek_beyond_tolerance || is_forward_seek_beyond_cache } else { true }; @@ -395,7 +368,6 @@ impl AVAssetReaderDecoder { this.reset(requested_time); frames = this.inner.frames(); *last_sent_frame.borrow_mut() = None; - backward_stale_count = 0; cache.retain(|&f, _| f >= cache_min && f <= cache_max); } @@ -445,7 +417,6 @@ impl AVAssetReaderDecoder { } cache.insert(current_frame, cache_frame.clone()); - backward_stale_count = 0; let mut remaining_requests = Vec::with_capacity(pending_requests.len()); for req in pending_requests.drain(..) { @@ -456,13 +427,9 @@ impl AVAssetReaderDecoder { debug!("frame receiver dropped before send"); } } else if req.frame < current_frame { - let prev_frame_data = last_sent_frame.borrow().clone(); - if let Some(data) = prev_frame_data { - if req.sender.send(data.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } - } else { - let data = cache_frame.data().clone(); + if let Some(cached) = cache.get(&req.frame) { + let data = cached.data().clone(); + *last_sent_frame.borrow_mut() = Some(data.clone()); if req.sender.send(data.to_decoded_frame()).is_err() { debug!("frame receiver dropped before send"); } @@ -486,39 +453,11 @@ impl AVAssetReaderDecoder { this.is_done = true; for req in pending_requests.drain(..) { - let prev_data = last_sent_frame.borrow().clone(); - if let Some(data) = prev_data { + if let Some(cached) = cache.get(&req.frame) { + let data = cached.data().clone(); if req.sender.send(data.to_decoded_frame()).is_err() { debug!("frame receiver dropped before send"); } - } else if let Some(first_frame) = first_ever_frame.borrow().clone() { - debug!( - "Returning first decoded frame as fallback for request {}", - req.frame - ); - if req.sender.send(first_frame.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } - } else { - debug!( - "No frames available for request {}, returning black frame", - req.frame - ); - let black_frame_data = vec![0u8; (video_width * video_height * 4) as usize]; - let black_frame = ProcessedFrame { - _number: req.frame, - width: video_width, - height: video_height, - format: PixelFormat::Rgba, - frame_data: FrameData { - data: Arc::new(black_frame_data), - y_stride: video_width * 4, - uv_stride: 0, - }, - }; - if req.sender.send(black_frame.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } } } } From 9b03f4ed1b574f7cb904a29fe18c492d3a862128 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 15:26:44 +0000 Subject: [PATCH 09/37] Fix audio sync in fragmented m4s --- crates/recording/src/capture_pipeline.rs | 9 ++- crates/recording/src/output_pipeline/core.rs | 63 +++++++++++++++- .../recording/src/output_pipeline/ffmpeg.rs | 37 ++++++--- .../output_pipeline/macos_fragmented_m4s.rs | 75 +++++-------------- crates/recording/src/recovery.rs | 27 +++++-- crates/recording/src/studio_recording.rs | 39 +++++++--- crates/timestamp/src/lib.rs | 25 +++++++ crates/timestamp/src/macos.rs | 13 ++++ crates/timestamp/src/win.rs | 6 ++ 9 files changed, 211 insertions(+), 83 deletions(-) diff --git a/crates/recording/src/capture_pipeline.rs b/crates/recording/src/capture_pipeline.rs index d5575bc846..7f745d3470 100644 --- a/crates/recording/src/capture_pipeline.rs +++ b/crates/recording/src/capture_pipeline.rs @@ -1,4 +1,5 @@ use crate::{ + SharedPauseState, feeds::microphone::MicrophoneFeedLock, output_pipeline::*, sources, @@ -50,6 +51,7 @@ pub trait MakeCapturePipeline: ScreenCaptureFormat + std::fmt::Debug + 'static { output_path: PathBuf, start_time: Timestamps, fragmented: bool, + shared_pause_state: Option, #[cfg(windows)] encoder_preferences: EncoderPreferences, ) -> anyhow::Result where @@ -76,6 +78,7 @@ impl MakeCapturePipeline for screen_capture::CMSampleBufferCapture { output_path: PathBuf, start_time: Timestamps, fragmented: bool, + shared_pause_state: Option, ) -> anyhow::Result { if fragmented { let fragments_dir = output_path @@ -86,7 +89,10 @@ impl MakeCapturePipeline for screen_capture::CMSampleBufferCapture { OutputPipeline::builder(fragments_dir) .with_video::(screen_capture) .with_timestamps(start_time) - .build::(MacOSFragmentedM4SMuxerConfig::default()) + .build::(MacOSFragmentedM4SMuxerConfig { + shared_pause_state, + ..Default::default() + }) .await } else { OutputPipeline::builder(output_path.clone()) @@ -130,6 +136,7 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { output_path: PathBuf, start_time: Timestamps, fragmented: bool, + _shared_pause_state: Option, encoder_preferences: EncoderPreferences, ) -> anyhow::Result { let d3d_device = screen_capture.d3d_device.clone(); diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rs index 0946b8b1b0..13aca3ce2e 100644 --- a/crates/recording/src/output_pipeline/core.rs +++ b/crates/recording/src/output_pipeline/core.rs @@ -17,7 +17,7 @@ use std::{ path::{Path, PathBuf}, sync::{ Arc, - atomic::{self, AtomicBool}, + atomic::{self, AtomicBool, Ordering}, }, time::Duration, }; @@ -25,6 +25,67 @@ use tokio::task::JoinHandle; use tokio_util::sync::{CancellationToken, DropGuard}; use tracing::*; +struct SharedPauseStateInner { + paused_at: Option, + offset: Duration, +} + +#[derive(Clone)] +pub struct SharedPauseState { + flag: Arc, + inner: Arc>, +} + +impl SharedPauseState { + pub fn new(flag: Arc) -> Self { + Self { + flag, + inner: Arc::new(std::sync::Mutex::new(SharedPauseStateInner { + paused_at: None, + offset: Duration::ZERO, + })), + } + } + + pub fn adjust(&self, timestamp: Duration) -> anyhow::Result> { + let mut inner = self + .inner + .lock() + .map_err(|e| anyhow!("Lock poisoned: {e}"))?; + + if self.flag.load(Ordering::Relaxed) { + if inner.paused_at.is_none() { + inner.paused_at = Some(timestamp); + } + return Ok(None); + } + + if let Some(start) = inner.paused_at.take() { + let delta = timestamp.checked_sub(start).ok_or_else(|| { + anyhow!( + "Frame timestamp went backward during unpause (resume={start:?}, current={timestamp:?})" + ) + })?; + + inner.offset = inner.offset.checked_add(delta).ok_or_else(|| { + anyhow!( + "Pause offset overflow (offset={:?}, delta={delta:?})", + inner.offset + ) + })?; + } + + let adjusted = timestamp.checked_sub(inner.offset).ok_or_else(|| { + anyhow!( + "Adjusted timestamp underflow (timestamp={timestamp:?}, offset={:?})", + inner.offset + ) + })?; + + Ok(Some(adjusted)) + } +} + pub struct OnceSender(Option>); impl OnceSender { diff --git a/crates/recording/src/output_pipeline/ffmpeg.rs b/crates/recording/src/output_pipeline/ffmpeg.rs index 1c8e2fe694..8e94393317 100644 --- a/crates/recording/src/output_pipeline/ffmpeg.rs +++ b/crates/recording/src/output_pipeline/ffmpeg.rs @@ -1,5 +1,5 @@ use crate::{ - TaskPool, + SharedPauseState, TaskPool, output_pipeline::{AudioFrame, AudioMuxer, Muxer, VideoFrame, VideoMuxer}, }; use anyhow::{Context, anyhow}; @@ -197,16 +197,21 @@ impl AudioMuxer for FragmentedAudioMuxer { } } -pub struct SegmentedAudioMuxer(SegmentedAudioEncoder); +pub struct SegmentedAudioMuxer { + encoder: SegmentedAudioEncoder, + pause: Option, +} pub struct SegmentedAudioMuxerConfig { pub segment_duration: Duration, + pub shared_pause_state: Option, } impl Default for SegmentedAudioMuxerConfig { fn default() -> Self { Self { segment_duration: Duration::from_secs(3), + shared_pause_state: None, } } } @@ -228,14 +233,19 @@ impl Muxer for SegmentedAudioMuxer { let audio_config = audio_config.ok_or_else(|| anyhow!("No audio configuration provided"))?; - Ok(Self( - SegmentedAudioEncoder::init(output_path, audio_config, config.segment_duration) - .map_err(|e| anyhow!("Failed to initialize segmented audio encoder: {e}"))?, - )) + Ok(Self { + encoder: SegmentedAudioEncoder::init( + output_path, + audio_config, + config.segment_duration, + ) + .map_err(|e| anyhow!("Failed to initialize segmented audio encoder: {e}"))?, + pause: config.shared_pause_state, + }) } fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - self.0 + self.encoder .finish_with_timestamp(timestamp) .map_err(Into::into) .map(|_| Ok(())) @@ -244,8 +254,17 @@ impl Muxer for SegmentedAudioMuxer { impl AudioMuxer for SegmentedAudioMuxer { fn send_audio_frame(&mut self, frame: AudioFrame, timestamp: Duration) -> anyhow::Result<()> { - self.0 - .queue_frame(frame.inner, timestamp) + let adjusted_timestamp = if let Some(pause) = &self.pause { + match pause.adjust(timestamp)? { + Some(ts) => ts, + None => return Ok(()), + } + } else { + timestamp + }; + + self.encoder + .queue_frame(frame.inner, adjusted_timestamp) .map_err(|e| anyhow!("Failed to queue audio frame: {e}")) } } diff --git a/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs b/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs index 484171588c..b091b92789 100644 --- a/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs @@ -1,6 +1,6 @@ use crate::{ - AudioFrame, AudioMuxer, Muxer, TaskPool, VideoMuxer, output_pipeline::NativeCameraFrame, - screen_capture, + AudioFrame, AudioMuxer, Muxer, SharedPauseState, TaskPool, VideoMuxer, + output_pipeline::NativeCameraFrame, screen_capture, }; use anyhow::{Context, anyhow}; use cap_enc_ffmpeg::h264::{H264EncoderBuilder, H264Preset}; @@ -10,7 +10,7 @@ use std::{ path::PathBuf, sync::{ Arc, Mutex, - atomic::{AtomicBool, Ordering}, + atomic::AtomicBool, mpsc::{SyncSender, sync_channel}, }, thread::JoinHandle, @@ -25,55 +25,6 @@ fn get_muxer_buffer_size() -> usize { .unwrap_or(3) } -struct PauseTracker { - flag: Arc, - paused_at: Option, - offset: Duration, -} - -impl PauseTracker { - fn new(flag: Arc) -> Self { - Self { - flag, - paused_at: None, - offset: Duration::ZERO, - } - } - - fn adjust(&mut self, timestamp: Duration) -> anyhow::Result> { - if self.flag.load(Ordering::Relaxed) { - if self.paused_at.is_none() { - self.paused_at = Some(timestamp); - } - return Ok(None); - } - - if let Some(start) = self.paused_at.take() { - let delta = timestamp.checked_sub(start).ok_or_else(|| { - anyhow!( - "Frame timestamp went backward during unpause (resume={start:?}, current={timestamp:?})" - ) - })?; - - self.offset = self.offset.checked_add(delta).ok_or_else(|| { - anyhow!( - "Pause offset overflow (offset={:?}, delta={delta:?})", - self.offset - ) - })?; - } - - let adjusted = timestamp.checked_sub(self.offset).ok_or_else(|| { - anyhow!( - "Adjusted timestamp underflow (timestamp={timestamp:?}, offset={:?})", - self.offset - ) - })?; - - Ok(Some(adjusted)) - } -} - struct FrameDropTracker { drops_in_window: u32, frames_in_window: u32, @@ -146,7 +97,7 @@ pub struct MacOSFragmentedM4SMuxer { video_config: VideoInfo, segment_duration: Duration, state: Option, - pause: PauseTracker, + pause: SharedPauseState, frame_drops: FrameDropTracker, started: bool, } @@ -155,6 +106,7 @@ pub struct MacOSFragmentedM4SMuxerConfig { pub segment_duration: Duration, pub preset: H264Preset, pub output_size: Option<(u32, u32)>, + pub shared_pause_state: Option, } impl Default for MacOSFragmentedM4SMuxerConfig { @@ -163,6 +115,7 @@ impl Default for MacOSFragmentedM4SMuxerConfig { segment_duration: Duration::from_secs(3), preset: H264Preset::Ultrafast, output_size: None, + shared_pause_state: None, } } } @@ -187,12 +140,16 @@ impl Muxer for MacOSFragmentedM4SMuxer { std::fs::create_dir_all(&output_path) .with_context(|| format!("Failed to create segments directory: {output_path:?}"))?; + let pause = config + .shared_pause_state + .unwrap_or_else(|| SharedPauseState::new(pause_flag)); + Ok(Self { base_path: output_path, video_config, segment_duration: config.segment_duration, state: None, - pause: PauseTracker::new(pause_flag), + pause, frame_drops: FrameDropTracker::new(), started: false, }) @@ -602,7 +559,7 @@ pub struct MacOSFragmentedM4SCameraMuxer { video_config: VideoInfo, segment_duration: Duration, state: Option, - pause: PauseTracker, + pause: SharedPauseState, frame_drops: FrameDropTracker, started: bool, } @@ -611,6 +568,7 @@ pub struct MacOSFragmentedM4SCameraMuxerConfig { pub segment_duration: Duration, pub preset: H264Preset, pub output_size: Option<(u32, u32)>, + pub shared_pause_state: Option, } impl Default for MacOSFragmentedM4SCameraMuxerConfig { @@ -619,6 +577,7 @@ impl Default for MacOSFragmentedM4SCameraMuxerConfig { segment_duration: Duration::from_secs(3), preset: H264Preset::Ultrafast, output_size: None, + shared_pause_state: None, } } } @@ -644,12 +603,16 @@ impl Muxer for MacOSFragmentedM4SCameraMuxer { format!("Failed to create camera segments directory: {output_path:?}") })?; + let pause = config + .shared_pause_state + .unwrap_or_else(|| SharedPauseState::new(pause_flag)); + Ok(Self { base_path: output_path, video_config, segment_duration: config.segment_duration, state: None, - pause: PauseTracker::new(pause_flag), + pause, frame_drops: FrameDropTracker::new(), started: false, }) diff --git a/crates/recording/src/recovery.rs b/crates/recording/src/recovery.rs index 96f24578bb..c6512dac1a 100644 --- a/crates/recording/src/recovery.rs +++ b/crates/recording/src/recovery.rs @@ -766,6 +766,11 @@ impl RecoveryManager { fn build_recovered_meta( recording: &IncompleteRecording, ) -> Result { + let original_segments = match recording.meta.studio_meta() { + Some(StudioRecordingMeta::MultipleSegments { inner, .. }) => Some(&inner.segments), + _ => None, + }; + let segments: Vec = recording .recoverable_segments .iter() @@ -774,6 +779,9 @@ impl RecoveryManager { let segment_base = format!("content/segments/segment-{segment_index}"); let segment_dir = recording.project_path.join(&segment_base); + let original_segment = + original_segments.and_then(|segs| segs.get(segment_index as usize)); + let display_path = segment_dir.join("display.mp4"); let fps = get_video_fps(&display_path).unwrap_or(30); @@ -786,13 +794,18 @@ impl RecoveryManager { display: VideoMeta { path: RelativePathBuf::from(format!("{segment_base}/display.mp4")), fps, - start_time: None, + start_time: original_segment.and_then(|s| s.display.start_time), }, camera: if camera_path.exists() { Some(VideoMeta { path: RelativePathBuf::from(format!("{segment_base}/camera.mp4")), - fps: 30, - start_time: None, + fps: original_segment + .and_then(|s| s.camera.as_ref()) + .map(|c| c.fps) + .unwrap_or(30), + start_time: original_segment + .and_then(|s| s.camera.as_ref()) + .and_then(|c| c.start_time), }) } else { None @@ -800,7 +813,9 @@ impl RecoveryManager { mic: if mic_path.exists() { Some(AudioMeta { path: RelativePathBuf::from(format!("{segment_base}/audio-input.ogg")), - start_time: None, + start_time: original_segment + .and_then(|s| s.mic.as_ref()) + .and_then(|m| m.start_time), }) } else { None @@ -808,7 +823,9 @@ impl RecoveryManager { system_audio: if system_audio_path.exists() { Some(AudioMeta { path: RelativePathBuf::from(format!("{segment_base}/system_audio.ogg")), - start_time: None, + start_time: original_segment + .and_then(|s| s.system_audio.as_ref()) + .and_then(|a| a.start_time), }) } else { None diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index f93ab0c87a..ae2f50a6ec 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -1,5 +1,5 @@ use crate::{ - ActorError, MediaError, RecordingBaseInputs, RecordingError, + ActorError, MediaError, RecordingBaseInputs, RecordingError, SharedPauseState, capture_pipeline::{ MakeCapturePipeline, ScreenCaptureMethod, Stop, target_to_display_and_crop, }, @@ -626,11 +626,8 @@ async fn stop_recording( let segment_metas: Vec<_> = futures::stream::iter(segments) .then(async |s| { - let to_start_time = |timestamp: Timestamp| { - timestamp - .duration_since(s.pipeline.start_time) - .as_secs_f64() - }; + let to_start_time = + |timestamp: Timestamp| timestamp.signed_duration_since_secs(s.pipeline.start_time); MultipleSegment { display: VideoMeta { @@ -870,11 +867,24 @@ async fn create_segment_pipeline( trace!("preparing segment pipeline {index}"); + #[cfg(target_os = "macos")] + let shared_pause_state = if fragmented { + Some(SharedPauseState::new(Arc::new( + std::sync::atomic::AtomicBool::new(false), + ))) + } else { + None + }; + + #[cfg(windows)] + let shared_pause_state: Option = None; + let screen = ScreenCaptureMethod::make_studio_mode_pipeline( capture_source, screen_output_path.clone(), start_time, fragmented, + shared_pause_state.clone(), #[cfg(windows)] encoder_preferences.clone(), ) @@ -889,9 +899,10 @@ async fn create_segment_pipeline( OutputPipeline::builder(fragments_dir) .with_video::(camera_feed) .with_timestamps(start_time) - .build::( - MacOSFragmentedM4SCameraMuxerConfig::default(), - ) + .build::(MacOSFragmentedM4SCameraMuxerConfig { + shared_pause_state: shared_pause_state.clone(), + ..Default::default() + }) .instrument(error_span!("camera-out")) .await } else { @@ -942,7 +953,10 @@ async fn create_segment_pipeline( OutputPipeline::builder(fragments_dir) .with_audio_source::(mic_feed) .with_timestamps(start_time) - .build::(SegmentedAudioMuxerConfig::default()) + .build::(SegmentedAudioMuxerConfig { + shared_pause_state: shared_pause_state.clone(), + ..Default::default() + }) .instrument(error_span!("mic-out")) .await } else { @@ -964,7 +978,10 @@ async fn create_segment_pipeline( OutputPipeline::builder(fragments_dir) .with_audio_source::(system_audio_source) .with_timestamps(start_time) - .build::(SegmentedAudioMuxerConfig::default()) + .build::(SegmentedAudioMuxerConfig { + shared_pause_state: shared_pause_state.clone(), + ..Default::default() + }) .instrument(error_span!("system-audio-out")) .await } else { diff --git a/crates/timestamp/src/lib.rs b/crates/timestamp/src/lib.rs index 47b4b19013..4a37b8295c 100644 --- a/crates/timestamp/src/lib.rs +++ b/crates/timestamp/src/lib.rs @@ -45,6 +45,31 @@ impl Timestamp { } } + pub fn signed_duration_since_secs(&self, start: Timestamps) -> f64 { + match self { + Self::Instant(instant) => { + if let Some(duration) = instant.checked_duration_since(start.instant) { + duration.as_secs_f64() + } else { + let reverse = start.instant.duration_since(*instant); + -(reverse.as_secs_f64()) + } + } + Self::SystemTime(time) => match time.duration_since(start.system_time) { + Ok(duration) => duration.as_secs_f64(), + Err(e) => -(e.duration().as_secs_f64()), + }, + #[cfg(windows)] + Self::PerformanceCounter(counter) => { + counter.signed_duration_since_secs(start.performance_counter) + } + #[cfg(target_os = "macos")] + Self::MachAbsoluteTime(time) => { + time.signed_duration_since_secs(start.mach_absolute_time) + } + } + } + pub fn from_cpal(instant: cpal::StreamInstant) -> Self { #[cfg(windows)] { diff --git a/crates/timestamp/src/macos.rs b/crates/timestamp/src/macos.rs index b726533501..0a2e0539d2 100644 --- a/crates/timestamp/src/macos.rs +++ b/crates/timestamp/src/macos.rs @@ -39,6 +39,19 @@ impl MachAbsoluteTimestamp { Some(Duration::from_nanos((diff as f64 * freq) as u64)) } + pub fn signed_duration_since_secs(&self, other: Self) -> f64 { + let info = TimeBaseInfo::new(); + let freq = info.numer as f64 / info.denom as f64; + + let nanos = if self.0 >= other.0 { + ((self.0 - other.0) as f64 * freq) as i64 + } else { + -(((other.0 - self.0) as f64 * freq) as i64) + }; + + nanos as f64 / 1_000_000_000.0 + } + pub fn from_cpal(instant: cpal::StreamInstant) -> Self { use cpal::host::coreaudio::StreamInstantExt; diff --git a/crates/timestamp/src/win.rs b/crates/timestamp/src/win.rs index d3efe2d444..8a111489bd 100644 --- a/crates/timestamp/src/win.rs +++ b/crates/timestamp/src/win.rs @@ -65,6 +65,12 @@ impl PerformanceCounterTimestamp { } } + pub fn signed_duration_since_secs(&self, other: Self) -> f64 { + let freq = perf_freq() as f64; + let diff = self.0 as f64 - other.0 as f64; + diff / freq + } + pub fn now() -> Self { let mut value = 0; unsafe { QueryPerformanceCounter(&mut value).unwrap() }; From 94ad34c1e8dd80d561ce7790dafe2fd5fc14f6da Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 17:15:57 +0000 Subject: [PATCH 10/37] Add decode-benchmark example to editor crate --- crates/editor/Cargo.toml | 4 + crates/editor/examples/decode-benchmark.rs | 312 +++++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 crates/editor/examples/decode-benchmark.rs diff --git a/crates/editor/Cargo.toml b/crates/editor/Cargo.toml index 4410cd6fac..c612d1e33f 100644 --- a/crates/editor/Cargo.toml +++ b/crates/editor/Cargo.toml @@ -6,6 +6,10 @@ edition = "2024" [lints] workspace = true +[[example]] +name = "decode-benchmark" +path = "examples/decode-benchmark.rs" + [dependencies] cap-media = { path = "../media" } cap-project = { path = "../project" } diff --git a/crates/editor/examples/decode-benchmark.rs b/crates/editor/examples/decode-benchmark.rs new file mode 100644 index 0000000000..d77f7d1f1a --- /dev/null +++ b/crates/editor/examples/decode-benchmark.rs @@ -0,0 +1,312 @@ +use cap_rendering::decoder::{AsyncVideoDecoderHandle, spawn_decoder}; +use std::path::PathBuf; +use std::time::Instant; +use tokio::runtime::Runtime; + +#[derive(Debug, Clone)] +struct BenchmarkConfig { + video_path: PathBuf, + fps: u32, + iterations: usize, +} + +#[derive(Debug, Default)] +struct BenchmarkResults { + decoder_creation_ms: f64, + sequential_decode_times_ms: Vec, + sequential_fps: f64, + seek_times_by_distance: Vec<(f32, f64)>, + random_access_times_ms: Vec, + random_access_avg_ms: f64, + cache_hits: usize, + cache_misses: usize, +} + +impl BenchmarkResults { + fn print_report(&self) { + println!("\n{}", "=".repeat(60)); + println!(" VIDEO DECODE BENCHMARK RESULTS"); + println!("{}\n", "=".repeat(60)); + + println!("DECODER CREATION"); + println!( + " Time to create decoder: {:.2}ms", + self.decoder_creation_ms + ); + println!(); + + println!("SEQUENTIAL DECODE PERFORMANCE"); + if !self.sequential_decode_times_ms.is_empty() { + let avg: f64 = self.sequential_decode_times_ms.iter().sum::() + / self.sequential_decode_times_ms.len() as f64; + let min = self + .sequential_decode_times_ms + .iter() + .cloned() + .fold(f64::INFINITY, f64::min); + let max = self + .sequential_decode_times_ms + .iter() + .cloned() + .fold(f64::NEG_INFINITY, f64::max); + println!( + " Frames decoded: {}", + self.sequential_decode_times_ms.len() + ); + println!(" Avg decode time: {:.2}ms", avg); + println!(" Min decode time: {:.2}ms", min); + println!(" Max decode time: {:.2}ms", max); + println!(" Effective FPS: {:.1}", self.sequential_fps); + } + println!(); + + println!("SEEK PERFORMANCE (by distance)"); + if !self.seek_times_by_distance.is_empty() { + println!(" {:>10} | {:>12}", "Distance(s)", "Time(ms)"); + println!(" {}-+-{}", "-".repeat(10), "-".repeat(12)); + for (distance, time) in &self.seek_times_by_distance { + println!(" {:>10.1} | {:>12.2}", distance, time); + } + } + println!(); + + println!("RANDOM ACCESS PERFORMANCE"); + if !self.random_access_times_ms.is_empty() { + let avg = self.random_access_times_ms.iter().sum::() + / self.random_access_times_ms.len() as f64; + let min = self + .random_access_times_ms + .iter() + .cloned() + .fold(f64::INFINITY, f64::min); + let max = self + .random_access_times_ms + .iter() + .cloned() + .fold(f64::NEG_INFINITY, f64::max); + println!(" Samples: {}", self.random_access_times_ms.len()); + println!(" Avg access time: {:.2}ms", avg); + println!(" Min access time: {:.2}ms", min); + println!(" Max access time: {:.2}ms", max); + println!( + " P50: {:.2}ms", + percentile(&self.random_access_times_ms, 50.0) + ); + println!( + " P95: {:.2}ms", + percentile(&self.random_access_times_ms, 95.0) + ); + println!( + " P99: {:.2}ms", + percentile(&self.random_access_times_ms, 99.0) + ); + } + println!(); + + let total = self.cache_hits + self.cache_misses; + if total > 0 { + println!("CACHE STATISTICS"); + println!( + " Hits: {} ({:.1}%)", + self.cache_hits, + 100.0 * self.cache_hits as f64 / total as f64 + ); + println!( + " Misses: {} ({:.1}%)", + self.cache_misses, + 100.0 * self.cache_misses as f64 / total as f64 + ); + } + + println!("\n{}\n", "=".repeat(60)); + } +} + +fn percentile(data: &[f64], p: f64) -> f64 { + if data.is_empty() { + return 0.0; + } + let mut sorted: Vec = data.to_vec(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let idx = ((p / 100.0) * (sorted.len() - 1) as f64).round() as usize; + sorted[idx.min(sorted.len() - 1)] +} + +async fn benchmark_decoder_creation(path: &PathBuf, fps: u32, iterations: usize) -> f64 { + let mut total_ms = 0.0; + + for i in 0..iterations { + let start = Instant::now(); + let decoder = spawn_decoder("benchmark", path.clone(), fps, 0.0).await; + let elapsed = start.elapsed(); + + match decoder { + Ok(_) => { + total_ms += elapsed.as_secs_f64() * 1000.0; + } + Err(e) => { + if i == 0 { + eprintln!("Failed to create decoder: {}", e); + return -1.0; + } + } + } + } + + total_ms / iterations as f64 +} + +async fn benchmark_sequential_decode( + decoder: &AsyncVideoDecoderHandle, + fps: u32, + frame_count: usize, + start_time: f32, +) -> (Vec, f64) { + let mut times = Vec::with_capacity(frame_count); + let overall_start = Instant::now(); + + for i in 0..frame_count { + let time = start_time + (i as f32 / fps as f32); + let start = Instant::now(); + let _frame = decoder.get_frame(time).await; + let elapsed = start.elapsed(); + times.push(elapsed.as_secs_f64() * 1000.0); + } + + let overall_elapsed = overall_start.elapsed(); + let effective_fps = frame_count as f64 / overall_elapsed.as_secs_f64(); + + (times, effective_fps) +} + +async fn benchmark_seek( + decoder: &AsyncVideoDecoderHandle, + _fps: u32, + from_time: f32, + to_time: f32, +) -> f64 { + let _ = decoder.get_frame(from_time).await; + + let start = Instant::now(); + let _frame = decoder.get_frame(to_time).await; + let elapsed = start.elapsed(); + + elapsed.as_secs_f64() * 1000.0 +} + +async fn benchmark_random_access( + decoder: &AsyncVideoDecoderHandle, + _fps: u32, + duration_secs: f32, + sample_count: usize, +) -> Vec { + let mut times = Vec::with_capacity(sample_count); + + let golden_ratio = 1.618033988749895_f32; + let mut position = 0.0_f32; + + for _ in 0..sample_count { + position = (position + golden_ratio * duration_secs) % duration_secs; + let start = Instant::now(); + let _frame = decoder.get_frame(position).await; + let elapsed = start.elapsed(); + times.push(elapsed.as_secs_f64() * 1000.0); + } + + times +} + +async fn run_full_benchmark(config: BenchmarkConfig) -> BenchmarkResults { + let mut results = BenchmarkResults::default(); + + println!( + "Starting benchmark with video: {}", + config.video_path.display() + ); + println!("FPS: {}, Iterations: {}", config.fps, config.iterations); + println!(); + + println!("[1/5] Benchmarking decoder creation..."); + results.decoder_creation_ms = + benchmark_decoder_creation(&config.video_path, config.fps, 3).await; + if results.decoder_creation_ms < 0.0 { + eprintln!("Failed to benchmark decoder creation"); + return results; + } + println!(" Done: {:.2}ms avg", results.decoder_creation_ms); + + println!("[2/5] Creating decoder for remaining tests..."); + let decoder = match spawn_decoder("benchmark", config.video_path.clone(), config.fps, 0.0).await + { + Ok(d) => d, + Err(e) => { + eprintln!("Failed to create decoder: {}", e); + return results; + } + }; + println!(" Done"); + + println!("[3/5] Benchmarking sequential decode (100 frames from start)..."); + let (seq_times, seq_fps) = benchmark_sequential_decode(&decoder, config.fps, 100, 0.0).await; + results.sequential_decode_times_ms = seq_times; + results.sequential_fps = seq_fps; + println!(" Done: {:.1} effective FPS", seq_fps); + + println!("[4/5] Benchmarking seek performance..."); + let seek_distances = vec![0.5, 1.0, 2.0, 5.0, 10.0, 30.0]; + for distance in seek_distances { + let seek_time = benchmark_seek(&decoder, config.fps, 0.0, distance).await; + results.seek_times_by_distance.push((distance, seek_time)); + println!(" {:.1}s seek: {:.2}ms", distance, seek_time); + } + + println!("[5/5] Benchmarking random access (50 samples)..."); + let video_duration = 60.0f32; + results.random_access_times_ms = + benchmark_random_access(&decoder, config.fps, video_duration, 50).await; + results.random_access_avg_ms = if results.random_access_times_ms.is_empty() { + 0.0 + } else { + results.random_access_times_ms.iter().sum::() + / results.random_access_times_ms.len() as f64 + }; + println!(" Done: {:.2}ms avg", results.random_access_avg_ms); + + results +} + +fn main() { + let args: Vec = std::env::args().collect(); + + let video_path = args + .iter() + .position(|a| a == "--video") + .and_then(|i| args.get(i + 1)) + .map(PathBuf::from) + .expect("Usage: decode-benchmark --video [--fps ] [--iterations ]"); + + let fps = args + .iter() + .position(|a| a == "--fps") + .and_then(|i| args.get(i + 1)) + .and_then(|s| s.parse().ok()) + .unwrap_or(30); + + let iterations = args + .iter() + .position(|a| a == "--iterations") + .and_then(|i| args.get(i + 1)) + .and_then(|s| s.parse().ok()) + .unwrap_or(100); + + let config = BenchmarkConfig { + video_path, + fps, + iterations, + }; + + let rt = Runtime::new().expect("Failed to create Tokio runtime"); + let results = rt.block_on(run_full_benchmark(config)); + + results.print_report(); +} From ef81bbcca48aa4cbcc8554ecfe6efe55ad84be04 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 17:16:13 +0000 Subject: [PATCH 11/37] Refactor CameraLayer::prepare argument structure --- crates/rendering/src/layers/camera.rs | 21 ++++++++++++------- crates/rendering/src/lib.rs | 30 +++++++++++++-------------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/crates/rendering/src/layers/camera.rs b/crates/rendering/src/layers/camera.rs index 7a26d266be..c4c5dc7412 100644 --- a/crates/rendering/src/layers/camera.rs +++ b/crates/rendering/src/layers/camera.rs @@ -59,15 +59,24 @@ impl CameraLayer { &mut self, device: &wgpu::Device, queue: &wgpu::Queue, - data: Option<(CompositeVideoFrameUniforms, XY, &DecodedFrame, f32)>, + uniforms: Option, + frame_data: Option<(XY, &DecodedFrame, f32)>, ) { - self.hidden = data.is_none(); + let Some(uniforms) = uniforms else { + self.hidden = true; + return; + }; - let Some((uniforms, frame_size, camera_frame, recording_time)) = data else { + let has_previous_frame = self.last_recording_time.is_some(); + self.hidden = frame_data.is_none() && !has_previous_frame; + + queue.write_buffer(&self.uniforms_buffer, 0, bytemuck::cast_slice(&[uniforms])); + + let Some((frame_size, camera_frame, recording_time)) = frame_data else { return; }; - let frame_data = camera_frame.data(); + let frame_data_bytes = camera_frame.data(); let format = camera_frame.format(); let is_same_frame = self @@ -107,7 +116,7 @@ impl CameraLayer { origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All, }, - frame_data, + frame_data_bytes, wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(src_bytes_per_row), @@ -197,8 +206,6 @@ impl CameraLayer { self.last_recording_time = Some(recording_time); self.current_texture = next_texture; } - - queue.write_buffer(&self.uniforms_buffer, 0, bytemuck::cast_slice(&[uniforms])); } fn copy_from_yuv_output( diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 0f5a8594bc..08d7993d62 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -1694,27 +1694,25 @@ impl RendererLayers { self.camera.prepare( &constants.device, &constants.queue, - (|| { - Some(( - uniforms.camera?, - constants.options.camera_size?, - segment_frames.camera_frame.as_ref()?, - segment_frames.recording_time, - )) - })(), + uniforms.camera, + constants.options.camera_size.and_then(|size| { + segment_frames + .camera_frame + .as_ref() + .map(|frame| (size, frame, segment_frames.recording_time)) + }), ); self.camera_only.prepare( &constants.device, &constants.queue, - (|| { - Some(( - uniforms.camera_only?, - constants.options.camera_size?, - segment_frames.camera_frame.as_ref()?, - segment_frames.recording_time, - )) - })(), + uniforms.camera_only, + constants.options.camera_size.and_then(|size| { + segment_frames + .camera_frame + .as_ref() + .map(|frame| (size, frame, segment_frames.recording_time)) + }), ); self.text.prepare( From 4ff10db038aacc8c18469e212452660c94425dc7 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 18:43:14 +0000 Subject: [PATCH 12/37] Improve video decoding performance and add keyframe indexing --- .claude/settings.local.json | 3 +- crates/editor/src/playback.rs | 8 +- crates/rendering/src/decoder/avassetreader.rs | 139 ++++++++++++++++-- crates/rendering/src/decoder/mod.rs | 13 +- crates/rendering/src/lib.rs | 2 +- crates/video-decode/src/avassetreader.rs | 134 ++++++++++++++++- crates/video-decode/src/lib.rs | 2 +- 7 files changed, 281 insertions(+), 20 deletions(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 0f864d7abe..5babcbf0ad 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -50,7 +50,8 @@ "Bash(cargo build:*)", "Bash(footprint:*)", "Bash(RUST_LOG=info,cap_recording=debug ./target/release/examples/memory-leak-detector:*)", - "Bash(git rm:*)" + "Bash(git rm:*)", + "Bash(./target/release/examples/decode-benchmark:*)" ], "deny": [], "ask": [] diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index 9d333ec77f..a5ddeab9c0 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -30,11 +30,11 @@ use crate::{ segments::get_audio_segments, }; -const PREFETCH_BUFFER_SIZE: usize = 30; +const PREFETCH_BUFFER_SIZE: usize = 60; const PARALLEL_DECODE_TASKS: usize = 8; const MAX_PREFETCH_AHEAD: u32 = 90; const PREFETCH_BEHIND: u32 = 15; -const FRAME_CACHE_SIZE: usize = 30; +const FRAME_CACHE_SIZE: usize = 60; #[derive(Debug)] pub enum PlaybackStartError { @@ -356,8 +356,8 @@ impl Playback { let mut total_frames_rendered = 0u64; let mut _total_frames_skipped = 0u64; - let warmup_target_frames = 2usize; - let warmup_after_first_timeout = Duration::from_millis(50); + let warmup_target_frames = 1usize; + let warmup_after_first_timeout = Duration::from_millis(16); let mut first_frame_time: Option = None; while !*stop_rx.borrow() { diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index 61e0efcb69..43e96a7eaf 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -20,11 +20,22 @@ use crate::{DecodedFrame, PixelFormat}; use super::frame_converter::{copy_bgra_to_rgba, copy_rgba_plane}; use super::{DecoderInitResult, DecoderType, FRAME_CACHE_SIZE, VideoDecoderMessage, pts_to_frame}; +struct SendableImageBuf(R); +unsafe impl Send for SendableImageBuf {} +unsafe impl Sync for SendableImageBuf {} + +impl Clone for SendableImageBuf { + fn clone(&self) -> Self { + Self(self.0.retained()) + } +} + #[derive(Clone)] struct FrameData { data: Arc>, y_stride: u32, uv_stride: u32, + image_buf: Option>, } #[derive(Clone)] @@ -42,19 +53,32 @@ impl ProcessedFrame { data, y_stride, uv_stride, + image_buf, } = &self.frame_data; match self.format { PixelFormat::Rgba => { DecodedFrame::new_with_arc(Arc::clone(data), self.width, self.height) } - PixelFormat::Nv12 => DecodedFrame::new_nv12_with_arc( - Arc::clone(data), - self.width, - self.height, - *y_stride, - *uv_stride, - ), + PixelFormat::Nv12 => { + if let Some(img_buf) = image_buf { + DecodedFrame::new_nv12_zero_copy( + self.width, + self.height, + *y_stride, + *uv_stride, + img_buf.0.retained(), + ) + } else { + DecodedFrame::new_nv12_with_arc( + Arc::clone(data), + self.width, + self.height, + *y_stride, + *uv_stride, + ) + } + } PixelFormat::Yuv420p => DecodedFrame::new_yuv420p_with_arc( Arc::clone(data), self.width, @@ -201,11 +225,56 @@ impl ImageBufProcessor { } impl CachedFrame { - fn new(processor: &ImageBufProcessor, mut image_buf: R, number: u32) -> Self { + fn new(_processor: &ImageBufProcessor, image_buf: R, number: u32) -> Self { let width = image_buf.width() as u32; let height = image_buf.height() as u32; - let (data, format, y_stride, uv_stride) = processor.extract_raw(&mut image_buf); + let pixel_format = + cap_video_decode::avassetreader::pixel_format_to_pixel(image_buf.pixel_format()); + + let (format, y_stride, uv_stride, stored_image_buf) = match pixel_format { + format::Pixel::NV12 => { + let y_stride = image_buf.plane_bytes_per_row(0) as u32; + let uv_stride = image_buf.plane_bytes_per_row(1) as u32; + ( + PixelFormat::Nv12, + y_stride, + uv_stride, + Some(Arc::new(SendableImageBuf(image_buf))), + ) + } + format::Pixel::RGBA | format::Pixel::BGRA | format::Pixel::YUV420P => { + let mut img = image_buf; + let (data, fmt, y_str, uv_str) = _processor.extract_raw(&mut img); + return Self(ProcessedFrame { + _number: number, + width, + height, + format: fmt, + frame_data: FrameData { + data: Arc::new(data), + y_stride: y_str, + uv_stride: uv_str, + image_buf: None, + }, + }); + } + _ => { + let black_frame = vec![0u8; (width * height * 4) as usize]; + return Self(ProcessedFrame { + _number: number, + width, + height, + format: PixelFormat::Rgba, + frame_data: FrameData { + data: Arc::new(black_frame), + y_stride: width * 4, + uv_stride: 0, + image_buf: None, + }, + }); + } + }; let frame = ProcessedFrame { _number: number, @@ -213,9 +282,10 @@ impl CachedFrame { height, format, frame_data: FrameData { - data: Arc::new(data), + data: Arc::new(Vec::new()), y_stride, uv_stride, + image_buf: stored_image_buf, }, }; Self(frame) @@ -302,6 +372,12 @@ impl AVAssetReaderDecoder { match r { VideoDecoderMessage::GetFrame(requested_time, sender) => { let frame = (requested_time * fps as f32).floor() as u32; + debug!( + decoder = _name, + requested_time = requested_time, + frame = frame, + "GetFrame request received" + ); if !sender.is_closed() { pending_requests.push(PendingRequest { frame, sender }); } @@ -365,6 +441,13 @@ impl AVAssetReaderDecoder { }; if needs_reset { + debug!( + decoder = _name, + requested_frame = requested_frame, + cache_min = ?cache_frame_min, + cache_max = ?cache_frame_max, + "Triggering decoder reset" + ); this.reset(requested_time); frames = this.inner.frames(); *last_sent_frame.borrow_mut() = None; @@ -374,11 +457,13 @@ impl AVAssetReaderDecoder { last_active_frame = Some(requested_frame); let mut exit = false; + let mut frames_iterated = 0u32; for frame in &mut frames { let Ok(frame) = frame.map_err(|e| format!("read frame / {e}")) else { continue; }; + frames_iterated += 1; let current_frame = pts_to_frame(frame.pts().value, Rational::new(1, frame.pts().scale), fps); @@ -452,12 +537,46 @@ impl AVAssetReaderDecoder { this.is_done = true; + if !pending_requests.is_empty() { + debug!( + decoder = _name, + pending_count = pending_requests.len(), + frames_iterated = frames_iterated, + cache_size = cache.len(), + "Handling unfulfilled requests after frame iteration" + ); + } + for req in pending_requests.drain(..) { if let Some(cached) = cache.get(&req.frame) { let data = cached.data().clone(); if req.sender.send(data.to_decoded_frame()).is_err() { debug!("frame receiver dropped before send"); } + } else if let Some(last) = last_sent_frame.borrow().clone() { + debug!( + decoder = _name, + requested_frame = req.frame, + "Sending last known frame as fallback" + ); + if req.sender.send(last.to_decoded_frame()).is_err() { + debug!("frame receiver dropped before send"); + } + } else if let Some(first) = first_ever_frame.borrow().clone() { + debug!( + decoder = _name, + requested_frame = req.frame, + "Sending first ever frame as fallback" + ); + if req.sender.send(first.to_decoded_frame()).is_err() { + debug!("frame receiver dropped before send"); + } + } else { + debug!( + decoder = _name, + requested_frame = req.frame, + "No frame available to send - request dropped" + ); } } } diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs index 5a6e31db9b..5eb56ed798 100644 --- a/crates/rendering/src/decoder/mod.rs +++ b/crates/rendering/src/decoder/mod.rs @@ -547,7 +547,7 @@ pub fn pts_to_frame(pts: i64, time_base: Rational, fps: u32) -> u32 { .round() as u32 } -pub const FRAME_CACHE_SIZE: usize = 60; +pub const FRAME_CACHE_SIZE: usize = 150; #[derive(Clone)] pub struct AsyncVideoDecoderHandle { @@ -570,7 +570,16 @@ impl AsyncVideoDecoderHandle { return None; } - rx.await.ok() + match tokio::time::timeout(std::time::Duration::from_millis(500), rx).await { + Ok(result) => result.ok(), + Err(_) => { + debug!( + adjusted_time = adjusted_time, + "get_frame timed out after 500ms" + ); + None + } + } } pub fn get_time(&self, time: f32) -> f32 { diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 08d7993d62..84740ecd71 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -158,7 +158,7 @@ impl RecordingSegmentDecoders { segment.camera.as_ref().unwrap().fps } StudioRecordingMeta::MultipleSegments { inner, .. } => { - inner.segments[0].camera.as_ref().unwrap().fps + inner.segments[segment_i].camera.as_ref().unwrap().fps } }, match &meta { diff --git a/crates/video-decode/src/avassetreader.rs b/crates/video-decode/src/avassetreader.rs index 8471701bb9..efc9e9501b 100644 --- a/crates/video-decode/src/avassetreader.rs +++ b/crates/video-decode/src/avassetreader.rs @@ -10,6 +10,92 @@ use cidre::{ use ffmpeg::{codec as avcodec, format as avformat}; use tokio::runtime::Handle as TokioHandle; +pub struct KeyframeIndex { + keyframes: Vec<(u32, f64)>, + fps: f64, +} + +impl KeyframeIndex { + pub fn build(path: &PathBuf) -> Result { + let build_start = std::time::Instant::now(); + + let input = avformat::input(path) + .map_err(|e| format!("Failed to open video for keyframe scan: {e}"))?; + + let video_stream = input + .streams() + .best(ffmpeg::media::Type::Video) + .ok_or("No video stream found")?; + + let stream_index = video_stream.index(); + let time_base = video_stream.time_base(); + let fps = { + let rate = video_stream.avg_frame_rate(); + if rate.denominator() == 0 { + 30.0 + } else { + rate.numerator() as f64 / rate.denominator() as f64 + } + }; + + let mut keyframes = Vec::new(); + + let mut input = + avformat::input(path).map_err(|e| format!("Failed to reopen video for scan: {e}"))?; + + for (stream, packet) in input.packets() { + if stream.index() != stream_index { + continue; + } + + if packet.is_key() { + let pts = packet.pts().unwrap_or(0); + let time_secs = + pts as f64 * time_base.numerator() as f64 / time_base.denominator() as f64; + let frame_number = (time_secs * fps).round() as u32; + keyframes.push((frame_number, time_secs)); + } + } + + let elapsed = build_start.elapsed(); + tracing::info!( + path = %path.display(), + keyframe_count = keyframes.len(), + fps = fps, + build_ms = elapsed.as_millis(), + "Built keyframe index" + ); + + Ok(Self { keyframes, fps }) + } + + pub fn nearest_keyframe_before(&self, target_frame: u32) -> Option<(u32, f64)> { + if self.keyframes.is_empty() { + return None; + } + + let pos = self + .keyframes + .binary_search_by_key(&target_frame, |(frame, _)| *frame); + + let idx = match pos { + Ok(i) => i, + Err(0) => 0, + Err(i) => i - 1, + }; + + Some(self.keyframes[idx]) + } + + pub fn fps(&self) -> f64 { + self.fps + } + + pub fn keyframe_count(&self) -> usize { + self.keyframes.len() + } +} + pub struct AVAssetReaderDecoder { path: PathBuf, pixel_format: cv::PixelFormat, @@ -18,10 +104,23 @@ pub struct AVAssetReaderDecoder { reader: R, width: u32, height: u32, + keyframe_index: Option, } impl AVAssetReaderDecoder { pub fn new(path: PathBuf, tokio_handle: TokioHandle) -> Result { + let keyframe_index = match KeyframeIndex::build(&path) { + Ok(index) => Some(index), + Err(e) => { + tracing::warn!( + path = %path.display(), + error = %e, + "Failed to build keyframe index, seeking may be slower" + ); + None + } + }; + let (pixel_format, width, height) = { let input = ffmpeg::format::input(&path).unwrap(); @@ -55,23 +154,56 @@ impl AVAssetReaderDecoder { reader, width, height, + keyframe_index, }) } pub fn reset(&mut self, requested_time: f32) -> Result<(), String> { + let reset_start = std::time::Instant::now(); + self.reader.cancel_reading(); + + let (seek_time, keyframe_frame) = if let Some(ref keyframe_index) = self.keyframe_index { + let fps = keyframe_index.fps(); + let target_frame = (requested_time as f64 * fps).round() as u32; + + if let Some((kf_frame, keyframe_time)) = + keyframe_index.nearest_keyframe_before(target_frame) + { + (keyframe_time as f32, Some(kf_frame)) + } else { + (requested_time, None) + } + } else { + (requested_time, None) + }; + (self.track_output, self.reader) = Self::get_reader_track_output( &self.path, - requested_time, + seek_time, &self.tokio_handle, self.pixel_format, self.width, self.height, )?; + let elapsed = reset_start.elapsed(); + tracing::info!( + requested_time = requested_time, + seek_time = seek_time, + keyframe_frame = ?keyframe_frame, + reset_ms = elapsed.as_millis(), + has_keyframe_index = self.keyframe_index.is_some(), + "AVAssetReader reset completed" + ); + Ok(()) } + pub fn keyframe_index(&self) -> Option<&KeyframeIndex> { + self.keyframe_index.as_ref() + } + fn get_reader_track_output( path: &PathBuf, time: f32, diff --git a/crates/video-decode/src/lib.rs b/crates/video-decode/src/lib.rs index ae408bc902..d14bd82696 100644 --- a/crates/video-decode/src/lib.rs +++ b/crates/video-decode/src/lib.rs @@ -5,7 +5,7 @@ pub mod ffmpeg; pub mod media_foundation; #[cfg(target_os = "macos")] -pub use avassetreader::AVAssetReaderDecoder; +pub use avassetreader::{AVAssetReaderDecoder, KeyframeIndex}; pub use ffmpeg::FFmpegDecoder; #[cfg(target_os = "windows")] pub use media_foundation::{ From cf969693ccea328c9d56c914de100f622638a03a Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 18:56:40 +0000 Subject: [PATCH 13/37] Add multi-position decoder pool for AVAssetReader --- crates/rendering/src/decoder/avassetreader.rs | 302 +++++++++++++----- crates/rendering/src/decoder/mod.rs | 2 + .../rendering/src/decoder/multi_position.rs | 247 ++++++++++++++ crates/video-decode/src/avassetreader.rs | 136 +++++++- 4 files changed, 600 insertions(+), 87 deletions(-) create mode 100644 crates/rendering/src/decoder/multi_position.rs diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index 43e96a7eaf..db6ea9d382 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -18,6 +18,7 @@ use tokio::{runtime::Handle as TokioHandle, sync::oneshot}; use crate::{DecodedFrame, PixelFormat}; use super::frame_converter::{copy_bgra_to_rgba, copy_rgba_plane}; +use super::multi_position::{DecoderPoolManager, MultiPositionDecoderConfig, ScrubDetector}; use super::{DecoderInitResult, DecoderType, FRAME_CACHE_SIZE, VideoDecoderMessage, pts_to_frame}; struct SendableImageBuf(R); @@ -296,21 +297,141 @@ impl CachedFrame { } } -pub struct AVAssetReaderDecoder { +struct DecoderInstance { inner: cap_video_decode::AVAssetReaderDecoder, is_done: bool, + frames_iter_valid: bool, } -impl AVAssetReaderDecoder { - fn new(path: PathBuf, tokio_handle: TokioHandle) -> Result { +impl DecoderInstance { + fn new( + path: PathBuf, + tokio_handle: TokioHandle, + start_time: f32, + keyframe_index: Option, + ) -> Result { Ok(Self { - inner: cap_video_decode::AVAssetReaderDecoder::new(path, tokio_handle)?, + inner: cap_video_decode::AVAssetReaderDecoder::new_with_keyframe_index( + path, + tokio_handle, + start_time, + keyframe_index, + )?, is_done: false, + frames_iter_valid: true, }) } fn reset(&mut self, requested_time: f32) { let _ = self.inner.reset(requested_time); + self.is_done = false; + self.frames_iter_valid = true; + } + + fn current_position(&self) -> f32 { + self.inner.current_position_secs() + } +} + +pub struct AVAssetReaderDecoder { + decoders: Vec, + pool_manager: DecoderPoolManager, + active_decoder_idx: usize, + scrub_detector: ScrubDetector, +} + +impl AVAssetReaderDecoder { + fn new(path: PathBuf, tokio_handle: TokioHandle) -> Result { + let mut primary_decoder = + cap_video_decode::AVAssetReaderDecoder::new(path.clone(), tokio_handle.clone())?; + + let keyframe_index = primary_decoder.take_keyframe_index(); + let keyframe_index_arc: Option> = None; + + let fps = keyframe_index + .as_ref() + .map(|kf| kf.fps() as u32) + .unwrap_or(30); + let duration_secs = keyframe_index + .as_ref() + .map(|kf| kf.duration_secs()) + .unwrap_or(0.0); + + let config = MultiPositionDecoderConfig { + path: path.clone(), + tokio_handle: tokio_handle.clone(), + keyframe_index: keyframe_index_arc, + fps, + duration_secs, + }; + + let pool_manager = DecoderPoolManager::new(config); + + let primary_instance = DecoderInstance { + inner: primary_decoder, + is_done: false, + frames_iter_valid: true, + }; + + let mut decoders = vec![primary_instance]; + + let initial_positions = pool_manager.positions(); + for pos in initial_positions.iter().skip(1) { + let start_time = pos.position_secs; + match DecoderInstance::new(path.clone(), tokio_handle.clone(), start_time, None) { + Ok(instance) => { + decoders.push(instance); + tracing::info!( + position_secs = start_time, + decoder_index = decoders.len() - 1, + "Created additional decoder instance for multi-position pool" + ); + } + Err(e) => { + tracing::warn!( + position_secs = start_time, + error = %e, + "Failed to create additional decoder instance, continuing with fewer decoders" + ); + } + } + } + + tracing::info!( + decoder_count = decoders.len(), + fps = fps, + duration_secs = duration_secs, + "Initialized multi-position decoder pool" + ); + + Ok(Self { + decoders, + pool_manager, + active_decoder_idx: 0, + scrub_detector: ScrubDetector::new(), + }) + } + + fn select_best_decoder(&mut self, requested_time: f32) -> (usize, bool) { + let (best_id, distance, needs_reset) = + self.pool_manager.find_best_decoder_for_time(requested_time); + + let decoder_idx = best_id.min(self.decoders.len().saturating_sub(1)); + + if needs_reset && decoder_idx < self.decoders.len() { + tracing::debug!( + decoder_idx = decoder_idx, + requested_time = requested_time, + distance = distance, + "Resetting decoder for seek" + ); + self.decoders[decoder_idx].reset(requested_time); + self.pool_manager + .update_decoder_position(best_id, self.decoders[decoder_idx].current_position()); + } + + self.active_decoder_idx = decoder_idx; + (decoder_idx, needs_reset) } pub fn spawn( @@ -341,8 +462,8 @@ impl AVAssetReaderDecoder { } }; - let video_width = this.inner.width(); - let video_height = this.inner.height(); + let video_width = this.decoders[0].inner.width(); + let video_height = this.decoders[0].inner.height(); let init_result = DecoderInitResult { width: video_width, @@ -358,7 +479,6 @@ impl AVAssetReaderDecoder { let last_sent_frame = Rc::new(RefCell::new(None::)); let first_ever_frame = Rc::new(RefCell::new(None::)); - let mut frames = this.inner.frames(); let processor = ImageBufProcessor::new(); struct PendingRequest { @@ -397,6 +517,12 @@ impl AVAssetReaderDecoder { pending_requests.sort_by_key(|r| r.frame); + let is_scrubbing = if let Some(first_req) = pending_requests.first() { + this.scrub_detector.record_request(first_req.frame) + } else { + false + }; + let mut i = 0; while i < pending_requests.len() { let request = &pending_requests[i]; @@ -421,35 +547,16 @@ impl AVAssetReaderDecoder { let requested_frame = min_requested_frame; let requested_time = requested_frame as f32 / fps as f32; - const BACKWARD_SEEK_TOLERANCE: u32 = 120; + let (decoder_idx, was_reset) = this.select_best_decoder(requested_time); let cache_min = min_requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); - let cache_max = max_requested_frame + FRAME_CACHE_SIZE as u32 / 2; - - let cache_frame_min = cache.keys().next().copied(); - let cache_frame_max = cache.keys().next_back().copied(); - - let needs_reset = if let (Some(c_min), Some(c_max)) = (cache_frame_min, cache_frame_max) - { - let is_backward_seek_beyond_tolerance = - requested_frame + BACKWARD_SEEK_TOLERANCE < c_min; - let is_forward_seek_beyond_cache = - requested_frame > c_max + FRAME_CACHE_SIZE as u32 / 4; - is_backward_seek_beyond_tolerance || is_forward_seek_beyond_cache + let cache_max = if is_scrubbing { + max_requested_frame + FRAME_CACHE_SIZE as u32 / 4 } else { - true + max_requested_frame + FRAME_CACHE_SIZE as u32 / 2 }; - if needs_reset { - debug!( - decoder = _name, - requested_frame = requested_frame, - cache_min = ?cache_frame_min, - cache_max = ?cache_frame_max, - "Triggering decoder reset" - ); - this.reset(requested_time); - frames = this.inner.frames(); + if was_reset { *last_sent_frame.borrow_mut() = None; cache.retain(|&f, _| f >= cache_min && f <= cache_max); } @@ -458,84 +565,109 @@ impl AVAssetReaderDecoder { let mut exit = false; let mut frames_iterated = 0u32; + let mut last_decoded_position: Option = None; - for frame in &mut frames { - let Ok(frame) = frame.map_err(|e| format!("read frame / {e}")) else { - continue; - }; - frames_iterated += 1; + { + let decoder = &mut this.decoders[decoder_idx]; + let mut frames = decoder.inner.frames(); - let current_frame = - pts_to_frame(frame.pts().value, Rational::new(1, frame.pts().scale), fps); + for frame in &mut frames { + let Ok(frame) = frame.map_err(|e| format!("read frame / {e}")) else { + continue; + }; + frames_iterated += 1; - let Some(frame) = frame.image_buf() else { - continue; - }; + let current_frame = + pts_to_frame(frame.pts().value, Rational::new(1, frame.pts().scale), fps); - let cache_frame = CachedFrame::new(&processor, frame.retained(), current_frame); + let position_secs = current_frame as f32 / fps as f32; + last_decoded_position = Some(position_secs); - if first_ever_frame.borrow().is_none() { - *first_ever_frame.borrow_mut() = Some(cache_frame.data().clone()); - } + let Some(frame) = frame.image_buf() else { + continue; + }; - this.is_done = false; + let cache_frame = CachedFrame::new(&processor, frame.retained(), current_frame); - let exceeds_cache_bounds = current_frame > cache_max; - let too_small_for_cache_bounds = current_frame < cache_min; + if first_ever_frame.borrow().is_none() { + *first_ever_frame.borrow_mut() = Some(cache_frame.data().clone()); + } - if !too_small_for_cache_bounds { - if cache.len() >= FRAME_CACHE_SIZE { - if let Some(last_active) = &last_active_frame { - let frame_to_remove = if requested_frame > *last_active { - *cache.keys().next().unwrap() - } else if requested_frame < *last_active { - *cache.keys().next_back().unwrap() + decoder.is_done = false; + + let exceeds_cache_bounds = current_frame > cache_max; + let too_small_for_cache_bounds = current_frame < cache_min; + + if !too_small_for_cache_bounds { + if cache.len() >= FRAME_CACHE_SIZE { + if let Some(last_active) = &last_active_frame { + let frame_to_remove = if requested_frame > *last_active { + *cache.keys().next().unwrap() + } else if requested_frame < *last_active { + *cache.keys().next_back().unwrap() + } else { + let min = *cache.keys().min().unwrap(); + let max = *cache.keys().max().unwrap(); + if current_frame > max { min } else { max } + }; + cache.remove(&frame_to_remove); } else { - let min = *cache.keys().min().unwrap(); - let max = *cache.keys().max().unwrap(); - if current_frame > max { min } else { max } - }; - cache.remove(&frame_to_remove); - } else { - cache.clear() + cache.clear() + } } - } - cache.insert(current_frame, cache_frame.clone()); + cache.insert(current_frame, cache_frame.clone()); - let mut remaining_requests = Vec::with_capacity(pending_requests.len()); - for req in pending_requests.drain(..) { - if req.frame == current_frame { - let data = cache_frame.data().clone(); - *last_sent_frame.borrow_mut() = Some(data.clone()); - if req.sender.send(data.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } - } else if req.frame < current_frame { - if let Some(cached) = cache.get(&req.frame) { - let data = cached.data().clone(); + let mut remaining_requests = Vec::with_capacity(pending_requests.len()); + for req in pending_requests.drain(..) { + if req.frame == current_frame { + let data = cache_frame.data().clone(); *last_sent_frame.borrow_mut() = Some(data.clone()); if req.sender.send(data.to_decoded_frame()).is_err() { debug!("frame receiver dropped before send"); } + } else if req.frame < current_frame { + if let Some(cached) = cache.get(&req.frame) { + let data = cached.data().clone(); + *last_sent_frame.borrow_mut() = Some(data.clone()); + if req.sender.send(data.to_decoded_frame()).is_err() { + debug!("frame receiver dropped before send"); + } + } else if is_scrubbing { + let data = cache_frame.data().clone(); + *last_sent_frame.borrow_mut() = Some(data.clone()); + if req.sender.send(data.to_decoded_frame()).is_err() { + debug!( + "frame receiver dropped before send (scrub fallback)" + ); + } + } + } else { + remaining_requests.push(req); } - } else { - remaining_requests.push(req); } + pending_requests = remaining_requests; } - pending_requests = remaining_requests; - } - *last_sent_frame.borrow_mut() = Some(cache_frame.data().clone()); + *last_sent_frame.borrow_mut() = Some(cache_frame.data().clone()); - exit = exit || exceeds_cache_bounds; + exit = exit || exceeds_cache_bounds; + + if is_scrubbing && frames_iterated > 3 { + break; + } - if pending_requests.is_empty() || exit { - break; + if pending_requests.is_empty() || exit { + break; + } } + + decoder.is_done = true; } - this.is_done = true; + if let Some(pos) = last_decoded_position { + this.pool_manager.update_decoder_position(decoder_idx, pos); + } if !pending_requests.is_empty() { debug!( @@ -543,6 +675,7 @@ impl AVAssetReaderDecoder { pending_count = pending_requests.len(), frames_iterated = frames_iterated, cache_size = cache.len(), + is_scrubbing = is_scrubbing, "Handling unfulfilled requests after frame iteration" ); } @@ -557,6 +690,7 @@ impl AVAssetReaderDecoder { debug!( decoder = _name, requested_frame = req.frame, + is_scrubbing = is_scrubbing, "Sending last known frame as fallback" ); if req.sender.send(last.to_decoded_frame()).is_err() { diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs index 5eb56ed798..f02800b2d4 100644 --- a/crates/rendering/src/decoder/mod.rs +++ b/crates/rendering/src/decoder/mod.rs @@ -16,6 +16,8 @@ mod ffmpeg; mod frame_converter; #[cfg(target_os = "windows")] mod media_foundation; +#[cfg(target_os = "macos")] +pub mod multi_position; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DecoderType { diff --git a/crates/rendering/src/decoder/multi_position.rs b/crates/rendering/src/decoder/multi_position.rs new file mode 100644 index 0000000000..ce453a2ba8 --- /dev/null +++ b/crates/rendering/src/decoder/multi_position.rs @@ -0,0 +1,247 @@ +use std::{collections::BTreeMap, path::PathBuf, sync::Arc}; + +use tokio::runtime::Handle as TokioHandle; + +use cap_video_decode::avassetreader::KeyframeIndex; + +pub const MAX_DECODER_POOL_SIZE: usize = 3; +pub const REPOSITION_THRESHOLD_SECS: f32 = 5.0; + +pub struct DecoderPosition { + pub id: usize, + pub position_secs: f32, + pub last_access_time: std::time::Instant, + pub access_count: u64, +} + +impl DecoderPosition { + pub fn new(id: usize, position_secs: f32) -> Self { + Self { + id, + position_secs, + last_access_time: std::time::Instant::now(), + access_count: 0, + } + } + + pub fn touch(&mut self) { + self.last_access_time = std::time::Instant::now(); + self.access_count += 1; + } +} + +pub struct MultiPositionDecoderConfig { + pub path: PathBuf, + pub tokio_handle: TokioHandle, + pub keyframe_index: Option>, + pub fps: u32, + pub duration_secs: f64, +} + +pub struct DecoderPoolManager { + config: MultiPositionDecoderConfig, + positions: Vec, + access_history: BTreeMap, + total_accesses: u64, +} + +impl DecoderPoolManager { + pub fn new(config: MultiPositionDecoderConfig) -> Self { + let initial_positions = Self::calculate_initial_positions(&config); + + let positions: Vec = initial_positions + .into_iter() + .enumerate() + .map(|(id, pos)| DecoderPosition::new(id, pos)) + .collect(); + + Self { + config, + positions, + access_history: BTreeMap::new(), + total_accesses: 0, + } + } + + fn calculate_initial_positions(config: &MultiPositionDecoderConfig) -> Vec { + if let Some(ref kf_index) = config.keyframe_index { + let strategic = kf_index.get_strategic_positions(MAX_DECODER_POOL_SIZE); + strategic.into_iter().map(|t| t as f32).collect() + } else { + let duration = config.duration_secs as f32; + if duration <= 0.0 { + vec![0.0] + } else { + (0..MAX_DECODER_POOL_SIZE) + .map(|i| { + let frac = i as f32 / MAX_DECODER_POOL_SIZE as f32; + (duration * frac).min(duration) + }) + .collect() + } + } + } + + pub fn find_best_decoder_for_time(&mut self, requested_time: f32) -> (usize, f32, bool) { + self.total_accesses += 1; + + let frame = (requested_time * self.config.fps as f32).floor() as u32; + *self.access_history.entry(frame).or_insert(0) += 1; + + let mut best_decoder_id = 0; + let mut best_distance = f32::MAX; + let mut needs_reset = true; + + for position in &self.positions { + let distance = (position.position_secs - requested_time).abs(); + let is_usable = position.position_secs <= requested_time + && (requested_time - position.position_secs) < REPOSITION_THRESHOLD_SECS; + + if is_usable && distance < best_distance { + best_distance = distance; + best_decoder_id = position.id; + needs_reset = false; + } + } + + if needs_reset { + for position in &self.positions { + let distance = (position.position_secs - requested_time).abs(); + if distance < best_distance { + best_distance = distance; + best_decoder_id = position.id; + } + } + } + + if let Some(pos) = self.positions.iter_mut().find(|p| p.id == best_decoder_id) { + pos.touch(); + } + + (best_decoder_id, best_distance, needs_reset) + } + + pub fn update_decoder_position(&mut self, decoder_id: usize, new_position: f32) { + if let Some(pos) = self.positions.iter_mut().find(|p| p.id == decoder_id) { + pos.position_secs = new_position; + } + } + + pub fn should_rebalance(&self) -> bool { + self.total_accesses > 0 && self.total_accesses % 100 == 0 + } + + pub fn get_rebalance_positions(&self) -> Vec { + if self.access_history.is_empty() { + return self.positions.iter().map(|p| p.position_secs).collect(); + } + + let mut hotspots: Vec<(u32, u64)> = self + .access_history + .iter() + .map(|(&frame, &count)| (frame, count)) + .collect(); + hotspots.sort_by(|a, b| b.1.cmp(&a.1)); + + let top_hotspots: Vec = hotspots + .into_iter() + .take(MAX_DECODER_POOL_SIZE) + .map(|(frame, _)| frame as f32 / self.config.fps as f32) + .collect(); + + if top_hotspots.len() < MAX_DECODER_POOL_SIZE { + let mut result = top_hotspots; + let remaining = MAX_DECODER_POOL_SIZE - result.len(); + let duration = self.config.duration_secs as f32; + for i in 0..remaining { + let frac = (i + 1) as f32 / (remaining + 1) as f32; + result.push(duration * frac); + } + result + } else { + top_hotspots + } + } + + pub fn positions(&self) -> &[DecoderPosition] { + &self.positions + } + + pub fn config(&self) -> &MultiPositionDecoderConfig { + &self.config + } +} + +pub struct ScrubDetector { + last_request_time: std::time::Instant, + last_frame: u32, + request_rate: f64, + is_scrubbing: bool, + scrub_start_time: Option, +} + +impl ScrubDetector { + const SCRUB_THRESHOLD_RATE: f64 = 5.0; + const SCRUB_COOLDOWN_MS: u64 = 150; + + pub fn new() -> Self { + Self { + last_request_time: std::time::Instant::now(), + last_frame: 0, + request_rate: 0.0, + is_scrubbing: false, + scrub_start_time: None, + } + } + + pub fn record_request(&mut self, frame: u32) -> bool { + let now = std::time::Instant::now(); + let elapsed = now.duration_since(self.last_request_time); + let elapsed_secs = elapsed.as_secs_f64().max(0.001); + + let frame_delta = frame.abs_diff(self.last_frame); + + let instantaneous_rate = frame_delta as f64 / elapsed_secs; + self.request_rate = self.request_rate * 0.7 + instantaneous_rate * 0.3; + + let was_scrubbing = self.is_scrubbing; + + if self.request_rate > Self::SCRUB_THRESHOLD_RATE && frame_delta > 1 { + self.is_scrubbing = true; + if self.scrub_start_time.is_none() { + self.scrub_start_time = Some(now); + } + } else if elapsed.as_millis() as u64 > Self::SCRUB_COOLDOWN_MS { + self.is_scrubbing = false; + self.scrub_start_time = None; + } + + self.last_request_time = now; + self.last_frame = frame; + + if was_scrubbing != self.is_scrubbing { + tracing::debug!( + is_scrubbing = self.is_scrubbing, + request_rate = self.request_rate, + frame_delta = frame_delta, + "Scrub state changed" + ); + } + + self.is_scrubbing + } + + pub fn is_scrubbing(&self) -> bool { + self.is_scrubbing + } + + pub fn scrub_duration(&self) -> Option { + self.scrub_start_time.map(|start| start.elapsed()) + } +} + +impl Default for ScrubDetector { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/video-decode/src/avassetreader.rs b/crates/video-decode/src/avassetreader.rs index efc9e9501b..696fa2628a 100644 --- a/crates/video-decode/src/avassetreader.rs +++ b/crates/video-decode/src/avassetreader.rs @@ -13,6 +13,7 @@ use tokio::runtime::Handle as TokioHandle; pub struct KeyframeIndex { keyframes: Vec<(u32, f64)>, fps: f64, + duration_secs: f64, } impl KeyframeIndex { @@ -38,6 +39,15 @@ impl KeyframeIndex { } }; + let duration_secs = { + let duration = video_stream.duration(); + if duration > 0 { + duration as f64 * time_base.numerator() as f64 / time_base.denominator() as f64 + } else { + 0.0 + } + }; + let mut keyframes = Vec::new(); let mut input = @@ -62,11 +72,16 @@ impl KeyframeIndex { path = %path.display(), keyframe_count = keyframes.len(), fps = fps, + duration_secs = duration_secs, build_ms = elapsed.as_millis(), "Built keyframe index" ); - Ok(Self { keyframes, fps }) + Ok(Self { + keyframes, + fps, + duration_secs, + }) } pub fn nearest_keyframe_before(&self, target_frame: u32) -> Option<(u32, f64)> { @@ -87,13 +102,69 @@ impl KeyframeIndex { Some(self.keyframes[idx]) } + pub fn nearest_keyframe_after(&self, target_frame: u32) -> Option<(u32, f64)> { + if self.keyframes.is_empty() { + return None; + } + + let pos = self + .keyframes + .binary_search_by_key(&target_frame, |(frame, _)| *frame); + + let idx = match pos { + Ok(i) => { + if i + 1 < self.keyframes.len() { + i + 1 + } else { + i + } + } + Err(i) => { + if i < self.keyframes.len() { + i + } else { + return None; + } + } + }; + + Some(self.keyframes[idx]) + } + + pub fn get_strategic_positions(&self, num_positions: usize) -> Vec { + if self.keyframes.is_empty() || num_positions == 0 { + return vec![0.0]; + } + + let total_keyframes = self.keyframes.len(); + if total_keyframes <= num_positions { + return self.keyframes.iter().map(|(_, time)| *time).collect(); + } + + let step = total_keyframes / num_positions; + self.keyframes + .iter() + .step_by(step.max(1)) + .take(num_positions) + .map(|(_, time)| *time) + .collect() + } + pub fn fps(&self) -> f64 { self.fps } + pub fn duration_secs(&self) -> f64 { + self.duration_secs + } + pub fn keyframe_count(&self) -> usize { self.keyframes.len() } + + pub fn keyframes(&self) -> &[(u32, f64)] { + &self.keyframes + } } pub struct AVAssetReaderDecoder { @@ -105,10 +176,19 @@ pub struct AVAssetReaderDecoder { width: u32, height: u32, keyframe_index: Option, + current_position_secs: f32, } impl AVAssetReaderDecoder { pub fn new(path: PathBuf, tokio_handle: TokioHandle) -> Result { + Self::new_at_position(path, tokio_handle, 0.0) + } + + pub fn new_at_position( + path: PathBuf, + tokio_handle: TokioHandle, + start_time: f32, + ) -> Result { let keyframe_index = match KeyframeIndex::build(&path) { Ok(index) => Some(index), Err(e) => { @@ -121,6 +201,15 @@ impl AVAssetReaderDecoder { } }; + Self::new_with_keyframe_index(path, tokio_handle, start_time, keyframe_index) + } + + pub fn new_with_keyframe_index( + path: PathBuf, + tokio_handle: TokioHandle, + start_time: f32, + keyframe_index: Option, + ) -> Result { let (pixel_format, width, height) = { let input = ffmpeg::format::input(&path).unwrap(); @@ -143,8 +232,26 @@ impl AVAssetReaderDecoder { ) }; - let (track_output, reader) = - Self::get_reader_track_output(&path, 0.0, &tokio_handle, pixel_format, width, height)?; + let seek_time = if let Some(ref kf_index) = keyframe_index { + let fps = kf_index.fps(); + let target_frame = (start_time as f64 * fps).round() as u32; + if let Some((_, keyframe_time)) = kf_index.nearest_keyframe_before(target_frame) { + keyframe_time as f32 + } else { + start_time + } + } else { + start_time + }; + + let (track_output, reader) = Self::get_reader_track_output( + &path, + seek_time, + &tokio_handle, + pixel_format, + width, + height, + )?; Ok(Self { path, @@ -155,6 +262,7 @@ impl AVAssetReaderDecoder { width, height, keyframe_index, + current_position_secs: seek_time, }) } @@ -187,6 +295,8 @@ impl AVAssetReaderDecoder { self.height, )?; + self.current_position_secs = seek_time; + let elapsed = reset_start.elapsed(); tracing::info!( requested_time = requested_time, @@ -200,6 +310,26 @@ impl AVAssetReaderDecoder { Ok(()) } + pub fn current_position_secs(&self) -> f32 { + self.current_position_secs + } + + pub fn update_position(&mut self, position_secs: f32) { + self.current_position_secs = position_secs; + } + + pub fn path(&self) -> &PathBuf { + &self.path + } + + pub fn pixel_format(&self) -> cv::PixelFormat { + self.pixel_format + } + + pub fn take_keyframe_index(&mut self) -> Option { + self.keyframe_index.take() + } + pub fn keyframe_index(&self) -> Option<&KeyframeIndex> { self.keyframe_index.as_ref() } From 9232b71f115cff2eedca757c684e36b66ad8219f Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 19:11:54 +0000 Subject: [PATCH 14/37] Update audio playhead logic in MP4 export --- crates/export/src/mp4.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs index 080c53cb48..b582defd08 100644 --- a/crates/export/src/mp4.rs +++ b/crates/export/src/mp4.rs @@ -155,16 +155,18 @@ impl Mp4ExportSettings { if frame_count == 0 { first_frame = Some(frame.clone()); - if let Some(audio) = &mut audio_renderer { - audio.set_playhead(0.0, &project); - } + } + + let video_playhead = frame_number as f64 / fps as f64; + if let Some(audio) = &mut audio_renderer { + audio.set_playhead(video_playhead, &project); } let audio_frame = audio_renderer .as_mut() .and_then(|audio| audio.render_frame(audio_samples_per_frame, &project)) .map(|mut frame| { - let pts = ((frame_number * frame.rate()) as f64 / fps as f64) as i64; + let pts = ((frame_count * frame.rate()) as f64 / fps as f64) as i64; frame.set_pts(Some(pts)); frame }); From 89bb3a12c2be0e0884db2da44c0d2e8b7e35bd7d Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 22:58:20 +0000 Subject: [PATCH 15/37] Remove debug and trace logging statements --- crates/rendering/src/decoder/avassetreader.rs | 64 ++----------------- .../rendering/src/decoder/multi_position.rs | 9 --- crates/rendering/src/layers/display.rs | 14 ---- crates/rendering/src/lib.rs | 6 -- crates/video-decode/src/avassetreader.rs | 10 --- 5 files changed, 7 insertions(+), 96 deletions(-) diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index db6ea9d382..5b6a841895 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -419,12 +419,6 @@ impl AVAssetReaderDecoder { let decoder_idx = best_id.min(self.decoders.len().saturating_sub(1)); if needs_reset && decoder_idx < self.decoders.len() { - tracing::debug!( - decoder_idx = decoder_idx, - requested_time = requested_time, - distance = distance, - "Resetting decoder for seek" - ); self.decoders[decoder_idx].reset(requested_time); self.pool_manager .update_decoder_position(best_id, self.decoders[decoder_idx].current_position()); @@ -492,12 +486,6 @@ impl AVAssetReaderDecoder { match r { VideoDecoderMessage::GetFrame(requested_time, sender) => { let frame = (requested_time * fps as f32).floor() as u32; - debug!( - decoder = _name, - requested_time = requested_time, - frame = frame, - "GetFrame request received" - ); if !sender.is_closed() { pending_requests.push(PendingRequest { frame, sender }); } @@ -529,9 +517,7 @@ impl AVAssetReaderDecoder { if let Some(cached) = cache.get(&request.frame) { let data = cached.data().clone(); let req = pending_requests.remove(i); - if req.sender.send(data.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } + if req.sender.send(data.to_decoded_frame()).is_err() {} *last_sent_frame.borrow_mut() = Some(data); } else { i += 1; @@ -623,24 +609,16 @@ impl AVAssetReaderDecoder { if req.frame == current_frame { let data = cache_frame.data().clone(); *last_sent_frame.borrow_mut() = Some(data.clone()); - if req.sender.send(data.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } + if req.sender.send(data.to_decoded_frame()).is_err() {} } else if req.frame < current_frame { if let Some(cached) = cache.get(&req.frame) { let data = cached.data().clone(); *last_sent_frame.borrow_mut() = Some(data.clone()); - if req.sender.send(data.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } + if req.sender.send(data.to_decoded_frame()).is_err() {} } else if is_scrubbing { let data = cache_frame.data().clone(); *last_sent_frame.borrow_mut() = Some(data.clone()); - if req.sender.send(data.to_decoded_frame()).is_err() { - debug!( - "frame receiver dropped before send (scrub fallback)" - ); - } + if req.sender.send(data.to_decoded_frame()).is_err() {} } } else { remaining_requests.push(req); @@ -669,42 +647,14 @@ impl AVAssetReaderDecoder { this.pool_manager.update_decoder_position(decoder_idx, pos); } - if !pending_requests.is_empty() { - debug!( - decoder = _name, - pending_count = pending_requests.len(), - frames_iterated = frames_iterated, - cache_size = cache.len(), - is_scrubbing = is_scrubbing, - "Handling unfulfilled requests after frame iteration" - ); - } - for req in pending_requests.drain(..) { if let Some(cached) = cache.get(&req.frame) { let data = cached.data().clone(); - if req.sender.send(data.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } + if req.sender.send(data.to_decoded_frame()).is_err() {} } else if let Some(last) = last_sent_frame.borrow().clone() { - debug!( - decoder = _name, - requested_frame = req.frame, - is_scrubbing = is_scrubbing, - "Sending last known frame as fallback" - ); - if req.sender.send(last.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } + if req.sender.send(last.to_decoded_frame()).is_err() {} } else if let Some(first) = first_ever_frame.borrow().clone() { - debug!( - decoder = _name, - requested_frame = req.frame, - "Sending first ever frame as fallback" - ); - if req.sender.send(first.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } + if req.sender.send(first.to_decoded_frame()).is_err() {} } else { debug!( decoder = _name, diff --git a/crates/rendering/src/decoder/multi_position.rs b/crates/rendering/src/decoder/multi_position.rs index ce453a2ba8..62edb596a3 100644 --- a/crates/rendering/src/decoder/multi_position.rs +++ b/crates/rendering/src/decoder/multi_position.rs @@ -219,15 +219,6 @@ impl ScrubDetector { self.last_request_time = now; self.last_frame = frame; - if was_scrubbing != self.is_scrubbing { - tracing::debug!( - is_scrubbing = self.is_scrubbing, - request_rate = self.request_rate, - frame_delta = frame_delta, - "Scrub state changed" - ); - } - self.is_scrubbing } diff --git a/crates/rendering/src/layers/display.rs b/crates/rendering/src/layers/display.rs index 90f9cc34da..bf797a9b97 100644 --- a/crates/rendering/src/layers/display.rs +++ b/crates/rendering/src/layers/display.rs @@ -80,15 +80,6 @@ impl DisplayLayer { let format = segment_frames.screen_frame.format(); let current_recording_time = segment_frames.recording_time; - tracing::trace!( - format = ?format, - actual_width, - actual_height, - frame_data_len = frame_data.len(), - recording_time = current_recording_time, - "DisplayLayer::prepare - frame info" - ); - let skipped = self .last_recording_time .is_some_and(|last| (last - current_recording_time).abs() < 0.001); @@ -492,7 +483,6 @@ impl DisplayLayer { pub fn copy_to_texture(&mut self, encoder: &mut wgpu::CommandEncoder) { let Some(pending) = self.pending_copy.take() else { - tracing::trace!("copy_to_texture: no pending copy"); return; }; @@ -524,10 +514,6 @@ impl DisplayLayer { pub fn render(&self, pass: &mut wgpu::RenderPass<'_>) { if let Some(bind_group) = &self.bind_groups[self.current_texture] { - tracing::trace!( - current_texture_index = self.current_texture, - "DisplayLayer::render - rendering with bind group" - ); pass.set_pipeline(&self.pipeline.render_pipeline); pass.set_bind_group(0, bind_group, &[]); pass.draw(0..3, 0..1); diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 84740ecd71..48f2b20889 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -1780,12 +1780,6 @@ impl RendererLayers { } let should_render = uniforms.scene.should_render_screen(); - tracing::trace!( - should_render_screen = should_render, - screen_opacity = uniforms.scene.screen_opacity, - screen_blur = uniforms.scene.screen_blur, - "RendererLayers::render - checking should_render_screen" - ); if should_render { let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); diff --git a/crates/video-decode/src/avassetreader.rs b/crates/video-decode/src/avassetreader.rs index 696fa2628a..5f322342ea 100644 --- a/crates/video-decode/src/avassetreader.rs +++ b/crates/video-decode/src/avassetreader.rs @@ -297,16 +297,6 @@ impl AVAssetReaderDecoder { self.current_position_secs = seek_time; - let elapsed = reset_start.elapsed(); - tracing::info!( - requested_time = requested_time, - seek_time = seek_time, - keyframe_frame = ?keyframe_frame, - reset_ms = elapsed.as_millis(), - has_keyframe_index = self.keyframe_index.is_some(), - "AVAssetReader reset completed" - ); - Ok(()) } From c44ad7605c2fa0d048481627d3cee1e6d1418756 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 22:58:30 +0000 Subject: [PATCH 16/37] Fix audio sample calculation for MP4 export --- crates/export/src/mp4.rs | 67 +++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs index b582defd08..a6e22c2311 100644 --- a/crates/export/src/mp4.rs +++ b/crates/export/src/mp4.rs @@ -95,22 +95,15 @@ impl Mp4ExportSettings { info!("Created MP4File encoder"); - let mut encoded_frames = 0; while let Ok(frame) = frame_rx.recv() { encoder - .queue_video_frame( - frame.video, - Duration::from_secs_f32(encoded_frames as f32 / fps as f32), - ) + .queue_video_frame(frame.video, Duration::MAX) .map_err(|err| err.to_string())?; - encoded_frames += 1; if let Some(audio) = frame.audio { encoder.queue_audio_frame(audio); } } - info!("Encoded {encoded_frames} video frames"); - let res = encoder .finish() .map_err(|e| format!("Failed to finish encoding: {e}"))?; @@ -132,9 +125,9 @@ impl Mp4ExportSettings { async move { let mut frame_count = 0; let mut first_frame = None; - - let audio_samples_per_frame = - (f64::from(AudioRenderer::SAMPLE_RATE) / f64::from(fps)).ceil() as usize; + let sample_rate = u64::from(AudioRenderer::SAMPLE_RATE); + let fps_u64 = u64::from(fps); + let mut audio_sample_cursor = 0u64; loop { let (frame, frame_number) = @@ -155,21 +148,25 @@ impl Mp4ExportSettings { if frame_count == 0 { first_frame = Some(frame.clone()); + if let Some(audio) = &mut audio_renderer { + audio.set_playhead(0.0, &project); + } } - let video_playhead = frame_number as f64 / fps as f64; - if let Some(audio) = &mut audio_renderer { - audio.set_playhead(video_playhead, &project); - } - - let audio_frame = audio_renderer - .as_mut() - .and_then(|audio| audio.render_frame(audio_samples_per_frame, &project)) - .map(|mut frame| { - let pts = ((frame_count * frame.rate()) as f64 / fps as f64) as i64; + let audio_frame = audio_renderer.as_mut().and_then(|audio| { + let n = u64::from(frame_number); + let end = ((n + 1) * sample_rate) / fps_u64; + if end <= audio_sample_cursor { + return None; + } + let pts = audio_sample_cursor as i64; + let samples = (end - audio_sample_cursor) as usize; + audio_sample_cursor = end; + audio.render_frame(samples, &project).map(|mut frame| { frame.set_pts(Some(pts)); frame - }); + }) + }); if frame_tx .send(MP4Input { @@ -260,3 +257,29 @@ impl Mp4ExportSettings { Ok(output_path) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn sum_samples(sample_rate: u64, fps: u64, frames: u64) -> u64 { + (0..frames) + .map(|n| { + let start = (n * sample_rate) / fps; + let end = ((n + 1) * sample_rate) / fps; + end - start + }) + .sum() + } + + #[test] + fn audio_samples_match_duration_across_fps() { + let sample_rate = u64::from(AudioRenderer::SAMPLE_RATE); + + for fps in [24u64, 30, 60, 90, 120, 144] { + let frames = fps * 10; + let expected = (frames * sample_rate) / fps; + assert_eq!(sum_samples(sample_rate, fps, frames), expected); + } + } +} From cdb9b15f917027d6be9e0ca8f3af763e68e28110 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 23:03:19 +0000 Subject: [PATCH 17/37] clippy --- crates/editor/examples/decode-benchmark.rs | 30 +++++++++---------- crates/rendering/src/decoder/avassetreader.rs | 12 ++++---- .../rendering/src/decoder/multi_position.rs | 2 +- crates/video-decode/src/avassetreader.rs | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/crates/editor/examples/decode-benchmark.rs b/crates/editor/examples/decode-benchmark.rs index d77f7d1f1a..8cde55b949 100644 --- a/crates/editor/examples/decode-benchmark.rs +++ b/crates/editor/examples/decode-benchmark.rs @@ -1,5 +1,5 @@ use cap_rendering::decoder::{AsyncVideoDecoderHandle, spawn_decoder}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::time::Instant; use tokio::runtime::Runtime; @@ -53,9 +53,9 @@ impl BenchmarkResults { " Frames decoded: {}", self.sequential_decode_times_ms.len() ); - println!(" Avg decode time: {:.2}ms", avg); - println!(" Min decode time: {:.2}ms", min); - println!(" Max decode time: {:.2}ms", max); + println!(" Avg decode time: {avg:.2}ms"); + println!(" Min decode time: {min:.2}ms"); + println!(" Max decode time: {max:.2}ms"); println!(" Effective FPS: {:.1}", self.sequential_fps); } println!(); @@ -65,7 +65,7 @@ impl BenchmarkResults { println!(" {:>10} | {:>12}", "Distance(s)", "Time(ms)"); println!(" {}-+-{}", "-".repeat(10), "-".repeat(12)); for (distance, time) in &self.seek_times_by_distance { - println!(" {:>10.1} | {:>12.2}", distance, time); + println!(" {distance:>10.1} | {time:>12.2}"); } } println!(); @@ -85,9 +85,9 @@ impl BenchmarkResults { .cloned() .fold(f64::NEG_INFINITY, f64::max); println!(" Samples: {}", self.random_access_times_ms.len()); - println!(" Avg access time: {:.2}ms", avg); - println!(" Min access time: {:.2}ms", min); - println!(" Max access time: {:.2}ms", max); + println!(" Avg access time: {avg:.2}ms"); + println!(" Min access time: {min:.2}ms"); + println!(" Max access time: {max:.2}ms"); println!( " P50: {:.2}ms", percentile(&self.random_access_times_ms, 50.0) @@ -132,12 +132,12 @@ fn percentile(data: &[f64], p: f64) -> f64 { sorted[idx.min(sorted.len() - 1)] } -async fn benchmark_decoder_creation(path: &PathBuf, fps: u32, iterations: usize) -> f64 { +async fn benchmark_decoder_creation(path: &Path, fps: u32, iterations: usize) -> f64 { let mut total_ms = 0.0; for i in 0..iterations { let start = Instant::now(); - let decoder = spawn_decoder("benchmark", path.clone(), fps, 0.0).await; + let decoder = spawn_decoder("benchmark", path.to_path_buf(), fps, 0.0).await; let elapsed = start.elapsed(); match decoder { @@ -146,7 +146,7 @@ async fn benchmark_decoder_creation(path: &PathBuf, fps: u32, iterations: usize) } Err(e) => { if i == 0 { - eprintln!("Failed to create decoder: {}", e); + eprintln!("Failed to create decoder: {e}"); return -1.0; } } @@ -202,7 +202,7 @@ async fn benchmark_random_access( ) -> Vec { let mut times = Vec::with_capacity(sample_count); - let golden_ratio = 1.618033988749895_f32; + let golden_ratio = 1.618_034_f32; let mut position = 0.0_f32; for _ in 0..sample_count { @@ -240,7 +240,7 @@ async fn run_full_benchmark(config: BenchmarkConfig) -> BenchmarkResults { { Ok(d) => d, Err(e) => { - eprintln!("Failed to create decoder: {}", e); + eprintln!("Failed to create decoder: {e}"); return results; } }; @@ -250,14 +250,14 @@ async fn run_full_benchmark(config: BenchmarkConfig) -> BenchmarkResults { let (seq_times, seq_fps) = benchmark_sequential_decode(&decoder, config.fps, 100, 0.0).await; results.sequential_decode_times_ms = seq_times; results.sequential_fps = seq_fps; - println!(" Done: {:.1} effective FPS", seq_fps); + println!(" Done: {seq_fps:.1} effective FPS"); println!("[4/5] Benchmarking seek performance..."); let seek_distances = vec![0.5, 1.0, 2.0, 5.0, 10.0, 30.0]; for distance in seek_distances { let seek_time = benchmark_seek(&decoder, config.fps, 0.0, distance).await; results.seek_times_by_distance.push((distance, seek_time)); - println!(" {:.1}s seek: {:.2}ms", distance, seek_time); + println!(" {distance:.1}s seek: {seek_time:.2}ms"); } println!("[5/5] Benchmarking random access (50 samples)..."); diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index 5b6a841895..f3b3f0a44a 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -413,7 +413,7 @@ impl AVAssetReaderDecoder { } fn select_best_decoder(&mut self, requested_time: f32) -> (usize, bool) { - let (best_id, distance, needs_reset) = + let (best_id, _distance, needs_reset) = self.pool_manager.find_best_decoder_for_time(requested_time); let decoder_idx = best_id.min(self.decoders.len().saturating_sub(1)); @@ -517,7 +517,7 @@ impl AVAssetReaderDecoder { if let Some(cached) = cache.get(&request.frame) { let data = cached.data().clone(); let req = pending_requests.remove(i); - if req.sender.send(data.to_decoded_frame()).is_err() {} + let _ = req.sender.send(data.to_decoded_frame()); *last_sent_frame.borrow_mut() = Some(data); } else { i += 1; @@ -609,16 +609,16 @@ impl AVAssetReaderDecoder { if req.frame == current_frame { let data = cache_frame.data().clone(); *last_sent_frame.borrow_mut() = Some(data.clone()); - if req.sender.send(data.to_decoded_frame()).is_err() {} + let _ = req.sender.send(data.to_decoded_frame()); } else if req.frame < current_frame { if let Some(cached) = cache.get(&req.frame) { let data = cached.data().clone(); *last_sent_frame.borrow_mut() = Some(data.clone()); - if req.sender.send(data.to_decoded_frame()).is_err() {} + let _ = req.sender.send(data.to_decoded_frame()); } else if is_scrubbing { let data = cache_frame.data().clone(); *last_sent_frame.borrow_mut() = Some(data.clone()); - if req.sender.send(data.to_decoded_frame()).is_err() {} + let _ = req.sender.send(data.to_decoded_frame()); } } else { remaining_requests.push(req); @@ -650,7 +650,7 @@ impl AVAssetReaderDecoder { for req in pending_requests.drain(..) { if let Some(cached) = cache.get(&req.frame) { let data = cached.data().clone(); - if req.sender.send(data.to_decoded_frame()).is_err() {} + let _ = req.sender.send(data.to_decoded_frame()); } else if let Some(last) = last_sent_frame.borrow().clone() { if req.sender.send(last.to_decoded_frame()).is_err() {} } else if let Some(first) = first_ever_frame.borrow().clone() { diff --git a/crates/rendering/src/decoder/multi_position.rs b/crates/rendering/src/decoder/multi_position.rs index 62edb596a3..182d6860fe 100644 --- a/crates/rendering/src/decoder/multi_position.rs +++ b/crates/rendering/src/decoder/multi_position.rs @@ -204,7 +204,7 @@ impl ScrubDetector { let instantaneous_rate = frame_delta as f64 / elapsed_secs; self.request_rate = self.request_rate * 0.7 + instantaneous_rate * 0.3; - let was_scrubbing = self.is_scrubbing; + let _was_scrubbing = self.is_scrubbing; if self.request_rate > Self::SCRUB_THRESHOLD_RATE && frame_delta > 1 { self.is_scrubbing = true; diff --git a/crates/video-decode/src/avassetreader.rs b/crates/video-decode/src/avassetreader.rs index 5f322342ea..922cda6e61 100644 --- a/crates/video-decode/src/avassetreader.rs +++ b/crates/video-decode/src/avassetreader.rs @@ -267,11 +267,11 @@ impl AVAssetReaderDecoder { } pub fn reset(&mut self, requested_time: f32) -> Result<(), String> { - let reset_start = std::time::Instant::now(); + let _reset_start = std::time::Instant::now(); self.reader.cancel_reading(); - let (seek_time, keyframe_frame) = if let Some(ref keyframe_index) = self.keyframe_index { + let (seek_time, _keyframe_frame) = if let Some(ref keyframe_index) = self.keyframe_index { let fps = keyframe_index.fps(); let target_frame = (requested_time as f64 * fps).round() as u32; From 962df11a3d4ae8bcdd1a2a59a74990370c34f1ba Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 23:06:22 +0000 Subject: [PATCH 18/37] clippy --- apps/desktop/src-tauri/src/lib.rs | 7 ++++--- crates/rendering/src/yuv_converter.rs | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 6bdb84b4e6..7622b69ff7 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -106,11 +106,12 @@ use crate::{ }; use crate::{recording::start_recording, upload::build_video_meta}; +type FinalizingRecordingsMap = + std::collections::HashMap, watch::Receiver)>; + #[derive(Default)] pub struct FinalizingRecordings { - recordings: std::sync::Mutex< - std::collections::HashMap, watch::Receiver)>, - >, + recordings: std::sync::Mutex, } impl FinalizingRecordings { diff --git a/crates/rendering/src/yuv_converter.rs b/crates/rendering/src/yuv_converter.rs index 36d3749022..a4fe8e58ec 100644 --- a/crates/rendering/src/yuv_converter.rs +++ b/crates/rendering/src/yuv_converter.rs @@ -157,6 +157,7 @@ impl BindGroupCache { self.cached_height = 0; } + #[allow(clippy::too_many_arguments)] fn get_or_create_nv12( &mut self, device: &wgpu::Device, @@ -199,6 +200,7 @@ impl BindGroupCache { self.nv12_bind_groups[output_index].as_ref().unwrap() } + #[allow(clippy::too_many_arguments)] fn get_or_create_yuv420p( &mut self, device: &wgpu::Device, From c79f8de1ff5f2ee2bca1ad04dc667c682cfad696 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 23:21:30 +0000 Subject: [PATCH 19/37] coderabbit bits --- .claude/settings.local.json | 3 +- apps/desktop/src-tauri/src/lib.rs | 4 +- apps/desktop/src/routes/editor/index.tsx | 4 +- crates/editor/examples/decode-benchmark.rs | 175 ++++++++++++++---- crates/enc-ffmpeg/src/remux.rs | 101 +++------- crates/enc-ffmpeg/src/video/h264.rs | 4 +- .../output_pipeline/macos_fragmented_m4s.rs | 50 +++-- crates/rendering/src/decoder/avassetreader.rs | 4 +- 8 files changed, 212 insertions(+), 133 deletions(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 5babcbf0ad..f2037c92c6 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -51,7 +51,8 @@ "Bash(footprint:*)", "Bash(RUST_LOG=info,cap_recording=debug ./target/release/examples/memory-leak-detector:*)", "Bash(git rm:*)", - "Bash(./target/release/examples/decode-benchmark:*)" + "Bash(./target/release/examples/decode-benchmark:*)", + "Bash(RUST_LOG=warn ./target/release/examples/decode-benchmark:*)" ], "deny": [], "ask": [] diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 7622b69ff7..2f0811a485 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -125,7 +125,9 @@ impl FinalizingRecordings { pub fn finish_finalizing(&self, path: &Path) { let mut recordings = self.recordings.lock().unwrap(); if let Some((tx, _)) = recordings.remove(path) { - tx.send(true).ok(); + if tx.send(true).is_err() { + debug!("Finalizing receiver dropped for path: {:?}", path); + } } } diff --git a/apps/desktop/src/routes/editor/index.tsx b/apps/desktop/src/routes/editor/index.tsx index 1d45af3caf..ecfb44763d 100644 --- a/apps/desktop/src/routes/editor/index.tsx +++ b/apps/desktop/src/routes/editor/index.tsx @@ -2,10 +2,10 @@ import { Effect, getCurrentWindow } from "@tauri-apps/api/window"; import { type as ostype } from "@tauri-apps/plugin-os"; import { cx } from "cva"; import { createEffect, Suspense } from "solid-js"; -import { AbsoluteInsetLoader } from "~/components/Loader"; import { generalSettingsStore } from "~/store"; import { commands } from "~/utils/tauri"; import { Editor } from "./Editor"; +import { EditorSkeleton } from "./EditorSkeleton"; export default function () { const generalSettings = generalSettingsStore.createQuery(); @@ -27,7 +27,7 @@ export default function () { ) && "bg-transparent-window", )} > - }> + }> diff --git a/crates/editor/examples/decode-benchmark.rs b/crates/editor/examples/decode-benchmark.rs index 8cde55b949..60106567b6 100644 --- a/crates/editor/examples/decode-benchmark.rs +++ b/crates/editor/examples/decode-benchmark.rs @@ -1,8 +1,38 @@ use cap_rendering::decoder::{AsyncVideoDecoderHandle, spawn_decoder}; use std::path::{Path, PathBuf}; +use std::process::Command; use std::time::Instant; use tokio::runtime::Runtime; +const DEFAULT_DURATION_SECS: f32 = 60.0; + +fn get_video_duration(path: &Path) -> f32 { + let output = Command::new("ffprobe") + .args([ + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + ]) + .arg(path) + .output(); + + match output { + Ok(output) if output.status.success() => { + let duration_str = String::from_utf8_lossy(&output.stdout); + duration_str.trim().parse().unwrap_or(DEFAULT_DURATION_SECS) + } + _ => { + eprintln!( + "Warning: Could not determine video duration via ffprobe, using default {DEFAULT_DURATION_SECS}s" + ); + DEFAULT_DURATION_SECS + } + } +} + #[derive(Debug, Clone)] struct BenchmarkConfig { video_path: PathBuf, @@ -15,9 +45,12 @@ struct BenchmarkResults { decoder_creation_ms: f64, sequential_decode_times_ms: Vec, sequential_fps: f64, + sequential_failures: usize, seek_times_by_distance: Vec<(f32, f64)>, + seek_failures: usize, random_access_times_ms: Vec, random_access_avg_ms: f64, + random_access_failures: usize, cache_hits: usize, cache_misses: usize, } @@ -36,9 +69,13 @@ impl BenchmarkResults { println!(); println!("SEQUENTIAL DECODE PERFORMANCE"); - if !self.sequential_decode_times_ms.is_empty() { - let avg: f64 = self.sequential_decode_times_ms.iter().sum::() - / self.sequential_decode_times_ms.len() as f64; + if !self.sequential_decode_times_ms.is_empty() || self.sequential_failures > 0 { + let avg: f64 = if self.sequential_decode_times_ms.is_empty() { + 0.0 + } else { + self.sequential_decode_times_ms.iter().sum::() + / self.sequential_decode_times_ms.len() as f64 + }; let min = self .sequential_decode_times_ms .iter() @@ -53,6 +90,9 @@ impl BenchmarkResults { " Frames decoded: {}", self.sequential_decode_times_ms.len() ); + if self.sequential_failures > 0 { + println!(" Frames failed: {}", self.sequential_failures); + } println!(" Avg decode time: {avg:.2}ms"); println!(" Min decode time: {min:.2}ms"); println!(" Max decode time: {max:.2}ms"); @@ -61,19 +101,26 @@ impl BenchmarkResults { println!(); println!("SEEK PERFORMANCE (by distance)"); - if !self.seek_times_by_distance.is_empty() { + if !self.seek_times_by_distance.is_empty() || self.seek_failures > 0 { println!(" {:>10} | {:>12}", "Distance(s)", "Time(ms)"); println!(" {}-+-{}", "-".repeat(10), "-".repeat(12)); for (distance, time) in &self.seek_times_by_distance { println!(" {distance:>10.1} | {time:>12.2}"); } + if self.seek_failures > 0 { + println!(" Seek failures: {}", self.seek_failures); + } } println!(); println!("RANDOM ACCESS PERFORMANCE"); - if !self.random_access_times_ms.is_empty() { - let avg = self.random_access_times_ms.iter().sum::() - / self.random_access_times_ms.len() as f64; + if !self.random_access_times_ms.is_empty() || self.random_access_failures > 0 { + let avg = if self.random_access_times_ms.is_empty() { + 0.0 + } else { + self.random_access_times_ms.iter().sum::() + / self.random_access_times_ms.len() as f64 + }; let min = self .random_access_times_ms .iter() @@ -85,6 +132,9 @@ impl BenchmarkResults { .cloned() .fold(f64::NEG_INFINITY, f64::max); println!(" Samples: {}", self.random_access_times_ms.len()); + if self.random_access_failures > 0 { + println!(" Failures: {}", self.random_access_failures); + } println!(" Avg access time: {avg:.2}ms"); println!(" Min access time: {min:.2}ms"); println!(" Max access time: {max:.2}ms"); @@ -123,11 +173,15 @@ impl BenchmarkResults { } fn percentile(data: &[f64], p: f64) -> f64 { - if data.is_empty() { + let filtered: Vec = data.iter().copied().filter(|x| !x.is_nan()).collect(); + if filtered.is_empty() { return 0.0; } - let mut sorted: Vec = data.to_vec(); - sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let mut sorted = filtered; + sorted.sort_by(|a, b| { + a.partial_cmp(b) + .expect("NaN values should have been filtered out") + }); let idx = ((p / 100.0) * (sorted.len() - 1) as f64).round() as usize; sorted[idx.min(sorted.len() - 1)] } @@ -161,22 +215,35 @@ async fn benchmark_sequential_decode( fps: u32, frame_count: usize, start_time: f32, -) -> (Vec, f64) { +) -> (Vec, f64, usize) { let mut times = Vec::with_capacity(frame_count); + let mut failures = 0; let overall_start = Instant::now(); for i in 0..frame_count { let time = start_time + (i as f32 / fps as f32); let start = Instant::now(); - let _frame = decoder.get_frame(time).await; - let elapsed = start.elapsed(); - times.push(elapsed.as_secs_f64() * 1000.0); + match decoder.get_frame(time).await { + Some(_frame) => { + let elapsed = start.elapsed(); + times.push(elapsed.as_secs_f64() * 1000.0); + } + None => { + failures += 1; + eprintln!("Failed to get frame at time {time:.3}s"); + } + } } let overall_elapsed = overall_start.elapsed(); - let effective_fps = frame_count as f64 / overall_elapsed.as_secs_f64(); + let successful_frames = frame_count - failures; + let effective_fps = if overall_elapsed.as_secs_f64() > 0.0 { + successful_frames as f64 / overall_elapsed.as_secs_f64() + } else { + 0.0 + }; - (times, effective_fps) + (times, effective_fps, failures) } async fn benchmark_seek( @@ -184,14 +251,23 @@ async fn benchmark_seek( _fps: u32, from_time: f32, to_time: f32, -) -> f64 { - let _ = decoder.get_frame(from_time).await; +) -> Option { + if decoder.get_frame(from_time).await.is_none() { + eprintln!("Failed to get initial frame at time {from_time:.3}s for seek benchmark"); + return None; + } let start = Instant::now(); - let _frame = decoder.get_frame(to_time).await; - let elapsed = start.elapsed(); - - elapsed.as_secs_f64() * 1000.0 + match decoder.get_frame(to_time).await { + Some(_frame) => { + let elapsed = start.elapsed(); + Some(elapsed.as_secs_f64() * 1000.0) + } + None => { + eprintln!("Failed to get frame at time {to_time:.3}s for seek benchmark"); + None + } + } } async fn benchmark_random_access( @@ -199,8 +275,9 @@ async fn benchmark_random_access( _fps: u32, duration_secs: f32, sample_count: usize, -) -> Vec { +) -> (Vec, usize) { let mut times = Vec::with_capacity(sample_count); + let mut failures = 0; let golden_ratio = 1.618_034_f32; let mut position = 0.0_f32; @@ -208,12 +285,19 @@ async fn benchmark_random_access( for _ in 0..sample_count { position = (position + golden_ratio * duration_secs) % duration_secs; let start = Instant::now(); - let _frame = decoder.get_frame(position).await; - let elapsed = start.elapsed(); - times.push(elapsed.as_secs_f64() * 1000.0); + match decoder.get_frame(position).await { + Some(_frame) => { + let elapsed = start.elapsed(); + times.push(elapsed.as_secs_f64() * 1000.0); + } + None => { + failures += 1; + eprintln!("Failed to get frame at position {position:.3}s during random access"); + } + } } - times + (times, failures) } async fn run_full_benchmark(config: BenchmarkConfig) -> BenchmarkResults { @@ -246,24 +330,44 @@ async fn run_full_benchmark(config: BenchmarkConfig) -> BenchmarkResults { }; println!(" Done"); + let video_duration = get_video_duration(&config.video_path); + println!("Detected video duration: {video_duration:.2}s"); + println!(); + println!("[3/5] Benchmarking sequential decode (100 frames from start)..."); - let (seq_times, seq_fps) = benchmark_sequential_decode(&decoder, config.fps, 100, 0.0).await; + let (seq_times, seq_fps, seq_failures) = + benchmark_sequential_decode(&decoder, config.fps, 100, 0.0).await; results.sequential_decode_times_ms = seq_times; results.sequential_fps = seq_fps; + results.sequential_failures = seq_failures; println!(" Done: {seq_fps:.1} effective FPS"); + if seq_failures > 0 { + println!(" Warning: {seq_failures} frames failed to decode"); + } println!("[4/5] Benchmarking seek performance..."); - let seek_distances = vec![0.5, 1.0, 2.0, 5.0, 10.0, 30.0]; + let seek_distances: Vec = vec![0.5, 1.0, 2.0, 5.0, 10.0, 30.0] + .into_iter() + .filter(|&d| d <= video_duration) + .collect(); for distance in seek_distances { - let seek_time = benchmark_seek(&decoder, config.fps, 0.0, distance).await; - results.seek_times_by_distance.push((distance, seek_time)); - println!(" {distance:.1}s seek: {seek_time:.2}ms"); + match benchmark_seek(&decoder, config.fps, 0.0, distance).await { + Some(seek_time) => { + results.seek_times_by_distance.push((distance, seek_time)); + println!(" {distance:.1}s seek: {seek_time:.2}ms"); + } + None => { + results.seek_failures += 1; + println!(" {distance:.1}s seek: FAILED"); + } + } } println!("[5/5] Benchmarking random access (50 samples)..."); - let video_duration = 60.0f32; - results.random_access_times_ms = + let (random_times, random_failures) = benchmark_random_access(&decoder, config.fps, video_duration, 50).await; + results.random_access_times_ms = random_times; + results.random_access_failures = random_failures; results.random_access_avg_ms = if results.random_access_times_ms.is_empty() { 0.0 } else { @@ -271,6 +375,9 @@ async fn run_full_benchmark(config: BenchmarkConfig) -> BenchmarkResults { / results.random_access_times_ms.len() as f64 }; println!(" Done: {:.2}ms avg", results.random_access_avg_ms); + if random_failures > 0 { + println!(" Warning: {random_failures} random accesses failed"); + } results } diff --git a/crates/enc-ffmpeg/src/remux.rs b/crates/enc-ffmpeg/src/remux.rs index 5c658fff9d..f7c4209056 100644 --- a/crates/enc-ffmpeg/src/remux.rs +++ b/crates/enc-ffmpeg/src/remux.rs @@ -103,16 +103,10 @@ fn open_input_with_format( } } -fn concatenate_with_concat_demuxer( - concat_list_path: &Path, - output: &Path, +fn remux_streams( + ictx: &mut avformat::context::Input, + octx: &mut avformat::context::Output, ) -> Result<(), RemuxError> { - let mut options = ffmpeg::Dictionary::new(); - options.set("safe", "0"); - - let mut ictx = open_input_with_format(concat_list_path, "concat", options)?; - let mut octx = avformat::output(output)?; - let mut stream_mapping: Vec> = Vec::new(); let mut output_stream_index = 0usize; @@ -171,7 +165,7 @@ fn concatenate_with_concat_demuxer( packet.set_stream(output_index); packet.set_position(-1); - packet.write_interleaved(&mut octx)?; + packet.write_interleaved(octx)?; } } @@ -180,6 +174,19 @@ fn concatenate_with_concat_demuxer( Ok(()) } +fn concatenate_with_concat_demuxer( + concat_list_path: &Path, + output: &Path, +) -> Result<(), RemuxError> { + let mut options = ffmpeg::Dictionary::new(); + options.set("safe", "0"); + + let mut ictx = open_input_with_format(concat_list_path, "concat", options)?; + let mut octx = avformat::output(output)?; + + remux_streams(&mut ictx, &mut octx) +} + pub fn concatenate_audio_to_ogg(fragments: &[PathBuf], output: &Path) -> Result<(), RemuxError> { if fragments.is_empty() { return Err(RemuxError::NoFragments); @@ -432,7 +439,13 @@ pub fn concatenate_m4s_segments_with_init( let result = remux_to_regular_mp4(&combined_path, output); - let _ = std::fs::remove_file(&combined_path); + if let Err(e) = std::fs::remove_file(&combined_path) { + tracing::warn!( + "failed to remove combined file {}: {}", + combined_path.display(), + e + ); + } result } @@ -441,69 +454,5 @@ fn remux_to_regular_mp4(input_path: &Path, output_path: &Path) -> Result<(), Rem let mut ictx = avformat::input(input_path)?; let mut octx = avformat::output(output_path)?; - let mut stream_mapping: Vec> = Vec::new(); - let mut output_stream_index = 0usize; - - for input_stream in ictx.streams() { - let codec_params = input_stream.parameters(); - let medium = codec_params.medium(); - - if medium == ffmpeg::media::Type::Video || medium == ffmpeg::media::Type::Audio { - stream_mapping.push(Some(output_stream_index)); - output_stream_index += 1; - - let mut output_stream = octx.add_stream(None)?; - output_stream.set_parameters(codec_params); - unsafe { - (*output_stream.as_mut_ptr()).time_base = (*input_stream.as_ptr()).time_base; - } - } else { - stream_mapping.push(None); - } - } - - octx.write_header()?; - - let mut last_dts: Vec = vec![i64::MIN; output_stream_index]; - let mut dts_offset: Vec = vec![0; output_stream_index]; - - for (input_stream, packet) in ictx.packets() { - let input_stream_index = input_stream.index(); - - if let Some(Some(output_index)) = stream_mapping.get(input_stream_index) { - let output_index = *output_index; - let mut packet = packet; - let input_time_base = input_stream.time_base(); - let output_time_base = octx.stream(output_index).unwrap().time_base(); - - packet.rescale_ts(input_time_base, output_time_base); - - let current_dts = packet.dts().unwrap_or(0); - - if last_dts[output_index] != i64::MIN && current_dts <= last_dts[output_index] { - dts_offset[output_index] = last_dts[output_index] - current_dts + 1; - } - - let adjusted_dts = current_dts + dts_offset[output_index]; - let adjusted_pts = packet.pts().map(|pts| pts + dts_offset[output_index]); - - unsafe { - (*packet.as_mut_ptr()).dts = adjusted_dts; - if let Some(pts) = adjusted_pts { - (*packet.as_mut_ptr()).pts = pts; - } - } - - last_dts[output_index] = adjusted_dts; - - packet.set_stream(output_index); - packet.set_position(-1); - - packet.write_interleaved(&mut octx)?; - } - } - - octx.write_trailer()?; - - Ok(()) + remux_streams(&mut ictx, &mut octx) } diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs index d8e76718ca..bd566f2e34 100644 --- a/crates/enc-ffmpeg/src/video/h264.rs +++ b/crates/enc-ffmpeg/src/video/h264.rs @@ -8,7 +8,7 @@ use ffmpeg::{ frame, threading::Config, }; -use tracing::{debug, error, trace}; +use tracing::{debug, error, trace, warn}; use crate::base::EncoderBase; @@ -130,7 +130,7 @@ impl H264EncoderBuilder { "Selected hardware H264 encoder" ); } else { - error!( + warn!( encoder = %codec_name, input_width = input_config.width, input_height = input_config.height, diff --git a/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs b/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs index b091b92789..6d7c7532c6 100644 --- a/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs @@ -96,6 +96,8 @@ pub struct MacOSFragmentedM4SMuxer { base_path: PathBuf, video_config: VideoInfo, segment_duration: Duration, + preset: H264Preset, + output_size: Option<(u32, u32)>, state: Option, pause: SharedPauseState, frame_drops: FrameDropTracker, @@ -148,6 +150,8 @@ impl Muxer for MacOSFragmentedM4SMuxer { base_path: output_path, video_config, segment_duration: config.segment_duration, + preset: config.preset, + output_size: config.output_size, state: None, pause, frame_drops: FrameDropTracker::new(), @@ -174,11 +178,17 @@ impl Muxer for MacOSFragmentedM4SMuxer { let start = std::time::Instant::now(); loop { if handle.is_finished() { - if let Err(panic_payload) = handle.join() { - warn!( - "M4S encoder thread panicked during finish: {:?}", - panic_payload - ); + match handle.join() { + Err(panic_payload) => { + warn!( + "M4S encoder thread panicked during finish: {:?}", + panic_payload + ); + } + Ok(Err(e)) => { + warn!("M4S encoder thread returned error: {e}"); + } + Ok(Ok(())) => {} } break; } @@ -218,9 +228,9 @@ impl MacOSFragmentedM4SMuxer { let encoder_config = SegmentedVideoEncoderConfig { segment_duration: self.segment_duration, - preset: H264Preset::Ultrafast, + preset: self.preset, bpp: H264EncoderBuilder::QUALITY_BPP, - output_size: None, + output_size: self.output_size, }; let encoder = @@ -550,7 +560,7 @@ impl<'a> BaseAddrLockGuard<'a> { impl Drop for BaseAddrLockGuard<'_> { fn drop(&mut self) { - let _ = unsafe { self.0.unlock_lock_base_addr(self.1) }; + unsafe { self.0.unlock_lock_base_addr(self.1) }; } } @@ -558,6 +568,8 @@ pub struct MacOSFragmentedM4SCameraMuxer { base_path: PathBuf, video_config: VideoInfo, segment_duration: Duration, + preset: H264Preset, + output_size: Option<(u32, u32)>, state: Option, pause: SharedPauseState, frame_drops: FrameDropTracker, @@ -611,6 +623,8 @@ impl Muxer for MacOSFragmentedM4SCameraMuxer { base_path: output_path, video_config, segment_duration: config.segment_duration, + preset: config.preset, + output_size: config.output_size, state: None, pause, frame_drops: FrameDropTracker::new(), @@ -637,11 +651,17 @@ impl Muxer for MacOSFragmentedM4SCameraMuxer { let start = std::time::Instant::now(); loop { if handle.is_finished() { - if let Err(panic_payload) = handle.join() { - warn!( - "M4S camera encoder thread panicked during finish: {:?}", - panic_payload - ); + match handle.join() { + Err(panic_payload) => { + warn!( + "M4S camera encoder thread panicked during finish: {:?}", + panic_payload + ); + } + Ok(Err(e)) => { + warn!("M4S camera encoder thread returned error: {e}"); + } + Ok(Ok(())) => {} } break; } @@ -681,9 +701,9 @@ impl MacOSFragmentedM4SCameraMuxer { let encoder_config = SegmentedVideoEncoderConfig { segment_duration: self.segment_duration, - preset: H264Preset::Ultrafast, + preset: self.preset, bpp: H264EncoderBuilder::QUALITY_BPP, - output_size: None, + output_size: self.output_size, }; let encoder = diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index f3b3f0a44a..c7dcf11890 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -226,7 +226,7 @@ impl ImageBufProcessor { } impl CachedFrame { - fn new(_processor: &ImageBufProcessor, image_buf: R, number: u32) -> Self { + fn new(processor: &ImageBufProcessor, image_buf: R, number: u32) -> Self { let width = image_buf.width() as u32; let height = image_buf.height() as u32; @@ -246,7 +246,7 @@ impl CachedFrame { } format::Pixel::RGBA | format::Pixel::BGRA | format::Pixel::YUV420P => { let mut img = image_buf; - let (data, fmt, y_str, uv_str) = _processor.extract_raw(&mut img); + let (data, fmt, y_str, uv_str) = processor.extract_raw(&mut img); return Self(ProcessedFrame { _number: number, width, From c1c98f171aaa66b9ebf4765c39f716e7ca41e74a Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 23:31:19 +0000 Subject: [PATCH 20/37] coderabbit --- crates/editor/examples/decode-benchmark.rs | 2 +- crates/enc-ffmpeg/src/remux.rs | 4 ++- crates/rendering/src/decoder/avassetreader.rs | 31 ++++++++++++++++--- crates/video-decode/src/avassetreader.rs | 26 ++++++---------- 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/crates/editor/examples/decode-benchmark.rs b/crates/editor/examples/decode-benchmark.rs index 60106567b6..a42a3f9a67 100644 --- a/crates/editor/examples/decode-benchmark.rs +++ b/crates/editor/examples/decode-benchmark.rs @@ -312,7 +312,7 @@ async fn run_full_benchmark(config: BenchmarkConfig) -> BenchmarkResults { println!("[1/5] Benchmarking decoder creation..."); results.decoder_creation_ms = - benchmark_decoder_creation(&config.video_path, config.fps, 3).await; + benchmark_decoder_creation(&config.video_path, config.fps, config.iterations).await; if results.decoder_creation_ms < 0.0 { eprintln!("Failed to benchmark decoder creation"); return results; diff --git a/crates/enc-ffmpeg/src/remux.rs b/crates/enc-ffmpeg/src/remux.rs index f7c4209056..027c7cefd0 100644 --- a/crates/enc-ffmpeg/src/remux.rs +++ b/crates/enc-ffmpeg/src/remux.rs @@ -399,7 +399,9 @@ pub fn probe_m4s_can_decode_with_init( let result = probe_video_can_decode(&temp_path); - let _ = std::fs::remove_file(&temp_path); + if let Err(e) = std::fs::remove_file(&temp_path) { + tracing::warn!("failed to remove temp file {}: {}", temp_path.display(), e); + } result } diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index c7dcf11890..8acf438ac6 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -323,9 +323,21 @@ impl DecoderInstance { } fn reset(&mut self, requested_time: f32) { - let _ = self.inner.reset(requested_time); - self.is_done = false; - self.frames_iter_valid = true; + match self.inner.reset(requested_time) { + Ok(()) => { + self.is_done = false; + self.frames_iter_valid = true; + } + Err(e) => { + tracing::error!( + requested_time = requested_time, + error = %e, + "Failed to reset decoder, marking as invalid" + ); + self.is_done = true; + self.frames_iter_valid = false; + } + } } fn current_position(&self) -> f32 { @@ -558,8 +570,17 @@ impl AVAssetReaderDecoder { let mut frames = decoder.inner.frames(); for frame in &mut frames { - let Ok(frame) = frame.map_err(|e| format!("read frame / {e}")) else { - continue; + let frame = match frame { + Ok(f) => f, + Err(e) => { + tracing::error!( + decoder_idx = decoder_idx, + frames_iterated = frames_iterated, + error = %e, + "Failed to read frame, skipping" + ); + continue; + } }; frames_iterated += 1; diff --git a/crates/video-decode/src/avassetreader.rs b/crates/video-decode/src/avassetreader.rs index 922cda6e61..17ddf6f95d 100644 --- a/crates/video-decode/src/avassetreader.rs +++ b/crates/video-decode/src/avassetreader.rs @@ -93,13 +93,11 @@ impl KeyframeIndex { .keyframes .binary_search_by_key(&target_frame, |(frame, _)| *frame); - let idx = match pos { - Ok(i) => i, - Err(0) => 0, - Err(i) => i - 1, - }; - - Some(self.keyframes[idx]) + match pos { + Ok(i) => Some(self.keyframes[i]), + Err(0) => None, + Err(i) => Some(self.keyframes[i - 1]), + } } pub fn nearest_keyframe_after(&self, target_frame: u32) -> Option<(u32, f64)> { @@ -267,23 +265,19 @@ impl AVAssetReaderDecoder { } pub fn reset(&mut self, requested_time: f32) -> Result<(), String> { - let _reset_start = std::time::Instant::now(); - self.reader.cancel_reading(); - let (seek_time, _keyframe_frame) = if let Some(ref keyframe_index) = self.keyframe_index { + let seek_time = if let Some(ref keyframe_index) = self.keyframe_index { let fps = keyframe_index.fps(); let target_frame = (requested_time as f64 * fps).round() as u32; - if let Some((kf_frame, keyframe_time)) = - keyframe_index.nearest_keyframe_before(target_frame) - { - (keyframe_time as f32, Some(kf_frame)) + if let Some((_, keyframe_time)) = keyframe_index.nearest_keyframe_before(target_frame) { + keyframe_time as f32 } else { - (requested_time, None) + requested_time } } else { - (requested_time, None) + requested_time }; (self.track_output, self.reader) = Self::get_reader_track_output( From 5fb8232ef2835ca79122a0098e6e80d2c2fdf310 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 23:49:03 +0000 Subject: [PATCH 21/37] Add EditorSkeleton loading component --- .../src/routes/editor/EditorSkeleton.tsx | 232 ++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 apps/desktop/src/routes/editor/EditorSkeleton.tsx diff --git a/apps/desktop/src/routes/editor/EditorSkeleton.tsx b/apps/desktop/src/routes/editor/EditorSkeleton.tsx new file mode 100644 index 0000000000..372bbfdfcf --- /dev/null +++ b/apps/desktop/src/routes/editor/EditorSkeleton.tsx @@ -0,0 +1,232 @@ +import { type as ostype } from "@tauri-apps/plugin-os"; +import { cx } from "cva"; + +const DEFAULT_TIMELINE_HEIGHT = 260; +const MIN_PLAYER_HEIGHT = 328; +const RESIZE_HANDLE_HEIGHT = 8; + +function SkeletonPulse(props: { class?: string }) { + return ( +
+ ); +} + +function SkeletonButton(props: { class?: string; width?: string }) { + return ( + + ); +} + +function HeaderSkeleton() { + return ( +
+
+ {ostype() === "macos" &&
} + + + + +
+ + +
+ +
+ +
+ +
+ + +
+ +
+
+ ); +} + +function PlayerToolbarSkeleton() { + return ( +
+
+ + +
+
+ + +
+
+ ); +} + +function VideoPreviewSkeleton() { + return ( +
+
+
+
+ +
+
+
+
+ ); +} + +function PlayerControlsSkeleton() { + return ( +
+
+ + + +
+
+ + + +
+
+
+ + + + + +
+
+ ); +} + +function PlayerSkeleton() { + return ( +
+
+ + + +
+
+
+
+
+
+
+ ); +} + +function SidebarSkeleton() { + return ( +
+
+ + + + + + +
+
+ +
+ + + + +
+ + +
+ + + + + + + + +
+
+
+ ); +} + +function TimelineTrackSkeleton() { + return ( +
+
+ +
+
+ +
+
+ ); +} + +function TimelineSkeleton() { + return ( +
+
+
+
+ + + + + +
+
+ +
+
+
+ + +
+
+
+ ); +} + +export function EditorSkeleton() { + return ( + <> + +
+
+
+ + +
+
+
+ +
+
+
+
+ + ); +} From 8df37a91257101969898bcc4ae00270f8cc6acd7 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 22 Dec 2025 23:52:59 +0000 Subject: [PATCH 22/37] Refactor crop dialog to use canvas frame instead of video --- apps/desktop/src/routes/editor/Editor.tsx | 138 +++++----------------- 1 file changed, 28 insertions(+), 110 deletions(-) diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index 90f2b96d74..ea9b5ccf35 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -44,15 +44,7 @@ import { ExportDialog } from "./ExportDialog"; import { Header } from "./Header"; import { PlayerContent } from "./Player"; import { Timeline } from "./Timeline"; -import { - Dialog, - DialogContent, - EditorButton, - Input, - Slider, - Subfield, -} from "./ui"; -import { formatTime } from "./utils"; +import { Dialog, DialogContent, EditorButton, Input, Subfield } from "./ui"; const DEFAULT_TIMELINE_HEIGHT = 260; const MIN_PLAYER_CONTENT_HEIGHT = 320; @@ -208,10 +200,10 @@ function Inner() { ); return ( - <> +
- +
); } @@ -422,61 +414,28 @@ function Dialogs() { })()} > {(dialog) => { - const { - setProject: setState, - editorInstance, - editorState, - totalDuration, - project, - } = useEditorContext(); + const { setProject: setState, editorInstance } = + useEditorContext(); const display = editorInstance.recordings.segments[0].display; let cropperRef: CropperRef | undefined; - let videoRef: HTMLVideoElement | undefined; const [crop, setCrop] = createSignal(CROP_ZERO); const [aspect, setAspect] = createSignal(null); - const [previewTime, setPreviewTime] = createSignal( - editorState.playbackTime, - ); - const [videoLoaded, setVideoLoaded] = createSignal(false); - - const currentSegment = createMemo(() => { - const time = previewTime(); - let elapsed = 0; - for (const seg of project.timeline?.segments ?? []) { - const segDuration = (seg.end - seg.start) / seg.timescale; - if (time < elapsed + segDuration) { - return { - index: seg.recordingSegment ?? 0, - localTime: seg.start / seg.timescale + (time - elapsed), - }; - } - elapsed += segDuration; - } - return { index: 0, localTime: 0 }; - }); - - const videoSrc = createMemo(() => - convertFileSrc( - `${editorInstance.path}/content/segments/segment-${currentSegment().index}/display.mp4`, - ), - ); - createEffect( - on( - () => currentSegment().index, - () => { - setVideoLoaded(false); - }, - { defer: true }, - ), - ); - - createEffect(() => { - if (videoRef && videoLoaded()) { - videoRef.currentTime = currentSegment().localTime; + const [frameDataUrl, setFrameDataUrl] = createSignal< + string | null + >(null); + + const playerCanvas = document.getElementById( + "canvas", + ) as HTMLCanvasElement | null; + if (playerCanvas) { + try { + setFrameDataUrl(playerCanvas.toDataURL("image/png")); + } catch { + setFrameDataUrl(null); } - }); + } const initialBounds = { x: dialog().position.x, @@ -638,60 +597,19 @@ function Dialogs() { allowLightMode={true} onContextMenu={(e) => showCropOptionsMenu(e, true)} > -
- screenshot -
+ ) + } + />
-
- - {formatTime(previewTime())} - - setPreviewTime(v)} - aria-label="Video timeline" - /> - - {formatTime(totalDuration())} - -
- ) - } - leftFooterContent={ -
- - - {(est) => ( -

- - - {(() => { - const totalSeconds = Math.round( - est().duration_seconds, - ); - const hours = Math.floor(totalSeconds / 3600); - const minutes = Math.floor( - (totalSeconds % 3600) / 60, - ); - const seconds = totalSeconds % 60; - - if (hours > 0) { - return `${hours}:${minutes - .toString() - .padStart(2, "0")}:${seconds - .toString() - .padStart(2, "0")}`; - } - return `${minutes}:${seconds - .toString() - .padStart(2, "0")}`; - })()} - - - - {settings.resolution.width}×{settings.resolution.height} - - - - {est().estimated_size_mb.toFixed(2)} MB - - - - {(() => { - const totalSeconds = Math.round( - est().estimated_time_seconds, - ); - const hours = Math.floor(totalSeconds / 3600); - const minutes = Math.floor( - (totalSeconds % 3600) / 60, - ); - const seconds = totalSeconds % 60; - - if (hours > 0) { - return `~${hours}:${minutes - .toString() - .padStart(2, "0")}:${seconds - .toString() - .padStart(2, "0")}`; - } - return `~${minutes}:${seconds - .toString() - .padStart(2, "0")}`; - })()} - -

- )} - - -
- } - > -
- {/* Export to */} -
-
-
-

Export to

- - 1 - } - > -
{ - const menu = await Menu.new({ - items: await Promise.all( - organisations().map((org) => - CheckMenuItem.new({ - text: org.name, - action: () => { - setSettings("organizationId", org.id); - }, - checked: settings.organizationId === org.id, - }), - ), - ), - }); - menu.popup(); - }} - > - Organization: - - { - ( - organisations().find( - (o) => o.id === settings.organizationId, - ) ?? organisations()[0] - )?.name - } - - -
- - -
-
- - {(option) => ( - - )} - -
-
-
- {/* Format */} -
-
-

Format

-
- - {(option) => { - const disabledReason = () => { - if ( - option.value === "Mp4" && - hasTransparentBackground() - ) - return "MP4 format does not support transparent backgrounds"; - if ( - option.value === "Gif" && - settings.exportTo === "link" - ) - return "Shareable links cannot be made from GIFs"; - }; - - return ( - - - - ); - }} - -
-
-
- {/* Frame rate */} -
-
-

Frame rate

- - options={ - settings.format === "Gif" ? GIF_FPS_OPTIONS : FPS_OPTIONS - } - optionValue="value" - optionTextValue="label" - placeholder="Select FPS" - value={(settings.format === "Gif" - ? GIF_FPS_OPTIONS - : FPS_OPTIONS - ).find((opt) => opt.value === settings.fps)} - onChange={(option) => { - if (!option) return; - trackEvent("export_fps_changed", { - fps: option.value, - }); - setSettings("fps", option.value); - }} - disallowEmptySelection - itemComponent={(props) => ( - - as={KSelect.Item} - item={props.item} - > - - {props.item.rawValue.label} - - - )} - > - - class="flex-1 text-sm text-left truncate tabular-nums text-[--gray-500]"> - {(state) => {state.selectedOption()?.label}} - - - as={(props) => ( - - )} - /> - - - - as={KSelect.Content} - class={cx(topSlideAnimateClasses, "z-50")} - > - - class="max-h-32 custom-scroll" - as={KSelect.Listbox} - /> - - - -
-
- {/* Compression */} -
-
-

Compression

-
- - {(option) => ( - - )} - -
-
-
- {/* Resolution */} -
-
-

Resolution

-
- - {(option) => ( - - )} - -
-
-
-
- - - - {(exportState) => { - const [copyPressed, setCopyPressed] = createSignal(false); - const [clipboardCopyPressed, setClipboardCopyPressed] = - createSignal(false); - const [showCompletionScreen, setShowCompletionScreen] = createSignal( - exportState.type === "done" && exportState.action === "save", - ); - - createEffect(() => { - if (exportState.type === "done" && exportState.action === "save") { - setShowCompletionScreen(true); - } - }); - - return ( - <> - -
- Export -
setDialog((d) => ({ ...d, open: false }))} - class="flex justify-center items-center p-1 rounded-full transition-colors cursor-pointer hover:bg-gray-3" - > - -
-
- - -
- - - {(copyState) => ( -
-

- {copyState.type === "starting" - ? "Preparing..." - : copyState.type === "rendering" - ? settings.format === "Gif" - ? "Rendering GIF..." - : "Rendering video..." - : copyState.type === "copying" - ? "Copying to clipboard..." - : "Copied to clipboard"} -

- - {(copyState) => ( - <> - - - - )} - -
- )} - - - {(saveState) => ( -
- -

- {saveState.type === "starting" - ? "Preparing..." - : saveState.type === "rendering" - ? settings.format === "Gif" - ? "Rendering GIF..." - : "Rendering video..." - : saveState.type === "copying" - ? "Exporting to file..." - : "Export completed"} -

- - {(copyState) => ( - <> - - - - )} - - - } - > -
-
-
- -
-
-

- Export Completed -

-

- Your{" "} - {settings.format === "Gif" - ? "GIF" - : "video"}{" "} - has successfully been exported -

-
-
-
- -
- )} - - - {(uploadState) => ( - - - {(uploadState) => ( -
-

- Uploading Cap... -

- - - {(uploadState) => ( - - )} - - - {(renderState) => ( - <> - - - - )} - - -
- )} - - -
-
-

- Upload Complete -

-

- Your Cap has been uploaded successfully -

-
-
- - - )} - - -
- - - - -
- - - -
- - - -
- - -
- - - - - ); - }} - - - ); -} - -function RenderProgress(props: { state: RenderState; format?: ExportFormat }) { - return ( - - ); -} - -function ProgressView(props: { amount: number; label?: string }) { - return ( - <> -
-
-
-

{props.label}

- - ); -} diff --git a/apps/desktop/src/routes/editor/ExportPage.tsx b/apps/desktop/src/routes/editor/ExportPage.tsx new file mode 100644 index 0000000000..de8cba1c38 --- /dev/null +++ b/apps/desktop/src/routes/editor/ExportPage.tsx @@ -0,0 +1,1392 @@ +import { Button } from "@cap/ui-solid"; +import { debounce } from "@solid-primitives/scheduled"; +import { makePersisted } from "@solid-primitives/storage"; +import { + createMutation, + createQuery, + keepPreviousData, +} from "@tanstack/solid-query"; +import { Channel } from "@tauri-apps/api/core"; +import { CheckMenuItem, Menu } from "@tauri-apps/api/menu"; +import { ask, save as saveDialog } from "@tauri-apps/plugin-dialog"; +import { remove } from "@tauri-apps/plugin-fs"; +import { type as ostype } from "@tauri-apps/plugin-os"; +import { cx } from "cva"; +import { + createEffect, + createSignal, + For, + type JSX, + Match, + mergeProps, + on, + onCleanup, + Show, + Suspense, + Switch, +} from "solid-js"; +import { createStore, produce, reconcile } from "solid-js/store"; +import toast from "solid-toast"; +import { SignInButton } from "~/components/SignInButton"; +import Tooltip from "~/components/Tooltip"; +import { authStore } from "~/store"; +import { trackEvent } from "~/utils/analytics"; +import { createSignInMutation } from "~/utils/auth"; +import { createExportTask } from "~/utils/export"; +import { createOrganizationsQuery } from "~/utils/queries"; +import { + commands, + type ExportCompression, + type ExportSettings, + type FramesRendered, + type UploadProgress, +} from "~/utils/tauri"; +import { type RenderState, useEditorContext } from "./context"; +import { RESOLUTION_OPTIONS } from "./Header"; +import { Dialog, Slider } from "./ui"; + +class SilentError extends Error {} + +export const COMPRESSION_OPTIONS: Array<{ + label: string; + value: ExportCompression; + bpp: number; +}> = [ + { label: "Minimal", value: "Minimal", bpp: 0.3 }, + { label: "Social Media", value: "Social", bpp: 0.15 }, + { label: "Web", value: "Web", bpp: 0.08 }, + { label: "Potato", value: "Potato", bpp: 0.04 }, +]; + +const BPP_TO_COMPRESSION: Record = { + 0.3: "Minimal", + 0.15: "Social", + 0.08: "Web", + 0.04: "Potato", +}; + +const COMPRESSION_TO_BPP: Record = { + Minimal: 0.3, + Social: 0.15, + Web: 0.08, + Potato: 0.04, +}; + +export const FPS_OPTIONS = [ + { label: "15 FPS", value: 15 }, + { label: "30 FPS", value: 30 }, + { label: "60 FPS", value: 60 }, +] satisfies Array<{ label: string; value: number }>; + +export const GIF_FPS_OPTIONS = [ + { label: "10 FPS", value: 10 }, + { label: "15 FPS", value: 15 }, + { label: "20 FPS", value: 20 }, + { label: "25 FPS", value: 25 }, + { label: "30 FPS", value: 30 }, +] satisfies Array<{ label: string; value: number }>; + +export const EXPORT_TO_OPTIONS = [ + { + label: "File", + value: "file", + icon: , + }, + { + label: "Clipboard", + value: "clipboard", + icon: , + }, + { + label: "Shareable link", + value: "link", + icon: , + }, +] as const; + +type ExportFormat = ExportSettings["format"]; + +const FORMAT_OPTIONS = [ + { label: "MP4", value: "Mp4" }, + { label: "GIF", value: "Gif" }, +] as { label: string; value: ExportFormat; disabled?: boolean }[]; + +type ExportToOption = (typeof EXPORT_TO_OPTIONS)[number]["value"]; + +interface Settings { + format: ExportFormat; + fps: number; + exportTo: ExportToOption; + resolution: { label: string; value: string; width: number; height: number }; + compression: ExportCompression; + organizationId?: string | null; +} + +export function ExportPage() { + const { + setDialog, + editorInstance, + editorState, + setExportState, + exportState, + meta, + refetchMeta, + } = useEditorContext(); + + const handleBack = () => { + setDialog((d) => ({ ...d, open: false })); + }; + + const projectPath = editorInstance.path; + + const auth = authStore.createQuery(); + const organisations = createOrganizationsQuery(); + + const hasTransparentBackground = () => { + const backgroundSource = + editorInstance.savedProjectConfig.background.source; + return ( + backgroundSource.type === "color" && + backgroundSource.alpha !== undefined && + backgroundSource.alpha < 255 + ); + }; + + const isCancellationError = (error: unknown) => + error instanceof SilentError || + error === "Export cancelled" || + (error instanceof Error && error.message === "Export cancelled"); + + const [_settings, setSettings] = makePersisted( + createStore({ + format: "Mp4", + fps: 30, + exportTo: "file", + resolution: { label: "720p", value: "720p", width: 1280, height: 720 }, + compression: "Minimal", + }), + { name: "export_settings" }, + ); + + const settings = mergeProps(_settings, () => { + const ret: Partial = {}; + if (hasTransparentBackground() && _settings.format === "Mp4") + ret.format = "Gif"; + // Ensure GIF is not selected when exportTo is "link" + else if (_settings.format === "Gif" && _settings.exportTo === "link") + ret.format = "Mp4"; + else if (!["Mp4", "Gif"].includes(_settings.format)) ret.format = "Mp4"; + + Object.defineProperty(ret, "organizationId", { + get() { + if (!_settings.organizationId && organisations().length > 0) + return organisations()[0].id; + + return _settings.organizationId; + }, + }); + + return ret; + }); + + const [previewUrl, setPreviewUrl] = createSignal(null); + const [previewLoading, setPreviewLoading] = createSignal(false); + const [previewDialogOpen, setPreviewDialogOpen] = createSignal(false); + const [compressionBpp, setCompressionBpp] = createSignal( + COMPRESSION_TO_BPP[_settings.compression] ?? 0.15, + ); + + createEffect( + on( + () => _settings.compression, + (compression) => { + const bpp = COMPRESSION_TO_BPP[compression]; + if (bpp !== undefined) setCompressionBpp(bpp); + }, + ), + ); + + const debouncedFetchPreview = debounce( + async ( + frameTime: number, + fps: number, + resWidth: number, + resHeight: number, + bpp: number, + ) => { + setPreviewLoading(true); + try { + const result = await commands.generateExportPreviewFast(frameTime, { + fps, + resolution_base: { x: resWidth, y: resHeight }, + compression_bpp: bpp, + }); + + const oldUrl = previewUrl(); + if (oldUrl) URL.revokeObjectURL(oldUrl); + + const byteArray = Uint8Array.from(atob(result.jpeg_base64), (c) => + c.charCodeAt(0), + ); + const blob = new Blob([byteArray], { type: "image/jpeg" }); + setPreviewUrl(URL.createObjectURL(blob)); + } catch (e) { + console.error("Failed to generate preview:", e); + } finally { + setPreviewLoading(false); + } + }, + 300, + ); + + createEffect( + on( + [ + () => settings.fps, + () => settings.resolution.width, + () => settings.resolution.height, + compressionBpp, + ], + ([fps, width, height, bpp]) => { + if (settings.format === "Gif") return; + const frameTime = editorState.playbackTime ?? 0; + debouncedFetchPreview(frameTime, fps, width, height, bpp); + }, + ), + ); + + onCleanup(() => { + const url = previewUrl(); + if (url) URL.revokeObjectURL(url); + }); + + let cancelCurrentExport: (() => void) | null = null; + + const exportWithSettings = ( + onProgress: (progress: FramesRendered) => void, + ) => { + const { promise, cancel } = createExportTask( + projectPath, + settings.format === "Mp4" + ? { + format: "Mp4", + fps: settings.fps, + resolution_base: { + x: settings.resolution.width, + y: settings.resolution.height, + }, + compression: settings.compression, + } + : { + format: "Gif", + fps: settings.fps, + resolution_base: { + x: settings.resolution.width, + y: settings.resolution.height, + }, + quality: null, + }, + onProgress, + ); + cancelCurrentExport = cancel; + return promise.finally(() => { + if (cancelCurrentExport === cancel) cancelCurrentExport = null; + }); + }; + + const [outputPath, setOutputPath] = createSignal(null); + const [isCancelled, setIsCancelled] = createSignal(false); + + const handleCancel = async () => { + if ( + await ask("Are you sure you want to cancel the export?", { + title: "Cancel Export", + kind: "warning", + }) + ) { + setIsCancelled(true); + cancelCurrentExport?.(); + cancelCurrentExport = null; + setExportState({ type: "idle" }); + const path = outputPath(); + if (path) { + try { + await remove(path); + } catch (e) { + console.error("Failed to delete cancelled file", e); + } + } + } + }; + + const exportEstimates = createQuery(() => ({ + placeholderData: keepPreviousData, + queryKey: [ + "exportEstimates", + { + format: settings.format, + resolution: { + x: settings.resolution.width, + y: settings.resolution.height, + }, + fps: settings.fps, + compression: settings.compression, + }, + ] as const, + queryFn: ({ queryKey: [_, { format, resolution, fps, compression }] }) => { + const exportSettings = + format === "Mp4" + ? { + format: "Mp4" as const, + fps, + resolution_base: resolution, + compression, + } + : { + format: "Gif" as const, + fps, + resolution_base: resolution, + quality: null, + }; + return commands.getExportEstimates(projectPath, exportSettings); + }, + })); + + const exportButtonIcon: Record<"file" | "clipboard" | "link", JSX.Element> = { + file: , + clipboard: , + link: , + }; + + const copy = createMutation(() => ({ + mutationFn: async () => { + setIsCancelled(false); + if (exportState.type !== "idle") return; + setExportState(reconcile({ action: "copy", type: "starting" })); + + const outputPath = await exportWithSettings((progress) => { + if (isCancelled()) throw new SilentError("Cancelled"); + setExportState({ type: "rendering", progress }); + }); + + if (isCancelled()) throw new SilentError("Cancelled"); + + setExportState({ type: "copying" }); + + await commands.copyVideoToClipboard(outputPath); + }, + onError: (error) => { + if (isCancelled() || isCancellationError(error)) { + setExportState(reconcile({ type: "idle" })); + return; + } + commands.globalMessageDialog( + error instanceof Error ? error.message : "Failed to copy recording", + ); + setExportState(reconcile({ type: "idle" })); + }, + onSuccess() { + setExportState({ type: "done" }); + toast.success( + `${ + settings.format === "Gif" ? "GIF" : "Recording" + } exported to clipboard`, + ); + }, + })); + + const save = createMutation(() => ({ + mutationFn: async () => { + setIsCancelled(false); + if (exportState.type !== "idle") return; + + const extension = settings.format === "Gif" ? "gif" : "mp4"; + const savePath = await saveDialog({ + filters: [ + { + name: `${extension.toUpperCase()} filter`, + extensions: [extension], + }, + ], + defaultPath: `~/Desktop/${meta().prettyName}.${extension}`, + }); + if (!savePath) { + setExportState(reconcile({ type: "idle" })); + return; + } + + setExportState(reconcile({ action: "save", type: "starting" })); + + setOutputPath(savePath); + + trackEvent("export_started", { + resolution: settings.resolution, + fps: settings.fps, + path: savePath, + }); + + const videoPath = await exportWithSettings((progress) => { + if (isCancelled()) throw new SilentError("Cancelled"); + setExportState({ type: "rendering", progress }); + }); + + if (isCancelled()) throw new SilentError("Cancelled"); + + setExportState({ type: "copying" }); + + await commands.copyFileToPath(videoPath, savePath); + + setExportState({ type: "done" }); + }, + onError: (error) => { + if (isCancelled() || isCancellationError(error)) { + setExportState({ type: "idle" }); + return; + } + commands.globalMessageDialog( + error instanceof Error + ? error.message + : `Failed to export recording: ${error}`, + ); + setExportState({ type: "idle" }); + }, + onSuccess() { + toast.success( + `${settings.format === "Gif" ? "GIF" : "Recording"} exported to file`, + ); + }, + })); + + const upload = createMutation(() => ({ + mutationFn: async () => { + setIsCancelled(false); + if (exportState.type !== "idle") return; + setExportState(reconcile({ action: "upload", type: "starting" })); + + // Check authentication first + const existingAuth = await authStore.get(); + if (!existingAuth) createSignInMutation(); + trackEvent("create_shareable_link_clicked", { + resolution: settings.resolution, + fps: settings.fps, + has_existing_auth: !!existingAuth, + }); + + const metadata = await commands.getVideoMetadata(projectPath); + const plan = await commands.checkUpgradedAndUpdate(); + const canShare = { + allowed: plan || metadata.duration < 300, + reason: !plan && metadata.duration >= 300 ? "upgrade_required" : null, + }; + + if (!canShare.allowed) { + if (canShare.reason === "upgrade_required") { + await commands.showWindow("Upgrade"); + // The window takes a little to show and this prevents the user seeing it glitch + await new Promise((resolve) => setTimeout(resolve, 1000)); + throw new SilentError(); + } + } + + const uploadChannel = new Channel((progress) => { + console.log("Upload progress:", progress); + setExportState( + produce((state) => { + if (state.type !== "uploading") return; + + state.progress = Math.round(progress.progress * 100); + }), + ); + }); + + await exportWithSettings((progress) => { + if (isCancelled()) throw new SilentError("Cancelled"); + setExportState({ type: "rendering", progress }); + }); + + if (isCancelled()) throw new SilentError("Cancelled"); + + setExportState({ type: "uploading", progress: 0 }); + + console.log({ organizationId: settings.organizationId }); + + // Now proceed with upload + const result = meta().sharing + ? await commands.uploadExportedVideo( + projectPath, + "Reupload", + uploadChannel, + settings.organizationId ?? null, + ) + : await commands.uploadExportedVideo( + projectPath, + { Initial: { pre_created_video: null } }, + uploadChannel, + settings.organizationId ?? null, + ); + + if (result === "NotAuthenticated") + throw new Error("You need to sign in to share recordings"); + else if (result === "PlanCheckFailed") + throw new Error("Failed to verify your subscription status"); + else if (result === "UpgradeRequired") + throw new Error("This feature requires an upgraded plan"); + }, + onSuccess: async () => { + await refetchMeta(); + setExportState({ type: "done" }); + }, + onError: (error) => { + if (isCancelled() || isCancellationError(error)) { + setExportState(reconcile({ type: "idle" })); + return; + } + console.error(error); + if (!(error instanceof SilentError)) { + commands.globalMessageDialog( + error instanceof Error ? error.message : "Failed to upload recording", + ); + } + + setExportState(reconcile({ type: "idle" })); + }, + })); + + return ( +
+
+
+ {ostype() === "macos" &&
} + +
+

Export Cap

+
+
+
+ + +
+
+
+
+
+
+

Export to

+ + 1 + } + > +
{ + const menu = await Menu.new({ + items: await Promise.all( + organisations().map((org) => + CheckMenuItem.new({ + text: org.name, + action: () => { + setSettings("organizationId", org.id); + }, + checked: settings.organizationId === org.id, + }), + ), + ), + }); + menu.popup(); + }} + > + Organization: + + { + ( + organisations().find( + (o) => o.id === settings.organizationId, + ) ?? organisations()[0] + )?.name + } + + +
+ + +
+
+ + {(option) => ( + + )} + +
+
+
+ +
+
+

Format

+
+ + {(option) => { + const disabledReason = () => { + if ( + option.value === "Mp4" && + hasTransparentBackground() + ) + return "MP4 format does not support transparent backgrounds"; + if ( + option.value === "Gif" && + settings.exportTo === "link" + ) + return "Shareable links cannot be made from GIFs"; + }; + + return ( + + + + ); + }} + +
+
+
+ +
+
+

Resolution

+
+ + {(option) => ( + + )} + +
+
+
+ +
+
+
+

Frame Rate

+ + {settings.fps} FPS + +
+ { + if (v === undefined) return; + trackEvent("export_fps_changed", { fps: v }); + setSettings("fps", v); + }} + history={{ pause: () => () => {} }} + /> +
+
+ + +
+
+
+

Quality

+ + {(() => { + const bpp = compressionBpp(); + if (bpp >= 0.25) return "Minimal compression"; + if (bpp >= 0.12) return "Social Media quality"; + if (bpp >= 0.06) return "Web optimized"; + return "Maximum compression"; + })()} + +
+ { + if (v === undefined) return; + setCompressionBpp(v); + const closest = Object.entries(BPP_TO_COMPRESSION) + .map(([bpp, comp]) => ({ + bpp: Number(bpp), + comp, + diff: Math.abs(Number(bpp) - v), + })) + .sort((a, b) => a.diff - b.diff)[0]; + if (closest) setSettings("compression", closest.comp); + }} + history={{ pause: () => () => {} }} + /> +
+ Smaller file + Higher quality +
+
+
+ +
+ + +
+
+

Quality Preview

+
+ + + + Preview loading... + + } + > +
+ +
+ +
+ } + > + {(url) => ( + <> + Export preview + +
+
+ +
+
+ + + + )} + +
+
+ + {settings.resolution.width}×{settings.resolution.height} + + + + {(est) => ( + {est().estimated_size_mb.toFixed(1)} MB + )} + + +
+
+
+ +
+
+

Quality Preview

+ +
+
+ + {(url) => ( + Export preview full size + )} + +
+
+ + {settings.resolution.width}×{settings.resolution.height} + + + + {(est) => ( + + Estimated size: {est().estimated_size_mb.toFixed(1)}{" "} + MB + + )} + + +
+
+ + +
+
+ +
+
+ + + {(est) => ( +
+ + + {(() => { + const totalSeconds = Math.round(est().duration_seconds); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + + if (hours > 0) { + return `${hours}:${minutes + .toString() + .padStart(2, "0")}:${seconds + .toString() + .padStart(2, "0")}`; + } + return `${minutes}:${seconds + .toString() + .padStart(2, "0")}`; + })()} + + + + {settings.resolution.width}×{settings.resolution.height} + + + + {est().estimated_size_mb.toFixed(2)} MB + + + + {(() => { + const totalSeconds = Math.round( + est().estimated_time_seconds, + ); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + + if (hours > 0) { + return `~${hours}:${minutes + .toString() + .padStart(2, "0")}:${seconds + .toString() + .padStart(2, "0")}`; + } + return `~${minutes}:${seconds + .toString() + .padStart(2, "0")}`; + })()} + +
+ )} + + +
+
+ {settings.exportTo === "link" && !auth.data ? ( + + {exportButtonIcon[settings.exportTo]} + Sign in to share + + ) : ( + + )} +
+
+ + + {(exportState) => { + const [copyPressed, setCopyPressed] = createSignal(false); + const [clipboardCopyPressed, setClipboardCopyPressed] = + createSignal(false); + const [showCompletionScreen, setShowCompletionScreen] = createSignal( + exportState.type === "done" && exportState.action === "save", + ); + + createEffect(() => { + if (exportState.type === "done" && exportState.action === "save") { + setShowCompletionScreen(true); + } + }); + + return ( +
+
+ + + {(copyState) => ( +
+

+ {copyState.type === "starting" + ? "Preparing..." + : copyState.type === "rendering" + ? settings.format === "Gif" + ? "Rendering GIF..." + : "Rendering video..." + : copyState.type === "copying" + ? "Copying to clipboard..." + : "Copied to clipboard"} +

+ + {(copyState) => ( + <> + + + + )} + +
+ )} + + + {(saveState) => ( +
+ +

+ {saveState.type === "starting" + ? "Preparing..." + : saveState.type === "rendering" + ? settings.format === "Gif" + ? "Rendering GIF..." + : "Rendering video..." + : saveState.type === "copying" + ? "Exporting to file..." + : "Export completed"} +

+ + {(copyState) => ( + <> + + + + )} + + + } + > +
+
+
+ +
+
+

+ Export Completed +

+

+ Your{" "} + {settings.format === "Gif" ? "GIF" : "video"}{" "} + has successfully been exported +

+
+
+
+ +
+ )} + + + {(uploadState) => ( + + + {(uploadState) => ( +
+

+ Uploading Cap... +

+ + + {(uploadState) => ( + + )} + + + {(renderState) => ( + <> + + + + )} + + +
+ )} + + +
+
+

+ Upload Complete +

+

+ Your Cap has been uploaded successfully +

+
+
+ + + )} + + +
+ +
+ +
+ + + +
+ + + +
+ + +
+ +
+ +
+ ); + }} + +
+ ); +} + +function RenderProgress(props: { state: RenderState; format?: ExportFormat }) { + return ( + + ); +} + +function ProgressView(props: { amount: number; label?: string }) { + return ( + <> +
+
+
+

{props.label}

+ + ); +} diff --git a/packages/ui-solid/src/auto-imports.d.ts b/packages/ui-solid/src/auto-imports.d.ts index 2e7c41dfb8..cce2545d19 100644 --- a/packages/ui-solid/src/auto-imports.d.ts +++ b/packages/ui-solid/src/auto-imports.d.ts @@ -6,6 +6,7 @@ // biome-ignore lint: disable export {} declare global { + const IconCapArrowLeft: typeof import('~icons/cap/arrow-left.jsx')['default'] const IconCapArrows: typeof import('~icons/cap/arrows.jsx')['default'] const IconCapAudioOn: typeof import('~icons/cap/audio-on.jsx')['default'] const IconCapAuto: typeof import('~icons/cap/auto.jsx')['default'] @@ -67,6 +68,7 @@ declare global { const IconCapZoomOut: typeof import('~icons/cap/zoom-out.jsx')['default'] const IconHugeiconsEaseCurveControlPoints: typeof import('~icons/hugeicons/ease-curve-control-points.jsx')['default'] const IconLucideAlertTriangle: typeof import('~icons/lucide/alert-triangle.jsx')['default'] + const IconLucideArrowLeft: typeof import('~icons/lucide/arrow-left.jsx')['default'] const IconLucideBell: typeof import('~icons/lucide/bell.jsx')['default'] const IconLucideBoxSelect: typeof import('~icons/lucide/box-select.jsx')['default'] const IconLucideBug: typeof import('~icons/lucide/bug.jsx')['default'] @@ -85,6 +87,7 @@ declare global { const IconLucideLoader2: typeof import('~icons/lucide/loader2.jsx')['default'] const IconLucideLoaderCircle: typeof import('~icons/lucide/loader-circle.jsx')['default'] const IconLucideMaximize: typeof import('~icons/lucide/maximize.jsx')['default'] + const IconLucideMaximize2: typeof import('~icons/lucide/maximize2.jsx')['default'] const IconLucideMessageSquarePlus: typeof import('~icons/lucide/message-square-plus.jsx')['default'] const IconLucideMicOff: typeof import('~icons/lucide/mic-off.jsx')['default'] const IconLucideMonitor: typeof import('~icons/lucide/monitor.jsx')['default'] From 9ecd43ee4168b8e0e041a623ae60c83850899208 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Tue, 23 Dec 2025 22:14:50 +0000 Subject: [PATCH 25/37] Redesign export page UI and improve export options --- apps/desktop/src/routes/editor/ExportPage.tsx | 906 +++++++++--------- packages/ui-solid/src/auto-imports.d.ts | 5 + 2 files changed, 448 insertions(+), 463 deletions(-) diff --git a/apps/desktop/src/routes/editor/ExportPage.tsx b/apps/desktop/src/routes/editor/ExportPage.tsx index de8cba1c38..b8871f2495 100644 --- a/apps/desktop/src/routes/editor/ExportPage.tsx +++ b/apps/desktop/src/routes/editor/ExportPage.tsx @@ -1,4 +1,5 @@ import { Button } from "@cap/ui-solid"; +import { RadioGroup as KRadioGroup } from "@kobalte/core/radio-group"; import { debounce } from "@solid-primitives/scheduled"; import { makePersisted } from "@solid-primitives/storage"; import { @@ -29,6 +30,7 @@ import { createStore, produce, reconcile } from "solid-js/store"; import toast from "solid-toast"; import { SignInButton } from "~/components/SignInButton"; import Tooltip from "~/components/Tooltip"; +import CaptionControlsWindows11 from "~/components/titlebar/controls/CaptionControlsWindows11"; import { authStore } from "~/store"; import { trackEvent } from "~/utils/analytics"; import { createSignInMutation } from "~/utils/auth"; @@ -43,7 +45,7 @@ import { } from "~/utils/tauri"; import { type RenderState, useEditorContext } from "./context"; import { RESOLUTION_OPTIONS } from "./Header"; -import { Dialog, Slider } from "./ui"; +import { Dialog, Field, Slider } from "./ui"; class SilentError extends Error {} @@ -90,17 +92,20 @@ export const EXPORT_TO_OPTIONS = [ { label: "File", value: "file", - icon: , + icon: IconCapFile, + description: "Save to your computer", }, { label: "Clipboard", value: "clipboard", - icon: , + icon: IconCapCopy, + description: "Copy to paste anywhere", }, { - label: "Shareable link", + label: "Link", value: "link", - icon: , + icon: IconCapLink, + description: "Share via Cap cloud", }, ] as const; @@ -172,7 +177,6 @@ export function ExportPage() { const ret: Partial = {}; if (hasTransparentBackground() && _settings.format === "Mp4") ret.format = "Gif"; - // Ensure GIF is not selected when exportTo is "link" else if (_settings.format === "Gif" && _settings.exportTo === "link") ret.format = "Mp4"; else if (!["Mp4", "Gif"].includes(_settings.format)) ret.format = "Mp4"; @@ -191,6 +195,11 @@ export function ExportPage() { const [previewUrl, setPreviewUrl] = createSignal(null); const [previewLoading, setPreviewLoading] = createSignal(false); + + const updateSettings: typeof setSettings = (...args) => { + setPreviewLoading(true); + return setSettings(...args); + }; const [previewDialogOpen, setPreviewDialogOpen] = createSignal(false); const [compressionBpp, setCompressionBpp] = createSignal( COMPRESSION_TO_BPP[_settings.compression] ?? 0.15, @@ -214,7 +223,6 @@ export function ExportPage() { resHeight: number, bpp: number, ) => { - setPreviewLoading(true); try { const result = await commands.generateExportPreviewFast(frameTime, { fps, @@ -242,15 +250,22 @@ export function ExportPage() { createEffect( on( [ - () => settings.fps, - () => settings.resolution.width, - () => settings.resolution.height, + () => _settings.format, + () => _settings.fps, + () => _settings.resolution.width, + () => _settings.resolution.height, compressionBpp, ], - ([fps, width, height, bpp]) => { - if (settings.format === "Gif") return; + () => { const frameTime = editorState.playbackTime ?? 0; - debouncedFetchPreview(frameTime, fps, width, height, bpp); + setPreviewLoading(true); + debouncedFetchPreview( + frameTime, + settings.fps, + settings.resolution.width, + settings.resolution.height, + compressionBpp(), + ); }, ), ); @@ -352,12 +367,6 @@ export function ExportPage() { }, })); - const exportButtonIcon: Record<"file" | "clipboard" | "link", JSX.Element> = { - file: , - clipboard: , - link: , - }; - const copy = createMutation(() => ({ mutationFn: async () => { setIsCancelled(false); @@ -463,7 +472,6 @@ export function ExportPage() { if (exportState.type !== "idle") return; setExportState(reconcile({ action: "upload", type: "starting" })); - // Check authentication first const existingAuth = await authStore.get(); if (!existingAuth) createSignInMutation(); trackEvent("create_shareable_link_clicked", { @@ -482,7 +490,6 @@ export function ExportPage() { if (!canShare.allowed) { if (canShare.reason === "upgrade_required") { await commands.showWindow("Upgrade"); - // The window takes a little to show and this prevents the user seeing it glitch await new Promise((resolve) => setTimeout(resolve, 1000)); throw new SilentError(); } @@ -510,7 +517,6 @@ export function ExportPage() { console.log({ organizationId: settings.organizationId }); - // Now proceed with upload const result = meta().sharing ? await commands.uploadExportedVideo( projectPath, @@ -552,494 +558,468 @@ export function ExportPage() { }, })); + const qualityLabel = () => { + const option = COMPRESSION_OPTIONS.find( + (opt) => opt.value === settings.compression, + ); + return option?.label ?? "Minimal"; + }; + + const formatDuration = (seconds: number) => { + const hours = Math.floor(seconds / 3600); + const minutes = Math.floor((seconds % 3600) / 60); + const secs = seconds % 60; + if (hours > 0) { + return `${hours}:${minutes.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`; + } + return `${minutes}:${secs.toString().padStart(2, "0")}`; + }; + return (
+

+ Export +

{ostype() === "macos" &&
} - -
-

Export Cap

+
+ {ostype() === "windows" && }
-
-
-
-
-
-
-

Export to

- - 1 - } - > -
{ - const menu = await Menu.new({ - items: await Promise.all( - organisations().map((org) => - CheckMenuItem.new({ - text: org.name, - action: () => { - setSettings("organizationId", org.id); - }, - checked: settings.organizationId === org.id, - }), - ), - ), - }); - menu.popup(); - }} - > - Organization: - - { - ( - organisations().find( - (o) => o.id === settings.organizationId, - ) ?? organisations()[0] - )?.name - } - - +
+
+
+ + + + Generating preview...
- - + } + > +
+
+
+ +
+ } + > + {(url) => ( + <> + Export preview + +
+
+
+ + + + )} + +
+ + + + {(est) => ( +
+ + + {formatDuration(Math.round(est().duration_seconds))} + + + + {settings.resolution.width}×{settings.resolution.height} + + + ~ + {est().estimated_size_mb.toFixed(1)} MB + + + ~ + {formatDuration(Math.round(est().estimated_time_seconds))} +
-
- - {(option) => ( + )} + + +
+ +
+
+ }> + { + setSettings( + produce((newSettings) => { + newSettings.exportTo = value as ExportToOption; + if (value === "link" && settings.format === "Gif") { + newSettings.format = "Mp4"; + } + }), + ); + }} + > + + {(option) => { + const Icon = option.icon; + return ( + + + + + +
+ + {option.label} + + + {option.description} + +
+ + + ); + }} + + + + + 1 + } + > + + + + + + }> +
+ + {(option) => { + const isDisabled = () => + (option.value === "Mp4" && + hasTransparentBackground()) || + (option.value === "Gif" && + settings.exportTo === "link"); + + const disabledReason = () => + option.value === "Mp4" && hasTransparentBackground() + ? "MP4 doesn't support transparency" + : option.value === "Gif" && + settings.exportTo === "link" + ? "Links require MP4 format" + : undefined; + + const button = ( - )} - -
-
-
+ ); -
-
-

Format

-
- - {(option) => { - const disabledReason = () => { - if ( - option.value === "Mp4" && - hasTransparentBackground() - ) - return "MP4 format does not support transparent backgrounds"; - if ( - option.value === "Gif" && - settings.exportTo === "link" - ) - return "Shareable links cannot be made from GIFs"; - }; - - return ( - - - - ); - }} - -
+ return disabledReason() ? ( + {button} + ) : ( + button + ); + }} +
-
+ -
-
-

Resolution

-
- - {(option) => ( - - )} - -
-
-
- -
-
-
-

Frame Rate

- - {settings.fps} FPS - -
- } + > +
+ + {(option) => ( + + )} + +
+ + + } + value={ + + {settings.fps} FPS + + } + > +
+ + [10, 15, 20, 30].includes(o.value), + ) + : FPS_OPTIONS } - step={settings.format === "Gif" ? 5 : 15} - value={[settings.fps]} + > + {(option) => ( + + )} + +
+ + + + } + value={ + {qualityLabel()} + } + > + opt.value === settings.compression, + ), + ]} + minValue={0} + maxValue={COMPRESSION_OPTIONS.length - 1} + step={1} onChange={([v]) => { if (v === undefined) return; - trackEvent("export_fps_changed", { fps: v }); - setSettings("fps", v); + const option = + COMPRESSION_OPTIONS[COMPRESSION_OPTIONS.length - 1 - v]; + if (option) { + setPreviewLoading(true); + setCompressionBpp(option.bpp); + setSettings("compression", option.value); + } }} history={{ pause: () => () => {} }} /> -
-
- - -
-
-
-

Quality

- - {(() => { - const bpp = compressionBpp(); - if (bpp >= 0.25) return "Minimal compression"; - if (bpp >= 0.12) return "Social Media quality"; - if (bpp >= 0.06) return "Web optimized"; - return "Maximum compression"; - })()} - -
- { - if (v === undefined) return; - setCompressionBpp(v); - const closest = Object.entries(BPP_TO_COMPRESSION) - .map(([bpp, comp]) => ({ - bpp: Number(bpp), - comp, - diff: Math.abs(Number(bpp) - v), - })) - .sort((a, b) => a.diff - b.diff)[0]; - if (closest) setSettings("compression", closest.comp); - }} - history={{ pause: () => () => {} }} - /> -
- Smaller file - Higher quality -
-
-
+
- -
-
-

Quality Preview

-
- - - - Preview loading... - - } - > -
- -
- -
- } - > - {(url) => ( - <> - Export preview - -
-
- -
-
- - - - )} - -
-
- - {settings.resolution.width}×{settings.resolution.height} - - - - {(est) => ( - {est().estimated_size_mb.toFixed(1)} MB - )} - - -
-
-
- -
-
-

Quality Preview

- -
-
- - {(url) => ( - Export preview full size - )} - -
-
- - {settings.resolution.width}×{settings.resolution.height} - - - - {(est) => ( - - Estimated size: {est().estimated_size_mb.toFixed(1)}{" "} - MB - - )} - - -
-
- - +
+ {settings.exportTo === "link" && !auth.data ? ( + + + Sign in to share + + ) : ( + + )} +
-
-
- - - {(est) => ( -
- - - {(() => { - const totalSeconds = Math.round(est().duration_seconds); - const hours = Math.floor(totalSeconds / 3600); - const minutes = Math.floor((totalSeconds % 3600) / 60); - const seconds = totalSeconds % 60; - - if (hours > 0) { - return `${hours}:${minutes - .toString() - .padStart(2, "0")}:${seconds - .toString() - .padStart(2, "0")}`; - } - return `${minutes}:${seconds - .toString() - .padStart(2, "0")}`; - })()} - - - - {settings.resolution.width}×{settings.resolution.height} - - - - {est().estimated_size_mb.toFixed(2)} MB - - - - {(() => { - const totalSeconds = Math.round( - est().estimated_time_seconds, - ); - const hours = Math.floor(totalSeconds / 3600); - const minutes = Math.floor((totalSeconds % 3600) / 60); - const seconds = totalSeconds % 60; - - if (hours > 0) { - return `~${hours}:${minutes - .toString() - .padStart(2, "0")}:${seconds - .toString() - .padStart(2, "0")}`; - } - return `~${minutes}:${seconds - .toString() - .padStart(2, "0")}`; - })()} - -
+ +
+
+

Quality Preview

+ +
+
+ + {(url) => ( + Export preview full size )} - -
-
- {settings.exportTo === "link" && !auth.data ? ( - - {exportButtonIcon[settings.exportTo]} - Sign in to share - - ) : ( - - )} +
+
+ + {settings.resolution.width}×{settings.resolution.height} + + + + {(est) => ( + + Estimated size: {est().estimated_size_mb.toFixed(1)} MB + + )} + + +
-
+ + {(exportState) => { const [copyPressed, setCopyPressed] = createSignal(false); @@ -1162,12 +1142,12 @@ export function ExportPage() {

- Export Completed + Export Complete

Your{" "} {settings.format === "Gif" ? "GIF" : "video"}{" "} - has successfully been exported + is ready

@@ -1189,7 +1169,9 @@ export function ExportPage() { {(uploadState) => (

- Uploading Cap... + {uploadState.type === "uploading" + ? "Uploading..." + : "Preparing..."}

( )} diff --git a/packages/ui-solid/src/auto-imports.d.ts b/packages/ui-solid/src/auto-imports.d.ts index cce2545d19..22a5a28077 100644 --- a/packages/ui-solid/src/auto-imports.d.ts +++ b/packages/ui-solid/src/auto-imports.d.ts @@ -27,6 +27,7 @@ declare global { const IconCapCursorWindows: typeof import('~icons/cap/cursor-windows.jsx')['default'] const IconCapEnlarge: typeof import('~icons/cap/enlarge.jsx')['default'] const IconCapFile: typeof import('~icons/cap/file.jsx')['default'] + const IconCapFilm: typeof import('~icons/cap/film.jsx')['default'] const IconCapFilmCut: typeof import('~icons/cap/film-cut.jsx')['default'] const IconCapGauge: typeof import('~icons/cap/gauge.jsx')['default'] const IconCapGear: typeof import('~icons/cap/gear.jsx')['default'] @@ -63,6 +64,7 @@ declare global { const IconCapStopCircle: typeof import('~icons/cap/stop-circle.jsx')['default'] const IconCapTrash: typeof import('~icons/cap/trash.jsx')['default'] const IconCapUndo: typeof import('~icons/cap/undo.jsx')['default'] + const IconCapUpload: typeof import('~icons/cap/upload.jsx')['default'] const IconCapX: typeof import('~icons/cap/x.jsx')['default'] const IconCapZoomIn: typeof import('~icons/cap/zoom-in.jsx')['default'] const IconCapZoomOut: typeof import('~icons/cap/zoom-out.jsx')['default'] @@ -80,6 +82,7 @@ declare global { const IconLucideEyeOff: typeof import('~icons/lucide/eye-off.jsx')['default'] const IconLucideFastForward: typeof import('~icons/lucide/fast-forward.jsx')['default'] const IconLucideFolder: typeof import('~icons/lucide/folder.jsx')['default'] + const IconLucideGauge: typeof import('~icons/lucide/gauge.jsx')['default'] const IconLucideGift: typeof import('~icons/lucide/gift.jsx')['default'] const IconLucideHardDrive: typeof import('~icons/lucide/hard-drive.jsx')['default'] const IconLucideImage: typeof import('~icons/lucide/image.jsx')['default'] @@ -97,6 +100,7 @@ declare global { const IconLucideRotateCcw: typeof import('~icons/lucide/rotate-ccw.jsx')['default'] const IconLucideSave: typeof import('~icons/lucide/save.jsx')['default'] const IconLucideSearch: typeof import('~icons/lucide/search.jsx')['default'] + const IconLucideSparkles: typeof import('~icons/lucide/sparkles.jsx')['default'] const IconLucideSquarePlay: typeof import('~icons/lucide/square-play.jsx')['default'] const IconLucideTimer: typeof import('~icons/lucide/timer.jsx')['default'] const IconLucideType: typeof import('~icons/lucide/type.jsx')['default'] @@ -104,6 +108,7 @@ declare global { const IconLucideVideo: typeof import('~icons/lucide/video.jsx')['default'] const IconLucideVolume2: typeof import('~icons/lucide/volume2.jsx')['default'] const IconLucideX: typeof import('~icons/lucide/x.jsx')['default'] + const IconLucideZap: typeof import('~icons/lucide/zap.jsx')['default'] const IconPhMonitorBold: typeof import('~icons/ph/monitor-bold.jsx')['default'] const IconPhRecordFill: typeof import('~icons/ph/record-fill.jsx')['default'] const IconPhWarningBold: typeof import('~icons/ph/warning-bold.jsx')['default'] From e4517f1b795d22bfd8600da2ebfc4eeb0ee14158 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Tue, 23 Dec 2025 22:14:58 +0000 Subject: [PATCH 26/37] Add shimmer animation to Tailwind config --- apps/desktop/tailwind.config.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/desktop/tailwind.config.js b/apps/desktop/tailwind.config.js index 8fd2b2602a..f9dacf551d 100644 --- a/apps/desktop/tailwind.config.js +++ b/apps/desktop/tailwind.config.js @@ -13,10 +13,15 @@ module.exports = { "0%, 100%": { transform: "translateY(0)" }, "50%": { transform: "translateY(-4px)" }, }, + shimmer: { + "0%": { transform: "translateX(-100%)" }, + "100%": { transform: "translateX(100%)" }, + }, }, animation: { ...baseConfig.theme?.extend?.animation, "gentle-bounce": "gentleBounce 1.5s ease-in-out infinite", + shimmer: "shimmer 2s ease-in-out infinite", }, }, }, From da6841bd0266a23bcfea9a7127dc19cab7eae842 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Tue, 23 Dec 2025 22:36:52 +0000 Subject: [PATCH 27/37] Add export preview render time and frame estimates --- apps/desktop/src-tauri/src/export.rs | 18 ++ apps/desktop/src/routes/editor/ExportPage.tsx | 160 +++++++++++------- apps/desktop/src/utils/tauri.ts | 2 +- 3 files changed, 119 insertions(+), 61 deletions(-) diff --git a/apps/desktop/src-tauri/src/export.rs b/apps/desktop/src-tauri/src/export.rs index 07b1dcff0b..612d11c5a0 100644 --- a/apps/desktop/src-tauri/src/export.rs +++ b/apps/desktop/src-tauri/src/export.rs @@ -184,6 +184,8 @@ pub struct ExportPreviewResult { pub estimated_size_mb: f64, pub actual_width: u32, pub actual_height: u32, + pub frame_render_time_ms: f64, + pub total_frames: u32, } fn bpp_to_jpeg_quality(bpp: f32) -> u8 { @@ -201,6 +203,7 @@ pub async fn generate_export_preview( ) -> Result { use base64::{Engine, engine::general_purpose::STANDARD}; use cap_editor::create_segments; + use std::time::Instant; let recording_meta = RecordingMeta::load_for_project(&project_path) .map_err(|e| format!("Failed to load recording meta: {e}"))?; @@ -248,6 +251,8 @@ pub async fn generate_export_preview( .iter() .find(|v| v.index == segment.recording_clip); + let render_start = Instant::now(); + let segment_frames = render_segment .decoders .get_frames( @@ -287,6 +292,8 @@ pub async fn generate_export_preview( .await .map_err(|e| format!("Failed to render frame: {e}"))?; + let frame_render_time_ms = render_start.elapsed().as_secs_f64() * 1000.0; + let width = frame.width; let height = frame.height; @@ -320,6 +327,7 @@ pub async fn generate_export_preview( } else { metadata.duration }; + let total_frames = (duration_seconds * fps_f64).ceil() as u32; let video_bitrate = total_pixels * settings.compression_bpp as f64 * fps_f64; let audio_bitrate = 192_000.0; @@ -331,6 +339,8 @@ pub async fn generate_export_preview( estimated_size_mb, actual_width: width, actual_height: height, + frame_render_time_ms, + total_frames, }) } @@ -343,6 +353,7 @@ pub async fn generate_export_preview_fast( settings: ExportPreviewSettings, ) -> Result { use base64::{Engine, engine::general_purpose::STANDARD}; + use std::time::Instant; let project_config = editor.project_config.1.borrow().clone(); @@ -356,6 +367,8 @@ pub async fn generate_export_preview_fast( .iter() .find(|v| v.index == segment.recording_clip); + let render_start = Instant::now(); + let segment_frames = segment_media .decoders .get_frames( @@ -390,6 +403,8 @@ pub async fn generate_export_preview_fast( .await .map_err(|e| format!("Failed to render frame: {e}"))?; + let frame_render_time_ms = render_start.elapsed().as_secs_f64() * 1000.0; + let width = frame.width; let height = frame.height; @@ -418,6 +433,7 @@ pub async fn generate_export_preview_fast( let fps_f64 = settings.fps as f64; let duration_seconds = editor.recordings.duration(); + let total_frames = (duration_seconds * fps_f64).ceil() as u32; let video_bitrate = total_pixels * settings.compression_bpp as f64 * fps_f64; let audio_bitrate = 192_000.0; @@ -429,5 +445,7 @@ pub async fn generate_export_preview_fast( estimated_size_mb, actual_width: width, actual_height: height, + frame_render_time_ms, + total_frames, }) } diff --git a/apps/desktop/src/routes/editor/ExportPage.tsx b/apps/desktop/src/routes/editor/ExportPage.tsx index b8871f2495..ddb95e411c 100644 --- a/apps/desktop/src/routes/editor/ExportPage.tsx +++ b/apps/desktop/src/routes/editor/ExportPage.tsx @@ -2,11 +2,7 @@ import { Button } from "@cap/ui-solid"; import { RadioGroup as KRadioGroup } from "@kobalte/core/radio-group"; import { debounce } from "@solid-primitives/scheduled"; import { makePersisted } from "@solid-primitives/storage"; -import { - createMutation, - createQuery, - keepPreviousData, -} from "@tanstack/solid-query"; +import { createMutation } from "@tanstack/solid-query"; import { Channel } from "@tauri-apps/api/core"; import { CheckMenuItem, Menu } from "@tauri-apps/api/menu"; import { ask, save as saveDialog } from "@tauri-apps/plugin-dialog"; @@ -102,7 +98,7 @@ export const EXPORT_TO_OPTIONS = [ description: "Copy to paste anywhere", }, { - label: "Link", + label: "Shareable Link", value: "link", icon: IconCapLink, description: "Share via Cap cloud", @@ -195,11 +191,33 @@ export function ExportPage() { const [previewUrl, setPreviewUrl] = createSignal(null); const [previewLoading, setPreviewLoading] = createSignal(false); - - const updateSettings: typeof setSettings = (...args) => { + const [renderEstimate, setRenderEstimate] = createSignal<{ + frameRenderTimeMs: number; + totalFrames: number; + estimatedSizeMb: number; + } | null>(null); + + type EstimateCacheKey = string; + const estimateCache = new Map< + EstimateCacheKey, + { frameRenderTimeMs: number; totalFrames: number; estimatedSizeMb: number } + >(); + + const getEstimateCacheKey = ( + fps: number, + width: number, + height: number, + bpp: number, + ): EstimateCacheKey => `${fps}-${width}-${height}-${bpp}`; + + const updateSettings: typeof setSettings = (( + ...args: Parameters + ) => { setPreviewLoading(true); - return setSettings(...args); - }; + return (setSettings as (...args: Parameters) => void)( + ...args, + ); + }) as typeof setSettings; const [previewDialogOpen, setPreviewDialogOpen] = createSignal(false); const [compressionBpp, setCompressionBpp] = createSignal( COMPRESSION_TO_BPP[_settings.compression] ?? 0.15, @@ -223,6 +241,13 @@ export function ExportPage() { resHeight: number, bpp: number, ) => { + const cacheKey = getEstimateCacheKey(fps, resWidth, resHeight, bpp); + const cachedEstimate = estimateCache.get(cacheKey); + + if (cachedEstimate) { + setRenderEstimate(cachedEstimate); + } + try { const result = await commands.generateExportPreviewFast(frameTime, { fps, @@ -238,6 +263,17 @@ export function ExportPage() { ); const blob = new Blob([byteArray], { type: "image/jpeg" }); setPreviewUrl(URL.createObjectURL(blob)); + + const newEstimate = { + frameRenderTimeMs: result.frame_render_time_ms, + totalFrames: result.total_frames, + estimatedSizeMb: result.estimated_size_mb, + }; + + if (!cachedEstimate) { + estimateCache.set(cacheKey, newEstimate); + } + setRenderEstimate(newEstimate); } catch (e) { console.error("Failed to generate preview:", e); } finally { @@ -334,39 +370,6 @@ export function ExportPage() { } }; - const exportEstimates = createQuery(() => ({ - placeholderData: keepPreviousData, - queryKey: [ - "exportEstimates", - { - format: settings.format, - resolution: { - x: settings.resolution.width, - y: settings.resolution.height, - }, - fps: settings.fps, - compression: settings.compression, - }, - ] as const, - queryFn: ({ queryKey: [_, { format, resolution, fps, compression }] }) => { - const exportSettings = - format === "Mp4" - ? { - format: "Mp4" as const, - fps, - resolution_base: resolution, - compression, - } - : { - format: "Gif" as const, - fps, - resolution_base: resolution, - quality: null, - }; - return commands.getExportEstimates(projectPath, exportSettings); - }, - })); - const copy = createMutation(() => ({ mutationFn: async () => { setIsCancelled(false); @@ -653,13 +656,48 @@ export function ExportPage() {
- - - {(est) => ( + + + + + + + + + + + + + + + + + +
+ } + > + {(est) => { + const data = est(); + const durationSeconds = data.totalFrames / settings.fps; + + const exportSpeedMultiplier = + settings.format === "Gif" ? 4 : 10; + const totalTimeMs = + (data.frameRenderTimeMs * data.totalFrames) / + exportSpeedMultiplier; + const estimatedTimeSeconds = Math.max(1, totalTimeMs / 1000); + + const sizeMultiplier = settings.format === "Gif" ? 0.7 : 0.5; + const estimatedSizeMb = data.estimatedSizeMb * sizeMultiplier; + + return (
- {formatDuration(Math.round(est().duration_seconds))} + {formatDuration(Math.round(durationSeconds))} @@ -667,19 +705,19 @@ export function ExportPage() { ~ - {est().estimated_size_mb.toFixed(1)} MB + {estimatedSizeMb.toFixed(1)} MB ~ - {formatDuration(Math.round(est().estimated_time_seconds))} + {formatDuration(Math.round(estimatedTimeSeconds))}
- )} - - + ); + }} +
-
+
}> {settings.resolution.width}×{settings.resolution.height} - - - {(est) => ( + + {(est) => { + const sizeMultiplier = settings.format === "Gif" ? 0.7 : 0.5; + return ( - Estimated size: {est().estimated_size_mb.toFixed(1)} MB + Estimated size:{" "} + {(est().estimatedSizeMb * sizeMultiplier).toFixed(1)} MB - )} - - + ); + }} +
diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index cdb3fd1a02..18ba0fbe66 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -414,7 +414,7 @@ export type DownloadProgress = { progress: number; message: string } export type EditorStateChanged = { playhead_position: number } export type ExportCompression = "Minimal" | "Social" | "Web" | "Potato" export type ExportEstimates = { duration_seconds: number; estimated_time_seconds: number; estimated_size_mb: number } -export type ExportPreviewResult = { jpeg_base64: string; estimated_size_mb: number; actual_width: number; actual_height: number } +export type ExportPreviewResult = { jpeg_base64: string; estimated_size_mb: number; actual_width: number; actual_height: number; frame_render_time_ms: number; total_frames: number } export type ExportPreviewSettings = { fps: number; resolution_base: XY; compression_bpp: number } export type ExportSettings = ({ format: "Mp4" } & Mp4ExportSettings) | ({ format: "Gif" } & GifExportSettings) export type FileType = "recording" | "screenshot" From e9809d18963e23da53d9d66206253e23dba65e44 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Tue, 23 Dec 2025 23:23:00 +0000 Subject: [PATCH 28/37] Refactor ExportPage layout and add new animations --- apps/desktop/src/routes/editor/ExportPage.tsx | 830 +++++++++--------- apps/desktop/tailwind.config.js | 39 + 2 files changed, 456 insertions(+), 413 deletions(-) diff --git a/apps/desktop/src/routes/editor/ExportPage.tsx b/apps/desktop/src/routes/editor/ExportPage.tsx index ddb95e411c..c2a1cb3dbc 100644 --- a/apps/desktop/src/routes/editor/ExportPage.tsx +++ b/apps/desktop/src/routes/editor/ExportPage.tsx @@ -608,457 +608,445 @@ export function ExportPage() {
- -
-
-
- - - - Generating preview... -
- } - > -
-
-
- -
- } - > - {(url) => ( - <> - Export preview - -
-
-
- - - - )} - -
- +
+
+
+ + + Generating preview... +
+ } + > +
+
+
+ +
+ } + > + {(url) => ( + <> + Export preview + +
+
+
+ + + + )} + +
+ + + + + + + + + + + + + + + + + + +
+ } + > + {(est) => { + const data = est(); + const durationSeconds = data.totalFrames / settings.fps; + + const exportSpeedMultiplier = settings.format === "Gif" ? 4 : 10; + const totalTimeMs = + (data.frameRenderTimeMs * data.totalFrames) / + exportSpeedMultiplier; + const estimatedTimeSeconds = Math.max(1, totalTimeMs / 1000); + + const sizeMultiplier = settings.format === "Gif" ? 0.7 : 0.5; + const estimatedSizeMb = data.estimatedSizeMb * sizeMultiplier; + + return (
- + {formatDuration(Math.round(durationSeconds))} - + {settings.resolution.width}×{settings.resolution.height} - - + ~ + {estimatedSizeMb.toFixed(1)} MB - - + ~ + {formatDuration(Math.round(estimatedTimeSeconds))}
- } - > - {(est) => { - const data = est(); - const durationSeconds = data.totalFrames / settings.fps; - - const exportSpeedMultiplier = - settings.format === "Gif" ? 4 : 10; - const totalTimeMs = - (data.frameRenderTimeMs * data.totalFrames) / - exportSpeedMultiplier; - const estimatedTimeSeconds = Math.max(1, totalTimeMs / 1000); - - const sizeMultiplier = settings.format === "Gif" ? 0.7 : 0.5; - const estimatedSizeMb = data.estimatedSizeMb * sizeMultiplier; - - return ( -
- - - {formatDuration(Math.round(durationSeconds))} - - - - {settings.resolution.width}×{settings.resolution.height} - - - ~ - {estimatedSizeMb.toFixed(1)} MB - - - ~ - {formatDuration(Math.round(estimatedTimeSeconds))} - -
- ); - }} - -
+ ); + }} + +
-
-
- }> - { - setSettings( - produce((newSettings) => { - newSettings.exportTo = value as ExportToOption; - if (value === "link" && settings.format === "Gif") { - newSettings.format = "Mp4"; - } - }), +
+
+ }> + { + setSettings( + produce((newSettings) => { + newSettings.exportTo = value as ExportToOption; + if (value === "link" && settings.format === "Gif") { + newSettings.format = "Mp4"; + } + }), + ); + }} + > + + {(option) => { + const Icon = option.icon; + return ( + + + + + +
+ + {option.label} + + + {option.description} + +
+ + ); }} - > - - {(option) => { - const Icon = option.icon; - return ( - - - - - -
- - {option.label} - - - {option.description} - -
- - - ); - }} - - + + - - 1 - } - > - - - - - - }> -
- - {(option) => { - const isDisabled = () => - (option.value === "Mp4" && - hasTransparentBackground()) || - (option.value === "Gif" && - settings.exportTo === "link"); - - const disabledReason = () => - option.value === "Mp4" && hasTransparentBackground() - ? "MP4 doesn't support transparency" - : option.value === "Gif" && - settings.exportTo === "link" - ? "Links require MP4 format" - : undefined; - - const button = ( - - ); - - return disabledReason() ? ( - {button} - ) : ( - button - ); + ), + }); + menu.popup(); }} - -
- - - } - > -
- - {(option) => ( + Organization + + { + ( + organisations().find( + (o) => o.id === settings.organizationId, + ) ?? organisations()[0] + )?.name + } + + + + + + + + }> +
+ + {(option) => { + const isDisabled = () => + (option.value === "Mp4" && hasTransparentBackground()) || + (option.value === "Gif" && settings.exportTo === "link"); + + const disabledReason = () => + option.value === "Mp4" && hasTransparentBackground() + ? "MP4 doesn't support transparency" + : option.value === "Gif" && settings.exportTo === "link" + ? "Links require MP4 format" + : undefined; + + const button = ( - )} - -
- + ); + return disabledReason() ? ( + {button} + ) : ( + button + ); + }} + +
+ + + } + > +
+ + {(option) => ( + + )} + +
+ + + } + value={ + + {settings.fps} FPS + + } + > +
+ + [10, 15, 20, 30].includes(o.value), + ) + : FPS_OPTIONS + } + > + {(option) => ( + + )} + +
+ + + } + name="Quality" + icon={} value={ - - {settings.fps} FPS - + {qualityLabel()} } > -
- - [10, 15, 20, 30].includes(o.value), - ) - : FPS_OPTIONS + opt.value === settings.compression, + ), + ]} + minValue={0} + maxValue={COMPRESSION_OPTIONS.length - 1} + step={1} + onChange={([v]) => { + if (v === undefined) return; + const option = + COMPRESSION_OPTIONS[COMPRESSION_OPTIONS.length - 1 - v]; + if (option) { + setPreviewLoading(true); + setCompressionBpp(option.bpp); + setSettings("compression", option.value); } - > - {(option) => ( - - )} - -
- - - - } - value={ - {qualityLabel()} - } - > - opt.value === settings.compression, - ), - ]} - minValue={0} - maxValue={COMPRESSION_OPTIONS.length - 1} - step={1} - onChange={([v]) => { - if (v === undefined) return; - const option = - COMPRESSION_OPTIONS[COMPRESSION_OPTIONS.length - 1 - v]; - if (option) { - setPreviewLoading(true); - setCompressionBpp(option.bpp); - setSettings("compression", option.value); - } - }} - history={{ pause: () => () => {} }} - /> - - -
- -
- {settings.exportTo === "link" && !auth.data ? ( - - - Sign in to share - - ) : ( - - )} -
+ history={{ pause: () => () => {} }} + /> + +
-
- -
-
-

Quality Preview

- -
-
- - {(url) => ( - Export preview full size + {settings.exportTo === "file" && } + {settings.exportTo === "clipboard" && ( + )} - -
-
- - {settings.resolution.width}×{settings.resolution.height} - - - {(est) => { - const sizeMultiplier = settings.format === "Gif" ? 0.7 : 0.5; - return ( - - Estimated size:{" "} - {(est().estimatedSizeMb * sizeMultiplier).toFixed(1)} MB - - ); - }} - -
+ {settings.exportTo === "link" && } + Export {settings.format === "Gif" ? "GIF" : "Video"} + + )}
- - +
+
+ + +
+
+

Quality Preview

+ +
+
+ + {(url) => ( + Export preview full size + )} + +
+
+ + {settings.resolution.width}×{settings.resolution.height} + + + {(est) => { + const sizeMultiplier = settings.format === "Gif" ? 0.7 : 0.5; + return ( + + Estimated size:{" "} + {(est().estimatedSizeMb * sizeMultiplier).toFixed(1)} MB + + ); + }} + +
+
+ {(exportState) => { @@ -1076,7 +1064,13 @@ export function ExportPage() { }); return ( -
+
+ + +
); }} diff --git a/apps/desktop/tailwind.config.js b/apps/desktop/tailwind.config.js index f9dacf551d..6ab4fa0e40 100644 --- a/apps/desktop/tailwind.config.js +++ b/apps/desktop/tailwind.config.js @@ -17,11 +17,50 @@ module.exports = { "0%": { transform: "translateX(-100%)" }, "100%": { transform: "translateX(100%)" }, }, + float: { + "0%, 100%": { + transform: "translateY(0) rotate(0deg)", + opacity: "0.7", + }, + "50%": { + transform: "translateY(-20px) rotate(180deg)", + opacity: "1", + }, + }, + floatSlow: { + "0%, 100%": { transform: "translateY(0) scale(1)" }, + "50%": { transform: "translateY(-30px) scale(1.1)" }, + }, + pulse3d: { + "0%, 100%": { transform: "scale(1)", opacity: "0.8" }, + "50%": { transform: "scale(1.05)", opacity: "1" }, + }, + spin3d: { + "0%": { transform: "rotateY(0deg)" }, + "100%": { transform: "rotateY(360deg)" }, + }, + gradientShift: { + "0%": { backgroundPosition: "0% 50%" }, + "50%": { backgroundPosition: "100% 50%" }, + "100%": { backgroundPosition: "0% 50%" }, + }, + dash: { + "0%": { strokeDasharray: "1, 150", strokeDashoffset: "0" }, + "50%": { strokeDasharray: "90, 150", strokeDashoffset: "-35" }, + "100%": { strokeDasharray: "90, 150", strokeDashoffset: "-124" }, + }, }, animation: { ...baseConfig.theme?.extend?.animation, "gentle-bounce": "gentleBounce 1.5s ease-in-out infinite", shimmer: "shimmer 2s ease-in-out infinite", + float: "float 6s ease-in-out infinite", + "float-slow": "floatSlow 8s ease-in-out infinite", + "float-delayed": "float 6s ease-in-out 2s infinite", + pulse3d: "pulse3d 2s ease-in-out infinite", + spin3d: "spin3d 3s linear infinite", + "gradient-shift": "gradientShift 3s ease infinite", + dash: "dash 1.5s ease-in-out infinite", }, }, }, From 1f1fb45d3ded6f4343fe450d28e42b957a27805d Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Tue, 23 Dec 2025 23:33:16 +0000 Subject: [PATCH 29/37] Update export and header button styles and labels --- apps/desktop/src/routes/editor/ExportPage.tsx | 54 +++++++++++++------ apps/desktop/src/routes/editor/Header.tsx | 4 +- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/apps/desktop/src/routes/editor/ExportPage.tsx b/apps/desktop/src/routes/editor/ExportPage.tsx index c2a1cb3dbc..f77f46ce5a 100644 --- a/apps/desktop/src/routes/editor/ExportPage.tsx +++ b/apps/desktop/src/routes/editor/ExportPage.tsx @@ -658,22 +658,22 @@ export function ExportPage() { +
- + - + - + - +
} @@ -692,22 +692,30 @@ export function ExportPage() { const estimatedSizeMb = data.estimatedSizeMb * sizeMultiplier; return ( -
+
- {formatDuration(Math.round(durationSeconds))} + + {formatDuration(Math.round(durationSeconds))} + - {settings.resolution.width}×{settings.resolution.height} + + {settings.resolution.width}×{settings.resolution.height} + - ~ - {estimatedSizeMb.toFixed(1)} MB + + + ~{estimatedSizeMb.toFixed(1)} MB + - ~ - {formatDuration(Math.round(estimatedTimeSeconds))} + + + ~{formatDuration(Math.round(estimatedTimeSeconds))} +
); @@ -981,7 +989,7 @@ export function ExportPage() { ) : ( )}
diff --git a/apps/desktop/src/routes/editor/Header.tsx b/apps/desktop/src/routes/editor/Header.tsx index 86f7ecba70..905813d14c 100644 --- a/apps/desktop/src/routes/editor/Header.tsx +++ b/apps/desktop/src/routes/editor/Header.tsx @@ -165,7 +165,7 @@ export function Header() { {ostype() === "windows" && } From d7edd423c0457128ea4451edd201b1dbe247b592 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 24 Dec 2025 00:45:48 +0000 Subject: [PATCH 30/37] coderabbit bits --- .claude/settings.local.json | 4 +- apps/desktop/src-tauri/src/lib.rs | 18 ++-- apps/desktop/src/routes/editor/Editor.tsx | 23 ++-- apps/desktop/src/routes/editor/ExportPage.tsx | 86 +++++++-------- ...EditorSkeleton.tsx => editor-skeleton.tsx} | 0 apps/desktop/src/routes/editor/index.tsx | 2 +- crates/editor/examples/decode-benchmark.rs | 4 +- crates/editor/src/playback.rs | 14 ++- crates/recording/src/output_pipeline/core.rs | 11 +- crates/rendering/src/decoder/avassetreader.rs | 98 +++++------------ crates/rendering/src/decoder/mod.rs | 101 ------------------ crates/rendering/src/layers/camera.rs | 29 ----- crates/rendering/src/layers/display.rs | 19 +--- crates/video-decode/src/avassetreader.rs | 42 +++----- 14 files changed, 137 insertions(+), 314 deletions(-) rename apps/desktop/src/routes/editor/{EditorSkeleton.tsx => editor-skeleton.tsx} (100%) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index f2037c92c6..f0e207bf26 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -52,7 +52,9 @@ "Bash(RUST_LOG=info,cap_recording=debug ./target/release/examples/memory-leak-detector:*)", "Bash(git rm:*)", "Bash(./target/release/examples/decode-benchmark:*)", - "Bash(RUST_LOG=warn ./target/release/examples/decode-benchmark:*)" + "Bash(RUST_LOG=warn ./target/release/examples/decode-benchmark:*)", + "Bash(git mv:*)", + "Bash(xargs cat:*)" ], "deny": [], "ask": [] diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index b27e255f81..2bc1daa89d 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -3225,15 +3225,15 @@ async fn wait_for_recording_ready(app: &AppHandle, path: &Path) -> Result<(), St } }; - if let Some(studio_meta) = meta.studio_meta() { - if recording::needs_fragment_remux(path, studio_meta) { - info!("Recording needs remux (crash recovery), starting remux..."); - let path = path.to_path_buf(); - tokio::task::spawn_blocking(move || recording::remux_fragmented_recording(&path)) - .await - .map_err(|e| format!("Remux task panicked: {e}"))??; - info!("Crash recovery remux completed"); - } + if let Some(studio_meta) = meta.studio_meta() + && recording::needs_fragment_remux(path, studio_meta) + { + info!("Recording needs remux (crash recovery), starting remux..."); + let path = path.to_path_buf(); + tokio::task::spawn_blocking(move || recording::remux_fragmented_recording(&path)) + .await + .map_err(|e| format!("Remux task panicked: {e}"))??; + info!("Crash recovery remux completed"); } Ok(()) diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index adf2260716..dd2bba4421 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -14,6 +14,7 @@ import { createSignal, Match, on, + onCleanup, Show, Switch, } from "solid-js"; @@ -428,7 +429,7 @@ function Dialogs() { const [crop, setCrop] = createSignal(CROP_ZERO); const [aspect, setAspect] = createSignal(null); - const [frameDataUrl, setFrameDataUrl] = createSignal< + const [frameBlobUrl, setFrameBlobUrl] = createSignal< string | null >(null); @@ -436,13 +437,21 @@ function Dialogs() { "canvas", ) as HTMLCanvasElement | null; if (playerCanvas) { - try { - setFrameDataUrl(playerCanvas.toDataURL("image/png")); - } catch { - setFrameDataUrl(null); - } + playerCanvas.toBlob((blob) => { + if (blob) { + const url = URL.createObjectURL(blob); + setFrameBlobUrl(url); + } + }, "image/png"); } + onCleanup(() => { + const url = frameBlobUrl(); + if (url) { + URL.revokeObjectURL(url); + } + }); + const initialBounds = { x: dialog().position.x, y: dialog().position.y, @@ -607,7 +616,7 @@ function Dialogs() { class="shadow pointer-events-none max-h-[70vh]" alt="Current frame" src={ - frameDataUrl() ?? + frameBlobUrl() ?? convertFileSrc( `${editorInstance.path}/screenshots/display.jpg`, ) diff --git a/apps/desktop/src/routes/editor/ExportPage.tsx b/apps/desktop/src/routes/editor/ExportPage.tsx index f77f46ce5a..b08240a81e 100644 --- a/apps/desktop/src/routes/editor/ExportPage.tsx +++ b/apps/desktop/src/routes/editor/ExportPage.tsx @@ -56,11 +56,11 @@ export const COMPRESSION_OPTIONS: Array<{ { label: "Potato", value: "Potato", bpp: 0.04 }, ]; -const BPP_TO_COMPRESSION: Record = { - 0.3: "Minimal", - 0.15: "Social", - 0.08: "Web", - 0.04: "Potato", +const BPP_TO_COMPRESSION: Record = { + "0.3": "Minimal", + "0.15": "Social", + "0.08": "Web", + "0.04": "Potato", }; const COMPRESSION_TO_BPP: Record = { @@ -286,10 +286,10 @@ export function ExportPage() { createEffect( on( [ - () => _settings.format, - () => _settings.fps, - () => _settings.resolution.width, - () => _settings.resolution.height, + () => settings.format, + () => settings.fps, + () => settings.resolution.width, + () => settings.resolution.height, compressionBpp, ], () => { @@ -918,11 +918,7 @@ export function ExportPage() {
- [10, 15, 20, 30].includes(o.value), - ) - : FPS_OPTIONS + settings.format === "Gif" ? GIF_FPS_OPTIONS : FPS_OPTIONS } > {(option) => ( @@ -1301,35 +1297,39 @@ export function ExportPage() { exportState.type === "done" } > -
- - - -
+ + {(link) => ( +
+ + + + +
+ )} + 0 { diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index a5ddeab9c0..4096abfb1b 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -631,7 +631,12 @@ impl Playback { frame_number = frame_number.saturating_add(1); let _ = playback_position_tx.send(frame_number); - let _ = audio_playhead_tx.send(frame_number as f64 / fps_f64); + if audio_playhead_tx + .send(frame_number as f64 / fps_f64) + .is_err() + { + break 'playback; + } let expected_frame = self.start_frame_number + (start.elapsed().as_secs_f64() * fps_f64).floor() as u32; @@ -651,7 +656,12 @@ impl Playback { prefetch_buffer.retain(|p| p.frame_number >= frame_number); let _ = frame_request_tx.send(frame_number); let _ = playback_position_tx.send(frame_number); - let _ = audio_playhead_tx.send(frame_number as f64 / fps_f64); + if audio_playhead_tx + .send(frame_number as f64 / fps_f64) + .is_err() + { + break 'playback; + } } } } diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rs index 13aca3ce2e..d4f5f6204f 100644 --- a/crates/recording/src/output_pipeline/core.rs +++ b/crates/recording/src/output_pipeline/core.rs @@ -553,12 +553,13 @@ fn spawn_video_encoder, TVideo: V let mut drained = 0u64; let mut skipped = 0u64; + let mut hit_limit = false; while let Some(frame) = video_rx.next().await { frame_count += 1; if drain_start.elapsed() > drain_timeout || drained >= max_drain_frames { - skipped += 1; - continue; + hit_limit = true; + break; } drained += 1; @@ -581,11 +582,13 @@ fn spawn_video_encoder, TVideo: V } } } - if drained > 0 || skipped > 0 { + + if drained > 0 || skipped > 0 || hit_limit { info!( - "mux-video drain complete: {} frames processed, {} skipped in {:?}", + "mux-video drain complete: {} frames processed, {} errors (limit hit: {}) in {:?}", drained, skipped, + hit_limit, drain_start.elapsed() ); } diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index 8acf438ac6..abd8d6f804 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -21,22 +21,11 @@ use super::frame_converter::{copy_bgra_to_rgba, copy_rgba_plane}; use super::multi_position::{DecoderPoolManager, MultiPositionDecoderConfig, ScrubDetector}; use super::{DecoderInitResult, DecoderType, FRAME_CACHE_SIZE, VideoDecoderMessage, pts_to_frame}; -struct SendableImageBuf(R); -unsafe impl Send for SendableImageBuf {} -unsafe impl Sync for SendableImageBuf {} - -impl Clone for SendableImageBuf { - fn clone(&self) -> Self { - Self(self.0.retained()) - } -} - #[derive(Clone)] struct FrameData { data: Arc>, y_stride: u32, uv_stride: u32, - image_buf: Option>, } #[derive(Clone)] @@ -54,32 +43,19 @@ impl ProcessedFrame { data, y_stride, uv_stride, - image_buf, } = &self.frame_data; match self.format { PixelFormat::Rgba => { DecodedFrame::new_with_arc(Arc::clone(data), self.width, self.height) } - PixelFormat::Nv12 => { - if let Some(img_buf) = image_buf { - DecodedFrame::new_nv12_zero_copy( - self.width, - self.height, - *y_stride, - *uv_stride, - img_buf.0.retained(), - ) - } else { - DecodedFrame::new_nv12_with_arc( - Arc::clone(data), - self.width, - self.height, - *y_stride, - *uv_stride, - ) - } - } + PixelFormat::Nv12 => DecodedFrame::new_nv12_with_arc( + Arc::clone(data), + self.width, + self.height, + *y_stride, + *uv_stride, + ), PixelFormat::Yuv420p => DecodedFrame::new_yuv420p_with_arc( Arc::clone(data), self.width, @@ -233,21 +209,14 @@ impl CachedFrame { let pixel_format = cap_video_decode::avassetreader::pixel_format_to_pixel(image_buf.pixel_format()); - let (format, y_stride, uv_stride, stored_image_buf) = match pixel_format { - format::Pixel::NV12 => { - let y_stride = image_buf.plane_bytes_per_row(0) as u32; - let uv_stride = image_buf.plane_bytes_per_row(1) as u32; - ( - PixelFormat::Nv12, - y_stride, - uv_stride, - Some(Arc::new(SendableImageBuf(image_buf))), - ) - } - format::Pixel::RGBA | format::Pixel::BGRA | format::Pixel::YUV420P => { + match pixel_format { + format::Pixel::NV12 + | format::Pixel::RGBA + | format::Pixel::BGRA + | format::Pixel::YUV420P => { let mut img = image_buf; let (data, fmt, y_str, uv_str) = processor.extract_raw(&mut img); - return Self(ProcessedFrame { + Self(ProcessedFrame { _number: number, width, height, @@ -256,13 +225,12 @@ impl CachedFrame { data: Arc::new(data), y_stride: y_str, uv_stride: uv_str, - image_buf: None, }, - }); + }) } _ => { let black_frame = vec![0u8; (width * height * 4) as usize]; - return Self(ProcessedFrame { + Self(ProcessedFrame { _number: number, width, height, @@ -271,25 +239,10 @@ impl CachedFrame { data: Arc::new(black_frame), y_stride: width * 4, uv_stride: 0, - image_buf: None, }, - }); + }) } - }; - - let frame = ProcessedFrame { - _number: number, - width, - height, - format, - frame_data: FrameData { - data: Arc::new(Vec::new()), - y_stride, - uv_stride, - image_buf: stored_image_buf, - }, - }; - Self(frame) + } } fn data(&self) -> &ProcessedFrame { @@ -523,18 +476,21 @@ impl AVAssetReaderDecoder { false }; - let mut i = 0; - while i < pending_requests.len() { - let request = &pending_requests[i]; + let mut unfulfilled = Vec::with_capacity(pending_requests.len()); + let mut last_sent_data = None; + for request in pending_requests.drain(..) { if let Some(cached) = cache.get(&request.frame) { let data = cached.data().clone(); - let req = pending_requests.remove(i); - let _ = req.sender.send(data.to_decoded_frame()); - *last_sent_frame.borrow_mut() = Some(data); + let _ = request.sender.send(data.to_decoded_frame()); + last_sent_data = Some(data); } else { - i += 1; + unfulfilled.push(request); } } + if let Some(data) = last_sent_data { + *last_sent_frame.borrow_mut() = Some(data); + } + pending_requests = unfulfilled; if pending_requests.is_empty() { continue; diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs index f02800b2d4..81b92997a3 100644 --- a/crates/rendering/src/decoder/mod.rs +++ b/crates/rendering/src/decoder/mod.rs @@ -70,31 +70,9 @@ pub struct DecoderInitResult { pub decoder_type: DecoderType, } -#[cfg(target_os = "macos")] -use cidre::{arc::R, cv}; - #[cfg(target_os = "windows")] use windows::Win32::{Foundation::HANDLE, Graphics::Direct3D11::ID3D11Texture2D}; -#[cfg(target_os = "macos")] -pub struct SendableImageBuf(R); - -#[cfg(target_os = "macos")] -unsafe impl Send for SendableImageBuf {} -#[cfg(target_os = "macos")] -unsafe impl Sync for SendableImageBuf {} - -#[cfg(target_os = "macos")] -impl SendableImageBuf { - pub fn new(image_buf: R) -> Self { - Self(image_buf) - } - - pub fn inner(&self) -> &cv::ImageBuf { - &self.0 - } -} - #[cfg(target_os = "windows")] pub struct SendableD3D11Texture { texture: ID3D11Texture2D, @@ -174,8 +152,6 @@ pub struct DecodedFrame { format: PixelFormat, y_stride: u32, uv_stride: u32, - #[cfg(target_os = "macos")] - iosurface_backing: Option>, #[cfg(target_os = "windows")] d3d11_texture_backing: Option>, } @@ -202,8 +178,6 @@ impl DecodedFrame { format: PixelFormat::Rgba, y_stride: width * 4, uv_stride: 0, - #[cfg(target_os = "macos")] - iosurface_backing: None, #[cfg(target_os = "windows")] d3d11_texture_backing: None, } @@ -217,8 +191,6 @@ impl DecodedFrame { format: PixelFormat::Rgba, y_stride: width * 4, uv_stride: 0, - #[cfg(target_os = "macos")] - iosurface_backing: None, #[cfg(target_os = "windows")] d3d11_texture_backing: None, } @@ -232,8 +204,6 @@ impl DecodedFrame { format: PixelFormat::Nv12, y_stride, uv_stride, - #[cfg(target_os = "macos")] - iosurface_backing: None, #[cfg(target_os = "windows")] d3d11_texture_backing: None, } @@ -253,8 +223,6 @@ impl DecodedFrame { format: PixelFormat::Nv12, y_stride, uv_stride, - #[cfg(target_os = "macos")] - iosurface_backing: None, #[cfg(target_os = "windows")] d3d11_texture_backing: None, } @@ -274,8 +242,6 @@ impl DecodedFrame { format: PixelFormat::Yuv420p, y_stride, uv_stride, - #[cfg(target_os = "macos")] - iosurface_backing: None, #[cfg(target_os = "windows")] d3d11_texture_backing: None, } @@ -295,78 +261,11 @@ impl DecodedFrame { format: PixelFormat::Yuv420p, y_stride, uv_stride, - #[cfg(target_os = "macos")] - iosurface_backing: None, #[cfg(target_os = "windows")] d3d11_texture_backing: None, } } - #[cfg(target_os = "macos")] - pub fn new_nv12_with_iosurface( - data: Vec, - width: u32, - height: u32, - y_stride: u32, - uv_stride: u32, - image_buf: R, - ) -> Self { - Self { - data: Arc::new(data), - width, - height, - format: PixelFormat::Nv12, - y_stride, - uv_stride, - iosurface_backing: Some(Arc::new(SendableImageBuf::new(image_buf))), - } - } - - #[cfg(target_os = "macos")] - pub fn new_nv12_with_iosurface_arc( - data: Arc>, - width: u32, - height: u32, - y_stride: u32, - uv_stride: u32, - image_buf: R, - ) -> Self { - Self { - data, - width, - height, - format: PixelFormat::Nv12, - y_stride, - uv_stride, - iosurface_backing: Some(Arc::new(SendableImageBuf::new(image_buf))), - } - } - - #[cfg(target_os = "macos")] - pub fn new_nv12_zero_copy( - width: u32, - height: u32, - y_stride: u32, - uv_stride: u32, - image_buf: R, - ) -> Self { - Self { - data: Arc::new(Vec::new()), - width, - height, - format: PixelFormat::Nv12, - y_stride, - uv_stride, - iosurface_backing: Some(Arc::new(SendableImageBuf::new(image_buf))), - } - } - - #[cfg(target_os = "macos")] - #[allow(clippy::redundant_closure)] - pub fn iosurface_backing(&self) -> Option<&cv::ImageBuf> { - self.iosurface_backing.as_ref().map(|b| b.inner()) - } - #[cfg(target_os = "windows")] pub fn new_nv12_with_d3d11_texture(width: u32, height: u32, texture: ID3D11Texture2D) -> Self { Self { diff --git a/crates/rendering/src/layers/camera.rs b/crates/rendering/src/layers/camera.rs index c4c5dc7412..a0c1d85c3c 100644 --- a/crates/rendering/src/layers/camera.rs +++ b/crates/rendering/src/layers/camera.rs @@ -130,35 +130,6 @@ impl CameraLayer { ); } PixelFormat::Nv12 => { - #[cfg(target_os = "macos")] - let iosurface_result = camera_frame.iosurface_backing().map(|image_buf| { - self.yuv_converter - .convert_nv12_from_iosurface(device, queue, image_buf) - }); - - #[cfg(target_os = "macos")] - if let Some(Ok(_)) = iosurface_result { - self.copy_from_yuv_output(device, queue, next_texture, frame_size); - } else if let (Some(y_data), Some(uv_data)) = - (camera_frame.y_plane(), camera_frame.uv_plane()) - && self - .yuv_converter - .convert_nv12( - device, - queue, - y_data, - uv_data, - frame_size.x, - frame_size.y, - camera_frame.y_stride(), - camera_frame.uv_stride(), - ) - .is_ok() - { - self.copy_from_yuv_output(device, queue, next_texture, frame_size); - } - - #[cfg(not(target_os = "macos"))] if let (Some(y_data), Some(uv_data)) = (camera_frame.y_plane(), camera_frame.uv_plane()) && self diff --git a/crates/rendering/src/layers/display.rs b/crates/rendering/src/layers/display.rs index bf797a9b97..c799f8dfad 100644 --- a/crates/rendering/src/layers/display.rs +++ b/crates/rendering/src/layers/display.rs @@ -134,26 +134,9 @@ impl DisplayLayer { PixelFormat::Nv12 => { let screen_frame = &segment_frames.screen_frame; - #[cfg(target_os = "macos")] - let iosurface_result = screen_frame.iosurface_backing().map(|image_buf| { - self.yuv_converter - .convert_nv12_from_iosurface(device, queue, image_buf) - }); - #[cfg(target_os = "macos")] if !self.prefer_cpu_conversion { - if let Some(Ok(_)) = iosurface_result { - if self.yuv_converter.output_texture().is_some() { - self.pending_copy = Some(PendingTextureCopy { - width: frame_size.x, - height: frame_size.y, - dst_texture_index: next_texture, - }); - true - } else { - false - } - } else if let (Some(y_data), Some(uv_data)) = + if let (Some(y_data), Some(uv_data)) = (screen_frame.y_plane(), screen_frame.uv_plane()) { let y_stride = screen_frame.y_stride(); diff --git a/crates/video-decode/src/avassetreader.rs b/crates/video-decode/src/avassetreader.rs index 17ddf6f95d..189d60e011 100644 --- a/crates/video-decode/src/avassetreader.rs +++ b/crates/video-decode/src/avassetreader.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use cidre::{ arc::{self, R}, @@ -17,7 +17,7 @@ pub struct KeyframeIndex { } impl KeyframeIndex { - pub fn build(path: &PathBuf) -> Result { + pub fn build(path: &Path) -> Result { let build_start = std::time::Instant::now(); let input = avformat::input(path) @@ -165,6 +165,17 @@ impl KeyframeIndex { } } +fn compute_seek_time(keyframe_index: Option<&KeyframeIndex>, requested_time: f32) -> f32 { + if let Some(kf_index) = keyframe_index { + let fps = kf_index.fps(); + let target_frame = (requested_time as f64 * fps).round() as u32; + if let Some((_, keyframe_time)) = kf_index.nearest_keyframe_before(target_frame) { + return keyframe_time as f32; + } + } + requested_time +} + pub struct AVAssetReaderDecoder { path: PathBuf, pixel_format: cv::PixelFormat, @@ -230,17 +241,7 @@ impl AVAssetReaderDecoder { ) }; - let seek_time = if let Some(ref kf_index) = keyframe_index { - let fps = kf_index.fps(); - let target_frame = (start_time as f64 * fps).round() as u32; - if let Some((_, keyframe_time)) = kf_index.nearest_keyframe_before(target_frame) { - keyframe_time as f32 - } else { - start_time - } - } else { - start_time - }; + let seek_time = compute_seek_time(keyframe_index.as_ref(), start_time); let (track_output, reader) = Self::get_reader_track_output( &path, @@ -267,18 +268,7 @@ impl AVAssetReaderDecoder { pub fn reset(&mut self, requested_time: f32) -> Result<(), String> { self.reader.cancel_reading(); - let seek_time = if let Some(ref keyframe_index) = self.keyframe_index { - let fps = keyframe_index.fps(); - let target_frame = (requested_time as f64 * fps).round() as u32; - - if let Some((_, keyframe_time)) = keyframe_index.nearest_keyframe_before(target_frame) { - keyframe_time as f32 - } else { - requested_time - } - } else { - requested_time - }; + let seek_time = compute_seek_time(self.keyframe_index.as_ref(), requested_time); (self.track_output, self.reader) = Self::get_reader_track_output( &self.path, @@ -319,7 +309,7 @@ impl AVAssetReaderDecoder { } fn get_reader_track_output( - path: &PathBuf, + path: &Path, time: f32, handle: &TokioHandle, pixel_format: cv::PixelFormat, From 4a076597fdaf0a5e3d5d613ec817b9fd87f6bbfd Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 24 Dec 2025 00:50:35 +0000 Subject: [PATCH 31/37] clippy --- apps/desktop/src-tauri/src/export.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/desktop/src-tauri/src/export.rs b/apps/desktop/src-tauri/src/export.rs index 612d11c5a0..e1c315f592 100644 --- a/apps/desktop/src-tauri/src/export.rs +++ b/apps/desktop/src-tauri/src/export.rs @@ -189,8 +189,7 @@ pub struct ExportPreviewResult { } fn bpp_to_jpeg_quality(bpp: f32) -> u8 { - let quality = ((bpp - 0.04) / (0.3 - 0.04) * (95.0 - 40.0) + 40.0).clamp(40.0, 95.0) as u8; - quality + ((bpp - 0.04) / (0.3 - 0.04) * (95.0 - 40.0) + 40.0).clamp(40.0, 95.0) as u8 } #[tauri::command] From bbd80f3259256de9ba00e91e694a73dfcc660698 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 24 Dec 2025 00:56:20 +0000 Subject: [PATCH 32/37] clippy --- crates/enc-ffmpeg/src/mux/segmented_stream.rs | 82 ++++++++--------- crates/recording/src/cursor.rs | 89 ++++++++++--------- 2 files changed, 85 insertions(+), 86 deletions(-) diff --git a/crates/enc-ffmpeg/src/mux/segmented_stream.rs b/crates/enc-ffmpeg/src/mux/segmented_stream.rs index 8dab6f9aea..91df680cc2 100644 --- a/crates/enc-ffmpeg/src/mux/segmented_stream.rs +++ b/crates/enc-ffmpeg/src/mux/segmented_stream.rs @@ -37,10 +37,10 @@ fn atomic_write_json(path: &Path, data: &T) -> std::io::Result<()> } fn sync_file(path: &Path) { - if let Ok(file) = std::fs::File::open(path) { - if let Err(e) = file.sync_all() { - tracing::warn!("File fsync failed for {}: {e}", path.display()); - } + if let Ok(file) = std::fs::File::open(path) + && let Err(e) = file.sync_all() + { + tracing::warn!("File fsync failed for {}: {e}", path.display()); } } @@ -161,7 +161,7 @@ impl SegmentedVideoEncoder { set_opt("media_seg_name", "segment_$Number%03d$.m4s"); set_opt( "seg_duration", - &config.segment_duration.as_secs().to_string(), + &config.segment_duration.as_secs_f64().to_string(), ); set_opt("use_timeline", "0"); set_opt("use_template", "1"); @@ -428,30 +428,29 @@ impl SegmentedVideoEncoder { for entry in entries.flatten() { let path = entry.path(); - if let Some(name) = path.file_name().and_then(|n| n.to_str()) { - if name.starts_with("segment_") && name.ends_with(".m4s.tmp") { - let final_name = name.trim_end_matches(".tmp"); - let final_path = self.base_path.join(final_name); - - if let Ok(metadata) = std::fs::metadata(&path) { - if metadata.len() > 0 { - if let Err(e) = std::fs::rename(&path, &final_path) { - tracing::warn!( - "Failed to rename tmp segment {} to {}: {}", - path.display(), - final_path.display(), - e - ); - } else { - tracing::debug!( - "Finalized pending segment: {} ({} bytes)", - final_path.display(), - metadata.len() - ); - sync_file(&final_path); - } - } - } + if let Some(name) = path.file_name().and_then(|n| n.to_str()) + && name.starts_with("segment_") + && name.ends_with(".m4s.tmp") + && let Ok(metadata) = std::fs::metadata(&path) + && metadata.len() > 0 + { + let final_name = name.trim_end_matches(".tmp"); + let final_path = self.base_path.join(final_name); + + if let Err(e) = std::fs::rename(&path, &final_path) { + tracing::warn!( + "Failed to rename tmp segment {} to {}: {}", + path.display(), + final_path.display(), + e + ); + } else { + tracing::debug!( + "Finalized pending segment: {} ({} bytes)", + final_path.display(), + metadata.len() + ); + sync_file(&final_path); } } } @@ -474,20 +473,17 @@ impl SegmentedVideoEncoder { for entry in entries.flatten() { let path = entry.path(); - if let Some(name) = path.file_name().and_then(|n| n.to_str()) { - if name.starts_with("segment_") && name.ends_with(".m4s") && !name.contains(".tmp") - { - if let Some(index_str) = name - .strip_prefix("segment_") - .and_then(|s| s.strip_suffix(".m4s")) - { - if let Ok(index) = index_str.parse::() { - if !completed_indices.contains(&index) { - orphaned.push((index, path)); - } - } - } - } + if let Some(name) = path.file_name().and_then(|n| n.to_str()) + && name.starts_with("segment_") + && name.ends_with(".m4s") + && !name.contains(".tmp") + && let Some(index_str) = name + .strip_prefix("segment_") + .and_then(|s| s.strip_suffix(".m4s")) + && let Ok(index) = index_str.parse::() + && !completed_indices.contains(&index) + { + orphaned.push((index, path)); } } diff --git a/crates/recording/src/cursor.rs b/crates/recording/src/cursor.rs index 405c97bf59..796093d035 100644 --- a/crates/recording/src/cursor.rs +++ b/crates/recording/src/cursor.rs @@ -72,11 +72,8 @@ pub fn spawn_cursor_recorder( use cap_utils::spawn_actor; use device_query::{DeviceQuery, DeviceState}; use futures::future::Either; - use std::{ - hash::{DefaultHasher, Hash, Hasher}, - pin::pin, - time::Duration, - }; + use sha2::{Digest, Sha256}; + use std::{pin::pin, time::Duration}; use tracing::{error, info}; let stop_token = CancellationToken::new(); @@ -100,7 +97,7 @@ pub fn spawn_cursor_recorder( let mut last_flush = Instant::now(); let flush_interval = Duration::from_secs(CURSOR_FLUSH_INTERVAL_SECS); - let mut last_cursor_id = "default".to_string(); + let mut last_cursor_id: Option = None; loop { let sleep = tokio::time::sleep(Duration::from_millis(16)); @@ -116,51 +113,57 @@ pub fn spawn_cursor_recorder( let position = cap_cursor_capture::RawCursorPosition::get(); let position_changed = position != last_position; - let cursor_id = if position_changed { + if position_changed { last_position = position; - if let Some(data) = get_cursor_data() { - let mut hasher = DefaultHasher::default(); - data.image.hash(&mut hasher); - let id = hasher.finish(); - - let cursor_id = if let Some(existing_id) = response.cursors.get(&id) { - existing_id.id.to_string() - } else { - let cursor_id = response.next_cursor_id.to_string(); - let file_name = format!("cursor_{cursor_id}.png"); - let cursor_path = cursors_dir.join(&file_name); - - if let Ok(image) = image::load_from_memory(&data.image) { - let rgba_image = image.into_rgba8(); - - if let Err(e) = rgba_image.save(&cursor_path) { - error!("Failed to save cursor image: {}", e); - } else { - info!("Saved cursor {cursor_id} image to: {:?}", file_name); - response.cursors.insert( - id, - Cursor { - file_name, - id: response.next_cursor_id, - hotspot: data.hotspot, - shape: data.shape, - }, - ); - response.next_cursor_id += 1; - } + } + + let cursor_id = if let Some(data) = get_cursor_data() { + let hash_bytes = Sha256::digest(&data.image); + let id = u64::from_le_bytes( + hash_bytes[..8] + .try_into() + .expect("sha256 produces at least 8 bytes"), + ); + + let cursor_id = if let Some(existing_id) = response.cursors.get(&id) { + existing_id.id.to_string() + } else { + let cursor_id = response.next_cursor_id.to_string(); + let file_name = format!("cursor_{cursor_id}.png"); + let cursor_path = cursors_dir.join(&file_name); + + if let Ok(image) = image::load_from_memory(&data.image) { + let rgba_image = image.into_rgba8(); + + if let Err(e) = rgba_image.save(&cursor_path) { + error!("Failed to save cursor image: {}", e); + } else { + info!("Saved cursor {cursor_id} image to: {:?}", file_name); + response.cursors.insert( + id, + Cursor { + file_name, + id: response.next_cursor_id, + hotspot: data.hotspot, + shape: data.shape, + }, + ); + response.next_cursor_id += 1; } + } - cursor_id - }; - last_cursor_id = cursor_id.clone(); cursor_id - } else { - last_cursor_id.clone() - } + }; + last_cursor_id = Some(cursor_id.clone()); + Some(cursor_id) } else { last_cursor_id.clone() }; + let Some(cursor_id) = cursor_id else { + continue; + }; + if position_changed { let cropped_norm_pos = position .relative_to_display(display) From 599c380d8f818648fa02293cea6d2b759a3d4d1b Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 24 Dec 2025 01:00:02 +0000 Subject: [PATCH 33/37] fmt --- apps/desktop/src/routes/editor/ExportPage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/desktop/src/routes/editor/ExportPage.tsx b/apps/desktop/src/routes/editor/ExportPage.tsx index b08240a81e..c4087f46bf 100644 --- a/apps/desktop/src/routes/editor/ExportPage.tsx +++ b/apps/desktop/src/routes/editor/ExportPage.tsx @@ -1384,7 +1384,7 @@ export function ExportPage() {