wip: Optimize macOS recording pipeline with M4S muxer and async finalization (+ optimisations)

apps/desktop/src-tauri/src/lib.rs

@@ -90,7 +90,7 @@ use tauri_plugin_notification::{NotificationExt, PermissionState};
 use tauri_plugin_opener::OpenerExt;
 use tauri_plugin_shell::ShellExt;
 use tauri_specta::Event;
-use tokio::sync::{RwLock, oneshot};
+use tokio::sync::{RwLock, oneshot, watch};
 use tracing::*;
 use upload::{create_or_get_video, upload_image, upload_video};
 use web_api::AuthedApiError;
@@ -106,6 +106,34 @@ use crate::{
 };
 use crate::{recording::start_recording, upload::build_video_meta};
 
+#[derive(Default)]
+pub struct FinalizingRecordings {
+    recordings: std::sync::Mutex<
+        std::collections::HashMap<PathBuf, (watch::Sender<bool>, watch::Receiver<bool>)>,
+    >,
+}
+
+impl FinalizingRecordings {
+    pub fn start_finalizing(&self, path: PathBuf) -> watch::Receiver<bool> {
+        let mut recordings = self.recordings.lock().unwrap();
+        let (tx, rx) = watch::channel(false);
+        recordings.insert(path, (tx, rx.clone()));
+        rx
+    }
+
+    pub fn finish_finalizing(&self, path: &Path) {
+        let mut recordings = self.recordings.lock().unwrap();
+        if let Some((tx, _)) = recordings.remove(path) {
+            tx.send(true).ok();
+        }
+    }
+
+    pub fn is_finalizing(&self, path: &Path) -> Option<watch::Receiver<bool>> {
+        let recordings = self.recordings.lock().unwrap();
+        recordings.get(path).map(|(_, rx)| rx.clone())
+    }
+}
+
 #[allow(clippy::large_enum_variant)]
 pub enum RecordingState {
     None,
@@ -2597,6 +2625,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) {
             app.manage(http_client::HttpClient::default());
             app.manage(http_client::RetryableHttpClient::default());
             app.manage(PendingScreenshots::default());
+            app.manage(FinalizingRecordings::default());
 
             gpu_context::prewarm_gpu();
 
@@ -3142,6 +3171,8 @@ async fn create_editor_instance_impl(
 ) -> Result<Arc<EditorInstance>, String> {
     let app = app.clone();
 
+    wait_for_recording_ready(&app, &path).await?;
+
     let instance = {
         let app = app.clone();
         EditorInstance::new(
@@ -3170,6 +3201,39 @@ async fn create_editor_instance_impl(
     Ok(instance)
 }
 
+async fn wait_for_recording_ready(app: &AppHandle, path: &Path) -> Result<(), String> {
+    let finalizing_state = app.state::<FinalizingRecordings>();
+
+    if let Some(mut rx) = finalizing_state.is_finalizing(path) {
+        info!("Recording is being finalized, waiting for completion...");
+        rx.wait_for(|&ready| ready)
+            .await
+            .map_err(|_| "Finalization was cancelled".to_string())?;
+        info!("Recording finalization completed");
+        return Ok(());
+    }
+
+    let meta = match RecordingMeta::load_for_project(path) {
+        Ok(meta) => meta,
+        Err(e) => {
+            return Err(format!("Failed to load recording meta: {e}"));
+        }
+    };
+
+    if let Some(studio_meta) = meta.studio_meta() {
+        if recording::needs_fragment_remux(path, studio_meta) {
+            info!("Recording needs remux (crash recovery), starting remux...");
+            let path = path.to_path_buf();
+            tokio::task::spawn_blocking(move || recording::remux_fragmented_recording(&path))
+                .await
+                .map_err(|e| format!("Remux task panicked: {e}"))??;
+            info!("Crash recovery remux completed");
+        }
+    }
+
+    Ok(())
+}
+
 fn recordings_path(app: &AppHandle) -> PathBuf {
     let path = app.path().app_data_dir().unwrap().join("recordings");
     std::fs::create_dir_all(&path).unwrap_or_default();

apps/desktop/src-tauri/src/recording.rs

@@ -52,8 +52,8 @@ use crate::camera::{CameraPreviewManager, CameraPreviewShape};
 use crate::general_settings;
 use crate::web_api::AuthedApiError;
 use crate::{
-    App, CurrentRecordingChanged, MutableState, NewStudioRecordingAdded, RecordingState,
-    RecordingStopped, VideoUploadInfo,
+    App, CurrentRecordingChanged, FinalizingRecordings, MutableState, NewStudioRecordingAdded,
+    RecordingState, RecordingStopped, VideoUploadInfo,
     api::PresignedS3PutRequestMethod,
     audio::AppSounds,
     auth::AuthStore,
@@ -1379,22 +1379,75 @@ async fn handle_recording_finish(
                     .map_err(|e| format!("Failed to save recording meta: {e}"))?;
             }
 
-            let updated_studio_meta = if needs_fragment_remux(&recording_dir, &recording.meta) {
-                info!("Recording has fragments that need remuxing");
-                if let Err(e) = remux_fragmented_recording(&recording_dir) {
-                    error!("Failed to remux fragmented recording: {e}");
-                    return Err(format!("Failed to remux fragmented recording: {e}"));
+            let needs_remux = needs_fragment_remux(&recording_dir, &recording.meta);
+
+            if needs_remux {
+                info!("Recording has fragments that need remuxing - opening editor immediately");
+
+                let finalizing_state = app.state::<FinalizingRecordings>();
+                finalizing_state.start_finalizing(recording_dir.clone());
+
+                let post_behaviour = GeneralSettingsStore::get(app)
+                    .ok()
+                    .flatten()
+                    .map(|v| v.post_studio_recording_behaviour)
+                    .unwrap_or(PostStudioRecordingBehaviour::OpenEditor);
+
+                match post_behaviour {
+                    PostStudioRecordingBehaviour::OpenEditor => {
+                        let _ = ShowCapWindow::Editor {
+                            project_path: recording_dir.clone(),
+                        }
+                        .show(app)
+                        .await;
+                    }
+                    PostStudioRecordingBehaviour::ShowOverlay => {
+                        let _ = ShowCapWindow::RecordingsOverlay.show(app).await;
+
+                        let app_clone = AppHandle::clone(app);
+                        let recording_dir_clone = recording_dir.clone();
+                        tokio::spawn(async move {
+                            tokio::time::sleep(Duration::from_millis(1000)).await;
+                            let _ = NewStudioRecordingAdded {
+                                path: recording_dir_clone,
+                            }
+                            .emit(&app_clone);
+                        });
+                    }
                 }
 
-                let updated_meta = RecordingMeta::load_for_project(&recording_dir)
-                    .map_err(|e| format!("Failed to reload recording meta: {e}"))?;
-                updated_meta
-                    .studio_meta()
-                    .ok_or_else(|| "Expected studio meta after remux".to_string())?
-                    .clone()
-            } else {
-                recording.meta.clone()
-            };
+                AppSounds::StopRecording.play();
+
+                let app = app.clone();
+                let recording_dir_for_finalize = recording_dir.clone();
+                let screenshots_dir = screenshots_dir.clone();
+                let default_preset = PresetsStore::get_default_preset(&app)
+                    .ok()
+                    .flatten()
+                    .map(|p| p.config);
+
+                tokio::spawn(async move {
+                    let result = finalize_studio_recording(
+                        &app,
+                        recording_dir_for_finalize.clone(),
+                        screenshots_dir,
+                        recording,
+                        default_preset,
+                    )
+                    .await;
+
+                    if let Err(e) = result {
+                        error!("Failed to finalize recording: {e}");
+                    }
+
+                    app.state::<FinalizingRecordings>()
+                        .finish_finalizing(&recording_dir_for_finalize);
+                });
+
+                return Ok(());
+            }
+
+            let updated_studio_meta = recording.meta.clone();
 
             let display_output_path = match &updated_studio_meta {
                 StudioRecordingMeta::SingleSegment { segment } => {
@@ -1588,6 +1641,72 @@ async fn handle_recording_finish(
     Ok(())
 }
 
+async fn finalize_studio_recording(
+    app: &AppHandle,
+    recording_dir: PathBuf,
+    screenshots_dir: PathBuf,
+    recording: cap_recording::studio_recording::CompletedRecording,
+    default_preset: Option<ProjectConfiguration>,
+) -> Result<(), String> {
+    info!("Starting background finalization for recording");
+
+    let recording_dir_for_remux = recording_dir.clone();
+    let remux_result =
+        tokio::task::spawn_blocking(move || remux_fragmented_recording(&recording_dir_for_remux))
+            .await
+            .map_err(|e| format!("Remux task panicked: {e}"))?;
+
+    if let Err(e) = remux_result {
+        error!("Failed to remux fragmented recording: {e}");
+        return Err(format!("Failed to remux fragmented recording: {e}"));
+    }
+
+    let updated_meta = RecordingMeta::load_for_project(&recording_dir)
+        .map_err(|e| format!("Failed to reload recording meta: {e}"))?;
+    let updated_studio_meta = updated_meta
+        .studio_meta()
+        .ok_or_else(|| "Expected studio meta after remux".to_string())?
+        .clone();
+
+    let display_output_path = match &updated_studio_meta {
+        StudioRecordingMeta::SingleSegment { segment } => {
+            segment.display.path.to_path(&recording_dir)
+        }
+        StudioRecordingMeta::MultipleSegments { inner, .. } => {
+            inner.segments[0].display.path.to_path(&recording_dir)
+        }
+    };
+
+    let display_screenshot = screenshots_dir.join("display.jpg");
+    tokio::spawn(create_screenshot(
+        display_output_path,
+        display_screenshot,
+        None,
+    ));
+
+    let recordings = ProjectRecordingsMeta::new(&recording_dir, &updated_studio_meta)
+        .map_err(|e| format!("Failed to create project recordings meta: {e}"))?;
+
+    let config = project_config_from_recording(
+        app,
+        &cap_recording::studio_recording::CompletedRecording {
+            project_path: recording.project_path,
+            meta: updated_studio_meta,
+            cursor_data: recording.cursor_data,
+        },
+        &recordings,
+        default_preset,
+    );
+
+    config
+        .write(&recording_dir)
+        .map_err(|e| format!("Failed to write project config: {e}"))?;
+
+    info!("Background finalization completed for recording");
+
+    Ok(())
+}
+
 /// Core logic for generating zoom segments based on mouse click events.
 /// This is an experimental feature that automatically creates zoom effects
 /// around user interactions to highlight important moments.
@@ -1873,7 +1992,7 @@ fn project_config_from_recording(
     config
 }
 
-fn needs_fragment_remux(recording_dir: &Path, meta: &StudioRecordingMeta) -> bool {
+pub fn needs_fragment_remux(recording_dir: &Path, meta: &StudioRecordingMeta) -> bool {
     let StudioRecordingMeta::MultipleSegments { inner, .. } = meta else {
         return false;
     };
@@ -1888,7 +2007,7 @@ fn needs_fragment_remux(recording_dir: &Path, meta: &StudioRecordingMeta) -> boo
     false
 }
 
-fn remux_fragmented_recording(recording_dir: &Path) -> Result<(), String> {
+pub fn remux_fragmented_recording(recording_dir: &Path) -> Result<(), String> {
     let meta = RecordingMeta::load_for_project(recording_dir)
         .map_err(|e| format!("Failed to load recording meta: {e}"))?;
 
@@ -1924,10 +2043,12 @@ fn analyze_recording_for_remux(
 
     for (index, segment) in inner.segments.iter().enumerate() {
         let display_path = segment.display.path.to_path(project_path);
-        let display_fragments = if display_path.is_dir() {
-            find_fragments_in_dir(&display_path)
+        let (display_fragments, display_init_segment) = if display_path.is_dir() {
+            let frags = find_fragments_in_dir(&display_path);
+            let init = display_path.join("init.mp4");
+            (frags, if init.exists() { Some(init) } else { None })
         } else if display_path.exists() {
-            vec![display_path]
+            (vec![display_path], None)
         } else {
             continue;
         };
@@ -1936,17 +2057,27 @@ fn analyze_recording_for_remux(
             continue;
         }
 
-        let camera_fragments = segment.camera.as_ref().and_then(|cam| {
-            let cam_path = cam.path.to_path(project_path);
-            if cam_path.is_dir() {
-                let frags = find_fragments_in_dir(&cam_path);
-                if frags.is_empty() { None } else { Some(frags) }
-            } else if cam_path.exists() {
-                Some(vec![cam_path])
-            } else {
-                None
-            }
-        });
+        let (camera_fragments, camera_init_segment) = segment
+            .camera
+            .as_ref()
+            .map(|cam| {
+                let cam_path = cam.path.to_path(project_path);
+                if cam_path.is_dir() {
+                    let frags = find_fragments_in_dir(&cam_path);
+                    let init = cam_path.join("init.mp4");
+                    let init_seg = if init.exists() { Some(init) } else { None };
+                    if frags.is_empty() {
+                        (None, None)
+                    } else {
+                        (Some(frags), init_seg)
+                    }
+                } else if cam_path.exists() {
+                    (Some(vec![cam_path]), None)
+                } else {
+                    (None, None)
+                }
+            })
+            .unwrap_or((None, None));
 
         let cursor_path = segment
             .cursor
@@ -1981,7 +2112,9 @@ fn analyze_recording_for_remux(
         recoverable_segments.push(RecoverableSegment {
             index: index as u32,
             display_fragments,
+            display_init_segment,
             camera_fragments,
+            camera_init_segment,
             mic_fragments,
             system_audio_fragments,
             cursor_path,

crates/editor/src/audio.rs

@@ -277,6 +277,10 @@ impl<T: FromSampleBytes> AudioPlaybackBuffer<T> {
         self.frame_buffer.set_playhead(playhead, project);
     }
 
+    pub fn current_playhead(&self) -> f64 {
+        self.frame_buffer.elapsed_samples_to_playhead()
+    }
+
     pub fn buffer_reaching_limit(&self) -> bool {
         self.resampled_buffer.vacant_len()
             <= 2 * (Self::PROCESSING_SAMPLES_COUNT as usize) * self.resampler.output.channels