From d5a705d29ded57d1d3206ac5d06ab1f5305efd7f Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Fri, 22 Aug 2025 02:41:59 +0800 Subject: [PATCH 1/7] audio actor? --- Cargo.lock | 11 + apps/desktop/src-tauri/Cargo.toml | 1 + apps/desktop/src-tauri/src/lib.rs | 132 ++++++---- apps/desktop/src-tauri/src/recording.rs | 6 +- .../src/routes/(window-chrome)/(main).tsx | 1 + crates/cpal-ffmpeg/Cargo.toml | 11 + crates/cpal-ffmpeg/src/lib.rs | 81 +++++++ crates/media/Cargo.toml | 1 + crates/media/src/feeds/microphone.rs | 225 ++++++++++++++++++ crates/media/src/feeds/mod.rs | 1 + crates/media/src/sources/audio_input.rs | 5 +- crates/recording/Cargo.toml | 1 + crates/recording/src/lib.rs | 10 +- 13 files changed, 428 insertions(+), 58 deletions(-) create mode 100644 crates/cpal-ffmpeg/Cargo.toml create mode 100644 crates/cpal-ffmpeg/src/lib.rs create mode 100644 crates/media/src/feeds/microphone.rs diff --git a/Cargo.lock b/Cargo.lock index 71b460c926..33c2b0309d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -947,6 +947,14 @@ dependencies = [ "windows-core 0.60.1", ] +[[package]] +name = "cap-cpal-ffmpeg" +version = "0.1.0" +dependencies = [ + "cpal 0.15.3 (git+https://github.com/RustAudio/cpal?rev=f43d36e55494993bbbde3299af0c53e5cdf4d4cf)", + "ffmpeg-next", +] + [[package]] name = "cap-cursor-capture" version = "0.1.0" @@ -1006,6 +1014,7 @@ dependencies = [ "futures-intrusive", "global-hotkey", "image 0.25.6", + "kameo", "keyed_priority_queue", "lazy_static", "log", @@ -1156,6 +1165,7 @@ dependencies = [ "cap-camera", "cap-camera-ffmpeg", "cap-camera-windows", + "cap-cpal-ffmpeg", "cap-fail", "cap-flags", "cap-media-encoders", @@ -1258,6 +1268,7 @@ dependencies = [ "futures", "hex", "image 0.25.6", + "kameo", "objc", "objc2-app-kit", "relative-path", diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index af38d91fb8..ccfc75f2ff 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -101,6 +101,7 @@ md5 = "0.7.0" tokio-util = "0.7.15" wgpu.workspace = true bytemuck = "1.23.1" +kameo = "0.17.2" [target.'cfg(target_os = "macos")'.dependencies] core-graphics = "0.24.0" diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 46dca64d57..15021515ad 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -25,50 +25,49 @@ mod windows; use audio::AppSounds; use auth::{AuthStore, AuthenticationInvalid, Plan}; use camera::{CameraPreview, CameraWindowState}; -use cap_editor::EditorInstance; -use cap_editor::EditorState; -use cap_media::feeds::RawCameraFrame; -use cap_media::feeds::{AudioInputFeed, AudioInputSamplesSender}; -use cap_media::platform::Bounds; -use cap_media::{feeds::CameraFeed, sources::ScreenCaptureTarget}; -use cap_project::RecordingMetaInner; -use cap_project::XY; +use cap_editor::{EditorInstance, EditorState}; +use cap_media::{ + feeds::{ + self, AudioInputFeed, AudioInputSamplesSender, CameraFeed, RawCameraFrame, + microphone::MicrophoneFeed, + }, + platform::Bounds, + sources::ScreenCaptureTarget, +}; use cap_project::{ - ProjectConfiguration, RecordingMeta, SharingMeta, StudioRecordingMeta, ZoomSegment, + ProjectConfiguration, RecordingMeta, RecordingMetaInner, SharingMeta, StudioRecordingMeta, XY, + ZoomSegment, }; use cap_rendering::ProjectRecordingsMeta; -use clipboard_rs::common::RustImage; -use clipboard_rs::{Clipboard, ClipboardContext}; -use editor_window::EditorInstances; -use editor_window::WindowEditorInstance; +use clipboard_rs::{Clipboard, ClipboardContext, common::RustImage}; +use editor_window::{EditorInstances, WindowEditorInstance}; use general_settings::GeneralSettingsStore; +use kameo::{Actor, actor::ActorRef}; use mp4::Mp4Reader; use notifications::NotificationType; use png::{ColorType, Encoder}; use recording::InProgressRecording; use relative_path::RelativePathBuf; - -use scap::capturer::Capturer; -use scap::frame::Frame; -use scap::frame::VideoFrame; +use scap::{ + capturer::Capturer, + frame::{Frame, VideoFrame}, +}; use serde::{Deserialize, Serialize}; use serde_json::json; use specta::Type; -use std::collections::BTreeMap; -use std::path::Path; -use std::time::Duration; use std::{ + collections::BTreeMap, fs::File, future::Future, io::{BufReader, BufWriter}, marker::PhantomData, - path::PathBuf, + path::{Path, PathBuf}, process::Command, str::FromStr, sync::Arc, + time::Duration, }; -use tauri::Window; -use tauri::{AppHandle, Manager, State, WindowEvent}; +use tauri::{AppHandle, Manager, State, Window, WindowEvent}; use tauri_plugin_deep_link::DeepLinkExt; use tauri_plugin_dialog::DialogExt; use tauri_plugin_global_shortcut::GlobalShortcutExt; @@ -76,17 +75,14 @@ use tauri_plugin_notification::{NotificationExt, PermissionState}; use tauri_plugin_opener::OpenerExt; use tauri_plugin_shell::ShellExt; use tauri_specta::Event; -use tokio::sync::mpsc; -use tokio::sync::{Mutex, RwLock}; -use tokio::time::timeout; -use tracing::debug; -use tracing::error; -use tracing::trace; +use tokio::{ + sync::{Mutex, RwLock, mpsc}, + time::timeout, +}; +use tracing::{debug, error, trace}; use upload::{S3UploadMeta, create_or_get_video, upload_image, upload_video}; use web_api::ManagerExt as WebManagerExt; -use windows::EditorWindowIds; -use windows::set_window_transparent; -use windows::{CapWindowId, ShowCapWindow}; +use windows::{CapWindowId, EditorWindowIds, ShowCapWindow, set_window_transparent}; #[allow(clippy::large_enum_variant)] pub enum RecordingState { @@ -107,8 +103,8 @@ pub struct App { camera_feed: Option>>, #[serde(skip)] camera_feed_initialization: Option>, - #[serde(skip)] - mic_feed: Option, + // #[serde(skip)] + // mic_feed: Option, #[serde(skip)] mic_samples_tx: AudioInputSamplesSender, #[serde(skip)] @@ -117,6 +113,8 @@ pub struct App { recording_state: RecordingState, #[serde(skip)] recording_logging_handle: LoggingHandle, + #[serde(skip)] + mic_feed_actor: Option>, server_url: String, } @@ -221,25 +219,58 @@ impl App { async fn set_mic_input(state: MutableState<'_, App>, label: Option) -> Result<(), String> { let mut app = state.write().await; - match (label, &mut app.mic_feed) { - (Some(label), None) => { - AudioInputFeed::init(&label) - .await - .map_err(|e| e.to_string()) - .map(async |feed| { - feed.add_sender(app.mic_samples_tx.clone()).await.unwrap(); - app.mic_feed = Some(feed); - }) - .transpose_async() - .await + match label { + None => { + app.mic_feed_actor.take(); } - (Some(label), Some(feed)) => feed.switch_input(&label).await.map_err(|e| e.to_string()), - (None, _) => { - debug!("removing mic in set_start_recording_options"); - app.mic_feed.take(); - Ok(()) + Some(label) => { + let mic_feed_actor = app.mic_feed_actor.get_or_insert_with(|| { + let (error_tx, error_rx) = flume::bounded(1); + + // TODO: make this part of a global actor one day + tokio::spawn(async move { + let Ok(err) = error_rx.recv_async().await else { + return; + }; + + error!("Mic feed actor error: {err}"); + }); + + MicrophoneFeed::spawn(MicrophoneFeed::new(error_tx)) + }); + + let _ = mic_feed_actor + .ask(feeds::microphone::AddSender(app.mic_samples_tx.clone())) + .await; + + drop(app); + + let _config = mic_feed_actor + .ask(feeds::microphone::SetInput { label }) + .await + .map_err(|e| e.to_string())?; } } + + // match (label, &mut app.mic_feed) { + // (Some(label), None) => { + // AudioInputFeed::init(&label) + // .await + // .map_err(|e| e.to_string()) + // .map(async |feed| { + // feed.add_sender(app.mic_samples_tx.clone()).await.unwrap(); + // app.mic_feed = Some(feed); + // }) + // .transpose_async() + // .await + // } + // (Some(label), Some(feed)) => feed.switch_input(&label).await.map_err(|e| e.to_string()), + // (None, _) => { + // debug!("removing mic in set_start_recording_options"); + // app.mic_feed.take(); + // Ok(()) + // } + // } } #[tauri::command] @@ -2116,6 +2147,7 @@ pub async fn run(recording_logging_handle: LoggingHandle) { mic_feed: None, recording_state: RecordingState::None, recording_logging_handle, + mic_feed_actor: None, server_url: GeneralSettingsStore::get(&app) .ok() .flatten() diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 5dcfb1b97f..ce7bfcaf0b 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -17,10 +17,10 @@ use crate::{ windows::{CapWindowId, ShowCapWindow}, }; use cap_fail::fail; -use cap_media::{feeds::CameraFeed, platform::display_for_window, sources::ScreenCaptureTarget}; use cap_media::{ - platform::Bounds, - sources::{CaptureScreen, CaptureWindow}, + feeds::CameraFeed, + platform::{Bounds, display_for_window}, + sources::{CaptureScreen, CaptureWindow, ScreenCaptureTarge}, }; use cap_project::{ CursorClickEvent, Platform, ProjectConfiguration, RecordingMeta, RecordingMetaInner, diff --git a/apps/desktop/src/routes/(window-chrome)/(main).tsx b/apps/desktop/src/routes/(window-chrome)/(main).tsx index 188a93f7e6..e973977834 100644 --- a/apps/desktop/src/routes/(window-chrome)/(main).tsx +++ b/apps/desktop/src/routes/(window-chrome)/(main).tsx @@ -856,6 +856,7 @@ function MicrophoneSelect(props: { return (