diff --git a/Cargo.lock b/Cargo.lock index 89f2cc83d4..a0fad2ad34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -947,6 +947,14 @@ dependencies = [ "windows-core 0.60.1", ] +[[package]] +name = "cap-cpal-ffmpeg" +version = "0.1.0" +dependencies = [ + "cpal 0.16.0", + "ffmpeg-next", +] + [[package]] name = "cap-cursor-capture" version = "0.1.0" @@ -1006,6 +1014,7 @@ dependencies = [ "futures-intrusive", "global-hotkey", "image 0.25.6", + "kameo", "keyed_priority_queue", "lazy_static", "log", @@ -1158,6 +1167,7 @@ dependencies = [ "cap-camera", "cap-camera-ffmpeg", "cap-camera-windows", + "cap-cpal-ffmpeg", "cap-displays", "cap-fail", "cap-flags", @@ -1275,6 +1285,7 @@ dependencies = [ "objc", "objc2-app-kit", "relative-path", + "replace_with", "ringbuf", "scap-cpal", "scap-direct3d", @@ -3773,7 +3784,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.58.0", + "windows-core 0.61.2", ] [[package]] @@ -4435,7 +4446,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.53.3", ] [[package]] @@ -6633,6 +6644,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" +[[package]] +name = "replace_with" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51743d3e274e2b18df81c4dc6caf8a5b8e15dbe799e0dca05c7617380094e884" + [[package]] name = "reqwest" version = "0.12.22" @@ -10146,7 +10163,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/apps/cli/src/record.rs b/apps/cli/src/record.rs index b8f3952aea..aed946e096 100644 --- a/apps/cli/src/record.rs +++ b/apps/cli/src/record.rs @@ -65,7 +65,7 @@ impl RecordStart { cap_recording::RecordingBaseInputs { capture_target: target_info, capture_system_audio: self.system_audio, - mic_feed: &None, + mic_feed: None, }, camera.map(|c| Arc::new(Mutex::new(c))), false, diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 4d13c21bf4..7c49e88664 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -101,6 +101,7 @@ md5 = "0.7.0" tokio-util = "0.7.15" wgpu.workspace = true bytemuck = "1.23.1" +kameo = "0.17.2" [target.'cfg(target_os = "macos")'.dependencies] core-graphics = "0.24.0" diff --git a/apps/desktop/src-tauri/src/audio_meter.rs b/apps/desktop/src-tauri/src/audio_meter.rs index b5bd335a7b..e7cd067dea 100644 --- a/apps/desktop/src-tauri/src/audio_meter.rs +++ b/apps/desktop/src-tauri/src/audio_meter.rs @@ -1,4 +1,4 @@ -use cap_recording::feeds::{AudioInputSamples, AudioInputSamplesReceiver}; +use cap_recording::feeds::microphone::MicrophoneSamples; use cpal::{SampleFormat, StreamInstant}; use keyed_priority_queue::KeyedPriorityQueue; use serde::{Deserialize, Serialize}; @@ -15,7 +15,10 @@ const MIN_DB: f64 = -96.0; #[derive(Deserialize, specta::Type, Serialize, tauri_specta::Event, Debug, Clone)] pub struct AudioInputLevelChange(f64); -pub fn spawn_event_emitter(app_handle: AppHandle, audio_input_rx: AudioInputSamplesReceiver) { +pub fn spawn_event_emitter( + app_handle: AppHandle, + audio_input_rx: flume::Receiver, +) { let mut time_window = VolumeMeter::new(0.2); tokio::spawn(async move { while let Ok(samples) = audio_input_rx.recv_async().await { @@ -106,7 +109,7 @@ fn db_fs(data: impl Iterator) -> f64 { (20.0 * (max as f64 / MAX_AMPLITUDE_F32).log10()).clamp(MIN_DB, 0.0) } -fn samples_to_f64(samples: &AudioInputSamples) -> impl Iterator + use<'_> { +fn samples_to_f64(samples: &MicrophoneSamples) -> impl Iterator + use<'_> { samples .data .chunks(samples.format.sample_size()) diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 2a68041111..38c15d50ea 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -27,54 +27,49 @@ use audio::AppSounds; use auth::{AuthStore, AuthenticationInvalid, Plan}; use camera::{CameraPreview, CameraWindowState}; use cap_displays::{DisplayId, WindowId, bounds::LogicalBounds}; -use cap_editor::EditorInstance; -use cap_editor::EditorState; -use cap_project::RecordingMetaInner; -use cap_project::XY; +use cap_editor::{EditorInstance, EditorState}; use cap_project::{ - ProjectConfiguration, RecordingMeta, SharingMeta, StudioRecordingMeta, ZoomSegment, + ProjectConfiguration, RecordingMeta, RecordingMetaInner, SharingMeta, StudioRecordingMeta, XY, + ZoomSegment, }; -use cap_recording::feeds::DeviceOrModelID; use cap_recording::{ - feeds::CameraFeed, - feeds::RawCameraFrame, - feeds::{AudioInputFeed, AudioInputSamplesSender}, + feeds::{ + self, CameraFeed, DeviceOrModelID, RawCameraFrame, + microphone::{self, MicrophoneFeed}, + }, sources::ScreenCaptureTarget, }; -use cap_rendering::ProjectRecordingsMeta; -use cap_rendering::RenderedFrame; +use cap_rendering::{ProjectRecordingsMeta, RenderedFrame}; use clipboard_rs::common::RustImage; use clipboard_rs::{Clipboard, ClipboardContext}; -use editor_window::EditorInstances; -use editor_window::WindowEditorInstance; +use editor_window::{EditorInstances, WindowEditorInstance}; use general_settings::GeneralSettingsStore; +use kameo::{Actor, actor::ActorRef}; use mp4::Mp4Reader; use notifications::NotificationType; use png::{ColorType, Encoder}; use recording::InProgressRecording; use relative_path::RelativePathBuf; - -use scap::capturer::Capturer; -use scap::frame::Frame; -use scap::frame::VideoFrame; +use scap::{ + capturer::Capturer, + frame::{Frame, VideoFrame}, +}; use serde::{Deserialize, Serialize}; use serde_json::json; use specta::Type; -use std::collections::BTreeMap; -use std::path::Path; -use std::time::Duration; use std::{ + collections::BTreeMap, fs::File, future::Future, io::{BufReader, BufWriter}, marker::PhantomData, - path::PathBuf, + path::{Path, PathBuf}, process::Command, str::FromStr, sync::Arc, + time::Duration, }; -use tauri::Window; -use tauri::{AppHandle, Manager, State, WindowEvent}; +use tauri::{AppHandle, Manager, State, Window, WindowEvent}; use tauri_plugin_deep_link::DeepLinkExt; use tauri_plugin_dialog::DialogExt; use tauri_plugin_global_shortcut::GlobalShortcutExt; @@ -82,17 +77,14 @@ use tauri_plugin_notification::{NotificationExt, PermissionState}; use tauri_plugin_opener::OpenerExt; use tauri_plugin_shell::ShellExt; use tauri_specta::Event; -use tokio::sync::mpsc; -use tokio::sync::{Mutex, RwLock}; -use tokio::time::timeout; -use tracing::debug; -use tracing::error; -use tracing::trace; +use tokio::{ + sync::{Mutex, RwLock, mpsc}, + time::timeout, +}; +use tracing::{error, trace}; use upload::{S3UploadMeta, create_or_get_video, upload_image, upload_video}; use web_api::ManagerExt as WebManagerExt; -use windows::EditorWindowIds; -use windows::set_window_transparent; -use windows::{CapWindowId, ShowCapWindow}; +use windows::{CapWindowId, EditorWindowIds, ShowCapWindow, set_window_transparent}; use crate::upload::build_video_meta; @@ -116,15 +108,13 @@ pub struct App { #[serde(skip)] camera_feed_initialization: Option>, #[serde(skip)] - mic_feed: Option, - #[serde(skip)] - mic_samples_tx: AudioInputSamplesSender, - #[serde(skip)] handle: AppHandle, #[serde(skip)] recording_state: RecordingState, #[serde(skip)] recording_logging_handle: LoggingHandle, + #[serde(skip)] + mic_feed: ActorRef, server_url: String, } @@ -227,27 +217,26 @@ impl App { #[tauri::command] #[specta::specta] async fn set_mic_input(state: MutableState<'_, App>, label: Option) -> Result<(), String> { - let mut app = state.write().await; + let mic_feed = state.read().await.mic_feed.clone(); - match (label, &mut app.mic_feed) { - (Some(label), None) => { - AudioInputFeed::init(&label) - .await - .map_err(|e| e.to_string()) - .map(async |feed| { - feed.add_sender(app.mic_samples_tx.clone()).await.unwrap(); - app.mic_feed = Some(feed); - }) - .transpose_async() + match label { + None => { + mic_feed + .ask(microphone::RemoveInput) .await + .map_err(|e| e.to_string())?; } - (Some(label), Some(feed)) => feed.switch_input(&label).await.map_err(|e| e.to_string()), - (None, _) => { - debug!("removing mic in set_start_recording_options"); - app.mic_feed.take(); - Ok(()) + Some(label) => { + mic_feed + .ask(feeds::microphone::SetInput { label }) + .await + .map_err(|e| e.to_string())? + .await + .map_err(|e| e.to_string())?; } } + + Ok(()) } #[tauri::command] @@ -1119,7 +1108,7 @@ async fn list_audio_devices() -> Result, ()> { return Ok(vec![]); } - Ok(AudioInputFeed::list_devices().keys().cloned().collect()) + Ok(MicrophoneFeed::list().keys().cloned().collect()) } #[derive(Serialize, Type, tauri_specta::Event, Debug, Clone)] @@ -2024,7 +2013,28 @@ pub async fn run(recording_logging_handle: LoggingHandle) { let (camera_tx, camera_ws_port, _shutdown) = camera_legacy::create_camera_preview_ws().await; - let (audio_input_tx, audio_input_rx) = AudioInputFeed::create_channel(); + let (mic_samples_tx, mic_samples_rx) = flume::bounded(8); + + let mic_feed = { + let (error_tx, error_rx) = flume::bounded(1); + + let mic_feed = MicrophoneFeed::spawn(MicrophoneFeed::new(error_tx)); + + // TODO: make this part of a global actor one day + tokio::spawn(async move { + let Ok(err) = error_rx.recv_async().await else { + return; + }; + + error!("Mic feed actor error: {err}"); + }); + + let _ = mic_feed + .ask(feeds::microphone::AddSender(mic_samples_tx)) + .await; + + mic_feed + }; tauri::async_runtime::set(tokio::runtime::Handle::current()); @@ -2121,10 +2131,9 @@ pub async fn run(recording_logging_handle: LoggingHandle) { handle: app.clone(), camera_feed: None, camera_feed_initialization: None, - mic_samples_tx: audio_input_tx, - mic_feed: None, recording_state: RecordingState::None, recording_logging_handle, + mic_feed, server_url: GeneralSettingsStore::get(&app) .ok() .flatten() @@ -2173,7 +2182,7 @@ pub async fn run(recording_logging_handle: LoggingHandle) { } }); - audio_meter::spawn_event_emitter(app.clone(), audio_input_rx); + audio_meter::spawn_event_emitter(app.clone(), mic_samples_rx); tray::create_tray(&app).unwrap(); @@ -2213,7 +2222,8 @@ pub async fn run(recording_logging_handle: LoggingHandle) { let app_state = &mut *state.write().await; if !app_state.is_recording_active_or_pending() { - app_state.mic_feed.take(); + let _ = + app_state.mic_feed.ask(microphone::RemoveInput).await; app_state.camera_feed.take(); if let Some(camera) = CapWindowId::Camera.get(&app) { @@ -2422,28 +2432,6 @@ trait EventExt: tauri_specta::Event { impl EventExt for T {} -trait TransposeAsync { - type Output; - - fn transpose_async(self) -> impl Future - where - Self: Sized; -} - -impl, T, E> TransposeAsync for Result { - type Output = Result; - - async fn transpose_async(self) -> Self::Output - where - Self: Sized, - { - match self { - Ok(f) => Ok(f.await), - Err(e) => Err(e), - } - } -} - fn open_project_from_path(path: &Path, app: AppHandle) -> Result<(), String> { let meta = RecordingMeta::load_for_project(path).map_err(|v| v.to_string())?; diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 918d156bd1..b9892b217f 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -6,7 +6,7 @@ use cap_project::{ }; use cap_recording::{ CompletedStudioRecording, RecordingError, RecordingMode, StudioRecordingHandle, - feeds::CameraFeed, + feeds::{CameraFeed, microphone}, instant_recording::{CompletedInstantRecording, InstantRecordingHandle}, sources::{CaptureDisplay, CaptureWindow, ScreenCaptureTarget, screen_capture}, }; @@ -348,15 +348,21 @@ pub async fn start_recording( spawn_actor({ let state_mtx = Arc::clone(&state_mtx); let general_settings = general_settings.cloned(); - let capture_target = inputs.capture_target.clone(); async move { fail!("recording::spawn_actor"); let mut state = state_mtx.write().await; + use kameo::error::SendError; + let mic_feed = match state.mic_feed.ask(microphone::Lock).await { + Ok(lock) => Some(Arc::new(lock)), + Err(SendError::HandlerError(microphone::LockFeedError::NoInput)) => None, + Err(e) => return Err(e.to_string()), + }; + let base_inputs = cap_recording::RecordingBaseInputs { - capture_target, + capture_target: inputs.capture_target.clone(), capture_system_audio: inputs.capture_system_audio, - mic_feed: &state.mic_feed, + mic_feed, }; let (actor, actor_done_rx) = match inputs.mode { @@ -647,7 +653,7 @@ async fn handle_recording_end( let _ = v.close(); } app.camera_feed.take(); - app.mic_feed.take(); + let _ = app.mic_feed.ask(microphone::RemoveInput).await; } CurrentRecordingChanged.emit(&handle).ok(); diff --git a/apps/desktop/src/routes/(window-chrome)/(main).tsx b/apps/desktop/src/routes/(window-chrome)/(main).tsx index 30bf74ca9c..f250974dfe 100644 --- a/apps/desktop/src/routes/(window-chrome)/(main).tsx +++ b/apps/desktop/src/routes/(window-chrome)/(main).tsx @@ -176,10 +176,6 @@ function Page() { micName: () => mics.data?.find((name) => name === rawOptions.micName), }; - createEffect(() => { - console.log(_windows()); - }); - // if target is window and no windows are available, switch to screen capture createEffect(() => { const screen = _screens()?.[0]; @@ -257,7 +253,8 @@ function Page() { const setCamera = createCameraMutation(); onMount(() => { - if (rawOptions.cameraID) setCamera.mutate(rawOptions.cameraID); + if (rawOptions.micName) commands.setMicInput(rawOptions.micName); + if (rawOptions.cameraID) setCamera.rawMutate(rawOptions.cameraID); }); return ( @@ -821,7 +818,6 @@ function MicrophoneSelect(props: { const currentRecording = createCurrentRecordingQuery(); const [dbs, setDbs] = createSignal(); - const [isInitialized, setIsInitialized] = createSignal(false); const requestPermission = useRequestPermission(); @@ -852,14 +848,6 @@ function MicrophoneSelect(props: { const audioLevel = () => (1 - Math.max((dbs() ?? 0) + DB_SCALE, 0) / DB_SCALE) ** 0.5; - // Initialize audio input if needed - only once when component mounts - onMount(() => { - if (!props.value || !permissionGranted() || isInitialized()) return; - - setIsInitialized(true); - handleMicrophoneChange({ name: props.value }); - }); - return (