diff --git a/.github/workflows/ai-runtime-release.yml b/.github/workflows/ai-runtime-release.yml index 438a3a0..93e8bbf 100644 --- a/.github/workflows/ai-runtime-release.yml +++ b/.github/workflows/ai-runtime-release.yml @@ -121,11 +121,66 @@ jobs: dist/ai-runtime/reports/macos-arm64-size-report.json if-no-files-found: error + build-linux-runtime-cpu: + runs-on: ubuntu-24.04 + env: + VERSION: ${{ github.event.inputs.runtime_version || github.ref_name }} + AI_RUNTIME_STANDALONE_RELEASE_TAG: ${{ vars.OPENSTUDIO_AI_RUNTIME_STANDALONE_RELEASE_TAG != '' && vars.OPENSTUDIO_AI_RUNTIME_STANDALONE_RELEASE_TAG || '20260325' }} + AI_RUNTIME_STANDALONE_PYTHON_VERSION: ${{ vars.OPENSTUDIO_AI_RUNTIME_STANDALONE_PYTHON_VERSION != '' && vars.OPENSTUDIO_AI_RUNTIME_STANDALONE_PYTHON_VERSION || '3.10.20' }} + AI_RUNTIME_STANDALONE_FLAVOR: ${{ vars.OPENSTUDIO_AI_RUNTIME_STANDALONE_FLAVOR != '' && vars.OPENSTUDIO_AI_RUNTIME_STANDALONE_FLAVOR || 'install_only' }} + steps: + - uses: actions/checkout@v5 + + - name: Normalize runtime version + shell: bash + run: echo "VERSION=${VERSION#ai-runtime-v}" | sed 's/^VERSION=v/VERSION=/' >> "$GITHUB_ENV" + + - name: Cache standalone Python and wheels + uses: actions/cache@v4 + with: + path: | + .cache/ai-runtime + ~/.cache/pip + key: linux-cpu-x64-${{ env.VERSION }}-${{ env.AI_RUNTIME_STANDALONE_RELEASE_TAG }}-${{ hashFiles('tools/ai-runtime-requirements-linux.txt') }} + + - name: Prepare Linux CPU runtime + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path dist/ai-runtime/reports | Out-Null + $runtimeRoot = Join-Path $env:RUNNER_TEMP "openstudio-ai-runtime-linux-cpu-x64" + ./tools/prepare-ai-runtime.ps1 ` + -Platform linux ` + -RuntimeRoot $runtimeRoot ` + -Architecture x64 ` + -RuntimeFamily "linux-cpu-x64" ` + -RequirementsFile "tools/ai-runtime-requirements-linux.txt" ` + -ExpectedRuntimeVersion $env:VERSION ` + -StandaloneReleaseTag $env:AI_RUNTIME_STANDALONE_RELEASE_TAG ` + -StandalonePythonVersion $env:AI_RUNTIME_STANDALONE_PYTHON_VERSION ` + -StandaloneFlavor $env:AI_RUNTIME_STANDALONE_FLAVOR ` + -StandaloneCacheDir ".cache/ai-runtime" ` + -SizeReportPath "dist/ai-runtime/reports/linux-cpu-x64-size-report.json" + ./tools/package-ai-runtime.ps1 ` + -Platform linux ` + -RuntimeRoot $runtimeRoot ` + -OutputPath "dist/ai-runtime/OpenStudio-AI-Runtime-linux-cpu-x64.zip" ` + -ExpectedRuntimeVersion $env:VERSION ` + -MaxArtifactSizeBytes 2100000000 + + - uses: actions/upload-artifact@v4 + with: + name: linux-cpu-x64-ai-runtime + path: | + dist/ai-runtime/OpenStudio-AI-Runtime-linux-cpu-x64.zip + dist/ai-runtime/reports/linux-cpu-x64-size-report.json + if-no-files-found: error + publish-ai-runtime: runs-on: ubuntu-latest needs: - build-windows-runtime-base - build-macos-runtime-arm64 + - build-linux-runtime-cpu permissions: contents: write env: @@ -147,13 +202,20 @@ jobs: name: macos-arm64-ai-runtime path: dist/ai-runtime + - uses: actions/download-artifact@v4 + with: + name: linux-cpu-x64-ai-runtime + path: dist/ai-runtime + - name: Verify downloaded AI runtime artifacts shell: bash run: | test -f "dist/ai-runtime/OpenStudio-AI-Runtime-windows-base-x64.zip" test -f "dist/ai-runtime/OpenStudio-AI-Runtime-macos-arm64.zip" + test -f "dist/ai-runtime/OpenStudio-AI-Runtime-linux-cpu-x64.zip" test -f "dist/ai-runtime/reports/windows-base-size-report.json" test -f "dist/ai-runtime/reports/macos-arm64-size-report.json" + test -f "dist/ai-runtime/reports/linux-cpu-x64-size-report.json" - name: Publish AI runtime release uses: softprops/action-gh-release@v2 @@ -167,5 +229,7 @@ jobs: files: | dist/ai-runtime/OpenStudio-AI-Runtime-windows-base-x64.zip dist/ai-runtime/OpenStudio-AI-Runtime-macos-arm64.zip + dist/ai-runtime/OpenStudio-AI-Runtime-linux-cpu-x64.zip dist/ai-runtime/reports/windows-base-size-report.json dist/ai-runtime/reports/macos-arm64-size-report.json + dist/ai-runtime/reports/linux-cpu-x64-size-report.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 07187dd..da18b28 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -297,11 +297,83 @@ jobs: path: dist/macos/OpenStudio-macOS.dmg if-no-files-found: error + build-linux: + runs-on: ubuntu-24.04 + env: + VERSION: ${{ github.event.inputs.version || github.ref_name }} + BUILD_DIR: build-release-linux + ONNXRUNTIME_VERSION: 1.24.4 + RELEASE_SITE_URL: https://openstudio.org.in + steps: + - uses: actions/checkout@v5 + + - name: Normalize version + shell: bash + run: echo "VERSION=${VERSION#v}" >> "$GITHUB_ENV" + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential cmake ninja-build pkg-config \ + libasound2-dev libjack-jackd2-dev \ + libwebkit2gtk-4.1-dev libgtk-3-dev \ + libgl1-mesa-dev libfreetype6-dev libfontconfig1-dev \ + libcurl4-openssl-dev \ + libx11-dev libxext-dev libxrandr-dev libxi-dev \ + libxinerama-dev libxcursor-dev libxcomposite-dev + + - uses: actions/setup-node@v5 + with: + node-version: 20 + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Build frontend + run: cd frontend && npm ci && npm run build + + - name: Download ONNX Runtime (Linux CPU) + run: | + URL="https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz" + wget -q "$URL" -O onnxruntime.tgz + mkdir -p thirdparty/onnxruntime + tar -xzf onnxruntime.tgz --strip-components=1 -C thirdparty/onnxruntime + + - name: Configure CMake + run: | + cmake -S . -B "$BUILD_DIR" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DOPENSTUDIO_APP_VERSION="$VERSION" \ + -DOPENSTUDIO_UPDATE_MANIFEST_URL_VALUE="$RELEASE_SITE_URL/releases/stable/latest.json" \ + -DOPENSTUDIO_UPDATE_APPCAST_URL_VALUE="$RELEASE_SITE_URL/appcast/linux-stable.xml" \ + -DOPENSTUDIO_RELEASES_PAGE_URL_VALUE="$RELEASE_SITE_URL/download" \ + -DOPENSTUDIO_UPDATE_CHANNEL_VALUE="stable" \ + -DOPENSTUDIO_ENABLE_EXTERNAL_PYTHON_AI_FALLBACK=OFF \ + -DFETCHCONTENT_UPDATES_DISCONNECTED=ON + + - name: Build OpenStudio + run: cmake --build "$BUILD_DIR" --config Release --target OpenStudio + + - name: Package AppImage + run: | + chmod +x ./tools/package-linux-release.sh + bash ./tools/package-linux-release.sh "$VERSION" "$BUILD_DIR" + + - name: Verify Linux release outputs + run: test -f "dist/linux/OpenStudio-${VERSION}-linux-x86_64.AppImage" + + - uses: actions/upload-artifact@v4 + with: + name: linux-release + path: dist/linux/OpenStudio-*-linux-x86_64.AppImage + if-no-files-found: error + publish: runs-on: ubuntu-latest needs: - build-windows - build-macos + - build-linux permissions: contents: write env: @@ -368,6 +440,11 @@ jobs: name: macos-release path: dist/macos + - uses: actions/download-artifact@v4 + with: + name: linux-release + path: dist/linux + - name: Download AI runtime release assets shell: bash env: @@ -378,6 +455,7 @@ jobs: --repo "${{ github.repository }}" \ --pattern "OpenStudio-AI-Runtime-windows-base-x64.zip" \ --pattern "OpenStudio-AI-Runtime-macos-arm64.zip" \ + --pattern "OpenStudio-AI-Runtime-linux-cpu-x64.zip" \ --dir dist/ai-runtime - name: Verify downloaded release assets @@ -387,7 +465,8 @@ jobs: "dist/windows/OpenStudio-Setup-x64.exe", "dist/macos/OpenStudio-macOS.dmg", "dist/ai-runtime/OpenStudio-AI-Runtime-windows-base-x64.zip", - "dist/ai-runtime/OpenStudio-AI-Runtime-macos-arm64.zip" + "dist/ai-runtime/OpenStudio-AI-Runtime-macos-arm64.zip", + "dist/ai-runtime/OpenStudio-AI-Runtime-linux-cpu-x64.zip" ) foreach ($file in $requiredFiles) { @@ -406,6 +485,7 @@ jobs: $macosUrl = "https://github.com/${{ github.repository }}/releases/download/$tag/OpenStudio-macOS.dmg" $windowsBaseAiRuntimeUrl = "https://github.com/${{ github.repository }}/releases/download/$env:AI_RUNTIME_RELEASE_TAG/OpenStudio-AI-Runtime-windows-base-x64.zip" $macosArm64AiRuntimeUrl = "https://github.com/${{ github.repository }}/releases/download/$env:AI_RUNTIME_RELEASE_TAG/OpenStudio-AI-Runtime-macos-arm64.zip" + $linuxUrl = "https://github.com/${{ github.repository }}/releases/download/$tag/OpenStudio-$env:VERSION-linux-x86_64.AppImage" ./tools/generate-release-metadata.ps1 ` -Version $env:VERSION ` -Channel stable ` @@ -417,6 +497,8 @@ jobs: -WindowsInstallerArguments "/SP- /NOICONS" ` -MacAssetPath "dist/macos/OpenStudio-macOS.dmg" ` -MacAssetUrl $macosUrl ` + -LinuxAssetPath "dist/linux/OpenStudio-$env:VERSION-linux-x86_64.AppImage" ` + -LinuxAssetUrl $linuxUrl ` -WindowsBaseAiRuntimeAssetPath "dist/ai-runtime/OpenStudio-AI-Runtime-windows-base-x64.zip" ` -WindowsBaseAiRuntimeAssetUrl $windowsBaseAiRuntimeUrl ` -WindowsCudaInstallPlanPath "tools/ai-runtime-install-plan-windows-cuda.json" ` @@ -433,6 +515,7 @@ jobs: -Channel stable ` -WindowsAssetPath "dist/windows/OpenStudio-Setup-x64.exe" ` -MacAssetPath "dist/macos/OpenStudio-macOS.dmg" ` + -LinuxAssetPath "dist/linux/OpenStudio-$env:VERSION-linux-x86_64.AppImage" ` -WindowsBaseAiRuntimeAssetPath "dist/ai-runtime/OpenStudio-AI-Runtime-windows-base-x64.zip" ` -WindowsCudaInstallPlanPath "tools/ai-runtime-install-plan-windows-cuda.json" ` -WindowsDirectmlInstallPlanPath "tools/ai-runtime-install-plan-windows-directml.json" ` @@ -456,6 +539,7 @@ jobs: files: | dist/windows/OpenStudio-Setup-x64.exe dist/macos/OpenStudio-macOS.dmg + dist/linux/OpenStudio-*-linux-x86_64.AppImage dist/release-publish-assets/OpenStudio-checksums.txt dist/release-publish-assets/OpenStudio-release-latest.json dist/release-publish-assets/OpenStudio-release-stable-latest.json @@ -463,6 +547,7 @@ jobs: dist/release-publish-assets/OpenStudio-ai-runtime-stable-latest.json dist/release-publish-assets/OpenStudio-appcast-windows-stable.xml dist/release-publish-assets/OpenStudio-appcast-macos-stable.xml + dist/release-publish-assets/OpenStudio-appcast-linux-stable.xml - name: Verify website dispatch configuration shell: bash diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 5c72532..5d27263 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -36,7 +36,40 @@ jobs: - name: Install Windows prerequisite installers shell: pwsh - run: ./tools/setup-windows-prereqs.ps1 + run: | + ./tools/setup-windows-prereqs.ps1 + + function Invoke-PrerequisiteInstaller { + param( + [Parameter(Mandatory = $true)] + [string]$Path, + + [Parameter(Mandatory = $true)] + [string[]]$Arguments, + + [Parameter(Mandatory = $true)] + [string]$Name + ) + + if (-not (Test-Path -LiteralPath $Path)) { + throw "$Name installer was not found at '$Path'." + } + + $process = Start-Process -FilePath $Path -ArgumentList $Arguments -Wait -PassThru + if ($process.ExitCode -notin @(0, 3010, 1638)) { + Write-Warning "$Name installer exited with code $($process.ExitCode). Continuing so the startup self-test can report the actual runtime state." + } + } + + $prereqDir = Join-Path $env:GITHUB_WORKSPACE "thirdparty/windows-prereqs" + Invoke-PrerequisiteInstaller ` + -Path (Join-Path $prereqDir "vc_redist.x64.exe") ` + -Arguments @("/install", "/quiet", "/norestart") ` + -Name "Visual C++ Redistributable" + Invoke-PrerequisiteInstaller ` + -Path (Join-Path $prereqDir "MicrosoftEdgeWebView2RuntimeInstallerX64.exe") ` + -Arguments @("/silent", "/install") ` + -Name "Microsoft Edge WebView2 Runtime" - name: Configure CMake shell: pwsh @@ -60,10 +93,21 @@ jobs: shell: pwsh run: | $report = Join-Path $env:RUNNER_TEMP "OpenStudio_StartupSelfTest.txt" - & "$env:BUILD_DIR/OpenStudio_artefacts/Release/OpenStudio.exe" --startup-self-test --report $report - if ($LASTEXITCODE -ne 0) { + if (Test-Path $report) { + Remove-Item -LiteralPath $report -Force -ErrorAction SilentlyContinue + } + + $exePath = Join-Path $env:GITHUB_WORKSPACE "$env:BUILD_DIR/OpenStudio_artefacts/Release/OpenStudio.exe" + $process = Start-Process -FilePath $exePath -ArgumentList @("--startup-self-test", "--report", "`"$report`"") -Wait -PassThru + if ($process.ExitCode -ne 0) { if (Test-Path $report) { - Get-Content $report + Get-Content -LiteralPath $report + } else { + Write-Host "No self-test report was written." + $startupLog = Join-Path $env:APPDATA "OpenStudio/logs/OpenStudio_Startup.log" + if (Test-Path $startupLog) { + Get-Content -LiteralPath $startupLog + } } throw "Windows startup self-test failed." } @@ -112,8 +156,8 @@ jobs: echo "APP_BUNDLE=$APP_BUNDLE" >> "$GITHUB_ENV" - name: Validate macOS runtime bundle - shell: bash - run: ./tools/validate-runtime-bundle.ps1 -Platform macos -BundlePath "$APP_BUNDLE" -ExpectedVersion "$VERSION" -EnforceLeanBundle + shell: pwsh + run: ./tools/validate-runtime-bundle.ps1 -Platform macos -BundlePath "$env:APP_BUNDLE" -ExpectedVersion "$env:VERSION" -EnforceLeanBundle - name: Run macOS startup self-test shell: bash @@ -126,3 +170,82 @@ jobs: fi exit 1 fi + + verify-linux: + runs-on: ubuntu-24.04 + env: + VERSION: 0.0.0 + BUILD_DIR: build-verify-linux + ONNXRUNTIME_VERSION: 1.24.4 + steps: + - uses: actions/checkout@v5 + + - name: Install system dependencies + shell: bash + run: | + bash ./tools/setup-linux-prereqs.sh + sudo apt-get install -y xvfb + + - uses: actions/setup-node@v5 + with: + node-version: 20 + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Build frontend + shell: bash + run: | + cd frontend + npm ci + npm run build + + - name: Download ONNX Runtime + shell: bash + run: | + URL="https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz" + wget -q "$URL" -O onnxruntime.tgz + mkdir -p thirdparty/onnxruntime + tar -xzf onnxruntime.tgz --strip-components=1 -C thirdparty/onnxruntime + + - name: Configure CMake + shell: bash + run: | + cmake -S . -B "$BUILD_DIR" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DOPENSTUDIO_APP_VERSION="$VERSION" \ + -DOPENSTUDIO_UPDATE_MANIFEST_URL_VALUE="https://openstudio.org.in/releases/stable/latest.json" \ + -DOPENSTUDIO_UPDATE_APPCAST_URL_VALUE="https://openstudio.org.in/appcast/linux-stable.xml" \ + -DOPENSTUDIO_RELEASES_PAGE_URL_VALUE="https://openstudio.org.in/download" \ + -DOPENSTUDIO_UPDATE_CHANNEL_VALUE="stable" \ + -DOPENSTUDIO_ENABLE_EXTERNAL_PYTHON_AI_FALLBACK=OFF \ + -DFETCHCONTENT_UPDATES_DISCONNECTED=ON + + - name: Build OpenStudio + shell: bash + run: cmake --build "$BUILD_DIR" --config Release --target OpenStudio + + - name: Validate Linux runtime bundle + shell: pwsh + run: ./tools/validate-runtime-bundle.ps1 -Platform linux -BundlePath "$env:BUILD_DIR/OpenStudio_artefacts/Release" -ExpectedVersion $env:VERSION -EnforceLeanBundle + + - name: Run Linux startup self-test + shell: bash + run: | + REPORT="$RUNNER_TEMP/OpenStudio_StartupSelfTest.txt" + xvfb-run -a "$BUILD_DIR/OpenStudio_artefacts/Release/OpenStudio" --startup-self-test --report "$REPORT" + if [ $? -ne 0 ]; then + if [ -f "$REPORT" ]; then + cat "$REPORT" + fi + exit 1 + fi + + - name: Package AppImage + shell: bash + run: | + chmod +x ./tools/package-linux-release.sh + bash ./tools/package-linux-release.sh "$VERSION" "$BUILD_DIR" + + - name: Verify Linux AppImage output + shell: bash + run: test -f "dist/linux/OpenStudio-${VERSION}-linux-x86_64.AppImage" diff --git a/.gitignore b/.gitignore index a809bc3..4896094 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /build /build-*/ /out +/tmp/ /cmake-build-* CMakeCache.txt CMakeFiles @@ -38,6 +39,50 @@ get-pip.py # Bundled Python environment (downloaded per-machine, not committed) /tools/python/ +/tools/ffmpeg.exe +/tools/ffprobe.exe +/tools/sox/ +/tools/praat/ +/tools/audio-analysis-cache/ +/tools/audio-analysis-runtime/ +/tools/openstudio_ace_backend/vendor_runtime/ +/tools/*.zip +/tools/*-download.* +/tools/*-extract/ +/tools/linuxdeploy-*.AppImage + +# Local pitch/render regression outputs +/tmp_pitch_runs/ +/pitch_debug_captures/ +/*_signal_chain_debug/ +/signal_chain_debug/ +/tests/fixtures/pitch-regression/runs/ +/tests/fixtures/pitch-regression/**/*.wav +/tests/fixtures/pitch-regression/**/*.png +/tests/fixtures/pitch-regression/**/*.jsonl +*_render_route_report.json +*_pitch_*_analysis.json +*_reference_match.json +*_parity.json +*_route.json +*_context.wav +*_candidate_context.wav + +# Local exploratory pitch tools. Keep only the lightweight source-owned runner tracked. +/tools/pitch_*.py +/tools/pitch_harness_common.ps1 +/tools/reference_audio_match.py +/tools/ml_restore_benchmark.py +/tools/register-pitch-artifact.ps1 +/tools/run-ui-pitch-*.ps1 +/tools/run-pitch-*.ps1 +!/tools/run-pitch-headless-regression.ps1 +/tools/run-own-engine-parity.ps1 +/tools/rubberband/ +/local_tools/ + +# Linux AppImage build output +*.AppImage # Release artifacts and local publish validation /dist/netlify-release-site/ @@ -94,6 +139,9 @@ get-pip.py /resources/models/*.bin /resources/models/*.ort +# Audio test samples +/audio_test_samples/ + # Logs *.log npm-debug.log* diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..207c02d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,356 @@ +# Studio13-v3 + +A hybrid DAW (Digital Audio Workstation) with a **JUCE C++ backend** for audio processing and a **React/TypeScript frontend** rendered in WebView2. + +## Architecture + +``` +C++ (JUCE) Backend React/TypeScript Frontend +┌─────────────────────┐ ┌──────────────────────────┐ +│ AudioEngine │◄───►│ NativeBridge.ts │ +│ PlaybackEngine │ │ (window.__JUCE__) │ +│ AudioRecorder │ ├──────────────────────────┤ +│ TrackProcessor │ │ useDAWStore.ts (Zustand) │ +│ PluginManager │ ├──────────────────────────┤ +│ MIDIManager │ │ Timeline.tsx (Konva) │ +│ Metronome │ │ MixerPanel / ChannelStrip │ +│ MainComponent │ │ TransportBar / MenuBar │ +│ (WebBrowserComponent) │ FXChainPanel / PianoRoll │ +└─────────────────────┘ └──────────────────────────┘ +``` + +- **C++ backend** handles: audio I/O, recording to disk, clip playback with sample-rate conversion, VST3 plugin hosting, MIDI device management, metering, offline render/export +- **React frontend** handles: all UI, state management (Zustand), canvas-based timeline (Konva/react-konva), keyboard shortcuts, drag-and-drop, project save/load +- **Communication**: synchronous bridge via `window.__JUCE__.backend.*` functions (defined in NativeBridge.ts, exposed in MainComponent.cpp) + +## Directory Structure + +``` +Studio13-v3/ +├── Source/ # C++ backend +│ ├── Main.cpp # JUCE app entry point +│ ├── MainComponent.h/cpp # Hosts WebBrowserComponent + AudioEngine, exposes native functions to JS +│ ├── AudioEngine.h/cpp # Core audio callback, device management, track graph, render, pitch correction entry point +│ ├── PlaybackEngine.h/cpp # Clip playback scheduling, sample-rate-aware mixing, replaceClipAudioFile +│ ├── AudioRecorder.h/cpp # Thread-safe recording via juce::AudioFormatWriter::ThreadedWriter +│ ├── TrackProcessor.h/cpp # Per-track juce::AudioProcessor: metering, FX chain, input monitoring +│ ├── PluginManager.h/cpp # VST3/CLAP/LV2 plugin scanning and loading +│ ├── PluginWindowManager.h/cpp# Native plugin editor window management +│ ├── CLAPPluginFormat.h/cpp # CLAP plugin hosting (parameter discovery, GUI, state) +│ ├── MIDIManager.h/cpp # MIDI device enumeration and input routing +│ ├── MIDIClip.h/cpp # MIDI note event storage and time-range queries +│ ├── Metronome.h/cpp # Click track generation (BPM, time sig, accent patterns) +│ ├── AudioConverter.h/cpp # Channel/sample-rate conversion utilities +│ ├── PeakCache.h/cpp # REAPER-style multi-resolution peak cache (.s13peaks sidecar files) +│ ├── AudioAnalyzer.h/cpp # Audio analysis utilities +│ ├── BuiltInEffects.h/cpp # Built-in audio effects (EQ, compressor, etc.) +│ ├── BuiltInEffects2.h/cpp # Additional built-in effects +│ │ +│ │ # Pitch Editor / Correction Pipeline (see "Pitch Editor Subsystem" below) +│ ├── PitchAnalyzer.h/cpp # YIN monophonic pitch detection, note segmentation, pitchDrift +│ ├── PitchDetector.h/cpp # Low-level YIN pitch detection algorithm +│ ├── PitchMapper.h/cpp # Maps detected pitch to corrected pitch (scale/key snapping) +│ ├── PitchShifter.h/cpp # Phase vocoder pitch shifter (FFT 2048, hop 512, FIFO-based) +│ ├── PitchResynthesizer.h/cpp # Offline graphical pitch correction via native VSF pitch-only renderer +│ ├── FormantPreserver.h/cpp # WORLD vocoder (DIO+StoneMask+CheapTrick+D4C) for formant-preserving pitch shift (fallback) +│ ├── PolyPitchDetector.h/cpp # Polyphonic pitch detection via Basic-Pitch ONNX model +│ ├── PolyResynthesizer.h/cpp # Polyphonic pipeline: STFT→Wiener masks→per-note shift→accumulate→ISTFT +│ ├── HarmonicMaskGenerator.h/cpp # Wiener-filter soft masks at harmonic positions for poly separation +│ ├── SpectralPitchShifter.h/cpp # Phase vocoder on masked spectrograms with cepstral formant preservation +│ ├── SpectralProcessor.h/cpp # STFT/ISTFT utilities for spectral processing +│ ├── S13PitchCorrector.h/cpp # Real-time inline pitch corrector (auto-tune style) +│ │ +│ │ # Plugin System +│ ├── S13FXProcessor.h/cpp # JSFX/Lua script-based audio processor (wraps YSFX) +│ ├── S13FXGfxEditor.h/cpp # JSFX @gfx rendering via juce::Image framebuffer at 30fps +│ ├── S13PluginEditors.h/cpp # Built-in plugin editor windows +│ ├── S13ScriptWindow.h/cpp # Lua gfx API framebuffer window +│ ├── ScriptEngine.h/cpp # Lua scripting engine (sol2) +│ │ +│ │ # Other Features +│ ├── StemSeparator.h/cpp # AI stem separation (vocals/drums/bass/other) +│ └── ARAHostController.h/cpp # ARA plugin hosting controller +│ +├── frontend/ +│ ├── src/ +│ │ ├── App.tsx # Main layout: MenuBar → MainToolbar → workspace(TCP + Timeline) → TransportBar → LowerZone → Modals +│ │ ├── main.tsx # React entry point +│ │ ├── index.css # Tailwind theme with daw-* custom colors +│ │ ├── store/ +│ │ │ ├── useDAWStore.ts # Zustand store — all app state and actions (~3400 lines) +│ │ │ ├── actionRegistry.ts # Centralized action defs for Command Palette, shortcuts, menus +│ │ │ ├── pitchEditorStore.ts # Pitch editor Zustand store: notes, viewport, tools, undo, poly mode +│ │ │ ├── automationParams.ts # Automation parameter definitions +│ │ │ └── commands/ # Undo/redo: CommandManager.ts, TrackCommands.ts, ClipCommands.ts +│ │ ├── services/ +│ │ │ └── NativeBridge.ts # Type-safe bridge to C++ backend (with mock fallbacks for dev) +│ │ ├── utils/ +│ │ │ └── snapToGrid.ts # Musical grid snapping (bar, beat, subdivisions) +│ │ └── components/ +│ │ ├── Timeline.tsx # Konva canvas: clips, waveforms, rulers, zoom, drag, selection (~2100 lines) +│ │ ├── TransportBar.tsx # Bottom bar: play/stop/record, BPM, time display, loop, metronome +│ │ ├── MixerPanel.tsx # Horizontal mixer with ChannelStrip components +│ │ ├── ChannelStrip.tsx # Track volume fader, pan, solo/mute, FX, meter +│ │ ├── ChannelStripEQModal.tsx # Per-channel EQ editor +│ │ ├── TrackHeader.tsx # Track name, arm, solo, mute, input selector +│ │ ├── TrackRoutingModal.tsx # Track I/O routing configuration +│ │ ├── SortableTrackHeader.tsx # @dnd-kit wrapper for track reordering + context menu +│ │ ├── FXChainPanel.tsx # Plugin browser + FX chain management (input FX + track FX) +│ │ ├── PianoRoll.tsx # MIDI note editor (Konva canvas) +│ │ ├── VirtualPianoKeyboard.tsx # 88-key on-screen MIDI keyboard +│ │ ├── MainToolbar.tsx # Top toolbar: transport, undo/redo, snap, mixer toggle, settings +│ │ ├── MenuBar.tsx # File/Edit/View/Insert/Help dropdown menus +│ │ ├── SettingsModal.tsx # Audio device configuration (driver, I/O, sample rate, buffer) +│ │ ├── RenderModal.tsx # Export dialog (format, bit depth, channels, normalize, tail) +│ │ ├── PreferencesModal.tsx # Tabbed prefs: General, Editing, Display, Backup +│ │ ├── ProjectSettingsModal.tsx # Project-level settings +│ │ ├── KeyboardShortcutsModal.tsx # Searchable action/shortcut reference (from actionRegistry) +│ │ ├── CommandPalette.tsx # Ctrl+Shift+P fuzzy action search +│ │ ├── PluginBrowser.tsx # VST3/CLAP/LV2 plugin selection UI +│ │ ├── LowerZone.tsx # Bottom panel container (pitch editor, clip properties, etc.) +│ │ ├── ClipPropertiesPanel.tsx # Clip property inspector +│ │ ├── ClipLauncherView.tsx # Ableton-style clip launcher grid +│ │ │ +│ │ │ # Pitch Editor UI +│ │ ├── PitchCorrectorPanel.tsx # Real-time inline corrector (auto-tune style, key/scale/retune) +│ │ ├── PitchEditorLowerZone.tsx # Graphical pitch editor: canvas host, tools, controls, interaction handlers +│ │ ├── PitchEditorCanvas.ts # Imperative canvas renderer (60fps RAF loop): notes, contour, grid, piano keys +│ │ ├── S13PitchEditor.tsx # Pitch editor wrapper/container +│ │ ├── pitchCorrectorPresets.ts # Preset definitions for real-time pitch corrector +│ │ │ +│ │ │ # Other +│ │ ├── StemSeparationModal.tsx # AI stem separation UI +│ │ ├── EnvelopeManagerModal.tsx # Automation envelope management +│ │ ├── TimecodeSettingsPanel.tsx # Timecode display settings +│ │ ├── ThemeEditor.tsx # Theme customization + REAPER theme import +│ │ ├── GettingStartedGuide.tsx # First-run guide +│ │ ├── HelpOverlay.tsx # Contextual help overlay +│ │ ├── MasterTrackHeader.tsx # Master track control strip +│ │ ├── Playhead.tsx # Timeline playhead cursor +│ │ ├── PeakMeter.tsx # Audio level meters +│ │ ├── icons.tsx # SVG icon components +│ │ ├── menus/ # MenuDropdown.tsx, EditMenu.tsx +│ │ └── ui/ # Base components: Button, Input, Select, NativeSelect, Slider, Modal, Checkbox, Textarea, TimeSignatureInput +│ ├── package.json +│ ├── vite.config.ts +│ └── tsconfig.json +│ +├── tools/ # ffmpeg.exe, stem_separator.py, setup scripts +├── resources/ # ONNX models, presets, resources +├── build/ # CMake build output +├── CMakeLists.txt # C++ build: JUCE 8.0.0, ASIO SDK, WebView2, VST3, ONNX Runtime +├── build.py # Python orchestrator: cmake + npm + vite dev server +├── pitch_corrector_feat_plan.md # Detailed pitch editor implementation plan (Melodyne/RePitch/VariAudio parity) +└── WORKFLOWS.md # Dev workflow docs +``` + +## Build & Dev + +```bash +# Full dev (installs deps, builds C++ Debug, starts Vite HMR, launches app) +python build.py dev --run + +# Frontend only (no C++ rebuild needed for UI changes) +cd frontend && npm run dev + +# C++ rebuild only — Debug (for Source/ changes, skips cmake configure if already done) +cmake --build build --config Debug + +# C++ rebuild only — Release +cmake --build build --config Release + +# Production (builds frontend + Release C++, single .exe with embedded frontend) +python build.py prod +``` + +**No feature flags** — all features (ASIO, WASAPI, DirectSound, VST3 hosting, WebView2) are always enabled via hardcoded `target_compile_definitions` in CMakeLists.txt. The `build.py dev` mode uses Debug config; `build.py prod` uses Release. + +## Key Technical Details + +### State Management +- **Zustand** store in `useDAWStore.ts` holds all application state (~3400 lines) +- `useShallow` selectors prevent unnecessary re-renders +- Undo/redo via CommandManager pattern (TrackCommands, ClipCommands) +- Multi-clip selection: `selectedClipIds: string[]`, multi-track: `selectedTrackIds: string[]` +- Clipboard supports single and multi-clip with track position info +- **Action Registry** (`actionRegistry.ts`): centralized list of all actions with id, name, category, shortcut, execute. Used by CommandPalette, KeyboardShortcutsModal, and menus +- **Modal state pattern**: each modal follows `showX: boolean` + `toggleX()` in store + useShallow selector in App.tsx + keyboard shortcut + menu item + action registry entry + +### Undo/Redo Requirement (IMPORTANT) + +**Every new action/function that modifies clip or track data MUST be tracked via `commandManager.push()` or `commandManager.execute()`.** This includes but is not limited to: + +- Adding, removing, moving, splitting, resizing clips +- Changing clip properties: volume, pan, fades, color, mute, lock, groupId, reverse +- Paste, nudge, quantize, normalize operations +- Time selection operations (cut, delete, insert silence) +- Razor edit content deletion +- Track property changes (name, color, volume, pan, mute, solo, armed) + +**Pattern for adding undo support:** + +1. **Before** the `set()` call, capture old state (snapshot or specific values) +2. **After** the `set()` call, capture new state +3. Call `commandManager.push({ type, description, timestamp, execute: () => set(newState), undo: () => set(oldState) })` +4. Call `set({ canUndo: commandManager.canUndo(), canRedo: commandManager.canRedo() })` + +For **continuous edits** (faders, knobs), use the begin/commit pattern: `beginXEdit()` captures initial state, intermediate `setX()` calls update live without undo, `commitXEdit()` pushes a single undo command covering the full range. + +**Do not skip undo tracking** — users expect Ctrl+Z to undo any data-modifying action. + +### Timeline Rendering +- **Konva** (react-konva) for canvas-based rendering +- Waveform peaks fetched from C++ via `getWaveformPeaks(filePath, samplesPerPixel, numPixels)` — backed by PeakCache (`.s13peaks` files), never reads audio files directly +- `samplesPerPixel` uses the clip's `sampleRate` (not hardcoded) with power-of-2 quantization for cache stability +- Zoom: exponential scaling via `Math.exp(-deltaY * sensitivity)`, anchored to cursor position +- Zoom debounce: suppresses waveform re-fetches during active zoom (`isZoomingRef`, 200ms timeout) +- Scroll debounce: suppresses waveform re-fetches during active scrolling (`isScrollingRef`, 200ms timeout) +- In-flight request dedup: `inFlightRef` prevents duplicate concurrent bridge calls for the same waveform cache key +- Zoom range: 1–1000 pixels/second (clamped in both Timeline.tsx and store's `setZoom`) +- Waveform peak data is parsed from flat C++ arrays into `WaveformPeak[]` objects via `parseFlatPeaks()` in NativeBridge.ts — format: `[numChannels, min_ch0_px0, max_ch0_px0, min_ch1_px0, max_ch1_px0, ...]` + +### Audio Engine +- Sample rate conversion in PlaybackEngine: linear interpolation when file rate != device rate +- Render uses same PlaybackEngine.fillTrackBuffer() — automatically handles rate conversion +- FX plugins: per-track input FX chain + track FX chain + master FX chain +- Render re-prepares plugins for offline block size, restores state after + +### Pitch Editor Subsystem + +The pitch editor enables vocal pitch correction with both real-time (auto-tune style) and graphical (Melodyne-style) modes. The implementation plan for reaching Melodyne/RePitch/VariAudio quality is in `pitch_corrector_feat_plan.md`. + +**Architecture**: +``` +Monophonic graphical pitch pipeline: + PitchAnalyzer (YIN) -> PitchNotes -> AudioEngine::applyPitchCorrection() -> selected native renderer branch + +Graphical offline apply: + Native VSF pitch-only rendering is the production path; historical renderer research lives in docs only. + +Polyphonic pipeline: + PolyPitchDetector (Basic-Pitch ONNX) -> PolyNotes -> HarmonicMaskGenerator (Wiener) -> SpectralPitchShifter -> PolyResynthesizer + +Real-time corrector: + S13PitchCorrector (per-block, key/scale aware) -> inserted as FX plugin on track +``` + +**Key data flow** (graphical editor): +1. User opens pitch editor → `pitchEditorStore.open()` → `nativeBridge.analyzePitchContourDirect()` +2. C++ `PitchAnalyzer` runs YIN detection, returns frames (per-hop pitch data) + notes (segmented note objects) +3. Frontend renders notes as blobs on canvas (`PitchEditorCanvas.ts`), user edits correctedPitch/drift/vibrato/etc. +4. On edit commit (400ms auto-apply debounce) → `nativeBridge.applyPitchCorrection(trackId, clipId, notes)` +5. C++ `AudioEngine::applyPitchCorrection()`: + - Loads original clip audio → extracts window around edited notes + - Runs the selected renderer branch and composites the corrected region back into the clip + - Deletes old output file, writes new rotating output file → `PlaybackEngine::replaceClipAudioFile()` swaps it in (resets `clip.offset = 0`) + +**PitchResynthesizer engine**: graphical offline pitch apply uses the native VSF pitch-only path. Preview/scrub audio and the realtime pitch-corrector FX use Signalsmith Stretch directly. + +**PitchShifter FIFO latency**: The phase vocoder has 2048-sample FIFO latency. The first fftSize output samples are zeros. `PitchResynthesizer` compensates by flushing the FIFO with silence and trimming the latency from the output. + +**WORLD vocoder** (`FormantPreserver`): Kept as fallback engine. Decomposes audio into F0 + spectral envelope + aperiodicity. Mono-only — when used via `processMultiChannel`, creates mono mix and duplicates to all channels. + +**Polyphonic detection**: Uses Spotify's Basic-Pitch ONNX model (22050 Hz, 256 hop). Thresholds are tunable: `noteThreshold=0.15`, `onsetThreshold=0.3`. Onset bypass accepts sustained notes ≥200ms even without detected onset. + +**Frontend state**: `pitchEditorStore.ts` (separate Zustand store from main DAW store) — holds notes, viewport, selectedNoteIds, tools (Select/Pitch/Drift/Vibrato/Transition), undo/redo stack, poly mode toggle, auto-apply debounce. + +### Audio QA Rules + +- Do not claim pitch/formant/stutter/artifact audio quality is fixed from harness metrics alone. User audition is the final veto for subjective audio quality. +- The lightweight pitch harness may assert only deterministic invariants: exact relative semitone request, renderer branch recorded, `formantCurveUsed=false` for pitch-only, route/source state, output file sanity, and explicit no-op render parity when that test is requested. +- Mel, formant, spectrogram, double-voice, naturalness, target-sample closeness, and "entry clean" reports are diagnostic only unless calibrated against user-approved examples. Mark these as `diagnostic_only` or `not_asserted`, never as proof of "done". +- Reference samples are QA oracles only. Production rendering must not read or depend on them. +- Work one audible issue per iteration. If app playback and rendered files disagree, debug route/render state before DSP. If render-only noise exists, pause pitch DSP and fix the render chain first. +- Exact `+4/-4` pitch tests must be relative semitone shifts from the detected note center, not chromatic snap targets. +- The default pitch harness is `tools/run-pitch-headless-regression.ps1`; it must stay headless and must not open an OpenStudio app window. Use visible UI harnesses only with explicit opt-in for UI integration debugging. +- Final audio-work summaries must say what is `pass`, `fail`, `diagnostic_only`, or `not_asserted`; never say "fixed", "natural", or "close" unless the user auditioned that exact artifact and agreed. + +### Audio Thread Safety (REAPER-inspired) + +- **PlaybackEngine::fillTrackBuffer()** uses `ScopedTryLock` (not `ScopedLock`) — returns silence if the message thread holds the lock (adding/removing clips). This is rare and inaudible. Never use blocking locks on the audio thread. +- **Pre-allocated buffers**: `PlaybackEngine` has a `reusableFileBuffer` member that is reused across clips/callbacks. Never heap-allocate (`new`, `AudioBuffer(...)`) on the audio thread. +- **Pre-loaded readers**: `addClip()` calls `preloadReader()` on the message thread so `AudioFormatReader` objects are cached before the audio thread needs them. The audio thread uses `getCachedReader()` which only does a map lookup — never creates readers or does disk I/O. +- **Cached pan gains**: `TrackProcessor` pre-computes `cos`/`sin` pan gains as `std::atomic` (`cachedPanL`, `cachedPanR`) when `setPan()` or `setVolume()` is called on the message thread. `processBlock()` on the audio thread loads these atomics cheaply — no trig computation per callback. +- **AudioRecorder::writeBlock()** also uses `ScopedTryLock` — same pattern. + +### PeakCache System (.s13peaks) + +- REAPER-inspired multi-resolution peak cache stored as `.s13peaks` sidecar files alongside audio files +- 4 mipmap levels at strides: 64, 256, 1024, 4096 samples per peak +- File format: `PeakFileHeader` (magic `0x53313350` / "S13P", version, source file size/timestamp for invalidation, sample rate, channels, level count) followed by flat float arrays per level +- `AudioEngine::getWaveformPeaks()` reads from PeakCache — never reads audio files directly. First call generates the cache synchronously; subsequent calls are instant (memory-cached mipmap lookup) +- Peak generation is triggered automatically in the background when recording stops (`peakCache.generateAsync()` for each completed clip) +- `PeakCache::buildPeaks()` reads the audio file in a single pass, computing all 4 mipmap levels simultaneously using per-level accumulators +- Background generation uses a `juce::ThreadPool` with 1 thread; completion callbacks fire on the message thread via `juce::MessageManager::callAsync` + +### Bridge Pattern +- `NativeBridge.ts` wraps `window.__JUCE__.backend.*` calls +- All methods have mock fallbacks for frontend-only development +- Real-time metering via event system (`addEventListener`/`removeEventListener`) +- Async: all bridge calls return Promises + +### Theme +- Tailwind CSS v4 with custom `daw-*` color tokens defined in `index.css` +- Dark theme: `daw-dark` (#121212), `daw-panel` (#1a1a1a), `daw-accent` (#0078d4) +- Semantic colors: `daw-record` (red), `daw-mute` (green), `daw-solo` (yellow), `daw-fx` (lime) +- UI components in `components/ui/` use variant pattern (default, primary, success, danger, etc.) + +## When You're Stuck + +**If you encounter the same error in a loop (3+ attempts) or need admin/system-level access you don't have (installing software, modifying system PATH, registry, etc.), STOP and immediately ask the user for help.** Do not keep retrying the same failing approach. Clearly explain: +1. What you're trying to do +2. What's failing and why +3. What the user needs to do (step by step) + +## Coding Preferences + +- Prefer targeted, minimal fixes over large refactors +- Frontend changes don't require C++ rebuild — just refresh the WebView +- C++ changes require `cmake --build build --config Debug` (or Release) +- C++ builds should compile with **zero warnings** (`/W4` is enabled) — use `juce::ignoreUnused()` for required-but-unused params, avoid C macro name collisions, use `const auto&` for rvalue refs +- TypeScript has some pre-existing errors in MenuBar, Playhead, ProjectSettingsModal, TrackHeader — these are known +- Use `npx tsc --noEmit` to check for new TS errors after changes + +## Known Pitfalls & Past Issues + +### useShallow Is Required for ALL useDAWStore Consumers + +**Every** component that calls `useDAWStore()` must use a `useShallow` selector. The RAF-based `setCurrentTime` loop fires at 60fps during playback, so any bare `useDAWStore()` (no selector) causes 60fps re-renders. With N tracks this multiplies. Always use `useShallow((s) => ({ ... }))` and only pick the fields you need. + +### Meter State Must Be Isolated from Tracks Array + +`meterLevels` and `peakLevels` are separate top-level maps in the store, NOT inside `tracks[]`. Never return a new `tracks` reference when only meters changed — otherwise every track-dependent component re-renders at 60fps. + +### syncBackend Must Pass Explicit Track IDs + +`App.tsx`'s `syncBackend` must call `nativeBridge.addTrack(track.id)` with the Zustand track ID. Without the ID, C++ creates tracks with auto-generated UUIDs that don't match Zustand's — causing `setTrackRecordArm` etc. to silently fail. + +### Dual Clamping on Zoom + +Zoom (`pixelsPerSecond`) is clamped in **two places**: `Timeline.tsx` constants AND `setZoom` in `useDAWStore.ts`. They must agree. + +### Konva Event Bubbling & Click Handlers + +In Konva, `onMouseDown` fires before `onClick`. Handle all selection logic in `onMouseDown` with modifier keys, not in `onClick`. To detect "background click" for deselection, check `e.target.name() === "timeline-bg"` — don't use `!targetName` as unnamed clip shapes also match. + +### Waveform samplesPerPixel Must Use File's Sample Rate + +`samplesPerPixel` must use the **file's** sample rate (`clip.sampleRate`), not a hardcoded 44100. For trimmed clips (`clip.offset > 0`), `numPeaks` must cover `(clip.offset + clip.duration) * sampleRate / samplesPerPixel`, not just `clip.duration` — peak data always starts from sample 0. + +### PlaybackEngine Sample Rate Mismatch + +`fillTrackBuffer()` uses the file's own `reader->sampleRate` for sample positions and linearly interpolates to the device rate. Without this, files at different rates play at wrong pitch/speed. + +### Plugin Channel Safety & Render State + +Some VST3 plugins (e.g., Amplitube) expect specific channel counts — `TrackProcessor` and render path expand buffers before `processBlock()` if needed (`safeRenderFX` lambda). The render path must also re-prepare all FX plugins with render block size (512) and `reset()` them, then restore original state after. Without this, plugins overflow internal buffers and produce noise. + +### Render Modal — What Actually Works in Backend + +- **Working**: format (wav/aiff/flac), bit depth (16/24/32), channels (stereo/mono), normalize, tail +- **Ignored**: sample rate (always renders at device rate) +- **Not implemented**: "selected_tracks"/"stems" source (always master mix), dither + +### C++ Naming Conflicts with C Standard Library Macros + +Never use `stderr`, `stdout`, `stdin`, or `errno` as C++ variable names — they are macros. Use names like `errOutput` instead. diff --git a/CLAUDE.md b/CLAUDE.md index 2994cfb..74d497e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,8 +51,7 @@ Studio13-v3/ │ ├── PitchDetector.h/cpp # Low-level YIN pitch detection algorithm │ ├── PitchMapper.h/cpp # Maps detected pitch to corrected pitch (scale/key snapping) │ ├── PitchShifter.h/cpp # Phase vocoder pitch shifter (FFT 2048, hop 512, FIFO-based) -│ ├── PitchResynthesizer.h/cpp # Offline pitch correction: builds correction curve, applies via RubberBand (default), WORLD, or phase vocoder -│ ├── RubberBandShifter.h/cpp # Rubber Band Library R3 wrapper: multi-channel, per-block pitch ratios, formant preservation +│ ├── PitchResynthesizer.h/cpp # Offline graphical pitch correction via native VSF pitch-only renderer │ ├── FormantPreserver.h/cpp # WORLD vocoder (DIO+StoneMask+CheapTrick+D4C) for formant-preserving pitch shift (fallback) │ ├── PolyPitchDetector.h/cpp # Polyphonic pitch detection via Basic-Pitch ONNX model │ ├── PolyResynthesizer.h/cpp # Polyphonic pipeline: STFT→Wiener masks→per-note shift→accumulate→ISTFT @@ -222,11 +221,8 @@ The pitch editor enables vocal pitch correction with both real-time (auto-tune s **Architecture**: ``` -Monophonic pipeline (default — Rubber Band): - PitchAnalyzer (YIN) → PitchNotes → PitchResynthesizer::processMultiChannel() → RubberBandShifter (R3 engine) - -Monophonic pipeline (fallback — WORLD): - PitchAnalyzer (YIN) → PitchNotes → PitchResynthesizer::process() → FormantPreserver (WORLD vocoder) +Monophonic graphical pitch pipeline: + PitchAnalyzer (YIN) → PitchNotes → PitchResynthesizer::processMultiChannel() → native VSF pitch-only renderer Polyphonic pipeline: PolyPitchDetector (Basic-Pitch ONNX) → PolyNotes → HarmonicMaskGenerator (Wiener) → SpectralPitchShifter → PolyResynthesizer @@ -242,13 +238,10 @@ Real-time corrector: 4. On edit commit (400ms auto-apply debounce) → `nativeBridge.applyPitchCorrection(trackId, clipId, notes)` 5. C++ `AudioEngine::applyPitchCorrection()`: - Loads original clip audio → extracts window around edited notes - - Passes all channels to `PitchResynthesizer::processMultiChannel()` with Rubber Band (native stereo) - - Crossfades corrected window back into clip at splice points (512-sample crossfade) + - Runs the native VSF pitch-only renderer and composites the corrected region back into the clip - Deletes old output file, writes new rotating output file → `PlaybackEngine::replaceClipAudioFile()` swaps it in (resets `clip.offset = 0`) -**Rubber Band Library** (default pitch engine): Uses R3 (Finer) engine in RealTime mode with `OptionPitchHighQuality`, `OptionFormantPreserved`, `OptionChannelsTogether`. RealTime mode is required for per-block `setPitchScale()` calls (Offline mode rejects mid-processing pitch changes). Has latency compensation: flush with silence after input, skip first `getLatency()` output samples. Built from `single/RubberBandSingle.cpp` with `USE_KISSFFT` + `NOMINMAX`. GPL v2+ license. - -**PitchResynthesizer engines** (`PitchEngine` enum): `RubberBand` (default, native stereo, high quality), `WorldVocoder` (WORLD, mono-only, speech-oriented fallback), `PhaseVocoder` (basic, no formant preservation). +**PitchResynthesizer engine** (`PitchEngine` enum): native VSF pitch-only offline apply. Preview/scrub audio and the realtime pitch-corrector FX use Signalsmith Stretch directly. **PitchShifter FIFO latency**: The phase vocoder has 2048-sample FIFO latency. The first fftSize output samples are zeros. `PitchResynthesizer` compensates by flushing the FIFO with silence and trimming the latency from the output. diff --git a/CMakeLists.txt b/CMakeLists.txt index c16f8f7..853d078 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -166,6 +166,8 @@ juce_add_gui_app(OpenStudio VERSION ${PROJECT_VERSION} ICON_BIG "${CMAKE_CURRENT_SOURCE_DIR}/assets/icon-256x256.png" ICON_SMALL "${CMAKE_CURRENT_SOURCE_DIR}/assets/icon-16x16.png" + MICROPHONE_PERMISSION_ENABLED TRUE + MICROPHONE_PERMISSION_TEXT "OpenStudio needs access to your audio input to record from microphones and audio interfaces." ) # Source files @@ -207,11 +209,13 @@ target_sources(OpenStudio PRIVATE Source/S13PitchCorrector.cpp Source/PitchAnalyzer.cpp Source/PitchResynthesizer.cpp - Source/SignalsmithShifter.cpp + Source/LpcEnvelopeTransfer.cpp + Source/OwnPitchEngine.cpp Source/TriggerEngine.cpp Source/SessionInterchange.cpp Source/PolyPitchDetector.cpp Source/StemSeparator.cpp + Source/AITrackEngine.cpp Source/ARAHostController.cpp ) @@ -292,7 +296,7 @@ if(WIN32) elseif(APPLE) set(_OPENSTUDIO_DEFAULT_APPCAST_URL "https://openstudio.org.in/appcast/macos-stable.xml") else() - set(_OPENSTUDIO_DEFAULT_APPCAST_URL "") + set(_OPENSTUDIO_DEFAULT_APPCAST_URL "https://openstudio.org.in/appcast/linux-stable.xml") endif() set(OPENSTUDIO_UPDATE_APPCAST_URL_VALUE "${_OPENSTUDIO_DEFAULT_APPCAST_URL}" CACHE STRING "Updater appcast URL compiled into OpenStudio") @@ -392,9 +396,15 @@ endif() # ============================================================================== if(UNIX AND NOT APPLE) find_package(PkgConfig REQUIRED) + find_package(CURL REQUIRED) pkg_check_modules(ALSA alsa) pkg_check_modules(JACK jack) - pkg_check_modules(WEBKIT webkit2gtk-4.0) + # webkit2gtk-4.1 is standard on Ubuntu 24.04+; fall back to 4.0 for older distros + pkg_check_modules(WEBKIT webkit2gtk-4.1) + if(NOT WEBKIT_FOUND) + pkg_check_modules(WEBKIT webkit2gtk-4.0) + endif() + pkg_check_modules(GTK gtk+-3.0) if(ALSA_FOUND) target_link_libraries(OpenStudio PRIVATE ${ALSA_LIBRARIES}) target_include_directories(OpenStudio PRIVATE ${ALSA_INCLUDE_DIRS}) @@ -407,6 +417,11 @@ if(UNIX AND NOT APPLE) target_link_libraries(OpenStudio PRIVATE ${WEBKIT_LIBRARIES}) target_include_directories(OpenStudio PRIVATE ${WEBKIT_INCLUDE_DIRS}) endif() + if(GTK_FOUND) + target_link_libraries(OpenStudio PRIVATE ${GTK_LIBRARIES}) + target_include_directories(OpenStudio PRIVATE ${GTK_INCLUDE_DIRS}) + endif() + target_link_libraries(OpenStudio PRIVATE CURL::libcurl) endif() # Generate the JuceHeader.h @@ -419,6 +434,14 @@ else() target_compile_options(OpenStudio PRIVATE -Wall -Wextra) endif() +# On Linux, set RPATH to $ORIGIN so the binary finds libonnxruntime.so etc. next to it +if(UNIX AND NOT APPLE) + set_target_properties(OpenStudio PROPERTIES + INSTALL_RPATH "$ORIGIN" + BUILD_WITH_INSTALL_RPATH TRUE + ) +endif() + # ============================================================================== # Post-build copy steps # ============================================================================== @@ -528,13 +551,31 @@ if(EXISTS "${SCRIPTS_DIR}") ) endif() -# Copy ONNX Runtime DLL + models to output directory +# Copy ONNX Runtime shared library + models to output directory if(HAS_ONNXRUNTIME AND WIN32) openstudio_copy_runtime_file( "${ONNXRUNTIME_ROOT}/lib/onnxruntime.dll" "onnxruntime.dll" "Copying ONNX Runtime DLL to output directory" ) +elseif(HAS_ONNXRUNTIME AND UNIX AND NOT APPLE) + file(GLOB OPENSTUDIO_ONNXRUNTIME_LINUX_LIBS "${ONNXRUNTIME_ROOT}/lib/libonnxruntime.so*") + foreach(OPENSTUDIO_ONNXRUNTIME_LINUX_LIB IN LISTS OPENSTUDIO_ONNXRUNTIME_LINUX_LIBS) + get_filename_component(OPENSTUDIO_ONNXRUNTIME_LINUX_LIB_NAME "${OPENSTUDIO_ONNXRUNTIME_LINUX_LIB}" NAME) + openstudio_copy_runtime_file( + "${OPENSTUDIO_ONNXRUNTIME_LINUX_LIB}" + "${OPENSTUDIO_ONNXRUNTIME_LINUX_LIB_NAME}" + "Copying ONNX Runtime Linux shared library ${OPENSTUDIO_ONNXRUNTIME_LINUX_LIB_NAME} to output directory" + ) + endforeach() + if(EXISTS "${ONNXRUNTIME_ROOT}/lib/libonnxruntime.so" AND NOT EXISTS "${ONNXRUNTIME_ROOT}/lib/libonnxruntime.so.1") + add_custom_command(TARGET OpenStudio POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${ONNXRUNTIME_ROOT}/lib/libonnxruntime.so" + "${OPENSTUDIO_RUNTIME_ASSET_ROOT}/libonnxruntime.so.1" + COMMENT "Copying ONNX Runtime Linux SONAME fallback libonnxruntime.so.1 to output directory" + ) + endif() endif() # Copy required core ML model(s) to output directory @@ -582,6 +623,56 @@ if(EXISTS "${AI_RUNTIME_PROBE_SCRIPT}") ) endif() +set(AI_MUSIC_GENERATION_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/tools/generate_music.py") +if(EXISTS "${AI_MUSIC_GENERATION_SCRIPT}") + openstudio_copy_runtime_file( + "${AI_MUSIC_GENERATION_SCRIPT}" + "scripts/generate_music.py" + "Copying music generation helper script to output directory" + ) +endif() + +set(AI_OPENSTUDIO_ACE_RUNNER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/tools/openstudio_ace_runner.py") +if(EXISTS "${AI_OPENSTUDIO_ACE_RUNNER_SCRIPT}") + openstudio_copy_runtime_file( + "${AI_OPENSTUDIO_ACE_RUNNER_SCRIPT}" + "scripts/openstudio_ace_runner.py" + "Copying OpenStudio ACE runner script to output directory" + ) +endif() + +set(AI_OPENSTUDIO_ACE_BACKEND_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tools/openstudio_ace_backend") +if(EXISTS "${AI_OPENSTUDIO_ACE_BACKEND_DIR}") + openstudio_copy_runtime_directory( + "${AI_OPENSTUDIO_ACE_BACKEND_DIR}" + "scripts/openstudio_ace_backend" + "Copying OpenStudio ACE backend package to output directory" + ) +endif() + +set(WINDOWS_AI_ACCELERATION_MANIFEST "${CMAKE_CURRENT_SOURCE_DIR}/tools/windows-ai-acceleration-manifest.json") +if(EXISTS "${WINDOWS_AI_ACCELERATION_MANIFEST}") + openstudio_copy_runtime_file( + "${WINDOWS_AI_ACCELERATION_MANIFEST}" + "scripts/windows-ai-acceleration-manifest.json" + "Copying Windows AI acceleration manifest to output directory" + ) +endif() + +# Copy Linux AI backend install plans to output directory +if(UNIX AND NOT APPLE) + foreach(LINUX_PLAN_FILE "ai-runtime-install-plan-linux-cuda.json" "ai-runtime-install-plan-linux-rocm.json") + set(LINUX_PLAN_SRC "${CMAKE_CURRENT_SOURCE_DIR}/tools/${LINUX_PLAN_FILE}") + if(EXISTS "${LINUX_PLAN_SRC}") + openstudio_copy_runtime_file( + "${LINUX_PLAN_SRC}" + "scripts/${LINUX_PLAN_FILE}" + "Copying Linux AI backend plan ${LINUX_PLAN_FILE} to output directory" + ) + endif() + endforeach() +endif() + # Remove legacy bundled AI runtime artifacts from reused build directories. openstudio_remove_runtime_path( "python" @@ -599,6 +690,28 @@ openstudio_remove_runtime_file( "models/download_checks.json" "Removing legacy bundled stem model download manifest from output directory" ) +openstudio_remove_runtime_file( + "scripts/ace_step_parity_replay.py" + "Removing ACE-Step parity replay script from output directory" +) +openstudio_remove_runtime_file( + "scripts/compare_ace_parity_artifacts.py" + "Removing ACE-Step parity comparison script from output directory" +) +openstudio_remove_runtime_file( + "scripts/run_ace_step_parity_capture.ps1" + "Removing ACE-Step parity capture helper from output directory" +) +openstudio_remove_runtime_file( + "scripts/openstudio_ace_backend/parity_debug.py" + "Removing ACE-Step parity backend helper from output directory" +) +set(LEGACY_EXTERNAL_ACE_RUNNER_PATH "scripts/co") +string(APPEND LEGACY_EXTERNAL_ACE_RUNNER_PATH "mfy_ace_runner.py") +openstudio_remove_runtime_file( + "${LEGACY_EXTERNAL_ACE_RUNNER_PATH}" + "Removing legacy external ACE runner from output directory" +) # Copy packaged frontend assets for release/offline builds set(FRONTEND_DIST_DIR "${CMAKE_CURRENT_SOURCE_DIR}/frontend/dist") diff --git a/README.md b/README.md index 6908e9a..46f23ea 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ C++ (JUCE) Backend React/TypeScript Frontend │ Metronome │ │ MixerPanel / ChannelStrip │ │ PitchAnalyzer │ │ PitchEditorLowerZone │ │ PitchResynthesizer │ │ PianoRoll / FXChainPanel │ -│ SignalsmithShifter │ │ TransportBar / MenuBar │ +│ Pitch preview/scrub │ │ TransportBar / MenuBar │ │ PitchCorrector │ │ PluginBrowser / RenderModal │ │ StemSeparator │ └──────────────────────────────┘ │ ARAHostController │ @@ -124,9 +124,8 @@ flowchart TD direction TB YIN[PitchAnalyzer\nYIN Detection · Note Segmentation] NOTES[PitchNotes\npitch · formant · vibrato · drift per note] - RSYNTH[PitchResynthesizer\nbuildCorrectionCurve · buildFormantCurve] - SS[SignalsmithShifter\nper-block setTransposeFactor · setFormantFactor] - XF[Crossfade Splice\n512-sample overlap-add] + VSF[Native VSF HQ\npitch-only note render] + PREVIEW[Signalsmith Stretch\nlive scrub/fast preview] end subgraph Polyphonic Path @@ -140,7 +139,9 @@ flowchart TD RTFX[PitchCorrector\nper-block · key/scale-aware\nSignalsmithStretch presetCheaper] end - AUDIO --> YIN --> NOTES --> RSYNTH --> SS --> XF --> OUT[Corrected Audio File\nreplaceClipAudioFile] + AUDIO --> YIN --> NOTES --> BACKEND --> RB --> OUT[Corrected Audio File\nreplaceClipAudioFile] + BACKEND --> NATIVE + BACKEND --> SS --> SPEAKERS AUDIO --> POLY --> MASKS --> SPECSHIFT --> OUT AUDIO --> RTFX --> SPEAKERS[Live Output] ``` @@ -207,7 +208,7 @@ sequenceDiagram UI->>NB: applyPitchCorrection(trackId, clipId, notes) NB->>MC: backend.applyPitchCorrection(json) MC->>AE: applyPitchCorrection(...) - AE->>AE: PitchResynthesizer → SignalsmithShifter + AE->>AE: native VSF note-HQ pitch-only render AE->>AE: replaceClipAudioFile(newPath) AE-->>MC: newFilePath MC-->>UI: newFilePath (React re-renders waveform) @@ -241,7 +242,7 @@ sequenceDiagram ### Pitch Editor - **Graphical pitch editor** (Melodyne/VariAudio-style) — analyze, display, and redraw pitch curves per note - **Real-time auto-tune** corrector (built-in pitch corrector FX) — key/scale-aware, inserted as an FX plugin -- **Signalsmith Stretch** (MIT) as the default pitch engine — native stereo, formant-preserving, offline quality +- **Native VSF pitch apply** for graphical note-HQ pitch-only edits; Signalsmith Stretch remains limited to live scrub/fast preview and the realtime pitch-corrector FX. - **Polyphonic pitch detection** via Spotify's Basic-Pitch ONNX model - Formant shift, vibrato, drift, and transition controls per note @@ -326,8 +327,7 @@ python build.py prod | Library | Purpose | |---------|---------| | [JUCE 8](https://juce.com/) | Audio engine, plugin hosting, WebBrowserComponent | -| [Signalsmith Stretch](https://github.com/Signalsmith-Audio/signalsmith-stretch) | MIT — pitch shifting with formant preservation | -| [RubberBand](https://breakfastquay.com/rubberband/) | R3 engine, alternative pitch shifter | +| [Signalsmith Stretch](https://github.com/Signalsmith-Audio/signalsmith-stretch) | MIT — pitch preview/scrub and realtime pitch-corrector FX | | [YSFX](https://github.com/jpcima/ysfx) | JSFX / Lua script processor | | [ONNX Runtime](https://onnxruntime.ai/) | Basic-Pitch polyphonic pitch detection | | [sol2](https://github.com/ThePhD/sol2) | Lua scripting engine bindings | @@ -366,7 +366,7 @@ repo-root/ ### In Progress - [ ] Graphical pitch editor — Melodyne/VariAudio parity (vibrato tool, multi-note operations) -- [ ] Polyphonic pitch correction rewrite with Signalsmith Stretch +- [ ] Polyphonic pitch correction rewrite - [ ] ARA2 deep integration ### Planned @@ -378,7 +378,7 @@ repo-root/ ### Completed - [x] VST3 / CLAP / LV2 plugin hosting - [x] ARA plugin hosting controller -- [x] Signalsmith Stretch pitch engine (formant-preserving, native stereo) +- [x] Signalsmith Stretch preview/scrub and realtime pitch-corrector support - [x] Real-time auto-tune pitch corrector - [x] Polyphonic pitch detection (Basic-Pitch ONNX) - [x] PRO DAW-style multi-resolution peak cache diff --git a/Source/AITrackEngine.cpp b/Source/AITrackEngine.cpp new file mode 100644 index 0000000..4299181 --- /dev/null +++ b/Source/AITrackEngine.cpp @@ -0,0 +1,1345 @@ +#include "AITrackEngine.h" + +namespace +{ +constexpr auto kPinnedMusicGenerationModelId = "acestep-v15-xl-turbo"; +constexpr auto kReaderSleepMs = 50; +constexpr auto kWorkerStartupTimeoutMs = 45000; +constexpr auto kWorkerRequestTimeoutMs = 10000; +constexpr auto kWorkerProtocolVersion = 2; +constexpr auto kMaxFramedPayloadBytes = 8 * 1024 * 1024; +constexpr auto kColdDecodeStallTimeoutMs = 90000; + +juce::String createSafeMusicGenerationTimestamp() +{ + const auto now = juce::Time::getCurrentTime(); + const auto milliseconds = juce::String(static_cast (now.toMilliseconds() % 1000)) + .paddedLeft('0', 3); + return now.formatted("%Y%m%d_%H%M%S") + "_" + milliseconds; +} + +juce::File getApplicationRuntimeDirectory() +{ + auto executableDir = juce::File::getSpecialLocation(juce::File::currentExecutableFile) + .getParentDirectory(); + + #if JUCE_MAC + auto resourcesDir = executableDir.getSiblingFile("Resources"); + if (resourcesDir.isDirectory()) + return resourcesDir; + #endif + + return executableDir; +} + +juce::File findPythonInRuntimeRoot(const juce::File& runtimeRoot) +{ + if (! runtimeRoot.isDirectory()) + return {}; + + #if JUCE_WINDOWS + for (const auto& relativePath : { + juce::String("python.exe"), + juce::String("python/python.exe"), + juce::String("Scripts/python.exe") + }) + { + auto candidate = runtimeRoot.getChildFile(relativePath); + if (candidate.existsAsFile()) + return candidate; + } + #else + for (const auto& relativePath : { + juce::String("python3"), + juce::String("python/bin/python3"), + juce::String("bin/python3"), + juce::String("python/bin/python"), + juce::String("bin/python") + }) + { + auto candidate = runtimeRoot.getChildFile(relativePath); + if (candidate.existsAsFile()) + return candidate; + } + #endif + + return {}; +} + +juce::String truncateForLog(const juce::String& text, int maxCharacters = 240) +{ + auto normalized = text.trim(); + if (normalized.length() <= maxCharacters) + return normalized; + return normalized.substring(0, maxCharacters) + "..."; +} + +juce::String computeScriptVersion(const juce::File& script) +{ + if (! script.existsAsFile()) + return {}; + + juce::MemoryBlock scriptBytes; + if (! script.loadFileAsData(scriptBytes)) + return {}; + + return juce::MD5(scriptBytes.getData(), scriptBytes.getSize()).toHexString().substring(0, 16); +} + +bool killWindowsProcessTree(int pid, juce::String* output = nullptr) +{ + #if JUCE_WINDOWS + if (pid <= 0) + return false; + + juce::StringArray command; + command.add("taskkill"); + command.add("/PID"); + command.add(juce::String(pid)); + command.add("/F"); + command.add("/T"); + + juce::ChildProcess killer; + if (! killer.start(command, juce::ChildProcess::wantStdOut | juce::ChildProcess::wantStdErr)) + return false; + + killer.waitForProcessToFinish(5000); + if (output != nullptr) + *output = killer.readAllProcessOutput().trim(); + return true; + #else + juce::ignoreUnused(pid, output); + return false; + #endif +} + +bool writeSocketFully(juce::StreamingSocket& socket, const char* data, int totalBytes, int timeoutMs) +{ + auto bytesWritten = 0; + const auto deadline = juce::Time::currentTimeMillis() + timeoutMs; + + while (bytesWritten < totalBytes && juce::Time::currentTimeMillis() < deadline) + { + if (socket.waitUntilReady(false, 250) <= 0) + continue; + + const auto chunkBytes = socket.write(data + bytesWritten, totalBytes - bytesWritten); + if (chunkBytes <= 0) + return false; + + bytesWritten += chunkBytes; + } + + return bytesWritten == totalBytes; +} + +bool readSocketFully(juce::StreamingSocket& socket, void* destination, int totalBytes, int timeoutMs) +{ + auto* writePtr = static_cast (destination); + auto bytesRead = 0; + const auto deadline = juce::Time::currentTimeMillis() + timeoutMs; + + while (bytesRead < totalBytes && juce::Time::currentTimeMillis() < deadline) + { + if (socket.waitUntilReady(true, 250) <= 0) + continue; + + const auto chunkBytes = socket.read(writePtr + bytesRead, totalBytes - bytesRead, false); + if (chunkBytes <= 0) + return false; + + bytesRead += chunkBytes; + } + + return bytesRead == totalBytes; +} + +bool writeFramedJson(juce::StreamingSocket& socket, + const juce::var& payload, + int timeoutMs, + int& payloadBytesWritten) +{ + auto json = juce::JSON::toString(payload, false); + payloadBytesWritten = static_cast (json.getNumBytesAsUTF8()); + if (payloadBytesWritten <= 0 || payloadBytesWritten > kMaxFramedPayloadBytes) + return false; + + char header[4] {}; + header[0] = static_cast ((payloadBytesWritten >> 24) & 0xFF); + header[1] = static_cast ((payloadBytesWritten >> 16) & 0xFF); + header[2] = static_cast ((payloadBytesWritten >> 8) & 0xFF); + header[3] = static_cast (payloadBytesWritten & 0xFF); + + return writeSocketFully(socket, header, 4, timeoutMs) + && writeSocketFully(socket, json.toRawUTF8(), payloadBytesWritten, timeoutMs); +} + +juce::var readFramedJson(juce::StreamingSocket& socket, int timeoutMs, int& payloadBytesRead) +{ + char header[4] {}; + payloadBytesRead = 0; + if (! readSocketFully(socket, header, 4, timeoutMs)) + return {}; + + payloadBytesRead = ((static_cast (header[0]) << 24) + | (static_cast (header[1]) << 16) + | (static_cast (header[2]) << 8) + | static_cast (header[3])); + + if (payloadBytesRead <= 0 || payloadBytesRead > kMaxFramedPayloadBytes) + return {}; + + juce::HeapBlock buffer(static_cast (payloadBytesRead + 1)); + zeromem(buffer.get(), static_cast (payloadBytesRead + 1)); + if (! readSocketFully(socket, buffer.get(), payloadBytesRead, timeoutMs)) + return {}; + + return juce::JSON::parse(juce::String::fromUTF8(buffer.get(), payloadBytesRead)); +} + +juce::String buildCommandLineForLog(const juce::StringArray& command) +{ + juce::StringArray escaped; + for (const auto& part : command) + { + if (part.containsAnyOf(" \t\"")) + escaped.add(part.quoted()); + else + escaped.add(part); + } + return escaped.joinIntoString(" "); +} +} + +AITrackEngine::~AITrackEngine() +{ + stopWorker(true, false); +} + +juce::File AITrackEngine::getUserDataRoot() const +{ + #if JUCE_WINDOWS + const auto localAppData = juce::SystemStats::getEnvironmentVariable("LOCALAPPDATA", {}); + if (localAppData.isNotEmpty()) + return juce::File(localAppData).getChildFile("OpenStudio"); + #elif JUCE_MAC + return juce::File::getSpecialLocation(juce::File::userHomeDirectory) + .getChildFile("Library") + .getChildFile("Application Support") + .getChildFile("OpenStudio"); + #elif JUCE_LINUX + const auto xdgDataHome = juce::SystemStats::getEnvironmentVariable("XDG_DATA_HOME", {}); + if (xdgDataHome.isNotEmpty()) + return juce::File(xdgDataHome).getChildFile("OpenStudio"); + return juce::File::getSpecialLocation(juce::File::userHomeDirectory) + .getChildFile(".local") + .getChildFile("share") + .getChildFile("OpenStudio"); + #endif + + return juce::File::getSpecialLocation(juce::File::userApplicationDataDirectory) + .getChildFile("OpenStudio"); +} + +juce::File AITrackEngine::getUserRuntimeRoot() const +{ + return getUserDataRoot().getChildFile("stem-runtime"); +} + +juce::File AITrackEngine::getMusicGenerationCheckpointRoot() const +{ + return juce::File::getSpecialLocation(juce::File::userHomeDirectory) + .getChildFile(".cache") + .getChildFile("ace-step") + .getChildFile("checkpoints"); +} + +juce::File AITrackEngine::findPython() const +{ + auto runtimePython = findPythonInRuntimeRoot(getUserRuntimeRoot()); + if (runtimePython.existsAsFile()) + return runtimePython; + + auto bundledRuntime = getApplicationRuntimeDirectory().getChildFile("python"); + runtimePython = findPythonInRuntimeRoot(bundledRuntime); + if (runtimePython.existsAsFile()) + return runtimePython; + + #if JUCE_WINDOWS + juce::ChildProcess probe; + if (probe.start("where python") && probe.waitForProcessToFinish(3000)) + { + auto output = probe.readAllProcessOutput().trim(); + if (output.isNotEmpty()) + { + auto firstLine = output.upToFirstOccurrenceOf("\n", false, false).trim(); + juce::File systemPython(firstLine); + if (systemPython.existsAsFile()) + return systemPython; + } + } + #else + juce::ChildProcess probe; + if (probe.start("which python3") && probe.waitForProcessToFinish(3000)) + { + auto output = probe.readAllProcessOutput().trim(); + if (output.isNotEmpty()) + { + juce::File systemPython(output); + if (systemPython.existsAsFile()) + return systemPython; + } + } + #endif + + return {}; +} + +juce::File AITrackEngine::findScript() const +{ + const auto runtimeDir = getApplicationRuntimeDirectory(); + const auto packagedScript = runtimeDir.getChildFile("scripts").getChildFile("generate_music.py"); + if (packagedScript.existsAsFile()) + return packagedScript; + + const auto bundledDevScript = runtimeDir.getChildFile("tools").getChildFile("generate_music.py"); + if (bundledDevScript.existsAsFile()) + return bundledDevScript; + + const auto workingCopyScript = juce::File::getCurrentWorkingDirectory() + .getChildFile("tools") + .getChildFile("generate_music.py"); + if (workingCopyScript.existsAsFile()) + return workingCopyScript; + + return {}; +} + +void AITrackEngine::cleanupLegacyWorkerProcesses(const juce::File& python, const juce::File& script) const +{ + #if JUCE_WINDOWS + auto escapedScriptPath = script.getFullPathName().replace("'", "''"); + auto escapedPythonPath = python.getFullPathName().replace("'", "''"); + juce::StringArray command; + command.add("powershell"); + command.add("-NoProfile"); + command.add("-ExecutionPolicy"); + command.add("Bypass"); + command.add("-Command"); + command.add( + "Get-CimInstance Win32_Process | " + "Where-Object { $_.CommandLine -like '*--worker*' " + " -and $_.CommandLine -like '*" + escapedScriptPath + "*' " + " -and $_.CommandLine -like '*" + escapedPythonPath + "*' } | " + "ForEach-Object { " + " & taskkill /PID $_.ProcessId /F /T 2>$null | Out-Null; " + " Write-Output ('stopped legacy ACE-Step worker pid ' + $_.ProcessId) " + "}"); + + juce::ChildProcess cleanup; + if (cleanup.start(command, juce::ChildProcess::wantStdOut | juce::ChildProcess::wantStdErr)) + { + cleanup.waitForProcessToFinish(5000); + auto output = cleanup.readAllProcessOutput().trim(); + if (output.isNotEmpty()) + juce::Logger::writeToLog("AITrackEngine: " + output); + } + #endif +} + +juce::String AITrackEngine::appendProcessDetailsLocked(const juce::String& message) const +{ + juce::String decorated = message; + + if (workerExitCode_ != 0) + decorated += " Exit code: " + juce::String(workerExitCode_) + "."; + + if (lastStderrLine_.isNotEmpty()) + decorated += " Last stderr: " + truncateForLog(lastStderrLine_) + "."; + else if (lastStdoutLine_.isNotEmpty()) + decorated += " Last stdout: " + truncateForLog(lastStdoutLine_) + "."; + + return decorated; +} + +void AITrackEngine::setProgressErrorLocked(const juce::String& phase, + const juce::String& message, + const juce::String& failureKind) +{ + currentProgress_.state = "error"; + currentProgress_.progress = 0.0f; + currentProgress_.phase = phase; + currentProgress_.message = message; + currentProgress_.error = message; + currentProgress_.failureKind = failureKind; + currentProgress_.workerExitCode = workerExitCode_; + currentProgress_.lastStdoutLine = lastStdoutLine_; + currentProgress_.lastStderrLine = lastStderrLine_; + currentProgress_.requestId = currentRequestId_; + currentProgress_.protocolVersion = std::max(workerProtocolVersion_, kWorkerProtocolVersion); + currentProgress_.scriptVersion = workerScriptVersion_.isNotEmpty() ? workerScriptVersion_ : expectedScriptVersion_; + if (currentProgress_.failureDetail.isEmpty()) + currentProgress_.failureDetail = message; +} + +bool AITrackEngine::waitForWorkerReady(int timeoutMs) +{ + const auto deadline = juce::Time::currentTimeMillis() + timeoutMs; + auto workerExitedBeforeReady = false; + + while (juce::Time::currentTimeMillis() < deadline) + { + { + const juce::ScopedLock sl(lock_); + if (cancelRequested_) + return false; + if (workerReady_ && workerPort_ > 0) + return true; + if (workerProtocolRejected_) + break; + + if (workerProcess_ == nullptr || ! workerProcess_->isRunning()) + { + workerExitedBeforeReady = true; + break; + } + } + + juce::Thread::sleep(50); + } + + { + const juce::ScopedLock sl(lock_); + if (cancelRequested_) + return false; + + if (currentProgress_.error.isEmpty()) + { + const auto message = workerExitedBeforeReady + ? appendProcessDetailsLocked("ACE-Step worker exited before reporting ready.") + : appendProcessDetailsLocked("ACE-Step worker did not become ready in time."); + setProgressErrorLocked(workerExitedBeforeReady ? "worker_start_failed" + : "worker_start_timeout", + message, + "worker_handshake"); + } + } + + return false; +} + +void AITrackEngine::joinGenerationThread() +{ + if (generationThread_.joinable()) + { + jassert(generationThread_.get_id() != std::this_thread::get_id()); + if (generationThread_.get_id() != std::this_thread::get_id()) + generationThread_.join(); + } +} + +void AITrackEngine::stopWorkerSession(bool clearProgress, bool userCancelled, bool keepGenerationActive) +{ + std::thread readerThreadToJoin; + bool shouldKillProcess = false; + int workerPidToKill = 0; + juce::ChildProcess* processToKill = nullptr; + + { + const juce::ScopedLock sl(lock_); + readerShouldExit_ = true; + expectedProcessExit_ = true; + cancelRequested_ = userCancelled; + if (! keepGenerationActive) + generationActive_ = false; + workerReady_ = false; + workerPort_ = 0; + workerPidToKill = workerPid_; + processToKill = workerProcess_.get(); + + if (workerProcess_ != nullptr && workerProcess_->isRunning()) + shouldKillProcess = true; + + if (readerThread_.joinable()) + readerThreadToJoin = std::move(readerThread_); + } + + if (shouldKillProcess && processToKill != nullptr && processToKill->isRunning()) + { + auto killedTree = false; + #if JUCE_WINDOWS + if (workerPidToKill > 0) + { + juce::String killOutput; + juce::Logger::writeToLog("AITrackEngine: stopping ACE-Step child process tree pid=" + + juce::String(workerPidToKill)); + killedTree = killWindowsProcessTree(workerPidToKill, &killOutput); + if (killOutput.isNotEmpty()) + juce::Logger::writeToLog("AITrackEngine: " + truncateForLog(killOutput, 512)); + } + #endif + + if (! killedTree && processToKill->isRunning()) + { + juce::Logger::writeToLog("AITrackEngine: stopping ACE-Step child process"); + processToKill->kill(); + } + } + + if (readerThreadToJoin.joinable()) + readerThreadToJoin.join(); + + const juce::ScopedLock sl(lock_); + workerProcess_.reset(); + resetProcessStateLocked(); + + if (clearProgress) + { + currentProgress_ = {}; + currentProgress_.state = "idle"; + currentProgress_.backend = "unknown"; + } +} + +bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce::File& script) +{ + const auto expectedScriptVersion = computeScriptVersion(script); + { + const juce::ScopedLock sl(lock_); + if (workerProcess_ != nullptr + && workerProcess_->isRunning() + && workerReady_ + && workerPort_ > 0 + && workerProtocolVersion_ == kWorkerProtocolVersion + && workerScriptVersion_ == expectedScriptVersion) + return true; + } + + stopWorkerSession(false, false, true); + cleanupLegacyWorkerProcesses(python, script); + + for (int launchAttempt = 0; launchAttempt < 2; ++launchAttempt) + { + juce::StringArray command; + command.add(python.getFullPathName()); + command.add(script.getFullPathName()); + command.add("--worker"); + command.add("--checkpoint-root"); + command.add(getMusicGenerationCheckpointRoot().getFullPathName()); + command.add("--music-gen-model"); + command.add(kPinnedMusicGenerationModelId); + + const auto commandLine = buildCommandLineForLog(command); + juce::Logger::writeToLog("AITrackEngine: launching persistent ACE-Step worker: " + commandLine + + " protocolVersion=" + juce::String(kWorkerProtocolVersion) + + " expectedScriptVersion=" + expectedScriptVersion + + " launchAttempt=" + juce::String(launchAttempt + 1)); + + { + const juce::ScopedLock sl(lock_); + workerProcess_ = std::make_unique(); + resetProcessStateLocked(); + readerShouldExit_ = false; + expectedProcessExit_ = false; + workerReady_ = false; + workerPort_ = 0; + workerProtocolRejected_ = false; + expectedScriptVersion_ = expectedScriptVersion; + workerLaunchAtMs_ = juce::Time::currentTimeMillis(); + + currentProgress_.state = "loading"; + currentProgress_.progress = 0.02f; + currentProgress_.phase = "starting_worker"; + currentProgress_.message = "Starting the ACE-Step runtime session..."; + currentProgress_.backend = "unknown"; + currentProgress_.error.clear(); + currentProgress_.runMode = "cold"; + currentProgress_.etaMs = -1.0; + currentProgress_.phaseProgress = -1.0; + currentProgress_.failureKind.clear(); + currentProgress_.sessionMode = "persistent"; + currentProgress_.workerExitCode = 0; + currentProgress_.protocolVersion = kWorkerProtocolVersion; + currentProgress_.scriptVersion = expectedScriptVersion; + currentProgress_.lastStdoutLine.clear(); + currentProgress_.lastStderrLine.clear(); + currentProgress_.tracePath.clear(); + currentProgress_.failureDetail.clear(); + currentProgress_.lmBackend.clear(); + currentProgress_.lmStage.clear(); + } + + if (! workerProcess_->start(command, juce::ChildProcess::wantStdOut | juce::ChildProcess::wantStdErr)) + { + const juce::ScopedLock sl(lock_); + workerProcess_.reset(); + setProgressErrorLocked("worker_start_failed", + "Failed to start the ACE-Step runtime session.", + "worker_start"); + return false; + } + + readerThread_ = std::thread([this]() { readerLoop(); }); + if (waitForWorkerReady(kWorkerStartupTimeoutMs)) + return true; + + auto shouldRetryLaunch = false; + { + const juce::ScopedLock sl(lock_); + shouldRetryLaunch = workerProtocolRejected_ && launchAttempt == 0; + } + + stopWorkerSession(false, false, true); + if (! shouldRetryLaunch) + return false; + + cleanupLegacyWorkerProcesses(python, script); + } + + return false; +} + +bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, + const juce::String& paramsJson, + const juce::File& outputFile) +{ + int port = 0; + juce::String requestId; + { + const juce::ScopedLock sl(lock_); + if (cancelRequested_) + return false; + port = workerPort_; + requestId = currentRequestId_; + } + + if (port <= 0) + { + const juce::ScopedLock sl(lock_); + setProgressErrorLocked("worker_connect_failed", + appendProcessDetailsLocked("ACE-Step worker did not provide a listening port."), + "worker_request"); + return false; + } + + juce::StreamingSocket socket; + if (! socket.connect("127.0.0.1", port, 4000)) + { + const juce::ScopedLock sl(lock_); + setProgressErrorLocked("worker_connect_failed", + appendProcessDetailsLocked("OpenStudio could not contact the ACE-Step runtime session."), + "worker_request"); + return false; + } + + auto request = std::make_unique(); + request->setProperty("command", "generate"); + request->setProperty("workflow", workflowId); + request->setProperty("params", paramsJson); + request->setProperty("output", outputFile.getFullPathName()); + request->setProperty("requestId", requestId); + request->setProperty("protocolVersion", kWorkerProtocolVersion); + request->setProperty("scriptVersion", expectedScriptVersion_); + + int payloadBytesWritten = 0; + if (! writeFramedJson(socket, juce::var(request.release()), kWorkerRequestTimeoutMs, payloadBytesWritten)) + { + const juce::ScopedLock sl(lock_); + if (cancelRequested_) + return false; + + setProgressErrorLocked("worker_request_failed", + appendProcessDetailsLocked("OpenStudio could not fully submit the generation request to the ACE-Step session."), + "worker_protocol"); + return false; + } + + juce::Logger::writeToLog("AITrackEngine: sent framed worker request" + " requestId=" + requestId + + " workerPid=" + juce::String(workerPid_) + + " payloadBytes=" + juce::String(payloadBytesWritten)); + + int ackPayloadBytes = 0; + auto parsed = readFramedJson(socket, kWorkerRequestTimeoutMs, ackPayloadBytes); + if (parsed.isVoid()) + { + const juce::ScopedLock sl(lock_); + if (cancelRequested_) + return false; + + setProgressErrorLocked("worker_request_timeout", + appendProcessDetailsLocked("ACE-Step did not acknowledge the generation request in time."), + "worker_protocol"); + return false; + } + + auto* object = parsed.getDynamicObject(); + if (object == nullptr || ! static_cast (object->getProperty("accepted"))) + { + auto error = object != nullptr + ? object->getProperty("error").toString() + : "ACE-Step returned an invalid worker response."; + + if (error.isEmpty()) + error = "ACE-Step rejected the generation request."; + + const juce::ScopedLock sl(lock_); + if (cancelRequested_) + return false; + + setProgressErrorLocked("worker_request_rejected", + appendProcessDetailsLocked(error), + object != nullptr && object->hasProperty("failureKind") + ? object->getProperty("failureKind").toString() + : "worker_protocol"); + return false; + } + + const auto ackRequestId = object->getProperty("requestId").toString(); + const auto ackProtocolVersion = static_cast (double (object->getProperty("protocolVersion"))); + const auto ackScriptVersion = object->getProperty("scriptVersion").toString(); + if (ackRequestId != requestId + || ackProtocolVersion != kWorkerProtocolVersion + || ackScriptVersion != expectedScriptVersion_) + { + const juce::ScopedLock sl(lock_); + setProgressErrorLocked("worker_protocol_failed", + appendProcessDetailsLocked("ACE-Step acknowledged the request with a mismatched protocol or request id."), + "worker_protocol"); + return false; + } + + juce::Logger::writeToLog("AITrackEngine: persistent worker accepted generation request" + " requestId=" + requestId + + " ackBytes=" + juce::String(ackPayloadBytes) + + " ackPid=" + object->getProperty("pid").toString()); + return true; +} + +void AITrackEngine::launchGenerationTask(const juce::File& python, + const juce::File& script, + const juce::String& workflowId, + const juce::String& paramsJson, + const juce::File& outputFile) +{ + if (! ensureWorkerAvailable(python, script)) + { + const juce::ScopedLock sl(lock_); + generationActive_ = false; + if (cancelRequested_) + { + currentProgress_.state = "cancelled"; + currentProgress_.message = "Music generation cancelled."; + currentProgress_.error.clear(); + } + return; + } + + { + const juce::ScopedLock sl(lock_); + if (cancelRequested_) + { + generationActive_ = false; + currentProgress_.state = "cancelled"; + currentProgress_.message = "Music generation cancelled."; + currentProgress_.error.clear(); + return; + } + + currentProgress_.state = "loading"; + currentProgress_.progress = 0.03f; + currentProgress_.phase = "submitting_request"; + currentProgress_.message = "Submitting the generation request to the ACE-Step session..."; + currentProgress_.outputFile.clear(); + currentProgress_.error.clear(); + currentProgress_.elapsedMs = 0.0; + currentProgress_.heartbeatTs = static_cast (lastHeartbeatAtMs_); + currentProgress_.phaseProgress = 0.0; + currentProgress_.etaMs = -1.0; + currentProgress_.sessionMode = "persistent"; + currentProgress_.workerExitCode = 0; + currentProgress_.failureKind.clear(); + } + + if (! sendGenerateRequest(workflowId, paramsJson, outputFile)) + { + const juce::ScopedLock sl(lock_); + generationActive_ = false; + if (cancelRequested_) + { + currentProgress_.state = "cancelled"; + currentProgress_.message = "Music generation cancelled."; + currentProgress_.error.clear(); + } + } +} + +bool AITrackEngine::startGeneration(const juce::String& workflowId, + const juce::String& paramsJson, + const juce::File& outputDir) +{ + joinGenerationThread(); + + { + const juce::ScopedLock sl(lock_); + if (generationActive_) + return false; + } + + const auto python = findPython(); + const auto script = findScript(); + + if (! python.existsAsFile() || ! script.existsAsFile()) + { + const juce::ScopedLock sl(lock_); + setProgressErrorLocked("runtime_missing", + "AI runtime is not ready. Install AI Tools first.", + "runtime_missing"); + return false; + } + + if (! outputDir.exists() && ! outputDir.createDirectory()) + { + const juce::ScopedLock sl(lock_); + setProgressErrorLocked("output_dir_failed", + "Failed to create the generated audio output folder.", + "output_dir_failed"); + return false; + } + + juce::File outputFile; + { + const juce::ScopedLock sl(lock_); + currentOutputFile_ = outputDir.getChildFile( + "generated_music_" + createSafeMusicGenerationTimestamp() + ".wav"); + outputFile = currentOutputFile_; + generationActive_ = true; + cancelRequested_ = false; + currentRequestId_ = juce::Uuid().toString(); + expectedScriptVersion_ = computeScriptVersion(script); + generationStartedAtMs_ = juce::Time::currentTimeMillis(); + lastHeartbeatAtMs_ = generationStartedAtMs_; + + currentProgress_.state = "loading"; + currentProgress_.progress = 0.01f; + currentProgress_.phase = "starting"; + currentProgress_.message = "Starting the ACE-Step runtime session..."; + currentProgress_.outputFile.clear(); + currentProgress_.error.clear(); + currentProgress_.elapsedMs = 0.0; + currentProgress_.heartbeatTs = static_cast (lastHeartbeatAtMs_); + currentProgress_.phaseProgress = 0.0; + currentProgress_.etaMs = -1.0; + currentProgress_.failureKind.clear(); + currentProgress_.sessionMode = "persistent"; + currentProgress_.workerExitCode = 0; + currentProgress_.lastStdoutLine.clear(); + currentProgress_.lastStderrLine.clear(); + currentProgress_.statusNote.clear(); + currentProgress_.attemptMode = "lm_dit"; + currentProgress_.attemptIndex = 1; + currentProgress_.protocolVersion = kWorkerProtocolVersion; + currentProgress_.scriptVersion = expectedScriptVersion_; + currentProgress_.requestId = currentRequestId_; + currentProgress_.priorFailure.clear(); + currentProgress_.lastProgressAgeMs = 0.0; + } + + generationThread_ = std::thread([this, python, script, workflowId, paramsJson, outputFile]() + { + launchGenerationTask(python, script, workflowId, paramsJson, outputFile); + }); + + return true; +} + +void AITrackEngine::parseOutputLine(const juce::String& line) +{ + auto trimmed = line.trim(); + if (trimmed.isEmpty()) + return; + + const juce::ScopedLock sl(lock_); + lastProcessOutputLine_ = trimmed; + + auto parsed = juce::JSON::parse(trimmed); + if (parsed.isVoid()) + { + lastStdoutLine_ = trimmed; + currentProgress_.lastStdoutLine = lastStdoutLine_; + currentProgress_.lastStderrLine = lastStderrLine_; + currentProgress_.workerExitCode = workerExitCode_; + juce::Logger::writeToLog("AITrackEngine: process output: " + truncateForLog(trimmed)); + return; + } + + auto* obj = parsed.getDynamicObject(); + if (obj == nullptr) + return; + + if (obj->hasProperty("event")) + { + const auto event = obj->getProperty("event").toString(); + + if (event == "stderr") + { + lastStderrLine_ = obj->getProperty("line").toString().trim(); + currentProgress_.lastStderrLine = lastStderrLine_; + juce::Logger::writeToLog("AITrackEngine: worker stderr: " + truncateForLog(lastStderrLine_)); + return; + } + + if (event == "ready") + { + workerProtocolVersion_ = obj->hasProperty("protocolVersion") + ? static_cast (double (obj->getProperty("protocolVersion"))) + : 0; + workerScriptVersion_ = obj->getProperty("scriptVersion").toString(); + workerScriptPath_ = obj->getProperty("scriptPath").toString(); + workerPid_ = obj->hasProperty("pid") + ? static_cast (double (obj->getProperty("pid"))) + : 0; + + currentProgress_.protocolVersion = workerProtocolVersion_; + currentProgress_.scriptVersion = workerScriptVersion_; + + if (workerProtocolVersion_ != kWorkerProtocolVersion + || (! expectedScriptVersion_.isEmpty() && workerScriptVersion_ != expectedScriptVersion_)) + { + workerProtocolRejected_ = true; + workerReady_ = false; + workerPort_ = 0; + setProgressErrorLocked("worker_protocol_failed", + appendProcessDetailsLocked("ACE-Step worker protocol or script version mismatch."), + "worker_protocol"); + currentProgress_.statusNote = "Rejecting stale worker session before generation starts."; + juce::Logger::writeToLog("AITrackEngine: rejecting worker ready handshake due to version mismatch" + " workerProtocol=" + juce::String(workerProtocolVersion_) + + " expectedProtocol=" + juce::String(kWorkerProtocolVersion) + + " workerScriptVersion=" + workerScriptVersion_ + + " expectedScriptVersion=" + expectedScriptVersion_); + return; + } + + workerReady_ = true; + workerPort_ = static_cast (double (obj->getProperty("port"))); + currentProgress_.state = "idle"; + currentProgress_.progress = 0.0f; + currentProgress_.phase = "worker_ready"; + currentProgress_.message = "ACE-Step runtime session is ready."; + currentProgress_.backend = obj->getProperty("backend").toString(); + currentProgress_.sessionMode = obj->hasProperty("sessionMode") + ? obj->getProperty("sessionMode").toString() + : "persistent"; + juce::Logger::writeToLog("AITrackEngine: worker ready handshake received on port " + + juce::String(workerPort_) + + " pid=" + juce::String(workerPid_) + + " protocolVersion=" + juce::String(workerProtocolVersion_) + + " scriptVersion=" + workerScriptVersion_ + + " after " + + juce::String(static_cast (juce::Time::currentTimeMillis() - workerLaunchAtMs_)) + + " ms"); + return; + } + } + + sawStructuredOutput_ = true; + lastStdoutLine_ = trimmed; + if (! loggedFirstStructuredOutput_) + { + loggedFirstStructuredOutput_ = true; + juce::Logger::writeToLog( + "AITrackEngine: first structured progress line state=" + + obj->getProperty("state").toString() + + " phase=" + obj->getProperty("phase").toString()); + } + + const auto incomingState = obj->hasProperty("state") + ? obj->getProperty("state").toString() + : juce::String(); + const auto currentStateIsTerminal = currentProgress_.state == "done" + || currentProgress_.state == "error" + || currentProgress_.state == "cancelled"; + const auto incomingStateIsTerminal = incomingState == "done" + || incomingState == "error" + || incomingState == "cancelled"; + if (currentStateIsTerminal && ! incomingStateIsTerminal) + { + juce::Logger::writeToLog("AITrackEngine: ignoring late non-terminal progress after terminal state" + " currentState=" + currentProgress_.state + + " incomingState=" + incomingState + + " requestId=" + currentProgress_.requestId); + return; + } + + if (obj->hasProperty("state")) + currentProgress_.state = obj->getProperty("state").toString(); + if (obj->hasProperty("progress")) + currentProgress_.progress = static_cast (double (obj->getProperty("progress"))); + if (obj->hasProperty("phase")) + { + currentProgress_.phase = obj->getProperty("phase").toString(); + currentProgress_.phaseProgress = -1.0; + } + if (obj->hasProperty("message")) + currentProgress_.message = obj->getProperty("message").toString(); + if (obj->hasProperty("backend")) + currentProgress_.backend = obj->getProperty("backend").toString(); + if (obj->hasProperty("outputFile")) + currentProgress_.outputFile = obj->getProperty("outputFile").toString(); + if (obj->hasProperty("error")) + currentProgress_.error = obj->getProperty("error").toString(); + if (obj->hasProperty("elapsedMs")) + currentProgress_.elapsedMs = double (obj->getProperty("elapsedMs")); + if (obj->hasProperty("heartbeatTs")) + currentProgress_.heartbeatTs = double (obj->getProperty("heartbeatTs")); + if (obj->hasProperty("phaseProgress")) + currentProgress_.phaseProgress = double (obj->getProperty("phaseProgress")); + if (obj->hasProperty("etaMs")) + currentProgress_.etaMs = double (obj->getProperty("etaMs")); + if (obj->hasProperty("runMode")) + currentProgress_.runMode = obj->getProperty("runMode").toString(); + if (obj->hasProperty("runtimeProfile")) + currentProgress_.runtimeProfile = obj->getProperty("runtimeProfile").toString(); + if (obj->hasProperty("lmModel")) + currentProgress_.lmModel = obj->getProperty("lmModel").toString(); + if (obj->hasProperty("statusNote")) + currentProgress_.statusNote = obj->getProperty("statusNote").toString(); + if (obj->hasProperty("failureKind")) + currentProgress_.failureKind = obj->getProperty("failureKind").toString(); + if (obj->hasProperty("failureDetail")) + currentProgress_.failureDetail = obj->getProperty("failureDetail").toString(); + if (obj->hasProperty("sessionMode")) + currentProgress_.sessionMode = obj->getProperty("sessionMode").toString(); + if (obj->hasProperty("workerExitCode")) + currentProgress_.workerExitCode = static_cast (double (obj->getProperty("workerExitCode"))); + if (obj->hasProperty("lastStdoutLine")) + currentProgress_.lastStdoutLine = obj->getProperty("lastStdoutLine").toString(); + if (obj->hasProperty("lastStderrLine")) + currentProgress_.lastStderrLine = obj->getProperty("lastStderrLine").toString(); + if (obj->hasProperty("attemptMode")) + currentProgress_.attemptMode = obj->getProperty("attemptMode").toString(); + if (obj->hasProperty("attemptIndex")) + currentProgress_.attemptIndex = static_cast (double (obj->getProperty("attemptIndex"))); + if (obj->hasProperty("protocolVersion")) + currentProgress_.protocolVersion = static_cast (double (obj->getProperty("protocolVersion"))); + if (obj->hasProperty("scriptVersion")) + currentProgress_.scriptVersion = obj->getProperty("scriptVersion").toString(); + if (obj->hasProperty("requestId")) + currentProgress_.requestId = obj->getProperty("requestId").toString(); + if (obj->hasProperty("priorFailure")) + currentProgress_.priorFailure = obj->getProperty("priorFailure").toString(); + if (obj->hasProperty("lastProgressAgeMs")) + currentProgress_.lastProgressAgeMs = double (obj->getProperty("lastProgressAgeMs")); + auto tracePathChanged = false; + juce::String incomingTracePath; + if (obj->hasProperty("tracePath")) + { + incomingTracePath = obj->getProperty("tracePath").toString(); + tracePathChanged = incomingTracePath.isNotEmpty() && incomingTracePath != currentProgress_.tracePath; + currentProgress_.tracePath = incomingTracePath; + } + if (obj->hasProperty("lmBackend")) + currentProgress_.lmBackend = obj->getProperty("lmBackend").toString(); + if (obj->hasProperty("lmStage")) + currentProgress_.lmStage = obj->getProperty("lmStage").toString(); + + lastHeartbeatAtMs_ = juce::Time::currentTimeMillis(); + currentProgress_.lastStdoutLine = lastStdoutLine_; + currentProgress_.lastStderrLine = lastStderrLine_; + currentProgress_.workerExitCode = workerExitCode_; + + if (currentProgress_.state == "error" && currentProgress_.failureKind.isEmpty()) + currentProgress_.failureKind = "generation"; + if (currentProgress_.state == "error" && currentProgress_.failureDetail.isEmpty()) + currentProgress_.failureDetail = currentProgress_.error.isNotEmpty() ? currentProgress_.error : currentProgress_.message; + + if (tracePathChanged) + juce::Logger::writeToLog("AITrackEngine: AI trace path " + currentProgress_.tracePath); + + if (currentProgress_.state == "done") + currentProgress_.failureKind.clear(); + + if (currentProgress_.state == "done" + || currentProgress_.state == "error" + || currentProgress_.state == "cancelled") + { + generationActive_ = false; + } +} + +void AITrackEngine::handleWorkerExit() +{ + juce::String finalError; + + { + const juce::ScopedLock sl(lock_); + const auto expectedExit = expectedProcessExit_; + expectedProcessExit_ = false; + workerReady_ = false; + workerPort_ = 0; + + workerExitCode_ = 0; + if (workerProcess_ != nullptr) + workerExitCode_ = static_cast (workerProcess_->getExitCode()); + + currentProgress_.workerExitCode = workerExitCode_; + currentProgress_.lastStdoutLine = lastStdoutLine_; + currentProgress_.lastStderrLine = lastStderrLine_; + + juce::Logger::writeToLog( + "AITrackEngine: ACE-Step child process exited with code " + + juce::String(workerExitCode_) + + " (sessionMode=" + currentProgress_.sessionMode + ")"); + + if (expectedExit) + return; + + if (! generationActive_) + return; + + if (currentProgress_.state == "done" + || currentProgress_.state == "error" + || currentProgress_.state == "cancelled") + { + generationActive_ = false; + return; + } + + if (currentOutputFile_.existsAsFile()) + { + currentProgress_.state = "done"; + currentProgress_.progress = 1.0f; + currentProgress_.outputFile = currentOutputFile_.getFullPathName(); + currentProgress_.phase = "done"; + currentProgress_.message = "Music generation completed."; + currentProgress_.failureKind.clear(); + generationActive_ = false; + return; + } + + if (! sawStructuredOutput_) + finalError = appendProcessDetailsLocked("The ACE-Step process exited before reporting progress."); + else if (currentProgress_.error.isNotEmpty()) + finalError = appendProcessDetailsLocked(currentProgress_.error); + else + finalError = appendProcessDetailsLocked("Generation stopped unexpectedly before writing audio."); + + setProgressErrorLocked("worker_exit", finalError, "worker_exit"); + generationActive_ = false; + } +} + +void AITrackEngine::readerLoop() +{ + juce::Logger::writeToLog("AITrackEngine: reader thread started"); + + for (;;) + { + if (readerShouldExit_) + break; + + auto* process = workerProcess_.get(); + if (process == nullptr) + break; + + char byte = 0; + const auto numRead = process->readProcessOutput(&byte, 1); + if (numRead > 0) + { + juce::StringArray linesToParse; + juce::String firstLineForLog; + bool shouldLogFirstByte = false; + bool shouldLogFirstLine = false; + juce::int64 firstByteDelayMs = 0; + juce::int64 firstLineDelayMs = 0; + + { + const juce::ScopedLock sl(lock_); + + if (! loggedFirstOutputByte_) + { + loggedFirstOutputByte_ = true; + firstOutputByteAtMs_ = juce::Time::currentTimeMillis(); + firstByteDelayMs = firstOutputByteAtMs_ - workerLaunchAtMs_; + shouldLogFirstByte = true; + } + + processOutputBuffer_.push_back(byte); + + while (true) + { + const auto newlinePos = processOutputBuffer_.find('\n'); + if (newlinePos == std::string::npos) + break; + + auto rawLine = processOutputBuffer_.substr(0, newlinePos); + processOutputBuffer_.erase(0, newlinePos + 1); + + auto parsedLine = juce::String::fromUTF8(rawLine.c_str(), static_cast (rawLine.size())).trim(); + if (parsedLine.isEmpty()) + continue; + + if (! loggedFirstOutputLine_) + { + loggedFirstOutputLine_ = true; + firstOutputLineAtMs_ = juce::Time::currentTimeMillis(); + firstLineDelayMs = firstOutputLineAtMs_ - workerLaunchAtMs_; + firstLineForLog = parsedLine; + shouldLogFirstLine = true; + } + + linesToParse.add(parsedLine); + } + } + + if (shouldLogFirstByte) + juce::Logger::writeToLog("AITrackEngine: first worker output byte received after " + + juce::String(static_cast (firstByteDelayMs)) + + " ms"); + + if (shouldLogFirstLine) + juce::Logger::writeToLog("AITrackEngine: first worker output line received after " + + juce::String(static_cast (firstLineDelayMs)) + + " ms: " + + truncateForLog(firstLineForLog)); + + for (const auto& parsedLine : linesToParse) + parseOutputLine(parsedLine); + } + else if (! process->isRunning()) + { + juce::String trailingLine; + { + const juce::ScopedLock sl(lock_); + if (! processOutputBuffer_.empty()) + { + trailingLine = juce::String::fromUTF8(processOutputBuffer_.data(), + static_cast (processOutputBuffer_.size())).trim(); + } + processOutputBuffer_.clear(); + } + + if (trailingLine.isNotEmpty()) + parseOutputLine(trailingLine); + + break; + } + else + { + juce::Thread::sleep(kReaderSleepMs); + } + } + + juce::Logger::writeToLog("AITrackEngine: reader thread exiting"); + handleWorkerExit(); +} + +void AITrackEngine::resetProcessStateLocked() +{ + processOutputBuffer_.clear(); + lastProcessOutputLine_.clear(); + lastStdoutLine_.clear(); + lastStderrLine_.clear(); + sawStructuredOutput_ = false; + loggedFirstStructuredOutput_ = false; + loggedFirstOutputByte_ = false; + loggedFirstOutputLine_ = false; + workerProtocolRejected_ = false; + workerExitCode_ = 0; + workerProtocolVersion_ = 0; + workerPid_ = 0; + workerLaunchAtMs_ = 0; + firstOutputByteAtMs_ = 0; + firstOutputLineAtMs_ = 0; + workerScriptVersion_.clear(); + workerScriptPath_.clear(); +} + +void AITrackEngine::stopWorker(bool clearProgress, bool userCancelled) +{ + stopWorkerSession(clearProgress, userCancelled, false); + joinGenerationThread(); +} + +AIGenerationProgress AITrackEngine::pollProgress() +{ + bool shouldStopForDecodeStall = false; + + { + const juce::ScopedLock sl(lock_); + + if (currentProgress_.state == "error" + && currentProgress_.failureKind == "decode_stalled" + && workerProcess_ != nullptr + && workerProcess_->isRunning()) + { + shouldStopForDecodeStall = true; + } + + if (generationActive_) + { + const auto nowMs = juce::Time::currentTimeMillis(); + currentProgress_.elapsedMs = static_cast (nowMs - generationStartedAtMs_); + + if (lastHeartbeatAtMs_ > 0) + { + currentProgress_.heartbeatTs = static_cast (lastHeartbeatAtMs_); + currentProgress_.lastProgressAgeMs = static_cast (nowMs - lastHeartbeatAtMs_); + } + + const auto decodeStallTimeoutMs = kColdDecodeStallTimeoutMs; + const auto inTerminalState = currentProgress_.state == "done" + || currentProgress_.state == "error" + || currentProgress_.state == "cancelled"; + if (! inTerminalState + && currentProgress_.phase == "decoding_audio" + && currentProgress_.lastProgressAgeMs >= static_cast (decodeStallTimeoutMs)) + { + auto message = "ACE-Step decode stalled while finalizing audio after " + + juce::String(static_cast (currentProgress_.lastProgressAgeMs / 1000.0)) + + " seconds."; + if (currentProgress_.priorFailure.isNotEmpty()) + message += " Prior failure: " + currentProgress_.priorFailure; + setProgressErrorLocked("decode_stalled", + appendProcessDetailsLocked(message), + "decode_stalled"); + currentProgress_.statusNote = "Stopping the stalled ACE-Step decode process."; + generationActive_ = false; + shouldStopForDecodeStall = true; + } + + if (workerProcess_ != nullptr && ! workerProcess_->isRunning() + && currentProgress_.state != "done" + && currentProgress_.state != "error" + && currentProgress_.state != "cancelled") + { + setProgressErrorLocked("worker_missing", + appendProcessDetailsLocked("Generation stopped unexpectedly before writing audio."), + "worker_exit"); + generationActive_ = false; + } + } + + currentProgress_.workerExitCode = workerExitCode_; + currentProgress_.lastStdoutLine = lastStdoutLine_; + currentProgress_.lastStderrLine = lastStderrLine_; + } + + if (shouldStopForDecodeStall) + stopWorkerSession(false, false, false); + + const juce::ScopedLock sl(lock_); + return currentProgress_; +} + +void AITrackEngine::cancel() +{ + stopWorker(true, true); +} + +bool AITrackEngine::isRunning() const +{ + const juce::ScopedLock sl(lock_); + return generationActive_; +} diff --git a/Source/AITrackEngine.h b/Source/AITrackEngine.h new file mode 100644 index 0000000..05fcacb --- /dev/null +++ b/Source/AITrackEngine.h @@ -0,0 +1,120 @@ +#pragma once + +#include +#include +#include +#include +#include + +struct AIGenerationProgress +{ + juce::String state { "idle" }; + float progress = 0.0f; + juce::String phase; + juce::String message; + juce::String backend { "unknown" }; + juce::String outputFile; + juce::String error; + double elapsedMs = 0.0; + double heartbeatTs = 0.0; + double phaseProgress = -1.0; + double etaMs = -1.0; + juce::String runMode; + juce::String runtimeProfile; + juce::String lmModel; + juce::String statusNote; + juce::String failureKind; + juce::String sessionMode; + int workerExitCode = 0; + juce::String lastStdoutLine; + juce::String lastStderrLine; + juce::String attemptMode; + int attemptIndex = 0; + int protocolVersion = 0; + juce::String scriptVersion; + juce::String requestId; + juce::String priorFailure; + double lastProgressAgeMs = -1.0; + juce::String tracePath; + juce::String failureDetail; + juce::String lmBackend; + juce::String lmStage; +}; + +class AITrackEngine +{ +public: + AITrackEngine() = default; + ~AITrackEngine(); + + bool startGeneration(const juce::String& workflowId, + const juce::String& paramsJson, + const juce::File& outputDir); + + AIGenerationProgress pollProgress(); + void cancel(); + bool isRunning() const; + +private: + juce::File getUserDataRoot() const; + juce::File getUserRuntimeRoot() const; + juce::File getMusicGenerationCheckpointRoot() const; + juce::File findPython() const; + juce::File findScript() const; + void cleanupLegacyWorkerProcesses(const juce::File& python, const juce::File& script) const; + bool ensureWorkerAvailable(const juce::File& python, const juce::File& script); + bool sendGenerateRequest(const juce::String& workflowId, + const juce::String& paramsJson, + const juce::File& outputFile); + void launchGenerationTask(const juce::File& python, + const juce::File& script, + const juce::String& workflowId, + const juce::String& paramsJson, + const juce::File& outputFile); + bool waitForWorkerReady(int timeoutMs); + void stopWorkerSession(bool clearProgress, bool userCancelled, bool keepGenerationActive); + void joinGenerationThread(); + void stopWorker(bool clearProgress, bool userCancelled); + void readerLoop(); + void parseOutputLine(const juce::String& line); + void handleWorkerExit(); + juce::String appendProcessDetailsLocked(const juce::String& message) const; + void setProgressErrorLocked(const juce::String& phase, + const juce::String& message, + const juce::String& failureKind); + void resetProcessStateLocked(); + + std::unique_ptr workerProcess_; + std::thread readerThread_; + std::thread generationThread_; + std::atomic readerShouldExit_ { false }; + mutable juce::CriticalSection lock_; + AIGenerationProgress currentProgress_; + std::string processOutputBuffer_; + juce::String lastProcessOutputLine_; + juce::String lastStdoutLine_; + juce::String lastStderrLine_; + juce::File currentOutputFile_; + juce::String currentRequestId_; + juce::String expectedScriptVersion_; + juce::String workerScriptVersion_; + juce::String workerScriptPath_; + bool generationActive_ = false; + bool expectedProcessExit_ = false; + bool cancelRequested_ = false; + bool sawStructuredOutput_ = false; + bool loggedFirstStructuredOutput_ = false; + bool loggedFirstOutputByte_ = false; + bool loggedFirstOutputLine_ = false; + bool workerReady_ = false; + bool workerProtocolRejected_ = false; + int workerPort_ = 0; + int workerExitCode_ = 0; + int workerProtocolVersion_ = 0; + int workerPid_ = 0; + juce::int64 generationStartedAtMs_ = 0; + juce::int64 lastHeartbeatAtMs_ = 0; + juce::int64 workerLaunchAtMs_ = 0; + juce::int64 firstOutputByteAtMs_ = 0; + juce::int64 firstOutputLineAtMs_ = 0; +}; diff --git a/Source/AudioEngine.cpp b/Source/AudioEngine.cpp index ad279ca..3df4e85 100644 --- a/Source/AudioEngine.cpp +++ b/Source/AudioEngine.cpp @@ -8,55 +8,2360 @@ namespace { +struct PitchPhraseRenderRegion +{ + double startSec = 0.0; + double endSec = 0.0; + bool startIsSafeBoundary = false; + bool endIsSafeBoundary = false; + bool expandedToFullClip = true; +}; + juce::File getOpenStudioDocumentsDirectory() { auto documentsDir = juce::File::getSpecialLocation(juce::File::userDocumentsDirectory); return documentsDir.getChildFile("OpenStudio"); } -juce::File getLegacyStudio13DocumentsDirectory() -{ - auto documentsDir = juce::File::getSpecialLocation(juce::File::userDocumentsDirectory); - return documentsDir.getChildFile("Studio13"); -} +juce::File getLegacyStudio13DocumentsDirectory() +{ + auto documentsDir = juce::File::getSpecialLocation(juce::File::userDocumentsDirectory); + return documentsDir.getChildFile("Studio13"); +} + +juce::File getOpenStudioApplicationDataDirectory() +{ + auto appDataDir = juce::File::getSpecialLocation(juce::File::userApplicationDataDirectory); + return appDataDir.getChildFile("OpenStudio"); +} + +juce::File getLegacyStudio13ApplicationDataDirectory() +{ + auto appDataDir = juce::File::getSpecialLocation(juce::File::userApplicationDataDirectory); + return appDataDir.getChildFile("Studio13"); +} + +juce::File getPreferredAppDataDirectory() +{ + auto openStudioDir = getOpenStudioDocumentsDirectory(); + auto legacyDir = getLegacyStudio13DocumentsDirectory(); + + if (!openStudioDir.exists() && legacyDir.exists()) + return legacyDir; + + return openStudioDir; +} + +juce::File getApplicationRuntimeDirectory() +{ + return juce::File::getSpecialLocation(juce::File::currentExecutableFile).getParentDirectory(); +} + +juce::File getPreferredApplicationDataDirectory() +{ + auto openStudioDir = getOpenStudioApplicationDataDirectory(); + auto legacyDir = getLegacyStudio13ApplicationDataDirectory(); + + if (!openStudioDir.exists() && legacyDir.exists()) + return legacyDir; + + return openStudioDir; +} + +float getPitchEnvFloat (const char* name, float fallback) +{ + auto value = juce::SystemStats::getEnvironmentVariable (name, {}).trim(); + if (value.isEmpty()) + return fallback; + + const float parsed = value.getFloatValue(); + return std::isfinite (parsed) ? parsed : fallback; +} + +bool getPitchEnvFlag (const char* name, bool fallback) +{ + const auto value = juce::SystemStats::getEnvironmentVariable (name, {}).trim().toLowerCase(); + if (value.isEmpty()) + return fallback; + + if (value == "1" || value == "true" || value == "yes" || value == "on") + return true; + + if (value == "0" || value == "false" || value == "no" || value == "off") + return false; + + return fallback; +} + +bool writeBufferToWavFile (const juce::AudioBuffer& buffer, + int numSamples, + double sampleRate, + const juce::File& outputFile) +{ + outputFile.deleteFile(); + + juce::WavAudioFormat wavFormat; + std::unique_ptr stream (outputFile.createOutputStream()); + if (! stream) + return false; + + std::unique_ptr writer ( + wavFormat.createWriterFor (stream.get(), sampleRate, + static_cast (buffer.getNumChannels()), + 32, {}, 0)); + if (! writer) + return false; + + stream.release(); + const bool ok = writer->writeFromAudioSampleBuffer (buffer, 0, numSamples); + writer.reset(); + return ok && outputFile.existsAsFile(); +} + +juce::var writePitchBakedContextWav (const juce::File& sourceFile, + double startSec, + double durationSec, + const juce::File& outputFile) +{ + auto* resultObj = new juce::DynamicObject(); + resultObj->setProperty ("success", false); + resultObj->setProperty ("sourceFile", sourceFile.getFullPathName()); + resultObj->setProperty ("filePath", outputFile.getFullPathName()); + resultObj->setProperty ("source", "baked_corrected_file"); + + if (! sourceFile.existsAsFile()) + { + resultObj->setProperty ("error", "source file missing"); + return juce::var (resultObj); + } + + juce::AudioFormatManager formatManager; + formatManager.registerBasicFormats(); + std::unique_ptr reader (formatManager.createReaderFor (sourceFile)); + if (! reader) + { + resultObj->setProperty ("error", "failed to create source reader"); + return juce::var (resultObj); + } + + const double sampleRate = reader->sampleRate > 0.0 ? reader->sampleRate : 44100.0; + const auto startSample = juce::jlimit ( + 0, + reader->lengthInSamples, + static_cast (std::llround (std::max (0.0, startSec) * sampleRate))); + const auto maxSamples = reader->lengthInSamples - startSample; + const int requestedSamples = std::max (1, static_cast (std::ceil (std::max (0.0, durationSec) * sampleRate))); + const int numSamples = static_cast (std::min (maxSamples, requestedSamples)); + if (numSamples <= 0) + { + resultObj->setProperty ("error", "empty baked context range"); + return juce::var (resultObj); + } + + juce::AudioBuffer buffer (static_cast (reader->numChannels), numSamples); + buffer.clear(); + reader->read (&buffer, 0, numSamples, startSample, true, true); + + outputFile.getParentDirectory().createDirectory(); + const bool wrote = writeBufferToWavFile (buffer, numSamples, sampleRate, outputFile); + if (! wrote) + { + resultObj->setProperty ("error", "failed to write baked context wav"); + return juce::var (resultObj); + } + + resultObj->setProperty ("success", true); + resultObj->setProperty ("filePath", outputFile.getFullPathName()); + resultObj->setProperty ("startSec", static_cast (startSample) / sampleRate); + resultObj->setProperty ("durationSec", static_cast (numSamples) / sampleRate); + resultObj->setProperty ("sampleRate", sampleRate); + resultObj->setProperty ("channels", static_cast (reader->numChannels)); + return juce::var (resultObj); +} + +bool readAudioFileForParity (const juce::File& file, + juce::AudioBuffer& buffer, + double& sampleRate) +{ + if (! file.existsAsFile()) + return false; + + juce::AudioFormatManager formatManager; + formatManager.registerBasicFormats(); + std::unique_ptr reader (formatManager.createReaderFor (file)); + if (! reader || reader->lengthInSamples <= 0) + return false; + + const int numSamples = static_cast (std::min ( + reader->lengthInSamples, + static_cast (std::numeric_limits::max()))); + buffer.setSize (static_cast (reader->numChannels), numSamples); + buffer.clear(); + reader->read (&buffer, 0, numSamples, 0, true, true); + sampleRate = reader->sampleRate; + return true; +} + +float sampleBufferCubicForParity (const juce::AudioBuffer& buffer, + int channel, + double position) +{ + const int availableSamples = buffer.getNumSamples(); + if (availableSamples <= 0 || buffer.getNumChannels() <= 0) + return 0.0f; + + const int safeChannel = juce::jlimit (0, buffer.getNumChannels() - 1, channel); + if (availableSamples == 1) + return buffer.getSample (safeChannel, 0); + + const double clampedPosition = juce::jlimit (0.0, + static_cast (availableSamples - 1), + position); + const int i1 = juce::jlimit (0, availableSamples - 1, + static_cast (std::floor (clampedPosition))); + const int i0 = juce::jlimit (0, availableSamples - 1, i1 - 1); + const int i2 = juce::jlimit (0, availableSamples - 1, i1 + 1); + const int i3 = juce::jlimit (0, availableSamples - 1, i1 + 2); + const float t = static_cast (clampedPosition - static_cast (i1)); + const float t2 = t * t; + const float t3 = t2 * t; + const float y0 = buffer.getSample (safeChannel, i0); + const float y1 = buffer.getSample (safeChannel, i1); + const float y2 = buffer.getSample (safeChannel, i2); + const float y3 = buffer.getSample (safeChannel, i3); + + return 0.5f * ((2.0f * y1) + + (-y0 + y2) * t + + (2.0f * y0 - 5.0f * y1 + 4.0f * y2 - y3) * t2 + + (-y0 + 3.0f * y1 - 3.0f * y2 + y3) * t3); +} + +juce::AudioBuffer resampleBufferCubicForParity (const juce::AudioBuffer& source, + double sourceSampleRate, + double targetSampleRate, + int targetSamples) +{ + const int channels = source.getNumChannels(); + juce::AudioBuffer result (channels, std::max (0, targetSamples)); + result.clear(); + if (channels <= 0 || targetSamples <= 0 || sourceSampleRate <= 0.0 || targetSampleRate <= 0.0) + return result; + + const double ratio = sourceSampleRate / targetSampleRate; + for (int ch = 0; ch < channels; ++ch) + { + for (int i = 0; i < targetSamples; ++i) + result.setSample (ch, i, sampleBufferCubicForParity (source, ch, static_cast (i) * ratio)); + } + return result; +} + +juce::var writePitchBakedResampledContextWav (const juce::File& sourceFile, + double startSec, + double durationSec, + double targetSampleRate, + const juce::File& outputFile) +{ + auto* resultObj = new juce::DynamicObject(); + resultObj->setProperty ("success", false); + resultObj->setProperty ("sourceFile", sourceFile.getFullPathName()); + resultObj->setProperty ("filePath", outputFile.getFullPathName()); + resultObj->setProperty ("source", "baked_corrected_file_resampled"); + resultObj->setProperty ("resampler", "shared_cubic_fractional"); + + if (! sourceFile.existsAsFile()) + { + resultObj->setProperty ("error", "source file missing"); + return juce::var (resultObj); + } + + if (targetSampleRate <= 0.0 || durationSec <= 0.0) + { + resultObj->setProperty ("error", "invalid target sample rate or duration"); + return juce::var (resultObj); + } + + juce::AudioFormatManager formatManager; + formatManager.registerBasicFormats(); + std::unique_ptr reader (formatManager.createReaderFor (sourceFile)); + if (! reader) + { + resultObj->setProperty ("error", "failed to create source reader"); + return juce::var (resultObj); + } + + const double sourceSampleRate = reader->sampleRate > 0.0 ? reader->sampleRate : targetSampleRate; + const double ratio = sourceSampleRate / targetSampleRate; + double exactFileStart = std::max (0.0, startSec) * sourceSampleRate; + const double roundedFileStart = std::round (exactFileStart); + if (std::abs (exactFileStart - roundedFileStart) < 1.0e-6) + exactFileStart = roundedFileStart; + + const auto fileStartSample = static_cast (std::floor (exactFileStart)); + const auto readStartSample = fileStartSample > 0 ? fileStartSample - 1 : fileStartSample; + const int readStartOffset = static_cast (fileStartSample - readStartSample); + const double fileStartFraction = exactFileStart - static_cast (fileStartSample); + const double bufferStartPosition = static_cast (readStartOffset) + fileStartFraction; + const int requestedSamples = std::max (1, static_cast (std::ceil (durationSec * targetSampleRate))); + int outputSamples = requestedSamples; + int sourceSamplesToRead = static_cast (std::ceil (bufferStartPosition + outputSamples * ratio)) + 3; + const auto sourceSamplesAvailable = reader->lengthInSamples - readStartSample; + if (sourceSamplesAvailable <= 0) + { + resultObj->setProperty ("error", "empty resampled context range"); + return juce::var (resultObj); + } + + if (sourceSamplesAvailable < sourceSamplesToRead) + { + sourceSamplesToRead = static_cast (sourceSamplesAvailable); + outputSamples = static_cast (std::floor ( + (static_cast (sourceSamplesToRead - 1) - bufferStartPosition) / ratio)); + } + + if (outputSamples <= 0 || sourceSamplesToRead <= 0) + { + resultObj->setProperty ("error", "empty resampled context output"); + return juce::var (resultObj); + } + + const int channels = static_cast (reader->numChannels); + juce::AudioBuffer sourceBuffer (channels, sourceSamplesToRead); + sourceBuffer.clear(); + reader->read (&sourceBuffer, 0, sourceSamplesToRead, readStartSample, true, true); + + juce::AudioBuffer outputBuffer (channels, outputSamples); + outputBuffer.clear(); + for (int ch = 0; ch < channels; ++ch) + { + for (int i = 0; i < outputSamples; ++i) + { + const double filePos = bufferStartPosition + static_cast (i) * ratio; + outputBuffer.setSample (ch, i, sampleBufferCubicForParity (sourceBuffer, ch, filePos)); + } + } + + outputFile.getParentDirectory().createDirectory(); + const bool wrote = writeBufferToWavFile (outputBuffer, outputSamples, targetSampleRate, outputFile); + if (! wrote) + { + resultObj->setProperty ("error", "failed to write resampled baked context wav"); + return juce::var (resultObj); + } + + resultObj->setProperty ("success", true); + resultObj->setProperty ("filePath", outputFile.getFullPathName()); + resultObj->setProperty ("startSec", std::max (0.0, startSec)); + resultObj->setProperty ("durationSec", static_cast (outputSamples) / targetSampleRate); + resultObj->setProperty ("nativeSampleRate", sourceSampleRate); + resultObj->setProperty ("sampleRate", targetSampleRate); + resultObj->setProperty ("channels", channels); + resultObj->setProperty ("sourceReadStartSample", static_cast (readStartSample)); + resultObj->setProperty ("sourceReadStartFraction", fileStartFraction); + return juce::var (resultObj); +} + +double safeDbRatio (double numerator, double denominator) +{ + constexpr double eps = 1.0e-12; + return 20.0 * std::log10 (std::max (numerator, eps) / std::max (denominator, eps)); +} + +double computeBufferRms (const juce::AudioBuffer& buffer, + int channels, + int startSample, + int numSamples) +{ + if (channels <= 0 || numSamples <= 0) + return 0.0; + + double energy = 0.0; + juce::int64 count = 0; + const int start = juce::jlimit (0, buffer.getNumSamples(), startSample); + const int end = juce::jlimit (start, buffer.getNumSamples(), start + numSamples); + for (int ch = 0; ch < std::min (channels, buffer.getNumChannels()); ++ch) + { + const float* data = buffer.getReadPointer (ch); + for (int i = start; i < end; ++i) + { + const double s = data[i]; + energy += s * s; + ++count; + } + } + + return count > 0 ? std::sqrt (energy / static_cast (count)) : 0.0; +} + +double computeShortRmsEnvelopeCorrelation (const juce::AudioBuffer& a, + const juce::AudioBuffer& b, + int channels, + double sampleRate) +{ + const int samples = std::min (a.getNumSamples(), b.getNumSamples()); + const int frameSamples = std::max (16, static_cast (std::round (0.020 * sampleRate))); + const int hopSamples = std::max (8, static_cast (std::round (0.010 * sampleRate))); + if (samples < frameSamples || channels <= 0) + return 0.0; + + std::vector envA; + std::vector envB; + for (int start = 0; start + frameSamples <= samples; start += hopSamples) + { + envA.push_back (computeBufferRms (a, channels, start, frameSamples)); + envB.push_back (computeBufferRms (b, channels, start, frameSamples)); + } + if (envA.size() < 2 || envA.size() != envB.size()) + return 0.0; + + double meanA = 0.0; + double meanB = 0.0; + for (size_t i = 0; i < envA.size(); ++i) + { + meanA += envA[i]; + meanB += envB[i]; + } + meanA /= static_cast (envA.size()); + meanB /= static_cast (envB.size()); + + double numerator = 0.0; + double denomA = 0.0; + double denomB = 0.0; + for (size_t i = 0; i < envA.size(); ++i) + { + const double da = envA[i] - meanA; + const double db = envB[i] - meanB; + numerator += da * db; + denomA += da * da; + denomB += db * db; + } + + const double denom = std::sqrt (denomA * denomB); + return denom > 1.0e-12 ? numerator / denom : 0.0; +} + +int findFirstActiveSample (const juce::AudioBuffer& buffer, + int channels, + double threshold) +{ + for (int i = 0; i < buffer.getNumSamples(); ++i) + { + double peak = 0.0; + for (int ch = 0; ch < std::min (channels, buffer.getNumChannels()); ++ch) + peak = std::max (peak, std::abs (static_cast (buffer.getSample (ch, i)))); + if (peak >= threshold) + return i; + } + + return -1; +} + +juce::var comparePitchParityFiles (const juce::File& bakedFile, + const juce::File& appPlaybackFile, + double captureStartClipSec, + double noteStartClipSec, + double noteEndClipSec) +{ + auto* resultObj = new juce::DynamicObject(); + resultObj->setProperty ("success", false); + resultObj->setProperty ("bakedContextPath", bakedFile.getFullPathName()); + resultObj->setProperty ("appPlaybackContextPath", appPlaybackFile.getFullPathName()); + + juce::AudioBuffer baked; + juce::AudioBuffer app; + double bakedSampleRate = 0.0; + double appSampleRate = 0.0; + if (! readAudioFileForParity (bakedFile, baked, bakedSampleRate) + || ! readAudioFileForParity (appPlaybackFile, app, appSampleRate)) + { + resultObj->setProperty ("error", "failed to read parity wav files"); + return juce::var (resultObj); + } + + const bool resampledReferenceForParity = std::abs (bakedSampleRate - appSampleRate) > 1.0e-6; + juce::AudioBuffer bakedComparison; + if (resampledReferenceForParity) + { + bakedComparison = resampleBufferCubicForParity (baked, + bakedSampleRate, + appSampleRate, + app.getNumSamples()); + } + else + { + bakedComparison.makeCopyOf (baked, true); + } + + const double comparisonSampleRate = resampledReferenceForParity ? appSampleRate : bakedSampleRate; + const int channels = std::min (bakedComparison.getNumChannels(), app.getNumChannels()); + const int samples = std::min (bakedComparison.getNumSamples(), app.getNumSamples()); + if (channels <= 0 || samples <= 0) + { + resultObj->setProperty ("error", "parity wav shape mismatch"); + resultObj->setProperty ("bakedSampleRate", bakedSampleRate); + resultObj->setProperty ("appSampleRate", appSampleRate); + resultObj->setProperty ("comparisonSampleRate", comparisonSampleRate); + resultObj->setProperty ("resampledReferenceForParity", resampledReferenceForParity); + return juce::var (resultObj); + } + + double bakedEnergy = 0.0; + double appEnergy = 0.0; + double diffEnergy = 0.0; + double maxAbsDiff = 0.0; + juce::int64 count = 0; + for (int ch = 0; ch < channels; ++ch) + { + const float* bakedData = bakedComparison.getReadPointer (ch); + const float* appData = app.getReadPointer (ch); + for (int i = 0; i < samples; ++i) + { + const double a = bakedData[i]; + const double b = appData[i]; + const double d = b - a; + bakedEnergy += a * a; + appEnergy += b * b; + diffEnergy += d * d; + maxAbsDiff = std::max (maxAbsDiff, std::abs (d)); + ++count; + } + } + + const double bakedRms = count > 0 ? std::sqrt (bakedEnergy / static_cast (count)) : 0.0; + const double appRms = count > 0 ? std::sqrt (appEnergy / static_cast (count)) : 0.0; + const double diffRms = count > 0 ? std::sqrt (diffEnergy / static_cast (count)) : 0.0; + const double residualRelativeDb = safeDbRatio (diffRms, bakedRms); + + const int entryStartSample = juce::jlimit (0, samples, + static_cast (std::round ((noteStartClipSec - captureStartClipSec) * comparisonSampleRate))); + const int entryEndSample = juce::jlimit (entryStartSample, samples, + static_cast (std::round ((std::min (noteEndClipSec, noteStartClipSec + 0.240) - captureStartClipSec) * comparisonSampleRate))); + double entryBakedEnergy = 0.0; + double entryDiffEnergy = 0.0; + juce::int64 entryCount = 0; + for (int ch = 0; ch < channels; ++ch) + { + const float* bakedData = bakedComparison.getReadPointer (ch); + const float* appData = app.getReadPointer (ch); + for (int i = entryStartSample; i < entryEndSample; ++i) + { + const double a = bakedData[i]; + const double d = static_cast (appData[i]) - a; + entryBakedEnergy += a * a; + entryDiffEnergy += d * d; + ++entryCount; + } + } + const double entryBakedRms = entryCount > 0 ? std::sqrt (entryBakedEnergy / static_cast (entryCount)) : 0.0; + const double entryDiffRms = entryCount > 0 ? std::sqrt (entryDiffEnergy / static_cast (entryCount)) : 0.0; + const double entryResidualRelativeDb = safeDbRatio (entryDiffRms, entryBakedRms); + + const double activeThreshold = std::max (1.0e-5, bakedRms * 0.02); + const int bakedActiveSample = findFirstActiveSample (bakedComparison, channels, activeThreshold); + const int appActiveSample = findFirstActiveSample (app, channels, activeThreshold); + const double activeStartDeltaMs = (bakedActiveSample >= 0 && appActiveSample >= 0) + ? (static_cast (appActiveSample - bakedActiveSample) * 1000.0 / comparisonSampleRate) + : 0.0; + const double envelopeCorrelation = computeShortRmsEnvelopeCorrelation (bakedComparison, app, channels, comparisonSampleRate); + + const bool passed = residualRelativeDb <= -70.0 + && entryResidualRelativeDb <= -60.0 + && std::abs (activeStartDeltaMs) <= 1.0 + && envelopeCorrelation >= 0.995; + + resultObj->setProperty ("success", true); + resultObj->setProperty ("parityPassed", passed); + resultObj->setProperty ("residualRelativeDb", residualRelativeDb); + resultObj->setProperty ("entryResidualRelativeDb", entryResidualRelativeDb); + resultObj->setProperty ("shortRmsEnvelopeCorrelation", envelopeCorrelation); + resultObj->setProperty ("activeStartDeltaMs", activeStartDeltaMs); + resultObj->setProperty ("maxAbsDiff", maxAbsDiff); + resultObj->setProperty ("bakedRms", bakedRms); + resultObj->setProperty ("appPlaybackRms", appRms); + resultObj->setProperty ("diffRms", diffRms); + resultObj->setProperty ("sampleRate", comparisonSampleRate); + resultObj->setProperty ("bakedNativeSampleRate", bakedSampleRate); + resultObj->setProperty ("appSampleRate", appSampleRate); + resultObj->setProperty ("resampledReferenceForParity", resampledReferenceForParity); + resultObj->setProperty ("referenceResampler", resampledReferenceForParity ? "shared_cubic_fractional" : "none"); + resultObj->setProperty ("channels", channels); + resultObj->setProperty ("samplesCompared", samples); + resultObj->setProperty ("entryStartSample", entryStartSample); + resultObj->setProperty ("entryEndSample", entryEndSample); + return juce::var (resultObj); +} + +PitchPhraseRenderRegion resolvePitchPhraseRenderRegion (const std::vector& frames, + double editStartSec, + double editEndSec, + double clipDurationSec) +{ + PitchPhraseRenderRegion region; + + constexpr double searchSec = 0.75; + constexpr float safeConfidence = 0.20f; + constexpr float safeRmsDb = -55.0f; + + bool foundLeft = false; + bool foundRight = false; + double left = std::max (0.0, editStartSec - searchSec); + double right = std::min (clipDurationSec, editEndSec + searchSec); + + for (auto it = frames.rbegin(); it != frames.rend(); ++it) + { + const double t = static_cast (it->time); + if (t >= editStartSec) + continue; + if (editStartSec - t > searchSec) + break; + if (! it->voiced || it->confidence < safeConfidence || it->rmsDB < safeRmsDb) + { + left = t; + foundLeft = true; + break; + } + } + + for (const auto& frame : frames) + { + const double t = static_cast (frame.time); + if (t <= editEndSec) + continue; + if (t - editEndSec > searchSec) + break; + if (! frame.voiced || frame.confidence < safeConfidence || frame.rmsDB < safeRmsDb) + { + right = t; + foundRight = true; + break; + } + } + + region.startSec = juce::jlimit (0.0, clipDurationSec, left); + region.endSec = juce::jlimit (region.startSec, clipDurationSec, right); + region.startIsSafeBoundary = foundLeft || region.startSec <= 0.0; + region.endIsSafeBoundary = foundRight || region.endSec >= clipDurationSec; + region.expandedToFullClip = region.startSec <= 0.0 && region.endSec >= clipDurationSec; + + return region; +} + +struct PitchCommitRange +{ + double startSec = 0.0; + double endSec = 0.0; + double bodyStartSec = 0.0; + double bodyEndSec = 0.0; + double pitchRatio = 1.0; + int pitchDirection = 0; + juce::String wordGroupId; + int editedNoteCount = 1; + bool variablePitchRatio = false; + juce::String entryBoundaryKind = "unknown"; + juce::String exitBoundaryKind = "unknown"; + double entryBoundaryScore = 0.0; + double exitBoundaryScore = 0.0; +}; + +struct PitchCompositeStats +{ + int dryProtectedSamples = 0; + int preBodyDryProtectedSamples = 0; + double audibleCommitStartSec = 0.0; + double audibleCommitEndSec = 0.0; + double entryInsideBodyFadeMs = 12.0; + double exitLeadInMs = 12.0; + double entryBridgeStartSec = 0.0; + double entryBridgeEndSec = 0.0; + double entryBridgeWetLagMs = 0.0; + double entryBridgeEnvelopeGainDb = 0.0; + bool entryBridgeUsed = false; + bool entrySimpleHandoffUsed = false; + bool entrySafeHandoffUsed = false; + double entryDryHoldMs = 0.0; + double entrySafeBridgeMs = 0.0; + double entryWetAlignmentMs = 0.0; + double entryWetGainDb = 0.0; + double entryWetVsDryRmsDb = 0.0; + bool entryEqualPowerBlendUsed = false; + bool entryRmsContinuityUsed = false; + double entryRmsContinuityGainDb = 0.0; + double entryRmsContinuityMs = 0.0; + bool entryTransientResidualUsed = false; + double entryTransientResidualMix = 0.0; + double entryTransientResidualMs = 0.0; + bool entryPhaseSafeUsed = false; + bool entryWetAlignmentAccepted = false; + double entryFirstCycleCorrelation = 0.0; + double entryZeroCrossOffsetMs = 0.0; + double entryBridgeGainRampDb = 0.0; + double entryTransientDryPreservedMs = 0.0; + bool exitDryRestoreUsed = false; + double exitDryRestoreStartSec = 0.0; + double exitDryRestoreEndSec = 0.0; + bool entryTimbreCorrectionUsed = false; + double entryRmsTrimDb = 0.0; + double entryTiltDb = 0.0; + bool downshiftCoreEnvelopePassUsed = false; + double downshiftCoreRmsTrimDb = 0.0; + double downshiftCoreEnvelopeMaxDb = 0.0; + int downshiftCoreEnvelopeFrames = 0; + juce::String entryBoundaryKind = "unknown"; + juce::String exitBoundaryKind = "unknown"; + double entryBoundaryScore = 0.0; + double exitBoundaryScore = 0.0; +}; + +bool hasPitchEditForDryCommit (const PitchAnalyzer::PitchNote& note) +{ + return std::abs (note.correctedPitch - note.detectedPitch) > 0.01f; +} + +std::vector buildDryPitchCommitRanges (const std::vector& notes, + double clipDurationSec) +{ + std::vector ranges; + for (const auto& note : notes) + { + if (! hasPitchEditForDryCommit (note)) + continue; + + PitchCommitRange range; + const double effectiveStart = std::min (static_cast (note.startTime), + static_cast (note.effectiveStartTime)); + const double effectiveEnd = std::max (static_cast (note.endTime), + static_cast (note.effectiveEndTime)); + range.startSec = juce::jlimit (0.0, clipDurationSec, effectiveStart); + range.endSec = juce::jlimit (range.startSec, clipDurationSec, effectiveEnd); + range.bodyStartSec = juce::jlimit (range.startSec, range.endSec, static_cast (note.startTime)); + range.bodyEndSec = juce::jlimit (range.bodyStartSec, range.endSec, static_cast (note.endTime)); + const double semitoneDelta = static_cast (note.correctedPitch - note.detectedPitch); + range.pitchRatio = std::pow (2.0, semitoneDelta / 12.0); + range.pitchDirection = semitoneDelta > 0.01 ? 1 : (semitoneDelta < -0.01 ? -1 : 0); + range.wordGroupId = note.wordGroupId.isEmpty() ? note.id : note.wordGroupId; + range.editedNoteCount = 1; + range.variablePitchRatio = false; + range.entryBoundaryKind = note.entryBoundaryKind.isEmpty() ? "unknown" : note.entryBoundaryKind; + range.exitBoundaryKind = note.exitBoundaryKind.isEmpty() ? "unknown" : note.exitBoundaryKind; + range.entryBoundaryScore = static_cast (note.entryBoundaryScore); + range.exitBoundaryScore = static_cast (note.exitBoundaryScore); + if (range.endSec - range.startSec >= 0.0005) + ranges.push_back (range); + } + + std::sort (ranges.begin(), ranges.end(), [] (const auto& a, const auto& b) + { + return a.startSec < b.startSec || (a.startSec == b.startSec && a.endSec < b.endSec); + }); + + std::vector merged; + for (const auto& range : ranges) + { + const bool sameWordGroup = ! merged.empty() + && ! merged.back().wordGroupId.isEmpty() + && merged.back().wordGroupId == range.wordGroupId; + const bool nearBodyGap = ! merged.empty() + && range.bodyStartSec - merged.back().bodyEndSec <= 0.080; + const bool overlapsContext = ! merged.empty() + && range.startSec <= merged.back().endSec + 0.001; + if (merged.empty() || (! overlapsContext && ! sameWordGroup && ! nearBodyGap)) + { + merged.push_back (range); + continue; + } + merged.back().endSec = std::max (merged.back().endSec, range.endSec); + merged.back().bodyStartSec = std::min (merged.back().bodyStartSec, range.bodyStartSec); + merged.back().bodyEndSec = std::max (merged.back().bodyEndSec, range.bodyEndSec); + merged.back().exitBoundaryKind = range.exitBoundaryKind; + merged.back().exitBoundaryScore = range.exitBoundaryScore; + merged.back().editedNoteCount += range.editedNoteCount; + if (merged.back().wordGroupId != range.wordGroupId) + merged.back().wordGroupId = {}; + if (merged.back().pitchDirection != range.pitchDirection) + { + merged.back().pitchDirection = 0; + merged.back().pitchRatio = 1.0; + merged.back().variablePitchRatio = true; + } + else if (std::abs (merged.back().pitchRatio - range.pitchRatio) > 1.0e-4) + { + merged.back().pitchRatio = 1.0; + merged.back().variablePitchRatio = true; + } + else + { + merged.back().pitchRatio = range.pitchRatio; + } + } + + return merged; +} + +double getCommitStartSec (const std::vector& ranges) +{ + return ranges.empty() ? 0.0 : ranges.front().startSec; +} + +double getCommitEndSec (const std::vector& ranges) +{ + return ranges.empty() ? 0.0 : ranges.back().endSec; +} + +double getCommitDurationSec (const std::vector& ranges) +{ + double duration = 0.0; + for (const auto& range : ranges) + duration += std::max (0.0, range.endSec - range.startSec); + return duration; +} + +double getAudibleCommitStartSec (const std::vector& ranges) +{ + if (ranges.empty()) + return 0.0; + + double start = std::numeric_limits::max(); + for (const auto& range : ranges) + { + const double maxBridgePreSec = range.entryBoundaryKind == "hard_word_like" ? 0.012 : 0.024; + start = std::min (start, std::max (range.startSec, range.bodyStartSec - maxBridgePreSec)); + } + return start == std::numeric_limits::max() ? 0.0 : start; +} + +double getAudibleCommitEndSec (const std::vector& ranges) +{ + double end = 0.0; + for (const auto& range : ranges) + end = std::max (end, range.endSec); + return end; +} + +double getAudibleCommitDurationSec (const std::vector& ranges) +{ + double duration = 0.0; + for (const auto& range : ranges) + { + const double maxBridgePreSec = range.entryBoundaryKind == "hard_word_like" ? 0.012 : 0.024; + duration += std::max (0.0, range.endSec - std::max (range.startSec, range.bodyStartSec - maxBridgePreSec)); + } + return duration; +} + +struct PitchEntryBridgePlan +{ + int startSample = 0; + int endSample = 0; + int wetLagSamples = 0; + double envelopeGainDb = 0.0; + double correlation = 0.0; + double transientDryPreserveMs = 0.0; + bool used = false; + bool safeHandoffUsed = false; + int dryHoldSamples = 0; + double safeBridgeMs = 0.0; + double wetAlignmentMs = 0.0; + double wetVsDryRmsDb = 0.0; + bool equalPowerBlend = false; + bool phaseSafeUsed = false; + bool wetAlignmentAccepted = false; + double firstCycleCorrelation = 0.0; + double zeroCrossOffsetMs = 0.0; + double bridgeGainRampDb = 0.0; +}; + +float smoothHalfCosine (float t) +{ + t = juce::jlimit (0.0f, 1.0f, t); + return 0.5f - 0.5f * std::cos (juce::MathConstants::pi * t); +} + +float monoOriginalAt (const juce::AudioBuffer& buffer, int sample) +{ + const int numChannels = buffer.getNumChannels(); + if (numChannels <= 0 || sample < 0 || sample >= buffer.getNumSamples()) + return 0.0f; + + double sum = 0.0; + for (int ch = 0; ch < numChannels; ++ch) + sum += buffer.getSample (ch, sample); + return static_cast (sum / static_cast (numChannels)); +} + +double computeOriginalHighbandRmsDb (const juce::AudioBuffer& buffer, + int startSample, + int endSample) +{ + if (buffer.getNumChannels() <= 0 || buffer.getNumSamples() <= 0) + return -240.0; + + const int start = juce::jlimit (0, buffer.getNumSamples(), startSample); + const int end = juce::jlimit (start, buffer.getNumSamples(), endSample); + if (end <= start) + return -240.0; + + double sumSq = 0.0; + int count = 0; + float previous = monoOriginalAt (buffer, start - 1); + for (int sample = start; sample < end; ++sample) + { + const float current = monoOriginalAt (buffer, sample); + const double high = static_cast (current) - 0.97 * static_cast (previous); + sumSq += high * high; + previous = current; + ++count; + } + + if (count <= 0) + return -240.0; + + const double rms = std::sqrt (sumSq / static_cast (count) + 1.0e-24); + return 20.0 * std::log10 (std::max (1.0e-12, rms)); +} + +double computeOriginalEntryHighbandBurstDb (const juce::AudioBuffer& buffer, + int bodyStartSample, + double sampleRate) +{ + if (sampleRate <= 0.0) + return 0.0; + + const int windowSamples = std::max (1, static_cast (std::round (0.024 * sampleRate))); + const double pre = computeOriginalHighbandRmsDb (buffer, bodyStartSample - windowSamples, bodyStartSample); + const double post = computeOriginalHighbandRmsDb (buffer, bodyStartSample, bodyStartSample + windowSamples); + return post - pre; +} + +float monoCorrectedAt (const std::vector>& correctedContext, + int contextIndex) +{ + if (correctedContext.empty()) + return 0.0f; + + double sum = 0.0; + int count = 0; + for (const auto& channel : correctedContext) + { + if (contextIndex >= 0 && contextIndex < static_cast (channel.size())) + { + sum += channel[static_cast (contextIndex)]; + ++count; + } + } + return count > 0 ? static_cast (sum / static_cast (count)) : 0.0f; +} + +double computeMonoRms (const juce::AudioBuffer& original, + const std::vector>& correctedContext, + int contextStartSample, + int startSample, + int numSamples, + int wetLagSamples, + bool useCorrected) +{ + double sumSq = 0.0; + int count = 0; + for (int i = 0; i < numSamples; ++i) + { + const int sample = startSample + i; + const float value = useCorrected + ? monoCorrectedAt (correctedContext, sample - contextStartSample + wetLagSamples) + : monoOriginalAt (original, sample); + sumSq += static_cast (value) * static_cast (value); + ++count; + } + return count > 0 ? std::sqrt (sumSq / static_cast (count)) : 0.0; +} + +double computeDryWetCorrelation (const juce::AudioBuffer& original, + const std::vector>& correctedContext, + int contextStartSample, + int startSample, + int numSamples, + int wetLagSamples) +{ + if (numSamples <= 4) + return 0.0; + + double dryMean = 0.0; + double wetMean = 0.0; + int count = 0; + for (int i = 0; i < numSamples; ++i) + { + const int sample = startSample + i; + dryMean += monoOriginalAt (original, sample); + wetMean += monoCorrectedAt (correctedContext, sample - contextStartSample + wetLagSamples); + ++count; + } + + if (count <= 0) + return 0.0; + + dryMean /= static_cast (count); + wetMean /= static_cast (count); + + double numerator = 0.0; + double dryEnergy = 0.0; + double wetEnergy = 0.0; + for (int i = 0; i < numSamples; ++i) + { + const int sample = startSample + i; + const double dry = static_cast (monoOriginalAt (original, sample)) - dryMean; + const double wet = static_cast (monoCorrectedAt (correctedContext, sample - contextStartSample + wetLagSamples)) - wetMean; + numerator += dry * wet; + dryEnergy += dry * dry; + wetEnergy += wet * wet; + } + + return numerator / (std::sqrt (dryEnergy * wetEnergy) + 1.0e-20); +} + +PitchEntryBridgePlan choosePitchEntryBridgePlan (const juce::AudioBuffer& original, + const std::vector>& correctedContext, + const PitchCommitRange& range, + int contextStartSample, + int contextRangeStartSample, + int bodyStartSample, + int bodyEndSample, + int endSample, + double sampleRate, + bool allowUpwardEntryAlignment) +{ + PitchEntryBridgePlan plan; + plan.startSample = bodyStartSample; + const bool isDownward = range.pitchDirection < 0 || range.pitchRatio < 0.999; + const bool hardEntryBoundary = range.entryBoundaryKind == "hard_word_like"; + const bool softEntryBoundary = range.entryBoundaryKind == "soft_legato"; + const bool constrainedUpwardEntry = ! isDownward && ! softEntryBoundary; + const double upwardBridgeEndSec = static_cast (juce::jlimit (10.0f, 40.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_BRIDGE_UP_ALIGN_END_MS", hardEntryBoundary ? 12.0f : 16.0f))) * 0.001; + const double bridgeEndSec = hardEntryBoundary + ? (isDownward ? 0.040 : upwardBridgeEndSec) + : (softEntryBoundary ? (isDownward ? 0.080 : 0.048) : (isDownward ? 0.080 : upwardBridgeEndSec)); + const int bridgeEndOffset = static_cast (std::round (bridgeEndSec * sampleRate)); + plan.endSample = juce::jlimit (bodyStartSample, endSample, bodyStartSample + bridgeEndOffset); + plan.transientDryPreserveMs = hardEntryBoundary ? 6.0 : (isDownward ? 10.0 : 8.0); + + const bool useUpwardEntryAlignment = ! isDownward + && ! softEntryBoundary + && allowUpwardEntryAlignment + && juce::SystemStats::getEnvironmentVariable ("OPENSTUDIO_PITCH_ENTRY_BRIDGE_UP_ALIGN", "1").trim() != "0"; + + if (! isDownward && ! softEntryBoundary && ! useUpwardEntryAlignment) + { + plan.startSample = bodyStartSample; + plan.endSample = juce::jlimit (bodyStartSample, endSample, + bodyStartSample + static_cast (std::round ((hardEntryBoundary ? 0.012 : 0.016) * sampleRate))); + plan.wetLagSamples = 0; + plan.envelopeGainDb = 0.0; + plan.correlation = 1.0; + plan.transientDryPreserveMs = 0.0; + plan.used = plan.endSample > plan.startSample; + return plan; + } + + if (constrainedUpwardEntry) + { + plan.transientDryPreserveMs = static_cast (juce::jlimit (0.0f, 12.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_BRIDGE_UP_ALIGN_DRY_PRESERVE_MS", 0.0f))); + } + + const double defaultPreSec = hardEntryBoundary ? 0.012 : (softEntryBoundary ? 0.024 : 0.024); + const double upwardPreSec = static_cast (juce::jlimit (0.0f, 12.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_BRIDGE_UP_ALIGN_PRE_MS", 0.0f))) * 0.001; + const double upwardPostSec = static_cast (juce::jlimit (2.0f, 14.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_BRIDGE_UP_ALIGN_POST_MS", 8.0f))) * 0.001; + const double upwardLagSec = static_cast (juce::jlimit (1.0f, 12.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_BRIDGE_UP_ALIGN_MAX_LAG_MS", 6.0f))) * 0.001; + const int maxPreSamples = static_cast (std::round ((constrainedUpwardEntry ? upwardPreSec : defaultPreSec) * sampleRate)); + const int postSearchSamples = static_cast (std::round ((constrainedUpwardEntry ? upwardPostSec : 0.008) * sampleRate)); + const int searchStart = juce::jlimit (contextRangeStartSample, endSample, bodyStartSample - maxPreSamples); + const int searchEnd = juce::jlimit (searchStart, std::min (bodyEndSample, endSample), bodyStartSample + postSearchSamples); + const int analysisSamples = std::max (32, static_cast (std::round (0.012 * sampleRate))); + const int energySamples = std::max (16, static_cast (std::round (0.006 * sampleRate))); + const int maxLagSamples = static_cast (std::round ((constrainedUpwardEntry ? upwardLagSec : (hardEntryBoundary ? 0.012 : 0.024)) * sampleRate)); + const int lagStep = std::max (1, static_cast (std::round (0.00025 * sampleRate))); + const int candidateStep = std::max (1, static_cast (std::round (0.001 * sampleRate))); + const double bodyRms = computeMonoRms (original, + correctedContext, + contextStartSample, + bodyStartSample, + std::max (analysisSamples, std::min (bodyEndSample - bodyStartSample, static_cast (std::round (0.040 * sampleRate)))), + 0, + false); + + double bestScore = std::numeric_limits::max(); + double bestCorrelation = -1.0; + int bestStart = bodyStartSample; + int bestLag = 0; + + for (int candidate = searchStart; candidate <= searchEnd; candidate += candidateStep) + { + const int available = std::min (analysisSamples, endSample - candidate); + if (available <= 16) + continue; + + int localBestLag = 0; + double localBestCorrelation = -1.0; + for (int lag = -maxLagSamples; lag <= maxLagSamples; lag += lagStep) + { + const int wetStart = candidate - contextStartSample + lag; + const int wetEnd = wetStart + available; + if (wetStart < 0) + continue; + bool hasEnoughCorrected = true; + for (const auto& channel : correctedContext) + { + if (wetEnd > static_cast (channel.size())) + { + hasEnoughCorrected = false; + break; + } + } + if (! hasEnoughCorrected) + continue; + + const double correlation = computeDryWetCorrelation ( + original, correctedContext, contextStartSample, candidate, available, lag); + if (correlation > localBestCorrelation) + { + localBestCorrelation = correlation; + localBestLag = lag; + } + } + + if (localBestCorrelation < -0.5) + continue; + + const double dryRms = computeMonoRms (original, + correctedContext, + contextStartSample, + candidate, + std::min (energySamples, endSample - candidate), + 0, + false); + const double wetRms = computeMonoRms (original, + correctedContext, + contextStartSample, + candidate, + std::min (energySamples, endSample - candidate), + localBestLag, + true); + const float dryPrev = monoOriginalAt (original, candidate - 1); + const float dryHere = monoOriginalAt (original, candidate); + const float wetHere = monoCorrectedAt (correctedContext, candidate - contextStartSample + localBestLag); + const double derivativeMismatch = std::abs ((static_cast (wetHere) - static_cast (dryPrev)) + - (static_cast (dryHere) - static_cast (dryPrev))); + const double rmsScale = std::max (1.0e-5, bodyRms); + const double energyPenalty = std::min (1.0, dryRms / rmsScale); + const double wetDryRmsPenalty = std::min (1.0, std::abs (wetRms - dryRms) / rmsScale); + const double preNoteBonus = (! constrainedUpwardEntry && candidate < bodyStartSample) ? -0.08 : 0.0; + const double score = (1.0 - juce::jlimit (-1.0, 1.0, localBestCorrelation)) * 2.0 + + energyPenalty * 0.35 + + wetDryRmsPenalty * 0.25 + + std::min (1.0, derivativeMismatch / rmsScale) * 0.25 + + preNoteBonus; + + if (score < bestScore) + { + bestScore = score; + bestCorrelation = localBestCorrelation; + bestStart = candidate; + bestLag = localBestLag; + } + } + + const int forcedUpwardPreSamples = constrainedUpwardEntry + ? static_cast (std::round (static_cast (juce::jlimit (0.0f, 24.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_BRIDGE_UP_FORCE_PRE_MS", + (! hardEntryBoundary + && computeOriginalEntryHighbandBurstDb (original, bodyStartSample, sampleRate) + <= static_cast (getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_BRIDGE_UP_MAX_SOURCE_BURST_DB", 1.0f))) + ? 24.0f + : 0.0f))) * 0.001 * sampleRate)) + : 0; + if (forcedUpwardPreSamples > 0) + bestStart = juce::jlimit (contextRangeStartSample, bodyStartSample, + bodyStartSample - forcedUpwardPreSamples); + + const int startLowerBound = forcedUpwardPreSamples > 0 ? contextRangeStartSample : searchStart; + plan.startSample = juce::jlimit (startLowerBound, std::max (bodyStartSample, searchEnd), bestStart); + if (plan.endSample <= plan.startSample) + plan.endSample = juce::jlimit (plan.startSample, endSample, plan.startSample + std::max (1, bridgeEndOffset)); + + const int gainSamples = std::min (std::max (16, static_cast (std::round (0.008 * sampleRate))), + std::max (1, plan.endSample - plan.startSample)); + const double dryRms = computeMonoRms (original, correctedContext, contextStartSample, plan.startSample, gainSamples, 0, false); + const double wetRms = computeMonoRms (original, correctedContext, contextStartSample, plan.startSample, gainSamples, bestLag, true); + const int bodyGainSamples = std::min (std::max (16, static_cast (std::round (0.024 * sampleRate))), + std::max (1, plan.endSample - bodyStartSample)); + const double dryBodyRms = computeMonoRms (original, correctedContext, contextStartSample, bodyStartSample, bodyGainSamples, 0, false); + const double wetBodyRms = computeMonoRms (original, correctedContext, contextStartSample, bodyStartSample, bodyGainSamples, 0, true); + const double bridgeGainDb = 20.0 * std::log10 ((dryRms + 1.0e-8) / (wetRms + 1.0e-8)); + const double bodyGainDb = 20.0 * std::log10 ((dryBodyRms + 1.0e-8) / (wetBodyRms + 1.0e-8)); + + const int downshiftTargetLag = -static_cast (std::round ((hardEntryBoundary ? 0.010 : 0.022) * sampleRate)); + plan.wetLagSamples = isDownward ? std::min (bestLag, downshiftTargetLag) : bestLag; + const double gainLimitDb = constrainedUpwardEntry + ? static_cast (juce::jlimit (0.0f, 1.2f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_BRIDGE_UP_ALIGN_GAIN_LIMIT_DB", 0.0f))) + : (hardEntryBoundary ? 1.2 : 1.7); + plan.envelopeGainDb = juce::jlimit (-gainLimitDb, gainLimitDb, + std::abs (bodyGainDb) > std::abs (bridgeGainDb) ? bodyGainDb : bridgeGainDb); + plan.correlation = bestCorrelation; + plan.used = plan.endSample > plan.startSample; + return plan; +} + +PitchEntryBridgePlan makeSimplePitchEntryHandoffPlan (int bodyStartSample, + int bodyEndSample, + int endSample, + double sampleRate) +{ + PitchEntryBridgePlan plan; + plan.startSample = bodyStartSample; + const float handoffMs = juce::jlimit (6.0f, 10.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_SIMPLE_ENTRY_HANDOFF_MS", 8.0f)); + const int handoffSamples = std::max (1, static_cast (std::round (handoffMs * 0.001f * static_cast (sampleRate)))); + const int maxEnd = std::max (bodyStartSample, std::min (bodyEndSample, endSample)); + plan.endSample = juce::jlimit (bodyStartSample, maxEnd, bodyStartSample + handoffSamples); + plan.wetLagSamples = 0; + plan.envelopeGainDb = 0.0; + plan.correlation = 1.0; + plan.transientDryPreserveMs = 0.0; + plan.used = plan.endSample > plan.startSample; + return plan; +} + +bool correctedContextHasRange (const std::vector>& correctedContext, + int startIndex, + int endIndex) +{ + if (correctedContext.empty() || startIndex < 0 || endIndex <= startIndex) + return false; + + for (const auto& channel : correctedContext) + { + if (endIndex > static_cast (channel.size())) + return false; + } + + return true; +} + +int chooseEntrySafeWetLag (const juce::AudioBuffer& original, + const std::vector>& correctedContext, + int contextStartSample, + int analysisStartSample, + int analysisSamples, + int maxLagSamples, + double sampleRate, + double& bestCorrelation) +{ + bestCorrelation = 0.0; + if (analysisSamples <= 16 || maxLagSamples <= 0 || sampleRate <= 0.0) + return 0; + + const int lagStep = std::max (1, static_cast (std::round (0.00025 * sampleRate))); + int bestLag = 0; + double bestScore = -2.0; + + for (int lag = -maxLagSamples; lag <= maxLagSamples; lag += lagStep) + { + const int wetStart = analysisStartSample - contextStartSample + lag; + if (! correctedContextHasRange (correctedContext, wetStart, wetStart + analysisSamples)) + continue; + + const double dryRms = computeMonoRms (original, + correctedContext, + contextStartSample, + analysisStartSample, + analysisSamples, + 0, + false); + const double wetRms = computeMonoRms (original, + correctedContext, + contextStartSample, + analysisStartSample, + analysisSamples, + lag, + true); + if (dryRms <= 1.0e-7 || wetRms <= 1.0e-7) + continue; + + const double correlation = computeDryWetCorrelation ( + original, correctedContext, contextStartSample, analysisStartSample, analysisSamples, lag); + if (correlation > bestScore) + { + bestScore = correlation; + bestLag = lag; + } + } + + bestCorrelation = bestScore > -1.5 ? bestScore : 0.0; + return bestCorrelation >= 0.10 ? bestLag : 0; +} + +int chooseEntryPhaseSafeDryHoldSamples (const juce::AudioBuffer& original, + const std::vector>& correctedContext, + int contextStartSample, + int bodyStartSample, + int bodyEndSample, + int wetLagSamples, + double sampleRate, + float defaultMinDryHoldMs = 0.0f, + float defaultMaxDryHoldMs = -1.0f) +{ + if (sampleRate <= 0.0 || bodyEndSample <= bodyStartSample) + return 0; + + const bool longBlendEnabled = getPitchEnvFlag ("OPENSTUDIO_PITCH_ENTRY_LONG_BLEND_ENABLE", false); + if (defaultMaxDryHoldMs < 0.0f) + defaultMaxDryHoldMs = longBlendEnabled ? 2.0f : 3.0f; + const int minOffset = std::max (0, static_cast (std::round (juce::jlimit (0.0f, 14.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_PHASE_SAFE_MIN_DRY_HOLD_MS", defaultMinDryHoldMs)) + * 0.001f * static_cast (sampleRate)))); + const int maxOffset = std::max (minOffset, static_cast (std::round (juce::jlimit (2.0f, 14.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_PHASE_SAFE_MAX_DRY_HOLD_MS", defaultMaxDryHoldMs)) + * 0.001f * static_cast (sampleRate)))); + const int searchEnd = std::min (bodyEndSample - 2, bodyStartSample + maxOffset); + if (searchEnd <= bodyStartSample) + return 0; + + const int searchStart = std::min (searchEnd, bodyStartSample + minOffset); + const int bodyAnalysisSamples = std::min (std::max (32, static_cast (std::round (0.030 * sampleRate))), + std::max (1, bodyEndSample - bodyStartSample)); + const double bodyRms = std::max (1.0e-5, computeMonoRms (original, + correctedContext, + contextStartSample, + bodyStartSample, + bodyAnalysisSamples, + 0, + false)); + + int bestOffset = searchStart - bodyStartSample; + double bestScore = std::numeric_limits::max(); + float prevOriginal = monoOriginalAt (original, bodyStartSample - 1); + float prevWet = monoCorrectedAt (correctedContext, bodyStartSample - contextStartSample - 1 + wetLagSamples); + + for (int sample = searchStart; sample <= searchEnd; ++sample) + { + const int wetIndex = sample - contextStartSample + wetLagSamples; + if (! correctedContextHasRange (correctedContext, wetIndex, wetIndex + 1)) + continue; + + const float dry = monoOriginalAt (original, sample); + const float wet = monoCorrectedAt (correctedContext, wetIndex); + const float dryPrev = sample == bodyStartSample ? prevOriginal : monoOriginalAt (original, sample - 1); + const float wetPrev = sample == bodyStartSample ? prevWet : monoCorrectedAt (correctedContext, wetIndex - 1); + + const double dryDerivative = static_cast (dry) - static_cast (dryPrev); + const double wetDerivative = static_cast (wet) - static_cast (wetPrev); + const double signBonus = ((dryPrev <= 0.0f && dry >= 0.0f) || (dryPrev >= 0.0f && dry <= 0.0f)) ? -0.20 : 0.0; + const double wetSignBonus = ((wetPrev <= 0.0f && wet >= 0.0f) || (wetPrev >= 0.0f && wet <= 0.0f)) ? -0.10 : 0.0; + const double offsetPenalty = static_cast (sample - bodyStartSample) / static_cast (std::max (1, maxOffset)) * 0.12; + const double score = std::abs (static_cast (dry)) / bodyRms + + std::abs (static_cast (wet)) / bodyRms * 0.55 + + std::abs (dryDerivative) / bodyRms * 0.45 + + std::abs (wetDerivative) / bodyRms * 0.25 + + std::abs (dryDerivative - wetDerivative) / bodyRms * 0.35 + + offsetPenalty + + signBonus + + wetSignBonus; + + if (score < bestScore) + { + bestScore = score; + bestOffset = sample - bodyStartSample; + } + + prevOriginal = dry; + prevWet = wet; + } + + return juce::jlimit (0, maxOffset, bestOffset); +} + +PitchEntryBridgePlan makeEntrySafePitchHandoffPlan (const juce::AudioBuffer& original, + const std::vector>& correctedContext, + int contextStartSample, + int bodyStartSample, + int bodyEndSample, + int endSample, + double sampleRate) +{ + PitchEntryBridgePlan plan; + plan.startSample = bodyStartSample; + + if (sampleRate <= 0.0) + return plan; + + const bool phaseSafeEnabled = ! getPitchEnvFlag ("OPENSTUDIO_PITCH_ENTRY_PHASE_SAFE_DISABLE", false); + const bool wetAlignmentEnabled = getPitchEnvFlag ("OPENSTUDIO_PITCH_ENTRY_ALIGN_ENABLE", false) + && ! getPitchEnvFlag ("OPENSTUDIO_PITCH_ENTRY_ALIGN_DISABLE", false); + const bool longBlendEnabled = getPitchEnvFlag ("OPENSTUDIO_PITCH_ENTRY_LONG_BLEND_ENABLE", false); + const float defaultDryHoldMs = phaseSafeEnabled + ? 0.0f + : 3.0f; + const float dryHoldMs = juce::jlimit (0.0f, 14.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_SAFE_DRY_HOLD_MS", defaultDryHoldMs)); + const float bridgeMs = phaseSafeEnabled + ? (longBlendEnabled + ? juce::jlimit (10.0f, 36.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_SAFE_BRIDGE_MS", 24.0f)) + : juce::jlimit (3.0f, 6.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_SAFE_BRIDGE_MS", 3.0f))) + : juce::jlimit (12.0f, 30.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_SAFE_BRIDGE_MS", 18.0f)); + const int requestedDryHoldSamples = std::max (0, static_cast (std::round ( + dryHoldMs * 0.001f * static_cast (sampleRate)))); + const int requestedBridgeSamples = std::max (1, static_cast (std::round ( + bridgeMs * 0.001f * static_cast (sampleRate)))); + const int maxEnd = std::max (bodyStartSample, std::min (bodyEndSample, endSample)); + plan.endSample = juce::jlimit (bodyStartSample, maxEnd, + bodyStartSample + requestedDryHoldSamples + requestedBridgeSamples); + const int totalSamples = plan.endSample - plan.startSample; + if (totalSamples <= 1) + return plan; + + const int requestedBridgeStartSample = std::min (plan.endSample - 1, + plan.startSample + std::min (requestedDryHoldSamples, std::max (0, totalSamples / 3))); + const int requestedBridgeSamplesAvailable = std::max (1, plan.endSample - requestedBridgeStartSample); + const int maxLagSamples = std::max (1, static_cast (std::round ( + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_SAFE_MAX_ALIGN_MS", 3.0f) + * 0.001f * static_cast (sampleRate)))); + const int analysisSamples = std::min (requestedBridgeSamplesAvailable, + std::max (32, static_cast (std::round (0.014 * sampleRate)))); + + double bestCorrelation = 0.0; + const double zeroLagCorrelation = computeDryWetCorrelation (original, + correctedContext, + contextStartSample, + requestedBridgeStartSample, + analysisSamples, + 0); + const int candidateLagSamples = wetAlignmentEnabled + ? chooseEntrySafeWetLag (original, + correctedContext, + contextStartSample, + requestedBridgeStartSample, + analysisSamples, + maxLagSamples, + sampleRate, + bestCorrelation) + : 0; + const double acceptedCorrelation = candidateLagSamples == 0 ? zeroLagCorrelation : bestCorrelation; + const double improvement = bestCorrelation - zeroLagCorrelation; + const bool acceptAlignment = phaseSafeEnabled + ? (wetAlignmentEnabled + && candidateLagSamples != 0 + && bestCorrelation >= static_cast (getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_ALIGN_MIN_CORRELATION", 0.25f)) + && improvement >= static_cast (getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_ALIGN_MIN_IMPROVEMENT", 0.08f))) + : (candidateLagSamples != 0); + plan.wetLagSamples = acceptAlignment ? candidateLagSamples : 0; + plan.correlation = acceptAlignment ? bestCorrelation : zeroLagCorrelation; + plan.wetAlignmentMs = static_cast (plan.wetLagSamples) * 1000.0 / sampleRate; + plan.wetAlignmentAccepted = acceptAlignment; + plan.firstCycleCorrelation = acceptedCorrelation; + + const int gainSamples = std::min (std::max (16, static_cast (std::round (0.045 * sampleRate))), + std::max (1, bodyEndSample - bodyStartSample)); + const double preliminaryDryRms = computeMonoRms (original, + correctedContext, + contextStartSample, + bodyStartSample, + gainSamples, + 0, + false); + const double preliminaryWetRms = computeMonoRms (original, + correctedContext, + contextStartSample, + bodyStartSample, + gainSamples, + plan.wetLagSamples, + true); + const double preliminaryWetVsDryRmsDb = 20.0 * std::log10 ((preliminaryWetRms + 1.0e-8) / (preliminaryDryRms + 1.0e-8)); + const bool weakWetDryShellEnabled = getPitchEnvFlag ("OPENSTUDIO_PITCH_ENTRY_WEAK_WET_DRY_SHELL_ENABLE", false); + const bool weakWetOnset = weakWetDryShellEnabled + && phaseSafeEnabled + && ! longBlendEnabled + && preliminaryWetVsDryRmsDb <= static_cast (getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_WEAK_WET_THRESHOLD_DB", -10.0f)); + + if (phaseSafeEnabled) + { + plan.phaseSafeUsed = true; + const float weakDryHoldMs = juce::jlimit (3.0f, 14.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_WEAK_WET_DRY_HOLD_MS", 8.0f)); + plan.dryHoldSamples = chooseEntryPhaseSafeDryHoldSamples (original, + correctedContext, + contextStartSample, + bodyStartSample, + bodyEndSample, + plan.wetLagSamples, + sampleRate, + weakWetOnset ? weakDryHoldMs : 0.0f, + weakWetOnset ? weakDryHoldMs : -1.0f); + const int phaseSafeBridgeStartSample = std::min (maxEnd - 1, plan.startSample + plan.dryHoldSamples); + const int phaseSafeBridgeEndSample = juce::jlimit (phaseSafeBridgeStartSample + 1, + maxEnd, + phaseSafeBridgeStartSample + requestedBridgeSamples); + plan.endSample = phaseSafeBridgeEndSample; + plan.zeroCrossOffsetMs = static_cast (plan.dryHoldSamples) * 1000.0 / sampleRate; + plan.firstCycleCorrelation = computeDryWetCorrelation (original, + correctedContext, + contextStartSample, + phaseSafeBridgeStartSample, + std::min (std::max (32, static_cast (std::round (0.012 * sampleRate))), + std::max (1, plan.endSample - phaseSafeBridgeStartSample)), + plan.wetLagSamples); + } + else + { + plan.dryHoldSamples = std::min (requestedDryHoldSamples, std::max (0, totalSamples / 3)); + plan.zeroCrossOffsetMs = static_cast (plan.dryHoldSamples) * 1000.0 / sampleRate; + } + + const int bridgeStartSample = std::min (plan.endSample - 1, plan.startSample + plan.dryHoldSamples); + const int bridgeSamples = std::max (1, plan.endSample - bridgeStartSample); + + const double dryRms = computeMonoRms (original, + correctedContext, + contextStartSample, + bodyStartSample, + gainSamples, + 0, + false); + const double wetRms = computeMonoRms (original, + correctedContext, + contextStartSample, + bodyStartSample, + gainSamples, + plan.wetLagSamples, + true); + plan.wetVsDryRmsDb = 20.0 * std::log10 ((wetRms + 1.0e-8) / (dryRms + 1.0e-8)); + plan.envelopeGainDb = (! phaseSafeEnabled && plan.wetVsDryRmsDb < -0.5) + ? juce::jlimit (0.0, 4.0, -plan.wetVsDryRmsDb) + : 0.0; + plan.bridgeGainRampDb = phaseSafeEnabled ? 0.0 : plan.envelopeGainDb; + plan.transientDryPreserveMs = static_cast (plan.dryHoldSamples) * 1000.0 / sampleRate; + plan.safeBridgeMs = static_cast (bridgeSamples) * 1000.0 / sampleRate; + plan.used = plan.endSample > plan.startSample; + plan.safeHandoffUsed = plan.used; + plan.equalPowerBlend = plan.used && getPitchEnvFlag ("OPENSTUDIO_PITCH_ENTRY_EQUAL_POWER_ENABLE", false); + return plan; +} + +struct PitchEntryTimbrePlan +{ + bool used = false; + float rmsTrimDb = 0.0f; + float tiltDb = 0.0f; +}; + +struct PitchEntryRmsContinuityPlan +{ + bool used = false; + double gainDb = 0.0; + double durationMs = 0.0; +}; + +struct PitchEntryTransientResidualPlan +{ + bool used = false; + double mix = 0.0; + double durationMs = 0.0; +}; + +struct PitchDownshiftCoreEnvelopePlan +{ + bool used = false; + double rmsTrimDb = 0.0; + double maxFrameGainDb = 0.0; + int envelopeFrames = 0; +}; + +PitchDownshiftCoreEnvelopePlan applyDownshiftCoreEnvelopePass (juce::AudioBuffer& destination, + const juce::AudioBuffer& original, + const PitchCommitRange& range, + double sampleRate) +{ + PitchDownshiftCoreEnvelopePlan plan; + if (range.pitchDirection >= 0 + || ! getPitchEnvFlag ("OPENSTUDIO_PITCH_DOWNSHIFT_CORE_ENVELOPE_ENABLE", true) + || getPitchEnvFlag ("OPENSTUDIO_PITCH_DOWNSHIFT_CORE_ENVELOPE_DISABLE", false) + || sampleRate <= 0.0 + || destination.getNumChannels() <= 0 + || destination.getNumSamples() <= 0) + return plan; + + const int numSamples = destination.getNumSamples(); + const int bodyStart = juce::jlimit (0, numSamples, static_cast (std::round (range.bodyStartSec * sampleRate))); + const int bodyEnd = juce::jlimit (bodyStart, numSamples, static_cast (std::round (range.bodyEndSec * sampleRate))); + const int edgeGuardSamples = std::max (1, static_cast (std::round ( + juce::jlimit (45.0f, 110.0f, getPitchEnvFloat ("OPENSTUDIO_PITCH_DOWNSHIFT_CORE_EDGE_GUARD_MS", 80.0f)) + * 0.001f * static_cast (sampleRate)))); + int coreStart = std::min (bodyEnd, bodyStart + edgeGuardSamples); + int coreEnd = std::max (coreStart, bodyEnd - edgeGuardSamples); + if (coreEnd - coreStart < static_cast (std::round (0.090 * sampleRate))) + { + const int fallbackGuard = std::max (1, static_cast (std::round (0.045 * sampleRate))); + coreStart = std::min (bodyEnd, bodyStart + fallbackGuard); + coreEnd = std::max (coreStart, bodyEnd - fallbackGuard); + } + if (coreEnd - coreStart < static_cast (std::round (0.070 * sampleRate))) + return plan; + + const float maxTrimDb = juce::jlimit (0.0f, 1.5f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_DOWNSHIFT_CORE_MAX_TRIM_DB", 1.5f)); + const float sourceHeadroomDb = juce::jlimit (0.0f, 2.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_DOWNSHIFT_CORE_SOURCE_HEADROOM_DB", 2.0f)); + const float frameMs = juce::jlimit (12.0f, 35.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_DOWNSHIFT_CORE_ENV_FRAME_MS", 24.0f)); + const float hopMs = juce::jlimit (4.0f, 16.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_DOWNSHIFT_CORE_ENV_HOP_MS", 8.0f)); + const int frameSamples = std::max (8, static_cast (std::round (frameMs * 0.001f * static_cast (sampleRate)))); + const int hopSamples = std::max (1, static_cast (std::round (hopMs * 0.001f * static_cast (sampleRate)))); + const int fadeSamples = std::max (1, static_cast (std::round (0.025 * sampleRate))); + const double sourceHeadroomGain = std::pow (10.0, -static_cast (sourceHeadroomDb) / 20.0); + + std::vector gainSum (static_cast (numSamples), 0.0f); + std::vector weightSum (static_cast (numSamples), 0.0f); + double globalSourceEnergy = 0.0; + double globalDestEnergy = 0.0; + int globalCount = 0; + + for (int frameStart = coreStart; frameStart < coreEnd; frameStart += hopSamples) + { + const int frameEnd = std::min (coreEnd, frameStart + frameSamples); + if (frameEnd - frameStart < 8) + continue; + + double sourceEnergy = 0.0; + double destEnergy = 0.0; + int count = 0; + for (int ch = 0; ch < destination.getNumChannels(); ++ch) + { + if (ch >= original.getNumChannels()) + continue; + + for (int sample = frameStart; sample < frameEnd; ++sample) + { + const double source = static_cast (original.getSample (ch, sample)) * sourceHeadroomGain; + const double dest = static_cast (destination.getSample (ch, sample)); + sourceEnergy += source * source; + destEnergy += dest * dest; + globalSourceEnergy += source * source; + globalDestEnergy += dest * dest; + ++count; + ++globalCount; + } + } + + if (count <= 0 || sourceEnergy <= 1.0e-12 || destEnergy <= 1.0e-12) + continue; + + const double frameGainDb = juce::jlimit (-static_cast (maxTrimDb), + static_cast (maxTrimDb), + 10.0 * std::log10 (sourceEnergy / destEnergy)); + const float frameGain = static_cast (std::pow (10.0, frameGainDb / 20.0)); + plan.maxFrameGainDb = std::abs (frameGainDb) > std::abs (plan.maxFrameGainDb) + ? frameGainDb + : plan.maxFrameGainDb; + ++plan.envelopeFrames; + + for (int sample = frameStart; sample < frameEnd; ++sample) + { + const float local = static_cast (sample - frameStart) / static_cast (std::max (1, frameEnd - frameStart - 1)); + const float window = std::sin (juce::MathConstants::pi * juce::jlimit (0.0f, 1.0f, local)); + const size_t index = static_cast (sample); + gainSum[index] += frameGain * window; + weightSum[index] += window; + } + } + + if (plan.envelopeFrames <= 0 || globalCount <= 0 || globalSourceEnergy <= 1.0e-12 || globalDestEnergy <= 1.0e-12) + return PitchDownshiftCoreEnvelopePlan(); + + plan.rmsTrimDb = juce::jlimit (-static_cast (maxTrimDb), + static_cast (maxTrimDb), + 10.0 * std::log10 (globalSourceEnergy / globalDestEnergy)); + const float globalGain = static_cast (std::pow (10.0, plan.rmsTrimDb / 20.0)); + + for (int ch = 0; ch < destination.getNumChannels(); ++ch) + { + for (int sample = coreStart; sample < coreEnd; ++sample) + { + const size_t index = static_cast (sample); + if (weightSum[index] <= 1.0e-5f) + continue; + + const float fromStart = static_cast (sample - coreStart) / static_cast (fadeSamples); + const float fromEnd = static_cast (coreEnd - 1 - sample) / static_cast (fadeSamples); + const float fade = smoothHalfCosine (std::min (fromStart, fromEnd)); + const float frameGain = gainSum[index] / weightSum[index]; + const float targetGain = std::sqrt (std::max (0.05f, frameGain * globalGain)); + const float gain = 1.0f + (targetGain - 1.0f) * fade; + destination.setSample (ch, sample, destination.getSample (ch, sample) * gain); + } + } + + plan.used = true; + return plan; +} + +PitchEntryRmsContinuityPlan applyPitchEntryRmsContinuity (juce::AudioBuffer& destination, + const juce::AudioBuffer& original, + const PitchCommitRange& range, + double sampleRate, + bool enabled, + int startSampleOverride = -1, + double maxGainDbOverride = -1.0, + double attackMsOverride = -1.0) +{ + PitchEntryRmsContinuityPlan plan; + if (! enabled + || getPitchEnvFlag ("OPENSTUDIO_PITCH_ENTRY_RMS_CONTINUITY_DISABLE", false) + || sampleRate <= 0.0 + || destination.getNumChannels() <= 0 + || destination.getNumSamples() <= 0) + return plan; + + const int numSamples = destination.getNumSamples(); + const int bodyStart = juce::jlimit (0, numSamples, static_cast (std::round (range.bodyStartSec * sampleRate))); + const int entryStart = juce::jlimit (bodyStart, numSamples, startSampleOverride >= 0 ? startSampleOverride : bodyStart); + const int entryLimit = juce::jlimit (entryStart, numSamples, static_cast (std::round (range.bodyEndSec * sampleRate))); + const float continuityDurationMs = juce::jlimit (30.0f, 120.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_RMS_CONTINUITY_DURATION_MS", 45.0f)); + const int entrySamples = std::min (entryLimit - entryStart, + std::max (1, static_cast (std::round (continuityDurationMs * 0.001f * static_cast (sampleRate))))); + if (entrySamples <= 16) + return plan; + const int measurementSamples = std::min (entrySamples, + std::max (1, static_cast (std::round (0.045 * sampleRate)))); + + double sourceEnergy = 0.0; + double destEnergy = 0.0; + int count = 0; + for (int ch = 0; ch < destination.getNumChannels(); ++ch) + { + if (ch >= original.getNumChannels()) + continue; -juce::File getOpenStudioApplicationDataDirectory() -{ - auto appDataDir = juce::File::getSpecialLocation(juce::File::userApplicationDataDirectory); - return appDataDir.getChildFile("OpenStudio"); -} + for (int i = 0; i < measurementSamples; ++i) + { + const int sample = entryStart + i; + const double source = static_cast (original.getSample (ch, sample)); + const double dest = static_cast (destination.getSample (ch, sample)); + sourceEnergy += source * source; + destEnergy += dest * dest; + ++count; + } + } -juce::File getLegacyStudio13ApplicationDataDirectory() -{ - auto appDataDir = juce::File::getSpecialLocation(juce::File::userApplicationDataDirectory); - return appDataDir.getChildFile("Studio13"); -} + if (count <= 0 || sourceEnergy <= 1.0e-10 || destEnergy <= 1.0e-10) + return plan; -juce::File getPreferredAppDataDirectory() -{ - auto openStudioDir = getOpenStudioDocumentsDirectory(); - auto legacyDir = getLegacyStudio13DocumentsDirectory(); + const double requestedGainDb = 10.0 * std::log10 (sourceEnergy / destEnergy); + const double maxGainDb = maxGainDbOverride >= 0.0 + ? juce::jlimit (0.0, 9.0, maxGainDbOverride) + : static_cast (juce::jlimit (0.0f, 9.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_RMS_CONTINUITY_MAX_GAIN_DB", 4.0f))); + plan.gainDb = juce::jlimit (0.0, maxGainDb, requestedGainDb); + if (plan.gainDb < 0.10) + return plan; - if (!openStudioDir.exists() && legacyDir.exists()) - return legacyDir; + const float targetGain = static_cast (std::pow (10.0, plan.gainDb / 20.0)); + const float attackMs = attackMsOverride >= 0.0 + ? static_cast (juce::jlimit (3.0, 45.0, attackMsOverride)) + : juce::jlimit (18.0f, 45.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_RMS_CONTINUITY_ATTACK_MS", 36.0f)); + const int fadeInSamples = std::max (1, static_cast (std::round (attackMs * 0.001f * static_cast (sampleRate)))); + const int fadeOutSamples = std::max (1, static_cast (std::round (0.018 * sampleRate))); + for (int ch = 0; ch < destination.getNumChannels(); ++ch) + { + for (int i = 0; i < entrySamples; ++i) + { + const float fromStart = static_cast (i) / static_cast (fadeInSamples); + const float fromEnd = static_cast (entrySamples - 1 - i) / static_cast (fadeOutSamples); + const float weight = smoothHalfCosine (std::min (fromStart, fromEnd)); + if (weight <= 0.0001f) + continue; - return openStudioDir; -} + const int sample = entryStart + i; + const float current = destination.getSample (ch, sample); + const float gain = 1.0f + (targetGain - 1.0f) * weight; + destination.setSample (ch, sample, current * gain); + } + } + + plan.used = true; + plan.durationMs = static_cast (entrySamples) * 1000.0 / sampleRate; + return plan; +} + +PitchEntryTransientResidualPlan applyPitchEntryTransientResidual (juce::AudioBuffer& destination, + const juce::AudioBuffer& original, + const PitchCommitRange& range, + const PitchEntryBridgePlan& entryBridge, + double sampleRate) +{ + PitchEntryTransientResidualPlan plan; + if (getPitchEnvFlag ("OPENSTUDIO_PITCH_ENTRY_TRANSIENT_RESIDUAL_DISABLE", false) + || sampleRate <= 0.0 + || destination.getNumChannels() <= 0 + || destination.getNumSamples() <= 0) + return plan; + + const int numSamples = destination.getNumSamples(); + const int bodyStart = juce::jlimit (0, numSamples, static_cast (std::round (range.bodyStartSec * sampleRate))); + const int bodyEnd = juce::jlimit (bodyStart, numSamples, static_cast (std::round (range.bodyEndSec * sampleRate))); + if (bodyEnd <= bodyStart + 16) + return plan; + + const int dryHoldSamples = entryBridge.safeHandoffUsed ? entryBridge.dryHoldSamples : 0; + const int residualStart = juce::jlimit (bodyStart, bodyEnd, bodyStart + dryHoldSamples); + const float durationMs = juce::jlimit (6.0f, 45.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_TRANSIENT_RESIDUAL_MS", + entryBridge.wetVsDryRmsDb <= -6.0 ? 24.0f : 14.0f)); + const int residualEnd = juce::jlimit (residualStart, bodyEnd, + residualStart + std::max (1, static_cast (std::round (durationMs * 0.001f * static_cast (sampleRate))))); + if (residualEnd <= residualStart + 8) + return plan; + + const float envMix = getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_TRANSIENT_RESIDUAL_MIX", -1.0f); + const float defaultMix = entryBridge.wetVsDryRmsDb <= -6.0 ? 0.36f : 0.24f; + const float mix = juce::jlimit (0.0f, 0.70f, envMix >= 0.0f ? envMix : defaultMix); + if (mix <= 0.0001f) + return plan; + + const float cutoffHz = juce::jlimit (900.0f, 4200.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_TRANSIENT_RESIDUAL_HIGHPASS_HZ", 1800.0f)); + const float alpha = static_cast (std::exp (-2.0 * juce::MathConstants::pi + * static_cast (cutoffHz) / sampleRate)); + const int fadeInSamples = std::max (1, static_cast (std::round (0.0025 * sampleRate))); + const int fadeOutSamples = std::max (1, static_cast (std::round (0.010 * sampleRate))); + + for (int ch = 0; ch < destination.getNumChannels(); ++ch) + { + if (ch >= original.getNumChannels()) + continue; -juce::File getApplicationRuntimeDirectory() -{ - return juce::File::getSpecialLocation(juce::File::currentExecutableFile).getParentDirectory(); -} + float sourceLow = original.getSample (ch, std::max (0, residualStart - 1)); + float destLow = destination.getSample (ch, std::max (0, residualStart - 1)); + for (int sample = residualStart; sample < residualEnd; ++sample) + { + const float source = original.getSample (ch, sample); + const float current = destination.getSample (ch, sample); + sourceLow = alpha * sourceLow + (1.0f - alpha) * source; + destLow = alpha * destLow + (1.0f - alpha) * current; + const float sourceHigh = source - sourceLow; + const float destHigh = current - destLow; + const float fromStart = static_cast (sample - residualStart) / static_cast (fadeInSamples); + const float fromEnd = static_cast (residualEnd - 1 - sample) / static_cast (fadeOutSamples); + const float weight = smoothHalfCosine (std::min (fromStart, fromEnd)); + if (weight <= 0.0001f) + continue; -juce::File getPreferredApplicationDataDirectory() -{ - auto openStudioDir = getOpenStudioApplicationDataDirectory(); - auto legacyDir = getLegacyStudio13ApplicationDataDirectory(); + const float highDelta = sourceHigh - destHigh; + destination.setSample (ch, sample, current + highDelta * mix * weight); + } + } + + plan.used = true; + plan.mix = mix; + plan.durationMs = static_cast (residualEnd - residualStart) * 1000.0 / sampleRate; + return plan; +} + +PitchEntryTimbrePlan applyPitchEntryTimbreCorrection (juce::AudioBuffer& destination, + const juce::AudioBuffer& original, + const PitchCommitRange& range, + double sampleRate) +{ + PitchEntryTimbrePlan plan; + if (destination.getNumChannels() <= 0 || destination.getNumSamples() <= 0 || sampleRate <= 0.0) + return plan; + + const bool downward = range.pitchDirection < 0 || range.pitchRatio < 0.999; + const float amount = juce::jlimit (0.0f, 1.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_TIMBRE_AMOUNT", 0.0f)); + if (amount <= 0.0001f) + return plan; + + const int numSamples = destination.getNumSamples(); + const int entryStart = juce::jlimit (0, numSamples, static_cast (std::round (range.bodyStartSec * sampleRate))); + const int entryLimit = juce::jlimit (entryStart, numSamples, static_cast (std::round (range.bodyEndSec * sampleRate))); + const float entryMs = juce::jlimit (35.0f, 95.0f, + getPitchEnvFloat (downward ? "OPENSTUDIO_PITCH_ENTRY_TIMBRE_DOWN_MS" : "OPENSTUDIO_PITCH_ENTRY_TIMBRE_UP_MS", 80.0f)); + const int entryEnd = juce::jlimit (entryStart, entryLimit, + entryStart + std::max (1, static_cast (std::round (entryMs * 0.001f * static_cast (sampleRate))))); + const int entryLen = entryEnd - entryStart; + if (entryLen <= 8) + return plan; + + const int fadeInSamples = std::max (1, std::min (entryLen / 3, + static_cast (std::round (getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_TIMBRE_FADE_IN_MS", 5.0f) * 0.001f * static_cast (sampleRate))))); + const int fadeOutSamples = std::max (1, std::min (entryLen / 2, + static_cast (std::round (getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_TIMBRE_FADE_OUT_MS", 24.0f) * 0.001f * static_cast (sampleRate))))); + + double sourceEnergy = 0.0; + double destEnergy = 0.0; + double weightSum = 0.0; + for (int s = entryStart; s < entryEnd; ++s) + { + const float fromStart = static_cast (s - entryStart) / static_cast (fadeInSamples); + const float fromEnd = static_cast (entryEnd - 1 - s) / static_cast (fadeOutSamples); + const float weight = smoothHalfCosine (std::min (fromStart, fromEnd)); + if (weight <= 0.0001f) + continue; - if (!openStudioDir.exists() && legacyDir.exists()) - return legacyDir; + for (int ch = 0; ch < destination.getNumChannels(); ++ch) + { + const float sourceSample = original.getSample (ch, s); + const float destSample = destination.getSample (ch, s); + sourceEnergy += static_cast (sourceSample) * static_cast (sourceSample) * weight; + destEnergy += static_cast (destSample) * static_cast (destSample) * weight; + weightSum += weight; + } + } - return openStudioDir; + if (weightSum <= 1.0 || sourceEnergy <= 1.0e-10 || destEnergy <= 1.0e-10) + return plan; + + const float sourceMatchScale = downward + ? juce::jlimit (0.0f, 1.0f, getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_TIMBRE_DOWN_SOURCE_SCALE", 0.30f)) + : juce::jlimit (0.0f, 1.0f, getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_TIMBRE_UP_SOURCE_SCALE", 0.55f)); + const float maxTrimDb = downward + ? juce::jlimit (0.0f, 2.0f, getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_TIMBRE_DOWN_MAX_TRIM_DB", 1.1f)) + : juce::jlimit (0.0f, 2.4f, getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_TIMBRE_UP_MAX_TRIM_DB", 1.7f)); + const float requestedTrimDb = static_cast (20.0 * std::log10 (std::sqrt (sourceEnergy / weightSum) + / std::sqrt (destEnergy / weightSum))) * sourceMatchScale; + const float trimDb = juce::jlimit (-maxTrimDb, maxTrimDb, requestedTrimDb) * amount; + const float tiltDb = (downward + ? juce::jlimit (-3.0f, 1.0f, getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_TIMBRE_DOWN_TILT_DB", -1.4f)) + : juce::jlimit (-1.0f, 3.5f, getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_TIMBRE_UP_TILT_DB", 1.7f))) * amount; + + if (std::abs (trimDb) < 0.02f && std::abs (tiltDb) < 0.02f) + return plan; + + const float lowpassCutoffHz = juce::jlimit (1200.0f, 4200.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_TIMBRE_CUTOFF_HZ", downward ? 2300.0f : 2600.0f)); + const float lowpassCoeff = static_cast (std::exp ( + -juce::MathConstants::twoPi * static_cast (lowpassCutoffHz) / sampleRate)); + const float lowGain = std::pow (10.0f, trimDb / 20.0f); + const float highGain = std::pow (10.0f, (trimDb + tiltDb) / 20.0f); + + for (int ch = 0; ch < destination.getNumChannels(); ++ch) + { + std::vector lowMid (static_cast (numSamples), 0.0f); + float state = 0.0f; + for (int s = 0; s < numSamples; ++s) + { + state = (1.0f - lowpassCoeff) * destination.getSample (ch, s) + lowpassCoeff * state; + lowMid[static_cast (s)] = state; + } + state = 0.0f; + for (int s = numSamples - 1; s >= 0; --s) + { + state = (1.0f - lowpassCoeff) * lowMid[static_cast (s)] + lowpassCoeff * state; + lowMid[static_cast (s)] = state; + } + + for (int s = entryStart; s < entryEnd; ++s) + { + const float fromStart = static_cast (s - entryStart) / static_cast (fadeInSamples); + const float fromEnd = static_cast (entryEnd - 1 - s) / static_cast (fadeOutSamples); + const float weight = smoothHalfCosine (std::min (fromStart, fromEnd)); + if (weight <= 0.0001f) + continue; + + const float current = destination.getSample (ch, s); + const float low = lowMid[static_cast (s)]; + const float high = current - low; + const float target = low * lowGain + high * highGain; + destination.setSample (ch, s, current + (target - current) * weight); + } + } + + plan.used = true; + plan.rmsTrimDb = trimDb; + plan.tiltDb = tiltDb; + return plan; +} + +PitchCompositeStats compositeDryProtectedPitchPatch (juce::AudioBuffer& destination, + const juce::AudioBuffer& original, + const std::vector>& correctedContext, + const std::vector& commitRanges, + int contextStartSample, + double sampleRate, + bool allowPitchOnlyEntryExperiments, + bool allowEntryBridge) +{ + destination.makeCopyOf (original, true); + + const int numChannels = destination.getNumChannels(); + const int clipNumSamples = destination.getNumSamples(); + PitchCompositeStats stats; + stats.audibleCommitStartSec = getAudibleCommitStartSec (commitRanges); + stats.audibleCommitEndSec = getAudibleCommitEndSec (commitRanges); + constexpr double exitLeadInSec = 0.012; + stats.entryInsideBodyFadeMs = 0.0; + stats.exitLeadInMs = exitLeadInSec * 1000.0; + int committedSamples = 0; + bool hasAudibleStart = false; + bool hasAudibleEnd = false; + const bool useLegacyEntryBridge = allowEntryBridge + && (allowPitchOnlyEntryExperiments + || juce::SystemStats::getEnvironmentVariable ("OPENSTUDIO_PITCH_LEGACY_ENTRY_BRIDGE_ENABLE", {}).trim() == "1"); + const bool useSimpleBoundaryHandoff = allowEntryBridge && ! useLegacyEntryBridge; + const bool disableEntrySafeHandoff = juce::SystemStats::getEnvironmentVariable ( + "OPENSTUDIO_PITCH_ENTRY_SAFE_HANDOFF_DISABLE", {}).trim() == "1"; + + for (const auto& range : commitRanges) + { + const int contextRangeStartSample = juce::jlimit (0, clipNumSamples, static_cast (std::floor (range.startSec * sampleRate))); + const int effectiveEndSample = juce::jlimit (contextRangeStartSample, clipNumSamples, static_cast (std::ceil (range.endSec * sampleRate))); + const int bodyStartSample = juce::jlimit (contextRangeStartSample, effectiveEndSample, static_cast (std::round (range.bodyStartSec * sampleRate))); + const int bodyEndSample = juce::jlimit (bodyStartSample, effectiveEndSample, static_cast (std::round (range.bodyEndSec * sampleRate))); + const int endSample = useSimpleBoundaryHandoff ? bodyEndSample : effectiveEndSample; + const auto entryBridge = useLegacyEntryBridge + ? choosePitchEntryBridgePlan ( + original, correctedContext, range, contextStartSample, + contextRangeStartSample, bodyStartSample, bodyEndSample, effectiveEndSample, sampleRate, + allowPitchOnlyEntryExperiments) + : (allowEntryBridge + ? (disableEntrySafeHandoff + ? makeSimplePitchEntryHandoffPlan (bodyStartSample, bodyEndSample, effectiveEndSample, sampleRate) + : makeEntrySafePitchHandoffPlan ( + original, correctedContext, contextStartSample, bodyStartSample, bodyEndSample, effectiveEndSample, sampleRate)) + : PitchEntryBridgePlan { bodyStartSample, bodyStartSample, 0, 0.0, 0.0, 0.0, false }); + const int startSample = entryBridge.used ? entryBridge.startSample : bodyStartSample; + const int rangeSamples = endSample - startSample; + if (rangeSamples <= 0) + continue; + + stats.preBodyDryProtectedSamples += std::max (0, startSample - contextRangeStartSample); + committedSamples += rangeSamples; + stats.entryBridgeUsed = stats.entryBridgeUsed || entryBridge.used; + stats.entrySimpleHandoffUsed = stats.entrySimpleHandoffUsed + || (entryBridge.used && useSimpleBoundaryHandoff && ! entryBridge.safeHandoffUsed); + stats.entrySafeHandoffUsed = stats.entrySafeHandoffUsed || entryBridge.safeHandoffUsed; + if (entryBridge.safeHandoffUsed) + { + stats.entryDryHoldMs = std::max (stats.entryDryHoldMs, entryBridge.transientDryPreserveMs); + stats.entrySafeBridgeMs = std::max (stats.entrySafeBridgeMs, entryBridge.safeBridgeMs); + stats.entryWetAlignmentMs = std::abs (entryBridge.wetAlignmentMs) > std::abs (stats.entryWetAlignmentMs) + ? entryBridge.wetAlignmentMs + : stats.entryWetAlignmentMs; + stats.entryWetGainDb = std::abs (entryBridge.envelopeGainDb) > std::abs (stats.entryWetGainDb) + ? entryBridge.envelopeGainDb + : stats.entryWetGainDb; + stats.entryWetVsDryRmsDb = std::abs (entryBridge.wetVsDryRmsDb) > std::abs (stats.entryWetVsDryRmsDb) + ? entryBridge.wetVsDryRmsDb + : stats.entryWetVsDryRmsDb; + stats.entryEqualPowerBlendUsed = stats.entryEqualPowerBlendUsed || entryBridge.equalPowerBlend; + stats.entryPhaseSafeUsed = stats.entryPhaseSafeUsed || entryBridge.phaseSafeUsed; + stats.entryWetAlignmentAccepted = stats.entryWetAlignmentAccepted || entryBridge.wetAlignmentAccepted; + stats.entryFirstCycleCorrelation = std::abs (entryBridge.firstCycleCorrelation) > std::abs (stats.entryFirstCycleCorrelation) + ? entryBridge.firstCycleCorrelation + : stats.entryFirstCycleCorrelation; + stats.entryZeroCrossOffsetMs = std::max (stats.entryZeroCrossOffsetMs, entryBridge.zeroCrossOffsetMs); + stats.entryBridgeGainRampDb = std::abs (entryBridge.bridgeGainRampDb) > std::abs (stats.entryBridgeGainRampDb) + ? entryBridge.bridgeGainRampDb + : stats.entryBridgeGainRampDb; + } + if (useSimpleBoundaryHandoff && effectiveEndSample > bodyEndSample) + { + stats.exitDryRestoreUsed = true; + const double exitStartSec = static_cast (bodyEndSample) / sampleRate; + const double exitEndSec = static_cast (effectiveEndSample) / sampleRate; + stats.exitDryRestoreStartSec = stats.exitDryRestoreStartSec == 0.0 + ? exitStartSec + : std::min (stats.exitDryRestoreStartSec, exitStartSec); + stats.exitDryRestoreEndSec = std::max (stats.exitDryRestoreEndSec, exitEndSec); + } + stats.audibleCommitEndSec = hasAudibleEnd + ? std::max (stats.audibleCommitEndSec, static_cast (endSample) / sampleRate) + : static_cast (endSample) / sampleRate; + hasAudibleEnd = true; + if (stats.entryBoundaryKind == "unknown" || range.entryBoundaryScore > stats.entryBoundaryScore) + { + stats.entryBoundaryKind = range.entryBoundaryKind.isEmpty() ? "unknown" : range.entryBoundaryKind; + stats.entryBoundaryScore = range.entryBoundaryScore; + } + if (stats.exitBoundaryKind == "unknown" || range.exitBoundaryScore > stats.exitBoundaryScore) + { + stats.exitBoundaryKind = range.exitBoundaryKind.isEmpty() ? "unknown" : range.exitBoundaryKind; + stats.exitBoundaryScore = range.exitBoundaryScore; + } + if (entryBridge.used) + { + if (! hasAudibleStart) + { + stats.entryBridgeStartSec = static_cast (entryBridge.startSample) / sampleRate; + stats.audibleCommitStartSec = stats.entryBridgeStartSec; + hasAudibleStart = true; + } + else + { + stats.entryBridgeStartSec = std::min (stats.entryBridgeStartSec, static_cast (entryBridge.startSample) / sampleRate); + stats.audibleCommitStartSec = std::min (stats.audibleCommitStartSec, static_cast (entryBridge.startSample) / sampleRate); + } + stats.entryBridgeEndSec = std::max (stats.entryBridgeEndSec, static_cast (entryBridge.endSample) / sampleRate); + stats.entryBridgeWetLagMs = static_cast (entryBridge.wetLagSamples) * 1000.0 / sampleRate; + stats.entryBridgeEnvelopeGainDb = std::abs (entryBridge.envelopeGainDb) > std::abs (stats.entryBridgeEnvelopeGainDb) + ? entryBridge.envelopeGainDb + : stats.entryBridgeEnvelopeGainDb; + stats.entryTransientDryPreservedMs = std::max (stats.entryTransientDryPreservedMs, entryBridge.transientDryPreserveMs); + } + + const bool upwardBridge = range.pitchDirection > 0 || range.pitchRatio > 1.001; + const bool autoUpPostBridgeLag = upwardBridge + && entryBridge.safeHandoffUsed + && entryBridge.firstCycleCorrelation < 0.0 + && entryBridge.wetVsDryRmsDb > -6.0 + && ! getPitchEnvFlag ("OPENSTUDIO_PITCH_ENTRY_UP_POST_BRIDGE_WET_LAG_AUTO_DISABLE", false); + const float defaultPostBridgeLagMs = autoUpPostBridgeLag ? 0.8f : 0.0f; + const float postBridgeLagMs = upwardBridge + ? juce::jlimit (-2.0f, 2.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_UP_POST_BRIDGE_WET_LAG_MS", defaultPostBridgeLagMs)) + : 0.0f; + const int postBridgeLagSamples = static_cast (std::round ( + postBridgeLagMs * 0.001f * static_cast (sampleRate))); + const float postBridgeLagTaperMs = juce::jlimit (4.0f, 60.0f, + getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_UP_POST_BRIDGE_WET_LAG_TAPER_MS", 24.0f)); + const int postBridgeLagEndSample = postBridgeLagSamples != 0 + ? juce::jlimit (entryBridge.endSample, endSample, + entryBridge.endSample + std::max (1, static_cast (std::round ( + postBridgeLagTaperMs * 0.001f * static_cast (sampleRate))))) + : entryBridge.endSample; + if (postBridgeLagSamples != 0) + { + const double postLagMs = static_cast (postBridgeLagSamples) * 1000.0 / sampleRate; + stats.entryWetAlignmentMs = std::abs (postLagMs) > std::abs (stats.entryWetAlignmentMs) + ? postLagMs + : stats.entryWetAlignmentMs; + } + + int fadeOutStartSample = juce::jlimit (startSample, endSample, + static_cast (std::round ((range.bodyEndSec - exitLeadInSec) * sampleRate))); + fadeOutStartSample = std::max (entryBridge.endSample, fadeOutStartSample); + const int dryPreserveSamples = static_cast (std::round ((entryBridge.transientDryPreserveMs / 1000.0) * sampleRate)); + + for (int ch = 0; ch < numChannels; ++ch) + { + if (static_cast (ch) >= correctedContext.size()) + continue; + + const bool downwardBridge = range.pitchDirection < 0 || range.pitchRatio < 0.999; + const float entryBodyBlend = downwardBridge + ? juce::jlimit (0.40f, 0.85f, getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_BRIDGE_DOWN_BODY_BLEND", 0.71f)) + : juce::jlimit (0.40f, 0.85f, getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_BRIDGE_UP_BODY_BLEND", 0.72f)); + const float entryStartBlend = downwardBridge + ? 0.0f + : juce::jlimit (0.0f, 0.85f, getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_BRIDGE_UP_START_BLEND", 0.0f)); + const auto& corrected = correctedContext[static_cast (ch)]; + for (int sample = startSample; sample < endSample; ++sample) + { + float entryProgress = 1.0f; + if (entryBridge.used && sample < entryBridge.endSample) + { + if (entryBridge.safeHandoffUsed) + { + const int dryHoldEndSample = std::min (entryBridge.endSample, + entryBridge.startSample + entryBridge.dryHoldSamples); + if (sample < dryHoldEndSample) + { + entryProgress = entryBridge.phaseSafeUsed + ? 0.0f + : 0.12f * smoothHalfCosine ( + static_cast (sample - entryBridge.startSample) + / static_cast (std::max (1, dryHoldEndSample - entryBridge.startSample))); + } + else + { + const float bridgeProgress = smoothHalfCosine ( + static_cast (sample - dryHoldEndSample) + / static_cast (std::max (1, entryBridge.endSample - dryHoldEndSample))); + entryProgress = entryBridge.phaseSafeUsed + ? bridgeProgress + : 0.12f + 0.88f * bridgeProgress; + } + } + else if (entryBridge.startSample >= bodyStartSample) + { + const float fadeProgress = smoothHalfCosine ( + static_cast (sample - entryBridge.startSample) + / static_cast (std::max (1, entryBridge.endSample - entryBridge.startSample))); + entryProgress = entryStartBlend + (1.0f - entryStartBlend) * fadeProgress; + } + else if (sample < bodyStartSample) + { + entryProgress = entryBodyBlend * smoothHalfCosine ( + static_cast (sample - entryBridge.startSample) + / static_cast (std::max (1, bodyStartSample - entryBridge.startSample))); + } + else + { + entryProgress = entryBodyBlend + (1.0f - entryBodyBlend) * smoothHalfCosine ( + static_cast (sample - bodyStartSample) + / static_cast (std::max (1, entryBridge.endSample - bodyStartSample))); + } + if (sample < entryBridge.startSample + dryPreserveSamples) + { + const float preserveProgress = static_cast (sample - entryBridge.startSample) + / static_cast (std::max (1, dryPreserveSamples)); + entryProgress *= smoothHalfCosine (preserveProgress); + } + } + + int taperedLag = 0; + if (entryBridge.used && sample < entryBridge.endSample) + { + if (postBridgeLagSamples != 0) + { + taperedLag = static_cast (std::round ( + static_cast (postBridgeLagSamples) * static_cast (entryProgress))); + } + else + { + taperedLag = static_cast (std::round ( + static_cast (entryBridge.wetLagSamples) * (1.0 - static_cast (entryProgress)))); + } + } + else if (postBridgeLagSamples != 0 && sample < postBridgeLagEndSample) + { + const float postLagProgress = smoothHalfCosine ( + static_cast (sample - entryBridge.endSample) + / static_cast (std::max (1, postBridgeLagEndSample - entryBridge.endSample))); + taperedLag = static_cast (std::round ( + static_cast (postBridgeLagSamples) * (1.0 - static_cast (postLagProgress)))); + } + const int contextIndex = sample - contextStartSample + taperedLag; + if (contextIndex < 0 || contextIndex >= static_cast (corrected.size())) + continue; + + float blend = entryBridge.used ? entryProgress : 1.0f; + if (sample > fadeOutStartSample) + { + const float t = static_cast (endSample - 1 - sample) + / static_cast (std::max (1, endSample - 1 - fadeOutStartSample)); + blend = std::min (blend, smoothHalfCosine (t)); + } + + const float dry = original.getSample (ch, sample); + float wet = corrected[static_cast (contextIndex)]; + if (entryBridge.used && sample < entryBridge.endSample && ! entryBridge.phaseSafeUsed) + { + const int gainStartSample = entryBridge.safeHandoffUsed + ? entryBridge.startSample + : bodyStartSample; + const float gainProgress = sample < gainStartSample + ? 0.0f + : smoothHalfCosine (static_cast (sample - gainStartSample) + / static_cast (std::max (1, entryBridge.endSample - gainStartSample))); + const double gainDb = entryBridge.envelopeGainDb * (1.0 - static_cast (gainProgress)); + wet *= static_cast (std::pow (10.0, gainDb / 20.0)); + } + if (entryBridge.equalPowerBlend && sample < entryBridge.endSample) + { + const float p = juce::jlimit (0.0f, 1.0f, blend); + const float theta = p * 0.5f * juce::MathConstants::pi; + const float dryGain = std::cos (theta); + const float wetGain = std::sin (theta); + destination.setSample (ch, sample, dry * dryGain + wet * wetGain); + } + else + { + destination.setSample (ch, sample, dry + (wet - dry) * blend); + } + } + } + + const bool autoEntryRmsContinuity = false; + const bool entryRmsContinuityEnabled = getPitchEnvFlag ( + "OPENSTUDIO_PITCH_ENTRY_RMS_CONTINUITY_ENABLE", autoEntryRmsContinuity); + PitchEntryRmsContinuityPlan entryContinuity; + if (entryRmsContinuityEnabled + && entryBridge.phaseSafeUsed + && entryBridge.safeHandoffUsed + && entryBridge.wetVsDryRmsDb <= -6.0 + && ! getPitchEnvFlag ("OPENSTUDIO_PITCH_ENTRY_SOURCE_RMS_CONTINUITY_DISABLE", false)) + { + const bool allowContinuityDuringBridge = getPitchEnvFlag ( + "OPENSTUDIO_PITCH_ENTRY_SOURCE_RMS_CONTINUITY_DURING_BRIDGE_ENABLE", false); + entryContinuity = applyPitchEntryRmsContinuity ( + destination, + original, + range, + sampleRate, + true, + allowContinuityDuringBridge ? entryBridge.startSample : entryBridge.endSample, + static_cast (getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_SOURCE_RMS_CONTINUITY_MAX_GAIN_DB", 3.0f)), + static_cast (getPitchEnvFloat ("OPENSTUDIO_PITCH_ENTRY_SOURCE_RMS_CONTINUITY_ATTACK_MS", 45.0f))); + } + + if (! entryContinuity.used && entryRmsContinuityEnabled) + { + entryContinuity = applyPitchEntryRmsContinuity ( + destination, + original, + range, + sampleRate, + entryBridge.safeHandoffUsed && entryBridge.wetVsDryRmsDb <= -6.0, + entryBridge.phaseSafeUsed ? entryBridge.endSample : -1); + } + + if (entryContinuity.used) + { + stats.entryRmsContinuityUsed = true; + stats.entryRmsContinuityGainDb = std::abs (entryContinuity.gainDb) > std::abs (stats.entryRmsContinuityGainDb) + ? entryContinuity.gainDb + : stats.entryRmsContinuityGainDb; + stats.entryRmsContinuityMs = std::max (stats.entryRmsContinuityMs, entryContinuity.durationMs); + } + + const auto entryTransientResidual = applyPitchEntryTransientResidual ( + destination, original, range, entryBridge, sampleRate); + if (entryTransientResidual.used) + { + stats.entryTransientResidualUsed = true; + stats.entryTransientResidualMix = std::max (stats.entryTransientResidualMix, entryTransientResidual.mix); + stats.entryTransientResidualMs = std::max (stats.entryTransientResidualMs, entryTransientResidual.durationMs); + } + + const auto entryTimbre = allowPitchOnlyEntryExperiments + ? applyPitchEntryTimbreCorrection (destination, original, range, sampleRate) + : PitchEntryTimbrePlan(); + if (entryTimbre.used) + { + stats.entryTimbreCorrectionUsed = true; + if (std::abs (entryTimbre.rmsTrimDb) > std::abs (stats.entryRmsTrimDb)) + stats.entryRmsTrimDb = entryTimbre.rmsTrimDb; + if (std::abs (entryTimbre.tiltDb) > std::abs (stats.entryTiltDb)) + stats.entryTiltDb = entryTimbre.tiltDb; + } + + const auto downshiftCore = applyDownshiftCoreEnvelopePass (destination, original, range, sampleRate); + if (downshiftCore.used) + { + stats.downshiftCoreEnvelopePassUsed = true; + if (std::abs (downshiftCore.rmsTrimDb) > std::abs (stats.downshiftCoreRmsTrimDb)) + stats.downshiftCoreRmsTrimDb = downshiftCore.rmsTrimDb; + if (std::abs (downshiftCore.maxFrameGainDb) > std::abs (stats.downshiftCoreEnvelopeMaxDb)) + stats.downshiftCoreEnvelopeMaxDb = downshiftCore.maxFrameGainDb; + stats.downshiftCoreEnvelopeFrames += downshiftCore.envelopeFrames; + } + } + + stats.dryProtectedSamples = std::max (0, clipNumSamples - committedSamples); + if (! stats.entryBridgeUsed) + { + stats.entryBridgeStartSec = stats.audibleCommitStartSec; + stats.entryBridgeEndSec = stats.audibleCommitStartSec; + } + return stats; } } @@ -68,23 +2373,26 @@ static void logToDisk(const juce::String& msg) f.appendText(juce::Time::getCurrentTime().toString(true, true) + ": " + msg + "\n"); } +static bool shouldEnablePitchEditorFormantDebugLogs() +{ #if JUCE_DEBUG -static constexpr bool kPitchEditorFormantDebugLogs = true; + return true; #else -static constexpr bool kPitchEditorFormantDebugLogs = false; + return juce::SystemStats::getEnvironmentVariable("OPENSTUDIO_PITCH_DEBUG", {}).trim() == "1"; #endif +} static void logPitchEditorFormant(const juce::String& msg) { - if (kPitchEditorFormantDebugLogs) + if (shouldEnablePitchEditorFormantDebugLogs()) juce::Logger::writeToLog("[pitchEditor.formant] " + msg); } -#if JUCE_DEBUG -static constexpr bool kAudioPathDebugLogs = true; -#else +// Per-callback audio path logging. Set to true ONLY when actively debugging audio +// path issues — it calls juce::Logger::writeToLog (disk write via FileLogger) from +// the audio thread every 50 callbacks (~33 ms at 32-sample blocks), which causes +// periodic xruns at low buffer sizes. Must be false in normal use. static constexpr bool kAudioPathDebugLogs = false; -#endif static void logAudioTransport(const juce::String& msg) { @@ -135,6 +2443,83 @@ static float peakFromFloatBuffer(const juce::AudioBuffer& buffer, int num return peak; } +static juce::var makeSignalChainBufferStats (const juce::AudioBuffer& buffer, + int numSamples) +{ + auto* obj = new juce::DynamicObject(); + const int safeSamples = juce::jlimit (0, buffer.getNumSamples(), numSamples); + double sumSquares = 0.0; + juce::int64 sampleCount = 0; + float peak = 0.0f; + float maxDerivative = 0.0f; + int nonFiniteCount = 0; + + for (int ch = 0; ch < buffer.getNumChannels(); ++ch) + { + const auto* data = buffer.getReadPointer (ch); + float previous = safeSamples > 0 ? data[0] : 0.0f; + for (int i = 0; i < safeSamples; ++i) + { + const float sample = data[i]; + if (! std::isfinite (sample)) + { + ++nonFiniteCount; + continue; + } + + peak = juce::jmax (peak, std::abs (sample)); + sumSquares += static_cast (sample) * static_cast (sample); + ++sampleCount; + + if (i > 0) + maxDerivative = juce::jmax (maxDerivative, std::abs (sample - previous)); + previous = sample; + } + } + + obj->setProperty ("samples", safeSamples); + obj->setProperty ("channels", buffer.getNumChannels()); + obj->setProperty ("peak", peak); + obj->setProperty ("rms", sampleCount > 0 ? std::sqrt (sumSquares / static_cast (sampleCount)) : 0.0); + obj->setProperty ("maxDerivative", maxDerivative); + obj->setProperty ("nonFiniteCount", nonFiniteCount); + return juce::var (obj); +} + +static void addToSignalChainCapture (juce::AudioBuffer& destination, + int destinationStart, + const juce::AudioBuffer& source, + int numSamples) +{ + if (destination.getNumSamples() <= 0 || destinationStart >= destination.getNumSamples()) + return; + + const int samplesToCopy = juce::jmin (numSamples, destination.getNumSamples() - destinationStart); + if (samplesToCopy <= 0) + return; + + const int channels = juce::jmin (destination.getNumChannels(), source.getNumChannels()); + for (int ch = 0; ch < channels; ++ch) + destination.addFrom (ch, destinationStart, source, ch, 0, samplesToCopy); +} + +static void copyToSignalChainCapture (juce::AudioBuffer& destination, + int destinationStart, + const juce::AudioBuffer& source, + int numSamples) +{ + if (destination.getNumSamples() <= 0 || destinationStart >= destination.getNumSamples()) + return; + + const int samplesToCopy = juce::jmin (numSamples, destination.getNumSamples() - destinationStart); + if (samplesToCopy <= 0) + return; + + const int channels = juce::jmin (destination.getNumChannels(), source.getNumChannels()); + for (int ch = 0; ch < channels; ++ch) + destination.copyFrom (ch, destinationStart, source, ch, 0, samplesToCopy); +} + static float peakFromDoubleBuffer(const juce::AudioBuffer& buffer, int numChannels, int numSamples) { float peak = 0.0f; @@ -732,26 +3117,68 @@ void AudioEngine::saveDeviceSettings() } void AudioEngine::loadDeviceSettings() +{ + #if JUCE_MAC + if (! isMicrophonePermissionGrantedForInput()) + { + juce::Logger::writeToLog("AudioEngine: macOS microphone permission is not granted; opening output-only audio first."); + loadDeviceSettingsWithChannelCounts(0, 2); + requestMicrophonePermissionIfNeeded([this] (bool granted) + { + juce::MessageManager::callAsync([this, granted] + { + if (granted) + { + juce::Logger::writeToLog("AudioEngine: macOS microphone permission granted; reopening audio inputs."); + loadDeviceSettingsWithChannelCounts(2, 2); + saveDeviceSettings(); + } + else + { + juce::Logger::writeToLog("AudioEngine: macOS microphone permission denied; keeping output-only audio configuration."); + } + }); + }); + return; + } + #endif + + loadDeviceSettingsWithChannelCounts(2, 2); +} + +void AudioEngine::loadDeviceSettingsWithChannelCounts(int inputChannels, int outputChannels) { auto settingsFile = getDeviceSettingsFile(); + const int requestedInputChannels = juce::jmax(0, inputChannels); + const int requestedOutputChannels = juce::jmax(0, outputChannels); + + auto activateAvailableChannels = [requestedInputChannels, requestedOutputChannels] (juce::AudioDeviceManager& manager) + { + juce::AudioDeviceManager::AudioDeviceSetup patchSetup; + manager.getAudioDeviceSetup (patchSetup); + patchSetup.useDefaultInputChannels = false; + patchSetup.inputChannels.clear(); + if (requestedInputChannels > 0) + patchSetup.inputChannels.setRange (0, 32, true); + patchSetup.useDefaultOutputChannels = false; + patchSetup.outputChannels.clear(); + if (requestedOutputChannels > 0) + patchSetup.outputChannels.setRange (0, 32, true); + manager.setAudioDeviceSetup (patchSetup, true); + }; + if (settingsFile.existsAsFile()) { auto xml = juce::XmlDocument::parse(settingsFile); if (xml) { - auto error = deviceManager.initialise(2, 2, xml.get(), true); + auto error = deviceManager.initialise(requestedInputChannels, requestedOutputChannels, xml.get(), true); if (error.isEmpty()) { // Patch: expand active channels to all available. // Existing XML may have been saved with only 2 channels active. // JUCE caps the bitmask at the device's actual channel count. - juce::AudioDeviceManager::AudioDeviceSetup patchSetup; - deviceManager.getAudioDeviceSetup (patchSetup); - patchSetup.useDefaultInputChannels = false; - patchSetup.inputChannels.setRange (0, 32, true); - patchSetup.useDefaultOutputChannels = false; - patchSetup.outputChannels.setRange (0, 32, true); - deviceManager.setAudioDeviceSetup (patchSetup, true); + activateAvailableChannels(deviceManager); juce::Logger::writeToLog("AudioEngine: Restored device settings from " + settingsFile.getFullPathName()); return; } @@ -760,17 +3187,34 @@ void AudioEngine::loadDeviceSettings() } // No saved settings or failed to load - use defaults - deviceManager.initialiseWithDefaultDevices(2, 2); + deviceManager.initialiseWithDefaultDevices(requestedInputChannels, requestedOutputChannels); // Patch: activate all channels on the default device too + activateAvailableChannels(deviceManager); +} + +bool AudioEngine::isMicrophonePermissionGrantedForInput() const +{ + #if JUCE_MAC + if (juce::RuntimePermissions::isRequired(juce::RuntimePermissions::recordAudio)) + return juce::RuntimePermissions::isGranted(juce::RuntimePermissions::recordAudio); + #endif + + return true; +} + +void AudioEngine::requestMicrophonePermissionIfNeeded(std::function completion) +{ + #if JUCE_MAC + if (juce::RuntimePermissions::isRequired(juce::RuntimePermissions::recordAudio) + && ! juce::RuntimePermissions::isGranted(juce::RuntimePermissions::recordAudio)) { - juce::AudioDeviceManager::AudioDeviceSetup patchSetup; - deviceManager.getAudioDeviceSetup (patchSetup); - patchSetup.useDefaultInputChannels = false; - patchSetup.inputChannels.setRange (0, 32, true); - patchSetup.useDefaultOutputChannels = false; - patchSetup.outputChannels.setRange (0, 32, true); - deviceManager.setAudioDeviceSetup (patchSetup, true); + juce::Logger::writeToLog("AudioEngine: requesting macOS microphone permission."); + juce::RuntimePermissions::request(juce::RuntimePermissions::recordAudio, std::move(completion)); + return; } + #endif + + completion(true); } void AudioEngine::audioDeviceAboutToStart (juce::AudioIODevice* device) @@ -984,9 +3428,15 @@ void AudioEngine::updateSpectrumAnalyzer (const float* const* outputChannelData, spectrumFFT.performFrequencyOnlyForwardTransform (fftData); { - const juce::ScopedLock specLock (spectrumLock); + const juce::ScopedTryLock specLock (spectrumLock); + if (! specLock.isLocked()) + { + spectrumFftLockMissCount.fetch_add (1, std::memory_order_relaxed); + continue; + } std::memcpy (spectrumOutputBuffer, fftData, sizeof (float) * static_cast (FFT_SIZE)); spectrumReady = true; + spectrumFftPublishCount.fetch_add (1, std::memory_order_relaxed); } } } @@ -1148,7 +3598,7 @@ void AudioEngine::processMonitoringFXChain (const std::shared_ptr= 2) @@ -1158,7 +3608,7 @@ void AudioEngine::applyMasterGainPanMono (float* const* outputChannelData, if (masterPanAutomation.shouldPlayback()) { - float autoPan = masterPanAutomation.eval (samplePosition); + float autoPan = masterPanAutomation.eval (currentTimeSeconds); computePanLawGains (currentPanLaw, autoPan, 1.0f, leftGain, rightGain); } @@ -1176,7 +3626,7 @@ void AudioEngine::applyMasterGainPanMono (float* const* outputChannelData, float effectiveMasterVol = masterVolume; if (masterVolumeAutomation.shouldPlayback()) { - float autoDb = masterVolumeAutomation.eval (samplePosition); + float autoDb = masterVolumeAutomation.eval (currentTimeSeconds); effectiveMasterVol = (autoDb <= -60.0f) ? 0.0f : std::pow (10.0f, autoDb / 20.0f); } if (hybrid64PostChainActive) @@ -1292,7 +3742,9 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha int numSamples, const juce::AudioIODeviceCallbackContext& context) { + juce::ScopedNoDenormals noDenormals; juce::ignoreUnused(context); + audioCallbackScopedNoDenormalsCount.fetch_add (1, std::memory_order_relaxed); const uint64 callbackCounter = audioCallbackCounter.fetch_add(1, std::memory_order_acq_rel) + 1; lastAudioCallbackCounter.store(callbackCounter, std::memory_order_relaxed); const bool firstCallbackAfterTransportStart = firstCallbackAfterTransportStartPending.exchange(false, std::memory_order_acq_rel); @@ -1312,7 +3764,11 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha // During offline rendering, skip ALL processing to avoid sharing FX plugin // instances between the audio callback and the render thread if (isRendering.load()) + { + const double callbackDurationMs = juce::Time::getMillisecondCounterHiRes() - callbackStartWallTimeMs; + lastAudioCallbackProcessMs.store (callbackDurationMs, std::memory_order_relaxed); return; + } auto rtTracks = std::atomic_load_explicit(&realtimeTrackSnapshot, std::memory_order_acquire); auto rtMasterFX = std::atomic_load_explicit(&realtimeMasterFXSnapshot, std::memory_order_acquire); @@ -1321,9 +3777,12 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha && (((callbackCounter % 50) == 1) || ((isPlaying || isRecordMode) && firstCallbackAfterTransportStart)); float postTrackPlaybackPeak = 0.0f; float postMonitoringInputPeak = 0.0f; - const float preTrackPeak = peakFromFloatChannels(outputChannelData, numOutputChannels, numSamples); + // preTrackPeak is only ever used in logging — compute it only when logging is active. + // (The output buffer is also all-zeros here, so the scan would return 0 anyway.) + float preTrackPeak = 0.0f; if (shouldLogCallbackSummary) { + preTrackPeak = peakFromFloatChannels(outputChannelData, numOutputChannels, numSamples); logAudioPlayback("callback start #" + juce::String(static_cast(callbackCounter)) + " isPlaying=" + juce::String(isPlaying.load() ? "true" : "false") + " isRecordMode=" + juce::String(isRecordMode.load() ? "true" : "false") @@ -1468,7 +3927,10 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha // happen on ASIO but guards against unusual driver behaviour), resize it here. // This heap-allocates only in that exceptional case. if (reusableTrackBuffer.getNumSamples() < numSamples || reusableTrackBuffer.getNumChannels() < 2) + { reusableTrackBuffer.setSize (2, numSamples, false, true); + audioCallbackTrackBufferResizeCount.fetch_add (1, std::memory_order_relaxed); + } float* trackChans[2] = { reusableTrackBuffer.getWritePointer (0), reusableTrackBuffer.getWritePointer (1) }; @@ -1555,7 +4017,7 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha } // Tell the track where we are on the timeline so automation can evaluate - track->setCurrentBlockPosition(currentSamplePosition, currentSampleRate); + track->setCurrentBlockPosition(currentSamplePosition / juce::jmax(1.0, currentSampleRate)); // ========== SIDECHAIN: provide source track's output to this track ========== // If this track has any sidechain-routed FX plugins, find the first sidechain @@ -1623,7 +4085,10 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha { auto& scOut = *trackEntry.sidechainOutputBuffer; if (scOut.getNumSamples() < numSamples) + { scOut.setSize(2, numSamples, false, false, true); + audioCallbackSidechainBufferResizeCount.fetch_add (1, std::memory_order_relaxed); + } for (int ch = 0; ch < juce::jmin(2, trackBuffer.getNumChannels()); ++ch) scOut.copyFrom(ch, 0, trackBuffer, ch, 0, numSamples); @@ -1710,6 +4175,40 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha } // ========== FX Chains (extracted helpers) ========== + if (reusablePitchScrubBuffer.getNumChannels() < numOutputChannels + || reusablePitchScrubBuffer.getNumSamples() < numSamples) + { + reusablePitchScrubBuffer.setSize (juce::jmax (1, numOutputChannels), + juce::jmax (1, numSamples), + false, + true); + audioCallbackPitchScrubBufferResizeCount.fetch_add (1, std::memory_order_relaxed); + } + reusablePitchScrubBuffer.clear(); + playbackEngine.renderPitchScrubPreview (reusablePitchScrubBuffer, currentSampleRate); + if (numOutputChannels > 0) + { + if (useHybrid64Summing) + { + for (int ch = 0; ch < juce::jmin (numOutputChannels, reusablePitchScrubBuffer.getNumChannels()); ++ch) + { + auto* dest = reusableMasterBufferDouble.getWritePointer (ch); + auto* src = reusablePitchScrubBuffer.getReadPointer (ch); + for (int sample = 0; sample < numSamples; ++sample) + dest[sample] += static_cast (src[sample]); + } + } + else + { + for (int ch = 0; ch < juce::jmin (numOutputChannels, reusablePitchScrubBuffer.getNumChannels()); ++ch) + { + juce::FloatVectorOperations::add (outputChannelData[ch], + reusablePitchScrubBuffer.getReadPointer (ch), + numSamples); + } + } + } + lastPostTrackPlaybackPeak.store(postTrackPlaybackPeak, std::memory_order_relaxed); lastPostMonitoringInputPeak.store(postMonitoringInputPeak, std::memory_order_relaxed); processMasterFXChain (rtMasterFX, outputChannelData, numOutputChannels, numSamples, useHybrid64Summing); @@ -1725,12 +4224,18 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha // ========== Master Gain, Pan, Mono, and Precision Conversion ========== applyMasterGainPanMono (outputChannelData, numOutputChannels, numSamples, - currentSamplePosition, hybrid64PostChainActive); + currentSamplePosition / juce::jmax(1.0, currentSampleRate), hybrid64PostChainActive); const float finalOutputPeak = peakFromFloatChannels(outputChannelData, numOutputChannels, numSamples); lastFinalOutputPeak.store(finalOutputPeak, std::memory_order_relaxed); - if (shouldLogCallbackSummary || (((isPlaying || isRecordMode) && finalOutputPeak <= 0.0001f) - && ((callbackCounter % 15) == 1))) - { + // Silence-detection log: only fires when kAudioPathDebugLogs is on. + // Previously also fired unconditionally in release builds; removed because + // it calls FileLogger (disk write + mutex) from the audio thread. + if (kAudioPathDebugLogs + && (shouldLogCallbackSummary || (((isPlaying || isRecordMode) && finalOutputPeak <= 0.0001f) + && ((callbackCounter % 15) == 1)))) + { + if (preTrackPeak == 0.0f) + preTrackPeak = peakFromFloatChannels(outputChannelData, numOutputChannels, numSamples); logAudioPlayback("callback peaks #" + juce::String(static_cast(callbackCounter)) + " preTrackPeak=" + juce::String(preTrackPeak, 4) + " postTrackPlaybackPeak=" + juce::String(postTrackPlaybackPeak, 4) @@ -1810,17 +4315,37 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha currentSamplePosition += numSamples; } - const double callbackDurationMs = juce::Time::getMillisecondCounterHiRes() - callbackStartWallTimeMs; - if (kEnableARADebugDiagnostics && anyActiveARAInCallback && callbackDurationMs > 10.0) + // Only measure callback duration when an ARA plugin is active — the QPC call is + // non-trivial at 32-sample block sizes (1500/sec) and the result is only used for + // ARA slow-block diagnostics. + if (kEnableARADebugDiagnostics && anyActiveARAInCallback) + { + const double callbackDurationMs = juce::Time::getMillisecondCounterHiRes() - callbackStartWallTimeMs; + if (callbackDurationMs > 10.0) + { + logToDisk("Audio callback slow: callback=" + juce::String(static_cast(callbackCounter)) + + " firstCallbackAfterTransportStart=" + juce::String(firstCallbackAfterTransportStart ? "true" : "false") + + " totalMs=" + juce::String(callbackDurationMs, 2) + + " tracksProcessed=" + juce::String(orderCount) + + " numSamples=" + juce::String(numSamples) + + " transportPlaying=" + juce::String(isPlaying.load() ? "true" : "false") + + " currentPositionSeconds=" + juce::String(currentSampleRate > 0.0 ? currentSamplePosition / currentSampleRate : 0.0, 3)); + } + } + + const double callbackProcessMs = juce::Time::getMillisecondCounterHiRes() - callbackStartWallTimeMs; + lastAudioCallbackProcessMs.store (callbackProcessMs, std::memory_order_relaxed); + double previousMaxMs = maxAudioCallbackProcessMs.load (std::memory_order_relaxed); + while (callbackProcessMs > previousMaxMs + && ! maxAudioCallbackProcessMs.compare_exchange_weak (previousMaxMs, + callbackProcessMs, + std::memory_order_relaxed)) { - logToDisk("Audio callback slow: callback=" + juce::String(static_cast(callbackCounter)) - + " firstCallbackAfterTransportStart=" + juce::String(firstCallbackAfterTransportStart ? "true" : "false") - + " totalMs=" + juce::String(callbackDurationMs, 2) - + " tracksProcessed=" + juce::String(orderCount) - + " numSamples=" + juce::String(numSamples) - + " transportPlaying=" + juce::String(isPlaying.load() ? "true" : "false") - + " currentPositionSeconds=" + juce::String(currentSampleRate > 0.0 ? currentSamplePosition / currentSampleRate : 0.0, 3)); } + + const double expectedBlockMs = lastAudioBlockDurationMs.load (std::memory_order_relaxed); + if (expectedBlockMs > 0.0 && callbackProcessMs > expectedBlockMs) + audioCallbackDeadlineMissCount.fetch_add (1, std::memory_order_relaxed); } juce::String AudioEngine::addTrack(const juce::String& explicitId) @@ -2499,6 +5024,7 @@ juce::var AudioEngine::getAudioDeviceSetup() // 1. Current Setup auto* currentSetup = new juce::DynamicObject(); auto currentDeviceType = deviceManager.getCurrentAudioDeviceType(); + const bool microphonePermissionGranted = isMicrophonePermissionGrantedForInput(); juce::Logger::writeToLog("AudioEngine: Current device type: " + currentDeviceType); @@ -2506,6 +5032,12 @@ juce::var AudioEngine::getAudioDeviceSetup() auto* device = deviceManager.getCurrentAudioDevice(); currentSetup->setProperty("audioDeviceType", currentDeviceType); + currentSetup->setProperty("microphonePermissionGranted", microphonePermissionGranted); + #if JUCE_MAC + currentSetup->setProperty("microphonePermissionRequired", juce::RuntimePermissions::isRequired(juce::RuntimePermissions::recordAudio)); + #else + currentSetup->setProperty("microphonePermissionRequired", false); + #endif if (device) { juce::Logger::writeToLog("AudioEngine: Current device: " + device->getName()); @@ -2636,11 +5168,34 @@ juce::var AudioEngine::getAudioDeviceSetup() data->setProperty("sampleRates", rates); data->setProperty("bufferSizes", buffers); - juce::Logger::writeToLog("AudioEngine: getAudioDeviceSetup() returning data"); - return data; + juce::Logger::writeToLog("AudioEngine: getAudioDeviceSetup() returning data"); + return data; +} + +void AudioEngine::setAudioDeviceSetup(const juce::String& type, const juce::String& input, const juce::String& output, double sampleRate, int bufferSize) +{ + #if JUCE_MAC + if (input.trim().isNotEmpty() && ! isMicrophonePermissionGrantedForInput()) + { + juce::Logger::writeToLog("AudioEngine: audio input setup requested before macOS microphone permission was granted."); + requestMicrophonePermissionIfNeeded([this, type, input, output, sampleRate, bufferSize] (bool granted) + { + juce::MessageManager::callAsync([this, type, input, output, sampleRate, bufferSize, granted] + { + if (granted) + applyAudioDeviceSetup(type, input, output, sampleRate, bufferSize); + else + juce::Logger::writeToLog("AudioEngine: macOS microphone permission denied; input device setup was not applied."); + }); + }); + return; + } + #endif + + applyAudioDeviceSetup(type, input, output, sampleRate, bufferSize); } -void AudioEngine::setAudioDeviceSetup(const juce::String& type, const juce::String& input, const juce::String& output, double sampleRate, int bufferSize) +void AudioEngine::applyAudioDeviceSetup(const juce::String& type, const juce::String& input, const juce::String& output, double sampleRate, int bufferSize) { juce::Logger::writeToLog("AudioEngine: Setting Audio Device..."); @@ -3263,7 +5818,7 @@ juce::var AudioEngine::runEngineBenchmarks() auto* track = it->second; trackBuffer.clear(); midi.clear(); - track->setCurrentBlockPosition(static_cast(iteration * benchmarkBlockSize), sr); + track->setCurrentBlockPosition(static_cast(iteration * benchmarkBlockSize) / sr); track->buildMidiBuffer(midi, 0.0, benchmarkBlockSize, sr, false); track->processBlock(trackBuffer, midi); if (benchmarkHybrid64) @@ -5306,7 +7861,7 @@ void AudioEngine::clearTrackPlaybackClips(const juce::String& trackId) juce::var AudioEngine::getWaveformPeaks(const juce::String& filePath, int samplesPerPixel, int startSample, int numPixels) { - // REAPER-inspired: read from pre-computed peak cache (.s13peaks) instead of audio file. + // REAPER-inspired: read from pre-computed peak cache (.ospeaks) instead of audio file. // If cache doesn't exist, kick off async generation and return empty. // Frontend will re-request on next render after cache is ready. juce::File audioFile(filePath); @@ -5833,6 +8388,15 @@ juce::var AudioEngine::getAudioDebugSnapshot() const root->setProperty("callbackInputChannels", lastCallbackInputChannels.load(std::memory_order_relaxed)); root->setProperty("callbackOutputChannels", lastCallbackOutputChannels.load(std::memory_order_relaxed)); root->setProperty("lastCallbackCounter", static_cast(lastAudioCallbackCounter.load(std::memory_order_relaxed))); + root->setProperty("lastAudioCallbackProcessMs", lastAudioCallbackProcessMs.load(std::memory_order_relaxed)); + root->setProperty("maxAudioCallbackProcessMs", maxAudioCallbackProcessMs.load(std::memory_order_relaxed)); + root->setProperty("audioCallbackDeadlineMissCount", static_cast(audioCallbackDeadlineMissCount.load(std::memory_order_relaxed))); + root->setProperty("audioCallbackScopedNoDenormalsCount", static_cast(audioCallbackScopedNoDenormalsCount.load(std::memory_order_relaxed))); + root->setProperty("audioCallbackTrackBufferResizeCount", audioCallbackTrackBufferResizeCount.load(std::memory_order_relaxed)); + root->setProperty("audioCallbackPitchScrubBufferResizeCount", audioCallbackPitchScrubBufferResizeCount.load(std::memory_order_relaxed)); + root->setProperty("audioCallbackSidechainBufferResizeCount", audioCallbackSidechainBufferResizeCount.load(std::memory_order_relaxed)); + root->setProperty("spectrumFftPublishCount", static_cast(spectrumFftPublishCount.load(std::memory_order_relaxed))); + root->setProperty("spectrumFftLockMissCount", static_cast(spectrumFftLockMissCount.load(std::memory_order_relaxed))); root->setProperty("postTrackPlaybackPeak", lastPostTrackPlaybackPeak.load(std::memory_order_relaxed)); root->setProperty("postMonitoringInputPeak", lastPostMonitoringInputPeak.load(std::memory_order_relaxed)); root->setProperty("postMasterFxPeak", lastPostMasterFXPeak.load(std::memory_order_relaxed)); @@ -5844,6 +8408,16 @@ juce::var AudioEngine::getAudioDebugSnapshot() const root->setProperty("lastOverlappingClipCount", playbackEngine.getLastOverlappingClipCount()); root->setProperty("lastMixedClipCount", playbackEngine.getLastMixedClipCount()); root->setProperty("lastTrackPlaybackPeak", playbackEngine.getLastTrackPlaybackPeak()); + root->setProperty("playbackFileBufferResizeCount", playbackEngine.getFileBufferResizeCount()); + root->setProperty("playbackPitchShiftWorkBufferResizeCount", playbackEngine.getPitchShiftWorkBufferResizeCount()); + root->setProperty("playbackRenderResampleScratchResizeCount", playbackEngine.getRenderResampleScratchResizeCount()); + root->setProperty("playbackChunkBoundaryReserveCount", playbackEngine.getChunkBoundaryReserveCount()); + const auto pitchRouting = playbackEngine.getPitchPreviewRoutingStatus({}); + root->setProperty("pitchRouteMonitorMode", pitchRouting.monitorMode); + root->setProperty("pitchRouteRenderedSegmentActive", pitchRouting.renderedSegmentActive); + root->setProperty("pitchRouteClipLivePreviewActive", pitchRouting.clipLivePreviewActive); + root->setProperty("pitchRouteScrubPreviewActive", pitchRouting.scrubPreviewActive); + root->setProperty("pitchRouteCorrectedSourceActive", pitchRouting.correctedSourceActive); root->setProperty("playbackTracks", playbackTracks); return juce::var(root); } @@ -6139,7 +8713,8 @@ bool AudioEngine::convertWithFFmpeg(const juce::File& inputFile, const juce::Fil bool AudioEngine::renderProject(const juce::String& source, double startTime, double endTime, const juce::String& filePath, const juce::String& format, double renderSampleRate, int bitDepth, int numChannels, - bool normalize, bool addTail, double tailLengthMs) + bool normalize, bool addTail, double tailLengthMs, + bool includeMetronome) { logToDisk("renderProject: START - file=" + filePath + " format=" + format + " range=" + juce::String(startTime) + "-" + juce::String(endTime) + @@ -6147,7 +8722,8 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do " ch=" + juce::String(numChannels) + " normalize=" + juce::String(normalize ? "true" : "false") + " addTail=" + juce::String(addTail ? "true" : "false") + - " tailMs=" + juce::String(tailLengthMs)); + " tailMs=" + juce::String(tailLengthMs) + + " includeMetronome=" + juce::String(includeMetronome ? "true" : "false")); // ========== 1. Validate inputs ========== if (endTime <= startTime) @@ -6197,9 +8773,48 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do isRendering = true; // Block audio callback from processing FX plugins juce::Thread::sleep(100); // Let audio callback finish current block + // Exports must be sourced from baked clip files only. Live pitch previews, + // scrub loops, and rendered preview segments are UI audition routes; letting + // them survive into the render snapshot can add bursts even for no-op renders. + playbackEngine.clearAllPitchPreviewRoutes(juce::String()); + // ========== 3. Snapshot clip data ========== auto clipSnapshot = playbackEngine.getClipSnapshot(); + auto renderedPreviewSegmentSnapshot = playbackEngine.getRenderedPreviewSegmentSnapshot(); + int staleRenderedPitchSegmentsDropped = 0; + const bool includePitchPreviewSegmentsInRender = getPitchEnvFlag ("OPENSTUDIO_RENDER_INCLUDE_PITCH_PREVIEW_SEGMENTS", false); + if (! includePitchPreviewSegmentsInRender && ! renderedPreviewSegmentSnapshot.empty()) + { + for (const auto& entry : renderedPreviewSegmentSnapshot) + staleRenderedPitchSegmentsDropped += static_cast(entry.second.size()); + renderedPreviewSegmentSnapshot.clear(); + } + for (const auto& clip : clipSnapshot) + { + if (clip.clipId.isEmpty()) + continue; + + const bool usingCorrectedSource = clip.originalAudioFile.existsAsFile() + && clip.audioFile.existsAsFile() + && clip.audioFile != clip.originalAudioFile; + if (! usingCorrectedSource) + continue; + + auto segmentIt = renderedPreviewSegmentSnapshot.find(clip.clipId); + if (segmentIt != renderedPreviewSegmentSnapshot.end()) + { + staleRenderedPitchSegmentsDropped += static_cast(segmentIt->second.size()); + renderedPreviewSegmentSnapshot.erase(segmentIt); + } + } logToDisk("renderProject: Clip snapshot has " + juce::String((int)clipSnapshot.size()) + " clips"); + logToDisk("renderProject: Rendered pitch segment snapshot has " + + juce::String((int)renderedPreviewSegmentSnapshot.size()) + " clips"); + if (staleRenderedPitchSegmentsDropped > 0) + logToDisk("renderProject: Dropped " + juce::String(staleRenderedPitchSegmentsDropped) + + " rendered pitch preview segment(s) before export" + + juce::String(includePitchPreviewSegmentsInRender ? " because corrected source files are active" + : " because exports must use baked/original clip sources")); if (clipSnapshot.empty()) { @@ -6225,9 +8840,23 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do float pan; bool muted; bool soloed; + int inputFxCount = 0; + int trackFxCount = 0; + int sendCount = 0; + int sidechainSourceCount = 0; + bool hasInstrument = false; + bool hasARA = false; + bool masterSendEnabled = true; + bool hasTrackAutomation = false; + bool requiresFullRender = false; }; std::vector trackSnapshots; + std::map trackSnapshotById; bool anySoloed = false; + auto automationIsActive = [] (const AutomationList& automation) + { + return automation.shouldPlayback() && automation.getNumPoints() > 0; + }; { const juce::ScopedLock sl(mainProcessorGraph->getCallbackLock()); for (const auto& trackId : trackOrder) @@ -6241,17 +8870,67 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do snap.pan = track->getPan(); snap.muted = track->getMute(); snap.soloed = track->getSolo(); + snap.inputFxCount = track->getNumInputFX(); + snap.trackFxCount = track->getNumTrackFX(); + snap.hasInstrument = track->getInstrument() != nullptr; + snap.hasARA = track->hasActiveARA(); + snap.masterSendEnabled = track->getMasterSendEnabled(); + snap.sendCount = static_cast(track->getRealtimeSendSnapshot().size()); + snap.sidechainSourceCount = static_cast(track->getSidechainSourceSnapshot().size()); + snap.hasTrackAutomation = + automationIsActive(track->getVolumeAutomation()) + || automationIsActive(track->getPanAutomation()) + || automationIsActive(track->getWidthAutomation()) + || automationIsActive(track->getPreFXVolumeAutomation()) + || automationIsActive(track->getPreFXPanAutomation()) + || automationIsActive(track->getPreFXWidthAutomation()) + || automationIsActive(track->getTrimVolumeAutomation()) + || automationIsActive(track->getMuteAutomation()); + snap.requiresFullRender = + snap.inputFxCount > 0 + || snap.trackFxCount > 0 + || snap.hasInstrument + || snap.hasARA + || snap.sendCount > 0 + || snap.sidechainSourceCount > 0 + || snap.hasTrackAutomation + || (!isStemRender && !snap.masterSendEnabled); if (snap.soloed) anySoloed = true; trackSnapshots.push_back(snap); + trackSnapshotById[snap.id] = snap; logToDisk(" Track " + trackId + ": vol=" + juce::String(snap.volumeDB) + "dB pan=" + juce::String(snap.pan) + " mute=" + juce::String(snap.muted ? "true" : "false") + - " solo=" + juce::String(snap.soloed ? "true" : "false")); + " solo=" + juce::String(snap.soloed ? "true" : "false") + + " fx=" + juce::String(snap.inputFxCount + snap.trackFxCount) + + " sends=" + juce::String(snap.sendCount) + + " sidechains=" + juce::String(snap.sidechainSourceCount) + + " automation=" + juce::String(snap.hasTrackAutomation ? "true" : "false")); } } logToDisk("renderProject: " + juce::String((int)trackSnapshots.size()) + " tracks, anySoloed=" + juce::String(anySoloed ? "true" : "false")); + struct ScopedTrackMuteBypass + { + TrackProcessor* track = nullptr; + ~ScopedTrackMuteBypass() + { + if (track != nullptr) + track->setIgnoreStaticMuteForProcessing(false); + } + } stemMuteBypass; + + if (isStemRender) + { + auto trackIt = trackMap.find(stemTrackId); + if (trackIt != trackMap.end() && trackIt->second != nullptr) + { + trackIt->second->setIgnoreStaticMuteForProcessing(true); + stemMuteBypass.track = trackIt->second; + } + } + // ========== 5. Create format writer ========== // For lossy formats (mp3/ogg) or sample rate conversion, render to temp WAV first juce::File outputFile(filePath); @@ -6327,6 +9006,58 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do "s totalSamples=" + juce::String(totalSamples) + " blockSize=" + juce::String(blockSize)); + const bool renderChainDebugEnabled = getPitchEnvFlag ("OPENSTUDIO_AUDIO_CHAIN_DEBUG", false) + || getPitchEnvFlag ("OPENSTUDIO_RENDER_CHAIN_DEBUG", false); + const double renderChainDebugMaxSec = juce::jlimit (0.25, 600.0, + static_cast (getPitchEnvFloat ("OPENSTUDIO_AUDIO_CHAIN_DEBUG_MAX_SEC", 12.0f))); + const int renderChainCaptureSamples = renderChainDebugEnabled + ? static_cast (std::min ( + totalSamples, + static_cast (std::ceil (renderChainDebugMaxSec * actualSampleRate)))) + : 0; + const auto renderChainDebugDirEnv = juce::SystemStats::getEnvironmentVariable ("OPENSTUDIO_AUDIO_CHAIN_DEBUG_DIR", {}).trim(); + const juce::File renderChainDebugDir = renderChainDebugDirEnv.isNotEmpty() + ? juce::File (renderChainDebugDirEnv) + : outputFile.getSiblingFile (outputFile.getFileNameWithoutExtension() + "_signal_chain_debug"); + juce::AudioBuffer chainPlaybackBlock; + juce::AudioBuffer chainTrackPostBlock; + juce::AudioBuffer chainMasterPreBlock; + juce::AudioBuffer chainMasterPostBlock; + juce::AudioBuffer chainWriterInputBlock; + juce::AudioBuffer chainPlaybackTap; + juce::AudioBuffer chainTrackPostTap; + juce::AudioBuffer chainMasterPreTap; + juce::AudioBuffer chainMasterPostTap; + juce::AudioBuffer chainWriterInputTap; + juce::Array renderChainBlockReports; + + if (renderChainDebugEnabled) + { + renderChainDebugDir.createDirectory(); + chainPlaybackBlock.setSize (2, blockSize); + chainTrackPostBlock.setSize (2, blockSize); + chainMasterPreBlock.setSize (2, blockSize); + chainMasterPostBlock.setSize (2, blockSize); + chainWriterInputBlock.setSize (2, blockSize); + if (renderChainCaptureSamples > 0) + { + chainPlaybackTap.setSize (2, renderChainCaptureSamples); + chainTrackPostTap.setSize (2, renderChainCaptureSamples); + chainMasterPreTap.setSize (2, renderChainCaptureSamples); + chainMasterPostTap.setSize (2, renderChainCaptureSamples); + chainWriterInputTap.setSize (2, renderChainCaptureSamples); + chainPlaybackTap.clear(); + chainTrackPostTap.clear(); + chainMasterPreTap.clear(); + chainMasterPostTap.clear(); + chainWriterInputTap.clear(); + } + + logToDisk ("renderProject: signal-chain debug enabled dir=" + renderChainDebugDir.getFullPathName() + + " captureSamples=" + juce::String (renderChainCaptureSamples) + + " maxSec=" + juce::String (renderChainDebugMaxSec, 2)); + } + // ========== 6b. Save plugin state & prepare plugins for offline rendering ========== // Plugins were prepared for the device's buffer size (e.g. 128/256). The render // uses 512-sample blocks. We must re-prepare all FX plugins with the render block @@ -6389,6 +9120,44 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do logToDisk("renderProject: Saved & prepared " + juce::String((int)pluginBackups.size()) + " FX plugins for render"); + int activeMasterFxCount = 0; + if (renderMasterStage) + { + for (const auto& slot : renderMasterStage->slots) + if (!slot.bypassed && slot.processor) + ++activeMasterFxCount; + } + + const bool hasMasterAutomation = !isStemRender + && (automationIsActive(masterVolumeAutomation) || automationIsActive(masterPanAutomation)); + + bool includedTracksCanUseSimplePath = true; + for (const auto& snap : trackSnapshots) + { + if (isStemRender && snap.id != stemTrackId) + continue; + if (!isStemRender && snap.muted) + continue; + if (!isStemRender && anySoloed && !snap.soloed) + continue; + + if (snap.requiresFullRender) + { + includedTracksCanUseSimplePath = false; + break; + } + } + + const bool useSimpleNoFxRenderLoop = + includedTracksCanUseSimplePath + && activeMasterFxCount == 0 + && !hasMasterAutomation; + + logToDisk("renderProject: renderLoopMode=" + + juce::String(useSimpleNoFxRenderLoop ? "simple_no_fx" : "full_realtime_snapshot") + + " activeMasterFx=" + juce::String(activeMasterFxCount) + + " masterAutomation=" + juce::String(hasMasterAutomation ? "true" : "false")); + // ========== 7 & 8. Render loop (with optional 2-pass normalization) ========== int numPasses = normalize ? 2 : 1; float normGain = 1.0f; @@ -6400,8 +9169,179 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do juce::Random ditherRng; float ditherErrorState[2] = { 0.0f, 0.0f }; // Per-channel error feedback for noise shaping - // Check if metronome was enabled for render - bool renderMetronomeAudio = metronome.isEnabled(); + // Renders are silent with respect to the click by default. Playback may + // have the metronome enabled, but exports must opt in explicitly. + bool renderMetronomeAudio = includeMetronome && metronome.isEnabled(); + const bool useRenderLagrangeSrc = getPitchEnvFlag("OPENSTUDIO_RENDER_USE_LAGRANGE_SRC", false); + auto rtTracks = std::atomic_load_explicit(&realtimeTrackSnapshot, std::memory_order_acquire); + + auto writeRenderRouteReport = [&]() + { + auto* root = new juce::DynamicObject(); + root->setProperty("schemaVersion", 1); + root->setProperty("purpose", "manual_render_route_report"); + root->setProperty("source", source); + root->setProperty("startTimeSec", startTime); + root->setProperty("endTimeSec", endTime); + root->setProperty("requestedOutputFile", outputFile.getFullPathName()); + root->setProperty("renderFile", renderFile.getFullPathName()); + root->setProperty("format", format); + root->setProperty("actualSampleRate", actualSampleRate); + root->setProperty("bitDepth", bitDepth); + root->setProperty("channels", numChannels); + root->setProperty("normalize", normalize); + root->setProperty("addTail", addTail); + root->setProperty("tailLengthMs", tailLengthMs); + root->setProperty("includeMetronome", includeMetronome); + root->setProperty("renderMetronomeAudio", renderMetronomeAudio); + root->setProperty("metronomeEnabledInProject", metronome.isEnabled()); + root->setProperty("renderPlaybackResampler", "shared_cubic_fractional"); + root->setProperty("renderPlaybackResamplerLegacyLagrangeRequested", useRenderLagrangeSrc); + root->setProperty("isStemRender", isStemRender); + root->setProperty("stemTrackId", stemTrackId); + root->setProperty("clipCount", static_cast(clipSnapshot.size())); + root->setProperty("trackCount", static_cast(trackSnapshots.size())); + root->setProperty("anySoloed", anySoloed); + root->setProperty("renderedPreviewSegmentClipCount", static_cast(renderedPreviewSegmentSnapshot.size())); + root->setProperty("renderedPitchSegmentsDroppedBeforeSnapshot", staleRenderedPitchSegmentsDropped); + root->setProperty("activeMasterFxCount", activeMasterFxCount); + root->setProperty("hasMasterAutomation", hasMasterAutomation); + root->setProperty("renderLoopMode", useSimpleNoFxRenderLoop ? "simple_no_fx" : "full_realtime_snapshot"); + root->setProperty("writtenAt", juce::Time::getCurrentTime().toISO8601(true)); + + juce::Array clipArray; + juce::AudioFormatManager reportFormatManager; + reportFormatManager.registerBasicFormats(); + for (const auto& clip : clipSnapshot) + { + auto* obj = new juce::DynamicObject(); + const bool usingCorrectedSource = clip.originalAudioFile.existsAsFile() + && clip.audioFile.existsAsFile() + && clip.audioFile != clip.originalAudioFile; + obj->setProperty("clipId", clip.clipId); + obj->setProperty("trackId", clip.trackId); + obj->setProperty("audioFile", clip.audioFile.getFullPathName()); + obj->setProperty("originalAudioFile", clip.originalAudioFile.getFullPathName()); + obj->setProperty("startTime", clip.startTime); + obj->setProperty("duration", clip.duration); + obj->setProperty("offset", clip.offset); + obj->setProperty("volumeDB", clip.volumeDB); + obj->setProperty("fadeIn", clip.fadeIn); + obj->setProperty("fadeOut", clip.fadeOut); + obj->setProperty("active", clip.isActive); + obj->setProperty("usingCorrectedSource", usingCorrectedSource); + if (auto reader = std::unique_ptr(reportFormatManager.createReaderFor(clip.audioFile))) + { + obj->setProperty("audioFileSampleRate", reader->sampleRate); + obj->setProperty("audioFileChannels", static_cast(reader->numChannels)); + obj->setProperty("audioFileLengthSamples", static_cast(reader->lengthInSamples)); + } + clipArray.add(juce::var(obj)); + } + root->setProperty("clips", juce::var(clipArray)); + + juce::Array trackArray; + for (const auto& snap : trackSnapshots) + { + auto* obj = new juce::DynamicObject(); + obj->setProperty("id", snap.id); + obj->setProperty("volumeDB", snap.volumeDB); + obj->setProperty("pan", snap.pan); + obj->setProperty("muted", snap.muted); + obj->setProperty("soloed", snap.soloed); + obj->setProperty("inputFxCount", snap.inputFxCount); + obj->setProperty("trackFxCount", snap.trackFxCount); + obj->setProperty("hasInstrument", snap.hasInstrument); + obj->setProperty("hasARA", snap.hasARA); + obj->setProperty("sendCount", snap.sendCount); + obj->setProperty("sidechainSourceCount", snap.sidechainSourceCount); + obj->setProperty("masterSendEnabled", snap.masterSendEnabled); + obj->setProperty("hasTrackAutomation", snap.hasTrackAutomation); + obj->setProperty("requiresFullRender", snap.requiresFullRender); + trackArray.add(juce::var(obj)); + } + root->setProperty("tracks", juce::var(trackArray)); + + const auto reportFile = outputFile.getSiblingFile(outputFile.getFileNameWithoutExtension() + "_render_route_report.json"); + reportFile.getParentDirectory().createDirectory(); + const bool wrote = reportFile.replaceWithText(juce::JSON::toString(juce::var(root), true)); + logToDisk("renderProject: route report=" + reportFile.getFullPathName() + + " wrote=" + juce::String(wrote ? "true" : "false")); + }; + + writeRenderRouteReport(); + + auto buildRenderChainRouteTrace = [&] (const juce::String& trackId, + double blockStartSec, + int blockSamples) -> juce::var + { + juce::Array routes; + const double blockEndSec = blockStartSec + static_cast (blockSamples) / actualSampleRate; + for (const auto& clip : clipSnapshot) + { + if (! clip.isActive || clip.trackId != trackId) + continue; + + const double clipEndSec = clip.startTime + clip.duration; + if (blockStartSec >= clipEndSec || blockEndSec <= clip.startTime) + continue; + + const double blockMidSec = (std::max (blockStartSec, clip.startTime) + + std::min (blockEndSec, clipEndSec)) * 0.5; + const double clipTimeSec = blockMidSec - clip.startTime; + juce::String sourceType = "original"; + juce::String sourceFile = clip.audioFile.getFullPathName(); + double playbackOffsetSec = clip.offset + clipTimeSec; + bool renderedSegmentActive = false; + bool correctedSourceActive = false; + + auto segmentIt = renderedPreviewSegmentSnapshot.find (clip.clipId); + if (segmentIt != renderedPreviewSegmentSnapshot.end()) + { + for (const auto& segment : segmentIt->second) + { + if (clipTimeSec >= segment.startSec && clipTimeSec < segment.endSec) + { + renderedSegmentActive = true; + sourceType = "rendered_segment"; + sourceFile = segment.audioFile.getFullPathName(); + playbackOffsetSec = segment.fileOffsetSec + (clipTimeSec - segment.startSec); + break; + } + } + } + + if (! renderedSegmentActive + && clip.originalAudioFile.existsAsFile() + && clip.audioFile.existsAsFile() + && clip.audioFile != clip.originalAudioFile) + { + correctedSourceActive = true; + sourceType = "corrected_source"; + } + + auto* routeObj = new juce::DynamicObject(); + routeObj->setProperty ("trackId", trackId); + routeObj->setProperty ("clipId", clip.clipId); + routeObj->setProperty ("sourceType", sourceType); + routeObj->setProperty ("audioFile", sourceFile); + routeObj->setProperty ("clipTimeSec", clipTimeSec); + routeObj->setProperty ("playbackOffsetSec", playbackOffsetSec); + routeObj->setProperty ("renderedSegmentActive", renderedSegmentActive); + routeObj->setProperty ("correctedSourceActive", correctedSourceActive); + routeObj->setProperty ("clipVolumeDb", clip.volumeDB); + routeObj->setProperty ("clipFadeInSec", clip.fadeIn); + routeObj->setProperty ("clipFadeOutSec", clip.fadeOut); + routes.add (juce::var (routeObj)); + } + + return juce::var (routes); + }; + + if (reusableMasterBuffer.getNumChannels() < 2 || reusableMasterBuffer.getNumSamples() < blockSize) + reusableMasterBuffer.setSize(2, blockSize, false, false, true); + if (reusableMasterBufferDouble.getNumChannels() < 2 || reusableMasterBufferDouble.getNumSamples() < blockSize) + reusableMasterBufferDouble.setSize(2, blockSize, false, false, true); for (int pass = 0; pass < numPasses; ++pass) { @@ -6422,12 +9362,27 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do // Create fresh offline playback engine for each pass (deterministic reads) PlaybackEngine passPlayback; - passPlayback.setRenderMode(true); // Use Lagrange interpolation for high-quality resampling + // The Lagrange render SRC path is stateful across blocks and currently + // unsafe for arbitrary render offsets/ranges. Keep exports on the + // exact-position stateless interpolation path unless explicitly opted in. + passPlayback.setRenderMode(useRenderLagrangeSrc); + for (const auto& entry : renderedPreviewSegmentSnapshot) + { + for (const auto& segment : entry.second) + { + passPlayback.setClipRenderedPreviewSegment(entry.first, + segment.audioFile, + segment.startSec, + segment.endSec, + segment.fileOffsetSec); + } + } for (const auto& clip : clipSnapshot) { passPlayback.addClip(clip.audioFile, clip.startTime, clip.duration, clip.trackId, clip.offset, clip.volumeDB, - clip.fadeIn, clip.fadeOut); + clip.fadeIn, clip.fadeOut, clip.clipId, + clip.originalAudioFile, clip.originalOffset); } // Create fresh metronome for each pass @@ -6442,6 +9397,25 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do renderMet.setEnabled(true); } + if (rtTracks) + { + for (const auto& entry : *rtTracks) + { + if (entry.sendAccumBuffer != nullptr) + { + if (entry.sendAccumBuffer->getNumChannels() < 2 || entry.sendAccumBuffer->getNumSamples() < blockSize) + entry.sendAccumBuffer->setSize(2, blockSize, false, false, true); + entry.sendAccumBuffer->clear(); + } + if (entry.sidechainOutputBuffer != nullptr) + { + if (entry.sidechainOutputBuffer->getNumChannels() < 2 || entry.sidechainOutputBuffer->getNumSamples() < blockSize) + entry.sidechainOutputBuffer->setSize(2, blockSize, false, false, true); + entry.sidechainOutputBuffer->clear(); + } + } + } + if (pass == 1) { // Second pass: calculate normalization gain @@ -6461,248 +9435,311 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do while (samplesRemaining > 0) { int samplesThisBlock = (int)std::min((juce::int64)blockSize, samplesRemaining); + const juce::int64 renderBlockStartSample = totalSamples - samplesRemaining; // Master buffer (always stereo internally) juce::AudioBuffer masterBuffer(2, samplesThisBlock); masterBuffer.clear(); - juce::AudioBuffer masterBufferDouble(2, samplesThisBlock); - masterBufferDouble.clear(); const bool renderHybrid64 = processingPrecisionMode == ProcessingPrecisionMode::Hybrid64; + const bool captureRenderChainBlock = renderChainDebugEnabled && pass == (numPasses - 1); + + if (renderHybrid64) + reusableMasterBufferDouble.clear(); + + if (captureRenderChainBlock) + { + chainPlaybackBlock.setSize (2, samplesThisBlock, false, false, true); + chainTrackPostBlock.setSize (2, samplesThisBlock, false, false, true); + chainMasterPreBlock.setSize (2, samplesThisBlock, false, false, true); + chainMasterPostBlock.setSize (2, samplesThisBlock, false, false, true); + chainWriterInputBlock.setSize (2, samplesThisBlock, false, false, true); + chainPlaybackBlock.clear(); + chainTrackPostBlock.clear(); + chainMasterPreBlock.clear(); + chainMasterPostBlock.clear(); + chainWriterInputBlock.clear(); + } + juce::Array chainBlockRoutes; // Add metronome if enabled if (renderMetronomeAudio) { renderMet.getNextAudioBlock(masterBuffer, samplePositionForMetronome); if (renderHybrid64) + copyFloatBufferToDoubleBuffer(masterBuffer, reusableMasterBufferDouble, 2, samplesThisBlock); + } + + if (rtTracks) + { + for (const auto& entry : *rtTracks) { - for (int ch = 0; ch < 2; ++ch) - { - auto* src = masterBuffer.getReadPointer(ch); - auto* dest = masterBufferDouble.getWritePointer(ch); - for (int sample = 0; sample < samplesThisBlock; ++sample) - dest[sample] = static_cast(src[sample]); - } + if (entry.sendAccumBuffer != nullptr) + entry.sendAccumBuffer->clear(); + if (entry.sidechainOutputBuffer != nullptr) + entry.sidechainOutputBuffer->clear(); } } - // Process each track - for (const auto& snap : trackSnapshots) + if (useSimpleNoFxRenderLoop) { - // Stem rendering: only process the specified track - if (isStemRender && snap.id != stemTrackId) continue; - - // Skip muted tracks (unless stem render — always render the target track) - if (!isStemRender && snap.muted) continue; - // If any track is soloed, skip non-soloed tracks (unless stem render) - if (!isStemRender && anySoloed && !snap.soloed) continue; - - // Fill track buffer from clips - juce::AudioBuffer trackBuffer(2, samplesThisBlock); - trackBuffer.clear(); - passPlayback.fillTrackBuffer(snap.id, trackBuffer, currentTimeSeconds, - samplesThisBlock, actualSampleRate); - - // Process track FX chain BEFORE volume/pan (matches real-time signal flow) - // Channel-safe: expand buffer if plugin needs more channels than our stereo buffer + for (const auto& snap : trackSnapshots) { - auto it = trackMap.find(snap.id); - if (it != trackMap.end() && it->second) - { - auto* track = it->second; - juce::MidiBuffer midiMessages = buildTrackMidiBlock(snap.id, currentTimeSeconds, - samplesThisBlock, actualSampleRate, true); - if (auto* instrument = track->getInstrument()) - instrument->setPlayHead(this); - int numInputFX = track->getNumInputFX(); - int numTrackFX = track->getNumTrackFX(); - const bool hasInstrument = track->getTrackType() == TrackType::Instrument - && track->getInstrument() != nullptr; - if (numInputFX > 0 || numTrackFX > 0 || hasInstrument) - { - auto safeRenderFX = [&](juce::AudioProcessor* proc) { - int pluginCh = juce::jmax(proc->getTotalNumInputChannels(), - proc->getTotalNumOutputChannels()); - const bool useDoublePrecision = renderHybrid64 - && proc->supportsDoublePrecisionProcessing(); - if (useDoublePrecision) - { - juce::AudioBuffer doubleBuffer(pluginCh, samplesThisBlock); - doubleBuffer.clear(); - for (int ch = 0; ch < juce::jmin(trackBuffer.getNumChannels(), pluginCh); ++ch) - { - auto* src = trackBuffer.getReadPointer(ch); - auto* dest = doubleBuffer.getWritePointer(ch); - for (int sample = 0; sample < samplesThisBlock; ++sample) - dest[sample] = static_cast(src[sample]); - } + if (isStemRender && snap.id != stemTrackId) + continue; + if (!isStemRender && snap.muted) + continue; + if (!isStemRender && anySoloed && !snap.soloed) + continue; - proc->processBlock(doubleBuffer, midiMessages); + juce::AudioBuffer trackBuffer(2, samplesThisBlock); + trackBuffer.clear(); + passPlayback.fillTrackBuffer(snap.id, trackBuffer, currentTimeSeconds, + samplesThisBlock, actualSampleRate); - const int availableDoubleChannels = juce::jmax(1, pluginCh); - for (int ch = 0; ch < trackBuffer.getNumChannels(); ++ch) - { - auto* dest = trackBuffer.getWritePointer(ch); - auto* src = doubleBuffer.getReadPointer(juce::jmin(ch, availableDoubleChannels - 1)); - for (int sample = 0; sample < samplesThisBlock; ++sample) - dest[sample] = static_cast(src[sample]); - } - } - else if (pluginCh <= trackBuffer.getNumChannels()) - { - proc->processBlock(trackBuffer, midiMessages); - // Mono plugin on stereo track: duplicate output to all channels - int outCh = proc->getTotalNumOutputChannels(); - if (outCh > 0 && outCh < trackBuffer.getNumChannels()) - { - for (int ch = outCh; ch < trackBuffer.getNumChannels(); ++ch) - trackBuffer.copyFrom (ch, 0, trackBuffer, 0, 0, samplesThisBlock); - } - } - else - { - // Expand buffer for this plugin (render thread — allocation OK) - juce::AudioBuffer expanded(pluginCh, samplesThisBlock); - expanded.clear(); - for (int ch = 0; ch < trackBuffer.getNumChannels(); ++ch) - expanded.copyFrom(ch, 0, trackBuffer, ch, 0, samplesThisBlock); - proc->processBlock(expanded, midiMessages); - for (int ch = 0; ch < trackBuffer.getNumChannels(); ++ch) - trackBuffer.copyFrom(ch, 0, expanded, ch, 0, samplesThisBlock); - } - }; - for (int fx = 0; fx < numInputFX; ++fx) - { - auto* proc = track->getInputFXProcessor(fx); - if (proc) safeRenderFX(proc); - } - if (hasInstrument) - safeRenderFX(track->getInstrument()); - for (int fx = 0; fx < numTrackFX; ++fx) - { - auto* proc = track->getTrackFXProcessor(fx); - if (proc) safeRenderFX(proc); - } + if (captureRenderChainBlock) + { + addToSignalChainCapture(chainPlaybackBlock, 0, trackBuffer, samplesThisBlock); + auto routeTrace = buildRenderChainRouteTrace(snap.id, currentTimeSeconds, samplesThisBlock); + if (auto* routeArray = routeTrace.getArray()) + { + for (const auto& route : *routeArray) + chainBlockRoutes.add(route); } } - } - // Apply per-track volume/pan AFTER FX (matches real-time signal flow) - float volumeGain = juce::Decibels::decibelsToGain(snap.volumeDB); - float leftGain = 1.0f; - float rightGain = 1.0f; - computePanLawGains(currentPanLaw, snap.pan, volumeGain, leftGain, rightGain); + const float volumeGain = juce::Decibels::decibelsToGain(snap.volumeDB); + float leftGain = 1.0f; + float rightGain = 1.0f; + computePanLawGains(currentPanLaw, snap.pan, volumeGain, leftGain, rightGain); + juce::FloatVectorOperations::multiply(trackBuffer.getWritePointer(0), leftGain, samplesThisBlock); + juce::FloatVectorOperations::multiply(trackBuffer.getWritePointer(1), rightGain, samplesThisBlock); - juce::FloatVectorOperations::multiply(trackBuffer.getWritePointer(0), leftGain, samplesThisBlock); - juce::FloatVectorOperations::multiply(trackBuffer.getWritePointer(1), rightGain, samplesThisBlock); + if (captureRenderChainBlock) + addToSignalChainCapture(chainTrackPostBlock, 0, trackBuffer, samplesThisBlock); - // Mix into master buffer - for (int ch = 0; ch < 2; ++ch) - { - if (renderHybrid64) - { - auto* dest = masterBufferDouble.getWritePointer(ch); - auto* src = trackBuffer.getReadPointer(ch); - for (int sample = 0; sample < samplesThisBlock; ++sample) - dest[sample] += static_cast(src[sample]); - } - else + for (int ch = 0; ch < 2; ++ch) { - masterBuffer.addFrom(ch, 0, trackBuffer, ch, 0, samplesThisBlock); + if (renderHybrid64) + { + auto* dest = reusableMasterBufferDouble.getWritePointer(ch); + auto* src = trackBuffer.getReadPointer(ch); + for (int sample = 0; sample < samplesThisBlock; ++sample) + dest[sample] += static_cast(src[sample]); + } + else + { + masterBuffer.addFrom(ch, 0, trackBuffer, ch, 0, samplesThisBlock); + } } } } - - // Process master FX chain (channel-safe, render thread — allocation OK) - // Skip master FX for stem renders (export raw track output) - if (!isStemRender && renderMasterStage && !renderMasterStage->slots.empty()) + else if (rtTracks && !rtTracks->empty()) { - juce::MidiBuffer dummyMidi; - for (const auto& slot : renderMasterStage->slots) + constexpr int kMaxOfflineTracks = 64; + int processedOrder[kMaxOfflineTracks]; + int orderCount = 0; + buildSidechainProcessingOrder(*rtTracks, processedOrder, orderCount, kMaxOfflineTracks); + + for (int orderIdx = 0; orderIdx < orderCount; ++orderIdx) { - if (!slot.processor || slot.bypassed) + const auto& trackEntry = (*rtTracks)[static_cast(processedOrder[orderIdx])]; + auto* track = dynamic_cast(trackEntry.node != nullptr ? trackEntry.node->getProcessor() : nullptr); + if (track == nullptr) continue; - auto* proc = slot.processor.get(); - int pluginCh = juce::jmax(proc->getTotalNumInputChannels(), - proc->getTotalNumOutputChannels()); - const bool useDoublePrecision = renderHybrid64 - && !slot.forceFloat - && slot.supportsDouble; - if (useDoublePrecision) + if (isStemRender && trackEntry.id != stemTrackId) + continue; + + auto snapshotIt = trackSnapshotById.find(trackEntry.id); + if (!isStemRender) { - juce::AudioBuffer expanded(juce::jmax(2, pluginCh), samplesThisBlock); - expanded.clear(); - for (int ch = 0; ch < juce::jmin(masterBufferDouble.getNumChannels(), expanded.getNumChannels()); ++ch) - { - auto* src = masterBufferDouble.getReadPointer(ch); - auto* dest = expanded.getWritePointer(ch); - for (int sample = 0; sample < samplesThisBlock; ++sample) - dest[sample] = src[sample]; - } + if (snapshotIt == trackSnapshotById.end()) + continue; + if (snapshotIt->second.muted) + continue; + if (anySoloed && !snapshotIt->second.soloed) + continue; + } + + juce::AudioBuffer trackBuffer(2, samplesThisBlock); + trackBuffer.clear(); - proc->processBlock(expanded, dummyMidi); + if (!isStemRender && trackEntry.sendAccumBuffer != nullptr && trackEntry.sendAccumBuffer->getNumSamples() >= samplesThisBlock) + { + auto& accumBuffer = *trackEntry.sendAccumBuffer; + for (int ch = 0; ch < juce::jmin(2, accumBuffer.getNumChannels()); ++ch) + juce::FloatVectorOperations::add(trackBuffer.getWritePointer(ch), + accumBuffer.getReadPointer(ch), + samplesThisBlock); + accumBuffer.clear(); + } - for (int ch = 0; ch < masterBufferDouble.getNumChannels(); ++ch) + if (!track->hasActiveARA()) + { + passPlayback.fillTrackBuffer(trackEntry.id, trackBuffer, currentTimeSeconds, + samplesThisBlock, actualSampleRate); + } + if (captureRenderChainBlock) + { + addToSignalChainCapture (chainPlaybackBlock, 0, trackBuffer, samplesThisBlock); + auto routeTrace = buildRenderChainRouteTrace (trackEntry.id, currentTimeSeconds, samplesThisBlock); + if (auto* routeArray = routeTrace.getArray()) { - auto* dest = masterBufferDouble.getWritePointer(ch); - auto* src = expanded.getReadPointer(juce::jmin(ch, expanded.getNumChannels() - 1)); - for (int sample = 0; sample < samplesThisBlock; ++sample) - dest[sample] = src[sample]; + for (const auto& route : *routeArray) + chainBlockRoutes.add (route); } } - else if (pluginCh <= masterBuffer.getNumChannels()) + + if (!isStemRender && !trackEntry.sidechainSourceIds.empty()) { - if (renderHybrid64) - copyDoubleBufferToFloatBuffer(masterBufferDouble, masterBuffer, masterBuffer.getNumChannels(), samplesThisBlock); - proc->processBlock(masterBuffer, dummyMidi); - if (renderHybrid64) - copyFloatBufferToDoubleBuffer(masterBuffer, masterBufferDouble, masterBuffer.getNumChannels(), samplesThisBlock); + const juce::AudioBuffer* sidechainBuffer = nullptr; + for (const auto& sourceId : trackEntry.sidechainSourceIds) + { + if (sourceId.isEmpty()) + continue; + + for (const auto& candidate : *rtTracks) + { + if (candidate.id == sourceId && candidate.sidechainOutputBuffer != nullptr) + { + sidechainBuffer = candidate.sidechainOutputBuffer.get(); + break; + } + } + + if (sidechainBuffer != nullptr) + break; + } + + track->setSidechainBuffer(sidechainBuffer); } else + { + track->setSidechainBuffer(nullptr); + } + + track->setCurrentBlockPosition(currentTimeSeconds); + if (auto* instrument = track->getInstrument()) + instrument->setPlayHead(this); + juce::MidiBuffer midiMessages = buildTrackMidiBlock(trackEntry.id, currentTimeSeconds, + samplesThisBlock, actualSampleRate, true); + if (!track->tryProcessBlock(trackBuffer, midiMessages)) + continue; + if (captureRenderChainBlock) + addToSignalChainCapture (chainTrackPostBlock, 0, trackBuffer, samplesThisBlock); + + if (!isStemRender && trackEntry.sidechainOutputBuffer != nullptr) + { + auto& sidechainOut = *trackEntry.sidechainOutputBuffer; + if (sidechainOut.getNumChannels() < 2 || sidechainOut.getNumSamples() < samplesThisBlock) + sidechainOut.setSize(2, samplesThisBlock, false, false, true); + for (int ch = 0; ch < juce::jmin(2, trackBuffer.getNumChannels()); ++ch) + sidechainOut.copyFrom(ch, 0, trackBuffer, ch, 0, samplesThisBlock); + } + + if (!isStemRender && !trackEntry.sends.empty()) + { + const auto& preFaderBuffer = track->getPreFaderBuffer(); + for (const auto& send : trackEntry.sends) + { + if (!send.enabled || send.level <= 0.0f || send.destTrackId.isEmpty()) + continue; + + for (const auto& destEntry : *rtTracks) + { + if (destEntry.id != send.destTrackId || destEntry.sendAccumBuffer == nullptr) + continue; + + auto& destBuffer = *destEntry.sendAccumBuffer; + if (destBuffer.getNumChannels() < 2 || destBuffer.getNumSamples() < samplesThisBlock) + destBuffer.setSize(2, samplesThisBlock, false, false, true); + + const auto& sourceBuffer = send.preFader ? preFaderBuffer : trackBuffer; + const int sourceChannels = sourceBuffer.getNumChannels(); + const int destChannels = destBuffer.getNumChannels(); + const float phaseMultiplier = send.phaseInvert ? -1.0f : 1.0f; + const float panAngle = (send.pan + 1.0f) * juce::MathConstants::pi / 4.0f; + const float leftGain = std::cos(panAngle) * send.level * phaseMultiplier; + const float rightGain = std::sin(panAngle) * send.level * phaseMultiplier; + + if (destChannels >= 2 && sourceChannels >= 2) + { + for (int sample = 0; sample < samplesThisBlock; ++sample) + { + destBuffer.getWritePointer(0)[sample] += sourceBuffer.getReadPointer(0)[sample] * leftGain; + destBuffer.getWritePointer(1)[sample] += sourceBuffer.getReadPointer(1)[sample] * rightGain; + } + } + else if (destChannels >= 1 && sourceChannels >= 1) + { + for (int sample = 0; sample < samplesThisBlock; ++sample) + destBuffer.getWritePointer(0)[sample] += sourceBuffer.getReadPointer(0)[sample] * send.level; + } + + break; + } + } + } + + if (!track->getMasterSendEnabled()) + continue; + + for (int ch = 0; ch < juce::jmin(2, trackBuffer.getNumChannels()); ++ch) { if (renderHybrid64) - copyDoubleBufferToFloatBuffer(masterBufferDouble, masterBuffer, masterBuffer.getNumChannels(), samplesThisBlock); - juce::AudioBuffer expanded(pluginCh, samplesThisBlock); - expanded.clear(); - for (int ch = 0; ch < masterBuffer.getNumChannels(); ++ch) - expanded.copyFrom(ch, 0, masterBuffer, ch, 0, samplesThisBlock); - proc->processBlock(expanded, dummyMidi); - for (int ch = 0; ch < masterBuffer.getNumChannels(); ++ch) - masterBuffer.copyFrom(ch, 0, expanded, ch, 0, samplesThisBlock); - if (renderHybrid64) - copyFloatBufferToDoubleBuffer(masterBuffer, masterBufferDouble, masterBuffer.getNumChannels(), samplesThisBlock); + { + auto* dest = reusableMasterBufferDouble.getWritePointer(ch); + auto* src = trackBuffer.getReadPointer(ch); + for (int sample = 0; sample < samplesThisBlock; ++sample) + dest[sample] += static_cast(src[sample]); + } + else + { + masterBuffer.addFrom(ch, 0, trackBuffer, ch, 0, samplesThisBlock); + } } } } - // Apply master pan (constant power law) — skip for stem renders - if (!isStemRender) + if (captureRenderChainBlock) + { + if (renderHybrid64) + copyDoubleBufferToFloatBuffer (reusableMasterBufferDouble, masterBuffer, 2, samplesThisBlock); + + const int chainCaptureOffset = static_cast (std::min ( + renderBlockStartSample, + static_cast (std::numeric_limits::max()))); + addToSignalChainCapture (chainPlaybackTap, chainCaptureOffset, chainPlaybackBlock, samplesThisBlock); + addToSignalChainCapture (chainTrackPostTap, chainCaptureOffset, chainTrackPostBlock, samplesThisBlock); + chainMasterPreBlock.copyFrom (0, 0, masterBuffer, 0, 0, samplesThisBlock); + chainMasterPreBlock.copyFrom (1, 0, masterBuffer, 1, 0, samplesThisBlock); + copyToSignalChainCapture (chainMasterPreTap, chainCaptureOffset, masterBuffer, samplesThisBlock); + } + + if (!isStemRender && renderMasterStage && !renderMasterStage->slots.empty()) { - float leftGain = 1.0f; - float rightGain = 1.0f; - computePanLawGains(currentPanLaw, masterPan, 1.0f, leftGain, rightGain); + float* masterChannelData[2] { masterBuffer.getWritePointer(0), masterBuffer.getWritePointer(1) }; + processMasterFXChain(std::static_pointer_cast(renderMasterStage), + masterChannelData, 2, samplesThisBlock, renderHybrid64); + } + if (captureRenderChainBlock) + { if (renderHybrid64) - applyStereoPanToDoubleBuffer(masterBufferDouble, samplesThisBlock, leftGain, rightGain); - else - { - juce::FloatVectorOperations::multiply(masterBuffer.getWritePointer(0), leftGain, samplesThisBlock); - juce::FloatVectorOperations::multiply(masterBuffer.getWritePointer(1), rightGain, samplesThisBlock); - } + copyDoubleBufferToFloatBuffer (reusableMasterBufferDouble, masterBuffer, 2, samplesThisBlock); + + const int chainCaptureOffset = static_cast (std::min ( + renderBlockStartSample, + static_cast (std::numeric_limits::max()))); + chainMasterPostBlock.copyFrom (0, 0, masterBuffer, 0, 0, samplesThisBlock); + chainMasterPostBlock.copyFrom (1, 0, masterBuffer, 1, 0, samplesThisBlock); + copyToSignalChainCapture (chainMasterPostTap, chainCaptureOffset, masterBuffer, samplesThisBlock); } - // Apply master volume — skip for stem renders if (!isStemRender) { - if (renderHybrid64) - { - applyGainToDoubleBuffer(masterBufferDouble, 2, samplesThisBlock, masterVolume); - } - else - { - for (int ch = 0; ch < 2; ++ch) - juce::FloatVectorOperations::multiply(masterBuffer.getWritePointer(ch), masterVolume, samplesThisBlock); - } + float* masterChannelData[2] { masterBuffer.getWritePointer(0), masterBuffer.getWritePointer(1) }; + applyMasterGainPanMono(masterChannelData, 2, samplesThisBlock, currentTimeSeconds, renderHybrid64); } // Apply normalization gain (pass 2 only) @@ -6710,7 +9747,7 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do { if (renderHybrid64) { - applyGainToDoubleBuffer(masterBufferDouble, 2, samplesThisBlock, normGain); + applyGainToDoubleBuffer(reusableMasterBufferDouble, 2, samplesThisBlock, normGain); } else { @@ -6719,25 +9756,10 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do } } - if (!isStemRender && masterMono.load(std::memory_order_relaxed)) - { - if (renderHybrid64) - downmixDoubleBufferToMono(masterBufferDouble, samplesThisBlock); - else - { - for (int sample = 0; sample < samplesThisBlock; ++sample) - { - float mono = (masterBuffer.getSample(0, sample) + masterBuffer.getSample(1, sample)) * 0.5f; - masterBuffer.setSample(0, sample, mono); - masterBuffer.setSample(1, sample, mono); - } - } - } - // Measure peak level if (renderHybrid64) { - passPeak = juce::jmax(passPeak, findPeakInDoubleBuffer(masterBufferDouble, 2, samplesThisBlock)); + passPeak = juce::jmax(passPeak, findPeakInDoubleBuffer(reusableMasterBufferDouble, 2, samplesThisBlock)); } else { @@ -6760,7 +9782,7 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do if (isFinalPass) { if (renderHybrid64) - copyDoubleBufferToFloatBuffer(masterBufferDouble, masterBuffer, 2, samplesThisBlock); + copyDoubleBufferToFloatBuffer(reusableMasterBufferDouble, masterBuffer, 2, samplesThisBlock); // Apply dither if requested (before bit-depth truncation by the writer) if (ditherMode > 0 && bitDepth < 32) @@ -6796,6 +9818,29 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do } } + if (captureRenderChainBlock) + { + const int chainCaptureOffset = static_cast (std::min ( + renderBlockStartSample, + static_cast (std::numeric_limits::max()))); + chainWriterInputBlock.copyFrom (0, 0, masterBuffer, 0, 0, samplesThisBlock); + chainWriterInputBlock.copyFrom (1, 0, masterBuffer, 1, 0, samplesThisBlock); + copyToSignalChainCapture (chainWriterInputTap, chainCaptureOffset, masterBuffer, samplesThisBlock); + + auto* blockObj = new juce::DynamicObject(); + blockObj->setProperty ("pass", pass + 1); + blockObj->setProperty ("blockStartSec", currentTimeSeconds); + blockObj->setProperty ("blockStartSample", static_cast (renderBlockStartSample)); + blockObj->setProperty ("numSamples", samplesThisBlock); + blockObj->setProperty ("playback", makeSignalChainBufferStats (chainPlaybackBlock, samplesThisBlock)); + blockObj->setProperty ("trackPost", makeSignalChainBufferStats (chainTrackPostBlock, samplesThisBlock)); + blockObj->setProperty ("masterPreFx", makeSignalChainBufferStats (chainMasterPreBlock, samplesThisBlock)); + blockObj->setProperty ("masterPostFx", makeSignalChainBufferStats (chainMasterPostBlock, samplesThisBlock)); + blockObj->setProperty ("writerInput", makeSignalChainBufferStats (chainWriterInputBlock, samplesThisBlock)); + blockObj->setProperty ("routes", juce::var (chainBlockRoutes)); + renderChainBlockReports.add (juce::var (blockObj)); + } + if (numChannels == 1) { // Mono downmix: average L+R @@ -6819,8 +9864,80 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do logToDisk("renderProject: Pass " + juce::String(pass + 1) + " complete. Peak level: " + juce::String(passPeak)); } - // Flush and close - writer.reset(); + // Flush and close + writer.reset(); + + if (renderChainDebugEnabled) + { + auto* reportObj = new juce::DynamicObject(); + reportObj->setProperty ("schemaVersion", 1); + reportObj->setProperty ("purpose", "signal_chain_first_render_debug"); + reportObj->setProperty ("source", source); + reportObj->setProperty ("renderFile", renderFile.getFullPathName()); + reportObj->setProperty ("requestedOutputFile", outputFile.getFullPathName()); + reportObj->setProperty ("format", format); + reportObj->setProperty ("actualSampleRate", actualSampleRate); + reportObj->setProperty ("bitDepth", bitDepth); + reportObj->setProperty ("channels", numChannels); + reportObj->setProperty ("normalize", normalize); + reportObj->setProperty ("ditherMode", ditherMode); + reportObj->setProperty ("startTimeSec", startTime); + reportObj->setProperty ("endTimeSec", endTime); + reportObj->setProperty ("totalSamples", static_cast (totalSamples)); + reportObj->setProperty ("capturedSamples", renderChainCaptureSamples); + reportObj->setProperty ("captureTruncated", renderChainCaptureSamples < totalSamples); + reportObj->setProperty ("pitchPreviewRoutesClearedBeforeSnapshot", true); + reportObj->setProperty ("renderedPitchSegmentsDroppedBeforeSnapshot", staleRenderedPitchSegmentsDropped); + reportObj->setProperty ("blockReports", juce::var (renderChainBlockReports)); + + auto* chainObj = new juce::DynamicObject(); + chainObj->setProperty ("live", "clip route resolution -> PlaybackEngine read/mix -> TrackProcessor -> sends/sidechain -> master/monitoring FX -> gain/pan/mono -> meters/spectrum -> audio device"); + chainObj->setProperty ("render", "renderProject -> render PlaybackEngine snapshot -> TrackProcessor -> master FX/gain -> writer"); + reportObj->setProperty ("chainReference", juce::var (chainObj)); + + auto* stageObj = new juce::DynamicObject(); + auto writeStage = [&] (const juce::String& name, + const juce::AudioBuffer& buffer) + { + auto* obj = new juce::DynamicObject(); + const auto wav = renderChainDebugDir.getChildFile (name + ".wav"); + const bool wrote = renderChainCaptureSamples > 0 + && writeBufferToWavFile (buffer, renderChainCaptureSamples, actualSampleRate, wav); + obj->setProperty ("filePath", wrote ? wav.getFullPathName() : juce::String()); + obj->setProperty ("wrote", wrote); + obj->setProperty ("stats", makeSignalChainBufferStats (buffer, renderChainCaptureSamples)); + stageObj->setProperty (name, juce::var (obj)); + }; + + writeStage ("playback_output", chainPlaybackTap); + writeStage ("track_post_processing", chainTrackPostTap); + writeStage ("master_pre_fx", chainMasterPreTap); + writeStage ("master_post_fx", chainMasterPostTap); + writeStage ("writer_input", chainWriterInputTap); + + juce::AudioBuffer writerOutputBuffer; + double writerOutputSampleRate = 0.0; + auto* writerOutputObj = new juce::DynamicObject(); + writerOutputObj->setProperty ("filePath", renderFile.getFullPathName()); + if (readAudioFileForParity (renderFile, writerOutputBuffer, writerOutputSampleRate)) + { + writerOutputObj->setProperty ("readable", true); + writerOutputObj->setProperty ("sampleRate", writerOutputSampleRate); + writerOutputObj->setProperty ("stats", makeSignalChainBufferStats ( + writerOutputBuffer, + juce::jmin (writerOutputBuffer.getNumSamples(), renderChainCaptureSamples))); + } + else + { + writerOutputObj->setProperty ("readable", false); + } + stageObj->setProperty ("writer_output", juce::var (writerOutputObj)); + reportObj->setProperty ("stages", juce::var (stageObj)); + + const auto reportFile = renderChainDebugDir.getChildFile ("render_chain_report.json"); + reportFile.replaceWithText (juce::JSON::toString (juce::var (reportObj), true)); + logToDisk ("renderProject: signal-chain debug report=" + reportFile.getFullPathName()); + } // ========== 9. FFmpeg post-processing (lossy encoding only) ========== if (needsFFmpegPostProcess) @@ -6876,11 +9993,142 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do return true; } +juce::var AudioEngine::capturePitchAuditionPlayback(const juce::String& trackId, + const juce::String& clipId, + double startTime, + double duration, + const juce::String& filePath, + double sampleRate, + bool offlineRenderMode) +{ + auto* resultObj = new juce::DynamicObject(); + resultObj->setProperty("success", false); + resultObj->setProperty("trackId", trackId); + resultObj->setProperty("clipId", clipId); + resultObj->setProperty("filePath", filePath); + resultObj->setProperty("offlineRenderMode", offlineRenderMode); + + if (trackId.isEmpty() || filePath.isEmpty()) + { + resultObj->setProperty("error", "missing trackId or output path"); + return juce::var(resultObj); + } + + const double safeSampleRate = sampleRate > 0.0 ? sampleRate : 44100.0; + const double safeStart = std::max(0.0, startTime); + const double safeDuration = std::max(0.0, duration); + if (safeDuration <= 0.0) + { + resultObj->setProperty("error", "duration must be positive"); + return juce::var(resultObj); + } + + const int totalSamples = juce::jmax(1, static_cast(std::ceil(safeDuration * safeSampleRate))); + juce::AudioBuffer capture(2, totalSamples); + capture.clear(); + + const bool wasRenderMode = isRendering.load(); + playbackEngine.setRenderMode(offlineRenderMode); + + int written = 0; + const int blockSize = 1024; + juce::AudioBuffer block(2, blockSize); + float peak = 0.0f; + double energy = 0.0; + juce::int64 energySamples = 0; + juce::Array routeTrace; + + while (written < totalSamples) + { + const int samplesThisBlock = juce::jmin(blockSize, totalSamples - written); + block.setSize(2, samplesThisBlock, false, false, true); + block.clear(); + + const double blockTime = safeStart + (static_cast(written) / safeSampleRate); + const auto sourceStatus = playbackEngine.getClipPlaybackSourceAtTime(trackId, clipId, blockTime); + auto* routeObj = new juce::DynamicObject(); + routeObj->setProperty("blockStartSec", blockTime); + routeObj->setProperty("blockDurationSec", static_cast(samplesThisBlock) / safeSampleRate); + routeObj->setProperty("clipFound", sourceStatus.clipFound); + routeObj->setProperty("clipTimeSec", sourceStatus.clipTime); + routeObj->setProperty("sourceType", sourceStatus.sourceType); + routeObj->setProperty("audioFile", sourceStatus.audioFile); + routeObj->setProperty("playbackOffsetSec", sourceStatus.playbackOffset); + routeObj->setProperty("renderedSegmentActiveAtTime", sourceStatus.renderedSegmentActiveAtTime); + routeObj->setProperty("correctedSourceActiveAtTime", sourceStatus.correctedSourceActiveAtTime); + routeTrace.add(juce::var(routeObj)); + playbackEngine.fillTrackBuffer(trackId, block, blockTime, samplesThisBlock, safeSampleRate); + + for (int ch = 0; ch < 2; ++ch) + { + capture.copyFrom(ch, written, block, ch, 0, samplesThisBlock); + const auto range = juce::FloatVectorOperations::findMinAndMax(block.getReadPointer(ch), samplesThisBlock); + peak = juce::jmax(peak, juce::jmax(std::abs(range.getStart()), std::abs(range.getEnd()))); + const float* data = block.getReadPointer(ch); + for (int i = 0; i < samplesThisBlock; ++i) + { + const double s = static_cast(data[i]); + energy += s * s; + ++energySamples; + } + } + + written += samplesThisBlock; + } + + playbackEngine.setRenderMode(wasRenderMode); + + const juce::File outputFile(filePath); + outputFile.getParentDirectory().createDirectory(); + const bool wrote = writeBufferToWavFile(capture, totalSamples, safeSampleRate, outputFile); + if (!wrote) + { + resultObj->setProperty("error", "failed to write audition capture wav"); + return juce::var(resultObj); + } + + resultObj->setProperty("success", true); + resultObj->setProperty("filePath", outputFile.getFullPathName()); + resultObj->setProperty("startSec", safeStart); + resultObj->setProperty("durationSec", static_cast(totalSamples) / safeSampleRate); + resultObj->setProperty("sampleRate", safeSampleRate); + resultObj->setProperty("peak", peak); + resultObj->setProperty("rms", energySamples > 0 ? std::sqrt(energy / static_cast(energySamples)) : 0.0); + resultObj->setProperty("source", offlineRenderMode ? "offline_render_engine" : "live_playback_engine"); + resultObj->setProperty("routeTrace", juce::var(routeTrace)); + return juce::var(resultObj); +} + +juce::var AudioEngine::capturePitchBakedContext(const juce::String& sourceFile, + double startSec, + double durationSec, + const juce::String& filePath) +{ + return writePitchBakedContextWav(juce::File(sourceFile), + startSec, + durationSec, + juce::File(filePath)); +} + +juce::var AudioEngine::comparePitchDebugAudioFiles(const juce::String& referenceFile, + const juce::String& candidateFile, + double captureStartClipSec, + double noteStartClipSec, + double noteEndClipSec) +{ + return comparePitchParityFiles(juce::File(referenceFile), + juce::File(candidateFile), + captureStartClipSec, + noteStartClipSec, + noteEndClipSec); +} + bool AudioEngine::renderProjectWithDither(const juce::String& source, double startTime, double endTime, const juce::String& filePath, const juce::String& format, double renderSampleRate, int bitDepth, int numChannels, bool normalize, bool addTail, double tailLengthMs, - const juce::String& ditherType) + const juce::String& ditherType, + bool includeMetronome) { // Map dither type string to mode: "tpdf" → 1, "shaped" → 2, else → 0 if (ditherType == "tpdf") @@ -6892,21 +10140,11 @@ bool AudioEngine::renderProjectWithDither(const juce::String& source, double sta return renderProject(source, startTime, endTime, filePath, format, renderSampleRate, bitDepth, numChannels, - normalize, addTail, tailLengthMs); + normalize, addTail, tailLengthMs, includeMetronome); } //============================================================================== -// Automation (Phase 1.1) - -static AutomationList* getAutomationListForParam(TrackProcessor* track, const juce::String& parameterId) -{ - if (parameterId == "volume") - return &track->getVolumeAutomation(); - if (parameterId == "pan") - return &track->getPanAutomation(); - // Future: plugin parameter automation would go here - return nullptr; -} +// Automation static AutomationMode parseAutomationMode(const juce::String& modeStr) { @@ -6933,25 +10171,25 @@ static juce::String automationModeToString(AutomationMode mode) void AudioEngine::setAutomationPoints(const juce::String& trackId, const juce::String& parameterId, const juce::String& pointsJSON) { - // Master automation special case AutomationList* list = nullptr; if (trackId == "master") { if (parameterId == "volume") list = &masterVolumeAutomation; else if (parameterId == "pan") list = &masterPanAutomation; - if (!list) return; + if (!list) + return; } else { auto it = trackMap.find(trackId); if (it == trackMap.end() || !it->second) return; - list = getAutomationListForParam(it->second, parameterId); - if (!list) + auto target = it->second->resolveAutomationTarget(parameterId, true); + if (!target.has_value() || target->list == nullptr) return; + list = target->list; } - // Parse JSON array of { time: , value: } auto parsed = juce::JSON::parse(pointsJSON); if (!parsed.isArray()) return; @@ -6964,9 +10202,7 @@ void AudioEngine::setAutomationPoints(const juce::String& trackId, const juce::S { double timeSec = item.getProperty("time", 0.0); float value = static_cast(static_cast(item.getProperty("value", 0.0))); - // Convert seconds to samples for the audio thread - double timeSamples = timeSec * currentSampleRate; - points.push_back({ timeSamples, value }); + points.push_back({ timeSec, value }); } list->setPoints(std::move(points)); @@ -6979,11 +10215,13 @@ void AudioEngine::setAutomationMode(const juce::String& trackId, const juce::Str const juce::String& modeStr) { AutomationList* list = nullptr; + float defaultValue = 0.0f; if (trackId == "master") { if (parameterId == "volume") list = &masterVolumeAutomation; else if (parameterId == "pan") list = &masterPanAutomation; - if (!list) return; + if (!list) + return; auto mode = parseAutomationMode(modeStr); list->setMode(mode); @@ -6991,9 +10229,10 @@ void AudioEngine::setAutomationMode(const juce::String& trackId, const juce::Str if (mode != AutomationMode::Off) { if (parameterId == "volume") - list->setDefaultValue(masterVolume); + defaultValue = masterVolume; else if (parameterId == "pan") - list->setDefaultValue(masterPan); + defaultValue = masterPan; + list->setDefaultValue(defaultValue); } } else @@ -7002,20 +10241,16 @@ void AudioEngine::setAutomationMode(const juce::String& trackId, const juce::Str if (it == trackMap.end() || !it->second) return; - list = getAutomationListForParam(it->second, parameterId); - if (!list) + auto target = it->second->resolveAutomationTarget(parameterId, true); + if (!target.has_value() || target->list == nullptr) return; + list = target->list; auto mode = parseAutomationMode(modeStr); list->setMode(mode); if (mode != AutomationMode::Off) - { - if (parameterId == "volume") - list->setDefaultValue(it->second->getVolume()); - else if (parameterId == "pan") - list->setDefaultValue(it->second->getPan()); - } + list->setDefaultValue(it->second->getAutomationDefaultValue(*target)); } juce::Logger::writeToLog("AudioEngine: Set automation mode for track " + trackId + @@ -7035,7 +10270,9 @@ juce::String AudioEngine::getAutomationMode(const juce::String& trackId, const j auto it = trackMap.find(trackId); if (it == trackMap.end() || !it->second) return "off"; - list = getAutomationListForParam(it->second, parameterId); + auto target = it->second->resolveAutomationTarget(parameterId, false); + if (target.has_value()) + list = target->list; } if (!list) return "off"; @@ -7054,8 +10291,11 @@ void AudioEngine::clearAutomation(const juce::String& trackId, const juce::Strin else { auto it = trackMap.find(trackId); - if (it == trackMap.end() || !it->second) return; - list = getAutomationListForParam(it->second, parameterId); + if (it == trackMap.end() || !it->second) + return; + auto target = it->second->resolveAutomationTarget(parameterId, false); + if (target.has_value()) + list = target->list; } if (list) list->clear(); @@ -7072,8 +10312,11 @@ void AudioEngine::beginTouchAutomation(const juce::String& trackId, const juce:: else { auto it = trackMap.find(trackId); - if (it == trackMap.end() || !it->second) return; - list = getAutomationListForParam(it->second, parameterId); + if (it == trackMap.end() || !it->second) + return; + auto target = it->second->resolveAutomationTarget(parameterId, false); + if (target.has_value()) + list = target->list; } if (list) list->beginTouch(); @@ -7090,8 +10333,11 @@ void AudioEngine::endTouchAutomation(const juce::String& trackId, const juce::St else { auto it = trackMap.find(trackId); - if (it == trackMap.end() || !it->second) return; - list = getAutomationListForParam(it->second, parameterId); + if (it == trackMap.end() || !it->second) + return; + auto target = it->second->resolveAutomationTarget(parameterId, false); + if (target.has_value()) + list = target->list; } if (list) list->endTouch(); @@ -7403,7 +10649,7 @@ juce::var AudioEngine::freezeTrack(const juce::String& trackId) playbackEngine.fillTrackBuffer(trackId, trackBuffer, samplePos / renderRate, blockSamples, renderRate); // Process through FX chain - track->setCurrentBlockPosition(samplePos, renderRate); + track->setCurrentBlockPosition(samplePos / renderRate); juce::MidiBuffer midiMessages = buildTrackMidiBlock(trackId, samplePos / renderRate, blockSamples, renderRate, true); juce::AudioBuffer fxBuffer(trackBuffer.getArrayOfWritePointers(), 2, blockSamples); @@ -8090,7 +11336,8 @@ juce::var AudioEngine::getPitchHistory(const juce::String& trackId, int fxIndex, // Pitch Corrector — Graphical Mode bridge methods // ============================================================================ -juce::var AudioEngine::analyzePitchContour(const juce::String& trackId, const juce::String& clipId) +juce::var AudioEngine::analyzePitchContour(const juce::String& trackId, const juce::String& clipId, + std::function shouldCancel) { // Find the clip's audio file auto clips = playbackEngine.getClipSnapshot(); @@ -8131,12 +11378,14 @@ juce::var AudioEngine::analyzePitchContour(const juce::String& trackId, const ju // Run analysis PitchAnalyzer analyzer; auto result = analyzer.analyzeClip(mono.getReadPointer(0), numSamples, - reader->sampleRate, clipId); + reader->sampleRate, clipId, shouldCancel); return PitchAnalyzer::resultToJSON(result); } -juce::var AudioEngine::analyzePitchContourDirect(const juce::String& filePath, double offset, double duration, const juce::String& clipId) +juce::var AudioEngine::analyzePitchContourDirect(const juce::String& filePath, double offset, double duration, + const juce::String& clipId, + std::function shouldCancel) { juce::File audioFile(filePath); if (!audioFile.existsAsFile()) @@ -8165,7 +11414,7 @@ juce::var AudioEngine::analyzePitchContourDirect(const juce::String& filePath, d PitchAnalyzer analyzer; auto result = analyzer.analyzeClip(mono.getReadPointer(0), numSamples, - reader->sampleRate, clipId); + reader->sampleRate, clipId, shouldCancel); return PitchAnalyzer::resultToJSON(result); } @@ -8173,10 +11422,12 @@ juce::var AudioEngine::analyzePitchContourDirect(const juce::String& filePath, d juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const juce::String& clipId, const juce::var& notesJson, const juce::var& framesJson, float globalFormantSemitones, - std::optional windowStartSecOverride, - std::optional windowEndSecOverride, - const juce::String& renderMode, - std::function shouldCancel) + std::optional windowStartSecOverride, + std::optional windowEndSecOverride, + const juce::String& renderMode, + std::function shouldCancel, + double jobStartDelayMs, + int previewRenderGenerationToken) { logPitchEditorFormant("AudioEngine::applyPitchCorrection begin clip=" + clipId + " track=" + trackId @@ -8250,10 +11501,10 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j const double sr = reader->sampleRate; const double clipDuration = static_cast(clipNumSamples) / sr; - const bool hasGlobalFormant = std::abs(globalFormantSemitones) > 0.01f; // Parse notes first so we can limit processing to only the edited region. auto editedNotes = PitchAnalyzer::notesFromJSON(notesJson); + const bool explicitGlobalFormantRequested = std::abs(globalFormantSemitones) > 0.01f; // Find the time range covered by ACTUALLY EDITED notes only (correctedPitch != detectedPitch). // WORLD vocoder is NOT transparent at ratio=1.0 — it degrades audio quality even when @@ -8268,6 +11519,9 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j int gainEditCount = 0; int driftEditCount = 0; int vibratoEditCount = 0; + bool hasUpwardPitchEdit = false; + bool hasDownwardPitchEdit = false; + bool explicitNoteFormantRequested = false; for (const auto& n : editedNotes) { const bool pitchEdited = std::abs(n.correctedPitch - n.detectedPitch) > editThreshold; @@ -8276,19 +11530,36 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j const bool driftEdited = n.driftCorrectionAmount > 0.01f; const bool vibratoEdited = std::abs(n.vibratoDepth - 1.0f) > 0.01f; - if (pitchEdited) ++pitchEditCount; + if (pitchEdited) + { + ++pitchEditCount; + const float pitchDelta = n.correctedPitch - n.detectedPitch; + hasUpwardPitchEdit = hasUpwardPitchEdit || pitchDelta > editThreshold; + hasDownwardPitchEdit = hasDownwardPitchEdit || pitchDelta < -editThreshold; + } if (gainEdited) ++gainEditCount; - if (noteFormantEdited) ++noteFormantEditCount; + if (noteFormantEdited) + { + ++noteFormantEditCount; + explicitNoteFormantRequested = true; + } if (driftEdited) ++driftEditCount; if (vibratoEdited) ++vibratoEditCount; if (pitchEdited || gainEdited || noteFormantEdited || driftEdited || vibratoEdited) { - notesStartSec = std::min(notesStartSec, static_cast(n.startTime)); - notesEndSec = std::max(notesEndSec, static_cast(n.endTime)); + const double noteStartSec = static_cast(n.effectiveStartTime); + const double noteEndSec = static_cast(n.effectiveEndTime); + notesStartSec = std::min(notesStartSec, noteStartSec); + notesEndSec = std::max(notesEndSec, noteEndSec); anyNoteEdited = true; } } + + const bool explicitFormantRequested = explicitGlobalFormantRequested || explicitNoteFormantRequested; + bool pitchOnlyFormantSuppressed = false; + + const bool hasGlobalFormant = std::abs(globalFormantSemitones) > 0.01f; const bool anyEdited = anyNoteEdited || hasGlobalFormant; juce::String requestMode = "none"; const bool hasPitchEdits = pitchEditCount > 0; @@ -8309,6 +11580,8 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j + " driftEdits=" + juce::String(driftEditCount) + " vibratoEdits=" + juce::String(vibratoEditCount) + " globalFormant=" + juce::String(hasGlobalFormant ? "true" : "false") + + " explicitFormantRequested=" + juce::String(explicitFormantRequested ? "true" : "false") + + " pitchOnlyFormantSuppressed=" + juce::String(pitchOnlyFormantSuppressed ? "true" : "false") + " mode=" + requestMode); // If no notes were actually edited, restore the original audio file @@ -8316,15 +11589,21 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j if (!anyEdited) { logPitchEditorFormant("no edits remain, restoring original clip=" + clipId); - if (foundClip->originalAudioFile.existsAsFile() - && foundClip->audioFile != foundClip->originalAudioFile) + playbackEngine.clearAllPitchPreviewRoutes(clipId); + if (foundClip->originalAudioFile.existsAsFile()) { playbackEngine.replaceClipAudioFile(clipId, foundClip->originalAudioFile); } + else + { + playbackEngine.clearPitchCorrectionFile(clipId); + } juce::DynamicObject::Ptr resultObj = new juce::DynamicObject(); resultObj->setProperty("outputFile", foundClip->originalAudioFile.getFullPathName()); resultObj->setProperty("success", true); resultObj->setProperty("restored", true); + resultObj->setProperty("processingMode", "none"); + resultObj->setProperty("formantCurveUsed", false); return juce::var(resultObj.get()); } @@ -8370,7 +11649,7 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j PitchAnalyzer analyzer; analysis = analyzer.analyzeClip(monoBuffer.getReadPointer(0), - clipNumSamples, sr, clipId); + clipNumSamples, sr, clipId, shouldCancel); logPitchEditorFormant("no frontend frames supplied, re-analyzed locally clip=" + clipId); } @@ -8378,9 +11657,226 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j int renderedOutputSamples = clipNumSamples; double previewCoverageStartSec = 0.0; double previewCoverageEndSec = clipDuration; + double candidateCoverageStartSec = 0.0; + double candidateCoverageEndSec = clipDuration; bool previewSegmentRender = false; bool fullClipHQRender = false; bool swapDeferred = false; + juce::String requestedRendererBranch = PitchResynthesizer::getRequestedPitchRendererBranchName(); + juce::String actualRendererBranch = requestedRendererBranch; + if (explicitFormantRequested) + { + const juce::String reason = "graphical pitch editor formant rendering is disabled; VSF pitch-only is the only offline apply path"; + juce::DynamicObject::Ptr resultObj = new juce::DynamicObject(); + resultObj->setProperty ("success", false); + resultObj->setProperty ("clipId", clipId); + resultObj->setProperty ("renderMode", renderMode); + resultObj->setProperty ("requestedRendererBranch", requestedRendererBranch); + resultObj->setProperty ("actualRendererBranch", actualRendererBranch); + resultObj->setProperty ("processingMode", requestMode); + resultObj->setProperty ("explicitFormantRequested", true); + resultObj->setProperty ("pitchOnlyFormantSuppressed", false); + resultObj->setProperty ("formantCurveUsed", false); + resultObj->setProperty ("hardFailReason", reason); + resultObj->setProperty ("fallbackReason", reason); + resultObj->setProperty ("pitchRenderStrategy", "unsupported_formant_request"); + return juce::var (resultObj.get()); + } + juce::String pitchOnlyRecoveryPath; + bool pitchOnlyNeutralFormantUsed = false; + bool usedRendererFallback = false; + juce::String rendererFallbackReason; + bool bridgeUsed = false; + bool bridgeFallbackUsed = false; + double bridgeStartSec = 0.0; + double bridgeLengthMs = 0.0; + int bridgeAlignmentLagSamples = 0; + float bridgeCorrelationScore = 0.0f; + float bridgeGainDeltaDb = 0.0f; + bool bodyReplacementUsed = false; + bool bodyReplacementFallbackUsed = false; + double entryLockStartSec = 0.0; + double entryLockLengthMs = 0.0; + double exitLockStartSec = 0.0; + double renderedBodyStartSec = 0.0; + double renderedBodyEndSec = 0.0; + bool islandNativeUsed = false; + bool islandNativeFallbackUsed = false; + double islandRenderStartSec = 0.0; + double islandRenderEndSec = 0.0; + float transientMaskPeak = 0.0f; + float voicedCoreMaskPeak = 0.0f; + bool hpssUsed = false; + bool hpssFallbackUsed = false; + float harmonicMaskPeak = 0.0f; + float aperiodicMaskPeak = 0.0f; + bool spectralEnvelopeCorrectionUsed = false; + bool pitchOnlyCoreTimbreCorrectionUsed = false; + double pitchOnlyCoreEnvelopeMix = 0.0; + double pitchOnlyCoreRmsTrimDb = 0.0; + int pitchOnlyCoreEnvelopeLifter = 0; + bool pitchOnlyEntryTimbreCorrectionUsed = false; + double pitchOnlyEntryRmsTrimDb = 0.0; + double pitchOnlyEntryTiltDb = 0.0; + bool pitchOnlyEntryHandoffUsed = false; + bool pitchOnlyExitHandoffUsed = false; + bool vocalSourceFilterUsed = false; + double vocalSourceFilterVoicedCoverage = 0.0; + double vocalSourceFilterResidualMix = 0.0; + bool vocalSourceFilterFallbackUsed = false; + juce::String vocalSourceFilterFallbackReason; + double vocalSourceFilterEntryDryMs = 0.0; + double vocalSourceFilterExitDryMs = 0.0; + double vocalSourceFilterResidualMixScale = 1.0; + bool vocalSourceFilterEpochInterpolationUsed = false; + double vocalSourceFilterEpochInterpolationStrength = 0.0; + double vocalSourceFilterGrainRadiusScale = 1.0; + double vocalSourceFilterUpPresenceTrimDb = 0.0; + double vocalSourceFilterUpPresenceHz = 0.0; + double vocalSourceFilterDownNasalTrimDb = 0.0; + double vocalSourceFilterDownNasalHz = 0.0; + double vocalSourceFilterDownBodyCompDb = 0.0; + double vocalSourceFilterDownBodyCompHz = 0.0; + bool wsolaUsed = false; + bool wsolaFallbackUsed = false; + int wsolaEntryLagSamples = 0; + int wsolaExitLagSamples = 0; + float wsolaCorrelationScore = 0.0f; + bool phaseLockUsed = false; + bool phaseLockFallbackUsed = false; + bool phaseAlignedEntry = false; + bool phaseAlignedExit = false; + int phasePeakCount = 0; + bool transitionHqUsed = false; + bool transitionHqFallbackUsed = false; + double transitionStartSec = 0.0; + double transitionEndSec = 0.0; + float transitionTransientPeak = 0.0f; + float transitionVoicedCorePeak = 0.0f; + float transitionResidualPeak = 0.0f; + bool transitionEnvelopeCorrectionUsed = false; + bool engineV2Used = false; + bool engineV2FallbackUsed = false; + int engineV2TransitionCount = 0; + double engineV2TransitionStartSec = 0.0; + double engineV2TransitionEndSec = 0.0; + float engineV2HarmonicSupportPeak = 0.0f; + float engineV2ResidualSupportPeak = 0.0f; + float engineV2EnvelopeSupportPeak = 0.0f; + bool transientBypassUsed = false; + bool residualCarryUsed = false; + int cepstralCutoffUsed = 0; + int engineV2FftSize = 0; + int engineV2HopSize = 0; + bool immediateLeftNeighborUsed = false; + bool immediateRightNeighborUsed = false; + int leftNeighborSamplesRendered = 0; + int rightNeighborSamplesRendered = 0; + double leftNeighborSmoothMs = 0.0; + double rightNeighborSmoothMs = 0.0; + bool nonImmediateNeighborTouched = false; + double entryAlignmentOffsetMs = 0.0; + double exitAlignmentOffsetMs = 0.0; + bool firstVoicedCyclesEntryUsed = false; + bool firstVoicedCyclesExitUsed = false; + bool v3TransitionPairUsed = false; + bool v3ContinuousRenderUsed = false; + bool formantCurveUsed = false; + bool phraseHqRenderUsed = false; + bool phraseHqExpandedToFullClip = false; + juce::String pitchRenderDirection = "none"; + bool downshiftFormantGuardUsed = false; + double downshiftFormantGuardAlpha = 0.0; + double noteHqEffectiveStartSec = 0.0; + double noteHqEffectiveEndSec = 0.0; + double noteHqContextStartSec = 0.0; + double noteHqContextEndSec = 0.0; + double noteHqAudibleCommitStartSec = 0.0; + double noteHqAudibleCommitEndSec = 0.0; + int noteHqPreBodyDryProtectedSamples = 0; + double noteHqEntryInsideBodyFadeMs = 12.0; + double noteHqExitLeadInMs = 12.0; + double noteHqEntryBridgeStartSec = 0.0; + double noteHqEntryBridgeEndSec = 0.0; + double noteHqEntryBridgeWetLagMs = 0.0; + double noteHqEntryBridgeEnvelopeGainDb = 0.0; + bool noteHqEntryBridgeUsed = false; + double noteHqEntryTransientDryPreservedMs = 0.0; + bool pitchOnlyEntrySimpleHandoffUsed = false; + bool pitchOnlyEntrySafeHandoffUsed = false; + double pitchOnlyEntryDryHoldMs = 0.0; + double pitchOnlyEntrySafeBridgeMs = 0.0; + double pitchOnlyEntryWetAlignmentMs = 0.0; + double pitchOnlyEntryWetGainDb = 0.0; + double pitchOnlyEntryWetVsDryRmsDb = 0.0; + bool pitchOnlyEntryEqualPowerBlendUsed = false; + bool pitchOnlyEntryRmsContinuityUsed = false; + double pitchOnlyEntryRmsContinuityGainDb = 0.0; + double pitchOnlyEntryRmsContinuityMs = 0.0; + bool pitchOnlyEntryTransientResidualUsed = false; + double pitchOnlyEntryTransientResidualMix = 0.0; + double pitchOnlyEntryTransientResidualMs = 0.0; + bool pitchOnlyEntryPhaseSafeUsed = false; + bool pitchOnlyEntryWetAlignmentAccepted = false; + double pitchOnlyEntryFirstCycleCorrelation = 0.0; + double pitchOnlyEntryZeroCrossOffsetMs = 0.0; + double pitchOnlyEntryBridgeGainRampDb = 0.0; + bool pitchOnlyDownshiftCoreEnvelopePassUsed = false; + double pitchOnlyDownshiftCoreRmsTrimDb = 0.0; + double pitchOnlyDownshiftCoreEnvelopeMaxDb = 0.0; + int pitchOnlyDownshiftCoreEnvelopeFrames = 0; + double pitchOnlyEntryWetLagMs = 0.0; + double pitchOnlyEntryBridgeDurationMs = 0.0; + bool pitchOnlyExitDryRestoreUsed = false; + double pitchOnlyExitDryRestoreStartSec = 0.0; + double pitchOnlyExitDryRestoreEndSec = 0.0; + juce::String noteHqEntryBoundaryKind = "unknown"; + juce::String noteHqExitBoundaryKind = "unknown"; + double noteHqEntryBoundaryScore = 0.0; + double noteHqExitBoundaryScore = 0.0; + juce::String noteHqRendererEntryBoundaryKind = "unknown"; + juce::String noteHqRendererExitBoundaryKind = "unknown"; + int noteHqEditIslandCount = 0; + int noteHqEditedNoteCount = 0; + juce::String pitchRenderProductPath = "preview"; + juce::String pitchRenderBackendId; + juce::String pitchRenderBackendVersion; + juce::String pitchRenderBackendFailureCode; + juce::var pitchRenderBackendCapabilities; + juce::var pitchRenderBackendDiagnostics; + bool pitchRenderBackendAvailable = false; + juce::String pitchRenderCommitPolicy; + int pitchRenderDryProtectedSamples = 0; + double pitchRenderContextDurationSec = 0.0; + double pitchRenderCommitDurationSec = 0.0; + double pitchRenderJobStartDelayMs = jobStartDelayMs; + juce::String hardFailReason; + double phraseHqStartSec = 0.0; + double phraseHqEndSec = clipDuration; + double v3EntryAnchorMs = 0.0; + double v3ExitAnchorMs = 0.0; + int v3FirstCyclesEntryCount = 0; + int v3FirstCyclesExitCount = 0; + double v3ShellDurationMs = 0.0; + double v3BodyDurationMs = 0.0; + double v3ResidualMix = 0.0; + juce::String v3FormantMode; + double v3NeighborLeftOverlapMs = 0.0; + double v3NeighborRightOverlapMs = 0.0; + bool noteHqEntryPitchHandoffUsed = false; + double noteHqEntryPitchHandoffStartSec = 0.0; + double noteHqEntryPitchHandoffEndSec = 0.0; + double noteHqEntryPitchHandoffPreMs = 0.0; + double noteHqEntryPitchHandoffBodyMs = 0.0; + double noteHqEntryPitchSlopeJumpStPerSec = 0.0; + bool noteHqEntryPitchAccelerationLimited = false; + + if (hasUpwardPitchEdit && hasDownwardPitchEdit) + pitchRenderDirection = "mixed"; + else if (hasDownwardPitchEdit) + pitchRenderDirection = "downward"; + else if (hasUpwardPitchEdit) + pitchRenderDirection = "upward"; // Process only the EDITED WINDOW (with padding), or a whole-clip render for explicit formants. { @@ -8389,30 +11885,72 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j const bool hasAnyFormantEdits = hasGlobalFormant || hasNoteFormantEdits; previewSegmentRender = renderMode == "preview_segment"; fullClipHQRender = renderMode == "full_clip_hq"; - const bool processFullClip = hasAnyFormantEdits && !previewSegmentRender; + const bool noteHqRender = renderMode == "note_hq"; + const bool phraseHqRender = noteHqRender + && requestMode == "pitch-only" + && ! hasAnyFormantEdits; + const auto phraseCommitRanges = phraseHqRender + ? buildDryPitchCommitRanges (editedNotes, clipDuration) + : std::vector(); + const auto phraseRegion = (phraseHqRender && ! phraseCommitRanges.empty()) + ? resolvePitchPhraseRenderRegion (analysis.frames, + getCommitStartSec (phraseCommitRanges), + getCommitEndSec (phraseCommitRanges), + clipDuration) + : PitchPhraseRenderRegion { 0.0, clipDuration, false, false, true }; + phraseHqRenderUsed = phraseHqRender; + phraseHqExpandedToFullClip = phraseHqRender && phraseRegion.expandedToFullClip; + phraseHqStartSec = phraseHqRender ? phraseRegion.startSec : 0.0; + phraseHqEndSec = phraseHqRender ? phraseRegion.endSec : clipDuration; + if (phraseHqRender) + { + pitchRenderCommitPolicy = "dry_protect_commit_ranges"; + pitchRenderCommitDurationSec = getAudibleCommitDurationSec (phraseCommitRanges); + pitchRenderContextDurationSec = std::max (0.0, phraseRegion.endSec - phraseRegion.startSec); + noteHqEditIslandCount = static_cast (phraseCommitRanges.size()); + for (const auto& range : phraseCommitRanges) + noteHqEditedNoteCount += range.editedNoteCount; + } + const bool processFullClip = (hasAnyFormantEdits && !previewSegmentRender) + || (phraseHqRender && phraseRegion.expandedToFullClip); + const bool strictNoteIslandRender = requestMode == "pitch-only" + && ! hasAnyFormantEdits + && ! phraseHqRender; // Preview segments must sound like the final formant result, not like a // watered-down proxy. Using the HQ formant path on 10-second staged // renders keeps preview character aligned with the final clip while // still avoiding a full-clip wait. const auto renderQuality = PitchResynthesizer::RenderQuality::FinalHQ; - const double kPaddingSec = previewSegmentRender ? 0.75 : 1.0; - double requestedWindowStartSec = processFullClip ? 0.0 : std::max (0.0, notesStartSec - kPaddingSec); - double requestedWindowEndSec = processFullClip ? clipDuration : std::min (clipDuration, notesEndSec + kPaddingSec); + const double kPaddingSec = strictNoteIslandRender ? 0.12 : (previewSegmentRender ? 0.75 : 1.0); + double requestedWindowStartSec = processFullClip ? 0.0 + : (phraseHqRender ? phraseRegion.startSec : std::max (0.0, notesStartSec - kPaddingSec)); + double requestedWindowEndSec = processFullClip ? clipDuration + : (phraseHqRender ? phraseRegion.endSec : std::min (clipDuration, notesEndSec + kPaddingSec)); double windowStartSec = processFullClip ? 0.0 : requestedWindowStartSec; double windowEndSec = processFullClip ? clipDuration : requestedWindowEndSec; + if (phraseHqRender) + { + noteHqEffectiveStartSec = phraseCommitRanges.empty() ? requestedWindowStartSec : getCommitStartSec (phraseCommitRanges); + noteHqEffectiveEndSec = phraseCommitRanges.empty() ? requestedWindowEndSec : getCommitEndSec (phraseCommitRanges); + noteHqContextStartSec = phraseRegion.startSec; + noteHqContextEndSec = phraseRegion.endSec; + noteHqAudibleCommitStartSec = phraseCommitRanges.empty() ? requestedWindowStartSec : getAudibleCommitStartSec (phraseCommitRanges); + noteHqAudibleCommitEndSec = phraseCommitRanges.empty() ? requestedWindowEndSec : getAudibleCommitEndSec (phraseCommitRanges); + } if (hasWindowOverride && processFullClip) { logPitchEditorFormant ("ignoring window override because explicit formant edits require full-clip render clip=" + clipId); } - if (hasWindowOverride && ! processFullClip) + if (hasWindowOverride && ! processFullClip && ! phraseHqRender) { requestedWindowStartSec = juce::jlimit (0.0, clipDuration, *windowStartSecOverride); requestedWindowEndSec = juce::jlimit (requestedWindowStartSec, clipDuration, *windowEndSecOverride); - windowStartSec = previewSegmentRender + const bool paddedPitchOnlyOverride = requestMode == "pitch-only" && ! hasAnyFormantEdits; + windowStartSec = (previewSegmentRender || paddedPitchOnlyOverride) ? std::max (0.0, requestedWindowStartSec - kPaddingSec) : requestedWindowStartSec; - windowEndSec = previewSegmentRender + windowEndSec = (previewSegmentRender || paddedPitchOnlyOverride) ? std::min (clipDuration, requestedWindowEndSec + kPaddingSec) : requestedWindowEndSec; } @@ -8423,9 +11961,13 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j previewCoverageEndSec = requestedWindowEndSec; logPitchEditorFormant (juce::String("processing scope=") - + (processFullClip ? "full-clip" : (previewSegmentRender ? "preview-segment" : (hasWindowOverride ? "playhead-window" : "edited-window"))) + + (phraseHqRender ? (phraseRegion.expandedToFullClip ? "phrase-hq-full-clip" : "phrase-hq-safe-region") + : (processFullClip ? "full-clip" : (previewSegmentRender ? "preview-segment" : (noteHqRender ? "note-hq" : (hasWindowOverride ? "playhead-window" : "edited-window"))))) + " clip=" + clipId + " renderMode=" + renderMode + + " noteIsland=" + juce::String (strictNoteIslandRender ? "true" : "false") + + " phraseHq=" + juce::String (phraseHqRender ? "true" : "false") + + " phraseFullClip=" + juce::String (phraseRegion.expandedToFullClip ? "true" : "false") + " requested=[" + juce::String (requestedWindowStartSec, 3) + "s - " + juce::String (requestedWindowEndSec, 3) + "s]" + " window=[" + juce::String (windowStartSec, 3) + "s - " + juce::String (windowEndSec, 3) + "s]" + " windowSamples=" + juce::String (windowNumSamples)); @@ -8460,20 +12002,165 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j { n.startTime = static_cast (static_cast (n.startTime) - windowStartSec); n.endTime = static_cast (static_cast (n.endTime) - windowStartSec); + n.effectiveStartTime = static_cast (static_cast (n.effectiveStartTime) - windowStartSec); + n.effectiveEndTime = static_cast (static_cast (n.effectiveEndTime) - windowStartSec); } - PitchResynthesizer resynth; - std::vector channelPtrs (static_cast (numChannels)); - for (int ch = 0; ch < numChannels; ++ch) - channelPtrs[static_cast (ch)] = windowBuffer.getReadPointer (ch); - - auto correctedWindow = resynth.processMultiChannel ( - channelPtrs.data(), numChannels, windowNumSamples, sr, - windowAnalysis.frames, windowNotes, - PitchResynthesizer::PitchEngine::Signalsmith, - globalFormantSemitones, - renderQuality, - shouldCancel); + std::vector> correctedWindow; + if (phraseHqRender) + { + pitchRenderProductPath = "native_vsf_note_hq"; + pitchRenderBackendId = "native_vsf"; + pitchRenderBackendVersion = "vsf_default"; + pitchRenderBackendAvailable = true; + pitchRenderCommitPolicy = "dry_protect_commit_ranges"; + logPitchEditorFormant ("phrase HQ using native VSF renderer clip=" + clipId); + } + + if (correctedWindow.empty()) + { + PitchResynthesizer resynth; + std::vector channelPtrs (static_cast (numChannels)); + for (int ch = 0; ch < numChannels; ++ch) + channelPtrs[static_cast (ch)] = windowBuffer.getReadPointer (ch); + + correctedWindow = resynth.processMultiChannel ( + channelPtrs.data(), numChannels, windowNumSamples, sr, + windowAnalysis.frames, windowNotes, + PitchResynthesizer::PitchEngine::NativeVsf, + globalFormantSemitones, + renderQuality, + shouldCancel); + const auto& renderDiagnostics = resynth.getLastRenderDiagnostics(); + if (renderDiagnostics.requestedRendererBranch.isNotEmpty()) + requestedRendererBranch = renderDiagnostics.requestedRendererBranch; + if (renderDiagnostics.actualRendererBranch.isNotEmpty()) + actualRendererBranch = renderDiagnostics.actualRendererBranch; + pitchOnlyRecoveryPath = renderDiagnostics.pitchOnlyRecoveryPath; + pitchOnlyNeutralFormantUsed = renderDiagnostics.pitchOnlyNeutralFormantUsed; + pitchRenderDirection = renderDiagnostics.pitchDirection; + downshiftFormantGuardUsed = renderDiagnostics.downshiftFormantGuardUsed; + downshiftFormantGuardAlpha = renderDiagnostics.downshiftFormantGuardAlpha; + noteHqRendererEntryBoundaryKind = renderDiagnostics.dominantEntryBoundaryKind; + noteHqRendererExitBoundaryKind = renderDiagnostics.dominantExitBoundaryKind; + formantCurveUsed = renderDiagnostics.formantCurveUsed; + usedRendererFallback = usedRendererFallback || renderDiagnostics.usedFallback; + if (rendererFallbackReason.isEmpty()) + rendererFallbackReason = renderDiagnostics.fallbackReason; + bridgeUsed = renderDiagnostics.bridgeUsed; + bridgeFallbackUsed = renderDiagnostics.bridgeFallbackUsed; + bridgeStartSec = renderDiagnostics.bridgeStartSec; + bridgeLengthMs = renderDiagnostics.bridgeLengthMs; + bridgeAlignmentLagSamples = renderDiagnostics.bridgeAlignmentLagSamples; + bridgeCorrelationScore = renderDiagnostics.bridgeCorrelationScore; + bridgeGainDeltaDb = renderDiagnostics.bridgeGainDeltaDb; + bodyReplacementUsed = renderDiagnostics.bodyReplacementUsed; + bodyReplacementFallbackUsed = renderDiagnostics.bodyReplacementFallbackUsed; + entryLockStartSec = renderDiagnostics.entryLockStartSec; + entryLockLengthMs = renderDiagnostics.entryLockLengthMs; + exitLockStartSec = renderDiagnostics.exitLockStartSec; + renderedBodyStartSec = renderDiagnostics.renderedBodyStartSec; + renderedBodyEndSec = renderDiagnostics.renderedBodyEndSec; + islandNativeUsed = renderDiagnostics.islandNativeUsed; + islandNativeFallbackUsed = renderDiagnostics.islandNativeFallbackUsed; + islandRenderStartSec = renderDiagnostics.islandRenderStartSec; + islandRenderEndSec = renderDiagnostics.islandRenderEndSec; + transientMaskPeak = renderDiagnostics.transientMaskPeak; + voicedCoreMaskPeak = renderDiagnostics.voicedCoreMaskPeak; + hpssUsed = renderDiagnostics.hpssUsed; + hpssFallbackUsed = renderDiagnostics.hpssFallbackUsed; + harmonicMaskPeak = renderDiagnostics.harmonicMaskPeak; + aperiodicMaskPeak = renderDiagnostics.aperiodicMaskPeak; + spectralEnvelopeCorrectionUsed = renderDiagnostics.spectralEnvelopeCorrectionUsed; + pitchOnlyCoreTimbreCorrectionUsed = renderDiagnostics.pitchOnlyCoreTimbreCorrectionUsed; + pitchOnlyCoreEnvelopeMix = renderDiagnostics.pitchOnlyCoreEnvelopeMix; + pitchOnlyCoreRmsTrimDb = renderDiagnostics.pitchOnlyCoreRmsTrimDb; + pitchOnlyCoreEnvelopeLifter = renderDiagnostics.pitchOnlyCoreEnvelopeLifter; + pitchOnlyEntryHandoffUsed = renderDiagnostics.pitchOnlyEntryHandoffUsed; + pitchOnlyExitHandoffUsed = renderDiagnostics.pitchOnlyExitHandoffUsed; + vocalSourceFilterUsed = renderDiagnostics.vocalSourceFilterUsed; + vocalSourceFilterVoicedCoverage = renderDiagnostics.vocalSourceFilterVoicedCoverage; + vocalSourceFilterResidualMix = renderDiagnostics.vocalSourceFilterResidualMix; + vocalSourceFilterFallbackUsed = renderDiagnostics.vocalSourceFilterFallbackUsed; + vocalSourceFilterFallbackReason = renderDiagnostics.vocalSourceFilterFallbackReason; + vocalSourceFilterEntryDryMs = renderDiagnostics.vocalSourceFilterEntryDryMs; + vocalSourceFilterExitDryMs = renderDiagnostics.vocalSourceFilterExitDryMs; + vocalSourceFilterResidualMixScale = renderDiagnostics.vocalSourceFilterResidualMixScale; + vocalSourceFilterEpochInterpolationUsed = renderDiagnostics.vocalSourceFilterEpochInterpolationUsed; + vocalSourceFilterEpochInterpolationStrength = renderDiagnostics.vocalSourceFilterEpochInterpolationStrength; + vocalSourceFilterGrainRadiusScale = renderDiagnostics.vocalSourceFilterGrainRadiusScale; + vocalSourceFilterUpPresenceTrimDb = renderDiagnostics.vocalSourceFilterUpPresenceTrimDb; + vocalSourceFilterUpPresenceHz = renderDiagnostics.vocalSourceFilterUpPresenceHz; + vocalSourceFilterDownNasalTrimDb = renderDiagnostics.vocalSourceFilterDownNasalTrimDb; + vocalSourceFilterDownNasalHz = renderDiagnostics.vocalSourceFilterDownNasalHz; + vocalSourceFilterDownBodyCompDb = renderDiagnostics.vocalSourceFilterDownBodyCompDb; + vocalSourceFilterDownBodyCompHz = renderDiagnostics.vocalSourceFilterDownBodyCompHz; + wsolaUsed = renderDiagnostics.wsolaUsed; + wsolaFallbackUsed = renderDiagnostics.wsolaFallbackUsed; + wsolaEntryLagSamples = renderDiagnostics.wsolaEntryLagSamples; + wsolaExitLagSamples = renderDiagnostics.wsolaExitLagSamples; + wsolaCorrelationScore = renderDiagnostics.wsolaCorrelationScore; + phaseLockUsed = renderDiagnostics.phaseLockUsed; + phaseLockFallbackUsed = renderDiagnostics.phaseLockFallbackUsed; + phaseAlignedEntry = renderDiagnostics.phaseAlignedEntry; + phaseAlignedExit = renderDiagnostics.phaseAlignedExit; + phasePeakCount = renderDiagnostics.phasePeakCount; + transitionHqUsed = renderDiagnostics.transitionHqUsed; + transitionHqFallbackUsed = renderDiagnostics.transitionHqFallbackUsed; + transitionStartSec = renderDiagnostics.transitionStartSec; + transitionEndSec = renderDiagnostics.transitionEndSec; + transitionTransientPeak = renderDiagnostics.transitionTransientPeak; + transitionVoicedCorePeak = renderDiagnostics.transitionVoicedCorePeak; + transitionResidualPeak = renderDiagnostics.transitionResidualPeak; + transitionEnvelopeCorrectionUsed = renderDiagnostics.transitionEnvelopeCorrectionUsed; + engineV2Used = renderDiagnostics.engineV2Used; + engineV2FallbackUsed = renderDiagnostics.engineV2FallbackUsed; + engineV2TransitionCount = renderDiagnostics.engineV2TransitionCount; + engineV2TransitionStartSec = renderDiagnostics.engineV2TransitionStartSec; + engineV2TransitionEndSec = renderDiagnostics.engineV2TransitionEndSec; + engineV2HarmonicSupportPeak = renderDiagnostics.engineV2HarmonicSupportPeak; + engineV2ResidualSupportPeak = renderDiagnostics.engineV2ResidualSupportPeak; + engineV2EnvelopeSupportPeak = renderDiagnostics.engineV2EnvelopeSupportPeak; + transientBypassUsed = renderDiagnostics.transientBypassUsed; + residualCarryUsed = renderDiagnostics.residualCarryUsed; + cepstralCutoffUsed = renderDiagnostics.cepstralCutoffUsed; + engineV2FftSize = renderDiagnostics.engineV2FftSize; + engineV2HopSize = renderDiagnostics.engineV2HopSize; + immediateLeftNeighborUsed = renderDiagnostics.immediateLeftNeighborUsed; + immediateRightNeighborUsed = renderDiagnostics.immediateRightNeighborUsed; + leftNeighborSamplesRendered = renderDiagnostics.leftNeighborSamplesRendered; + rightNeighborSamplesRendered = renderDiagnostics.rightNeighborSamplesRendered; + leftNeighborSmoothMs = renderDiagnostics.leftNeighborSmoothMs; + rightNeighborSmoothMs = renderDiagnostics.rightNeighborSmoothMs; + nonImmediateNeighborTouched = renderDiagnostics.nonImmediateNeighborTouched; + entryAlignmentOffsetMs = renderDiagnostics.entryAlignmentOffsetMs; + exitAlignmentOffsetMs = renderDiagnostics.exitAlignmentOffsetMs; + firstVoicedCyclesEntryUsed = renderDiagnostics.firstVoicedCyclesEntryUsed; + firstVoicedCyclesExitUsed = renderDiagnostics.firstVoicedCyclesExitUsed; + v3TransitionPairUsed = renderDiagnostics.v3TransitionPairUsed; + v3ContinuousRenderUsed = false; + v3EntryAnchorMs = renderDiagnostics.v3EntryAnchorMs; + v3ExitAnchorMs = renderDiagnostics.v3ExitAnchorMs; + v3FirstCyclesEntryCount = renderDiagnostics.v3FirstCyclesEntryCount; + v3FirstCyclesExitCount = renderDiagnostics.v3FirstCyclesExitCount; + v3ShellDurationMs = renderDiagnostics.v3ShellDurationMs; + v3BodyDurationMs = renderDiagnostics.v3BodyDurationMs; + v3ResidualMix = renderDiagnostics.v3ResidualMix; + v3FormantMode = renderDiagnostics.v3FormantMode; + v3NeighborLeftOverlapMs = renderDiagnostics.v3NeighborLeftOverlapMs; + v3NeighborRightOverlapMs = renderDiagnostics.v3NeighborRightOverlapMs; + noteHqEntryPitchHandoffUsed = renderDiagnostics.noteHqEntryPitchHandoffUsed; + noteHqEntryPitchHandoffStartSec = renderDiagnostics.noteHqEntryPitchHandoffStartSec > 0.0 + ? renderDiagnostics.noteHqEntryPitchHandoffStartSec + windowStartSec + : 0.0; + noteHqEntryPitchHandoffEndSec = renderDiagnostics.noteHqEntryPitchHandoffEndSec > 0.0 + ? renderDiagnostics.noteHqEntryPitchHandoffEndSec + windowStartSec + : 0.0; + noteHqEntryPitchHandoffPreMs = renderDiagnostics.noteHqEntryPitchHandoffPreMs; + noteHqEntryPitchHandoffBodyMs = renderDiagnostics.noteHqEntryPitchHandoffBodyMs; + noteHqEntryPitchSlopeJumpStPerSec = renderDiagnostics.noteHqEntryPitchSlopeJumpStPerSec; + noteHqEntryPitchAccelerationLimited = renderDiagnostics.noteHqEntryPitchAccelerationLimited; + } if (shouldCancel && shouldCancel()) return buildCancelledResult(); @@ -8514,6 +12201,8 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j windowNumSamples - trimStart)); renderedOutputBuffer.setSize (numChannels, segmentSamples); renderedOutputSamples = segmentSamples; + candidateCoverageStartSec = requestedWindowStartSec; + candidateCoverageEndSec = requestedWindowEndSec; const int xfadeLen = std::min (384, segmentSamples / 6); for (int ch = 0; ch < numChannels; ++ch) { @@ -8543,33 +12232,201 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j } else { - // Blend corrected window into full clip buffer with cosine crossfade at splice points. - const int xfadeLen = std::min (512, windowNumSamples / 4); - for (int ch = 0; ch < numChannels; ++ch) + const bool noteLocalSingleTrim = hasWindowOverride + && requestMode == "pitch-only" + && ! phraseHqRender + && ! hasAnyFormantEdits; + const bool useContinuousWindowOutput = false; + + if (phraseHqRender) { - if (static_cast (ch) >= correctedWindow.size()) continue; - const auto& corrected = correctedWindow[static_cast (ch)]; - int copyLen = std::min (windowNumSamples, static_cast (corrected.size())); - for (int i = 0; i < copyLen; ++i) + candidateCoverageStartSec = phraseCommitRanges.empty() ? requestedWindowStartSec : getCommitStartSec (phraseCommitRanges); + candidateCoverageEndSec = phraseCommitRanges.empty() ? requestedWindowEndSec : getCommitEndSec (phraseCommitRanges); + renderedOutputBuffer.setSize (numChannels, clipNumSamples); + renderedOutputSamples = clipNumSamples; + pitchRenderCommitPolicy = "dry_protect_commit_ranges"; + pitchRenderContextDurationSec = std::max (0.0, windowEndSec - windowStartSec); + pitchRenderCommitDurationSec = getAudibleCommitDurationSec (phraseCommitRanges); + const auto compositeStats = compositeDryProtectedPitchPatch ( + renderedOutputBuffer, + clipBuffer, + correctedWindow, + phraseCommitRanges, + windowStartSample, + sr, + false, + actualRendererBranch != "pitch_only_vocal_source_filter_hq" + && ! getPitchEnvFlag ("OPENSTUDIO_PITCH_ENTRY_BRIDGE_DISABLE", false)); + pitchRenderDryProtectedSamples = compositeStats.dryProtectedSamples; + noteHqPreBodyDryProtectedSamples = compositeStats.preBodyDryProtectedSamples; + noteHqAudibleCommitStartSec = compositeStats.audibleCommitStartSec; + noteHqAudibleCommitEndSec = compositeStats.audibleCommitEndSec; + noteHqEntryInsideBodyFadeMs = compositeStats.entryInsideBodyFadeMs; + noteHqExitLeadInMs = compositeStats.exitLeadInMs; + noteHqEntryBridgeStartSec = compositeStats.entryBridgeStartSec; + noteHqEntryBridgeEndSec = compositeStats.entryBridgeEndSec; + noteHqEntryBridgeWetLagMs = compositeStats.entryBridgeWetLagMs; + noteHqEntryBridgeEnvelopeGainDb = compositeStats.entryBridgeEnvelopeGainDb; + noteHqEntryBridgeUsed = compositeStats.entryBridgeUsed; + noteHqEntryTransientDryPreservedMs = compositeStats.entryTransientDryPreservedMs; + pitchOnlyEntrySimpleHandoffUsed = compositeStats.entrySimpleHandoffUsed; + pitchOnlyEntrySafeHandoffUsed = compositeStats.entrySafeHandoffUsed; + pitchOnlyEntryDryHoldMs = compositeStats.entryDryHoldMs; + pitchOnlyEntrySafeBridgeMs = compositeStats.entrySafeBridgeMs; + pitchOnlyEntryWetAlignmentMs = compositeStats.entryWetAlignmentMs; + pitchOnlyEntryWetGainDb = compositeStats.entryWetGainDb; + pitchOnlyEntryWetVsDryRmsDb = compositeStats.entryWetVsDryRmsDb; + pitchOnlyEntryEqualPowerBlendUsed = compositeStats.entryEqualPowerBlendUsed; + pitchOnlyEntryRmsContinuityUsed = compositeStats.entryRmsContinuityUsed; + pitchOnlyEntryRmsContinuityGainDb = compositeStats.entryRmsContinuityGainDb; + pitchOnlyEntryRmsContinuityMs = compositeStats.entryRmsContinuityMs; + pitchOnlyEntryTransientResidualUsed = compositeStats.entryTransientResidualUsed; + pitchOnlyEntryTransientResidualMix = compositeStats.entryTransientResidualMix; + pitchOnlyEntryTransientResidualMs = compositeStats.entryTransientResidualMs; + pitchOnlyEntryPhaseSafeUsed = compositeStats.entryPhaseSafeUsed; + pitchOnlyEntryWetAlignmentAccepted = compositeStats.entryWetAlignmentAccepted; + pitchOnlyEntryFirstCycleCorrelation = compositeStats.entryFirstCycleCorrelation; + pitchOnlyEntryZeroCrossOffsetMs = compositeStats.entryZeroCrossOffsetMs; + pitchOnlyEntryBridgeGainRampDb = compositeStats.entryBridgeGainRampDb; + pitchOnlyDownshiftCoreEnvelopePassUsed = compositeStats.downshiftCoreEnvelopePassUsed; + pitchOnlyDownshiftCoreRmsTrimDb = compositeStats.downshiftCoreRmsTrimDb; + pitchOnlyDownshiftCoreEnvelopeMaxDb = compositeStats.downshiftCoreEnvelopeMaxDb; + pitchOnlyDownshiftCoreEnvelopeFrames = compositeStats.downshiftCoreEnvelopeFrames; + pitchOnlyEntryWetLagMs = compositeStats.entryBridgeWetLagMs; + pitchOnlyEntryBridgeDurationMs = std::max (0.0, (compositeStats.entryBridgeEndSec - compositeStats.entryBridgeStartSec) * 1000.0); + pitchOnlyExitDryRestoreUsed = compositeStats.exitDryRestoreUsed; + pitchOnlyExitDryRestoreStartSec = compositeStats.exitDryRestoreStartSec; + pitchOnlyExitDryRestoreEndSec = compositeStats.exitDryRestoreEndSec; + noteHqEntryBoundaryKind = compositeStats.entryBoundaryKind; + noteHqExitBoundaryKind = compositeStats.exitBoundaryKind; + noteHqEntryBoundaryScore = compositeStats.entryBoundaryScore; + noteHqExitBoundaryScore = compositeStats.exitBoundaryScore; + pitchOnlyEntryTimbreCorrectionUsed = compositeStats.entryTimbreCorrectionUsed; + pitchOnlyEntryRmsTrimDb = compositeStats.entryRmsTrimDb; + pitchOnlyEntryTiltDb = compositeStats.entryTiltDb; + logPitchEditorFormant ("phrase HQ dry patch composited clip=" + clipId + + " context=[" + juce::String (windowStartSec, 3) + "s - " + juce::String (windowEndSec, 3) + "s]" + + " commitRanges=" + juce::String (static_cast (phraseCommitRanges.size())) + + " dryProtectedSamples=" + juce::String (pitchRenderDryProtectedSamples) + + " entryTimbre=" + juce::String (pitchOnlyEntryTimbreCorrectionUsed ? "true" : "false") + + " entryRmsTrimDb=" + juce::String (pitchOnlyEntryRmsTrimDb, 3) + + " entryTiltDb=" + juce::String (pitchOnlyEntryTiltDb, 3) + + " simpleEntryHandoff=" + juce::String (pitchOnlyEntrySimpleHandoffUsed ? "true" : "false") + + " safeEntryHandoff=" + juce::String (pitchOnlyEntrySafeHandoffUsed ? "true" : "false") + + " safeEntryAlignMs=" + juce::String (pitchOnlyEntryWetAlignmentMs, 3) + + " safeEntryGainDb=" + juce::String (pitchOnlyEntryWetGainDb, 3) + + " phaseSafeEntry=" + juce::String (pitchOnlyEntryPhaseSafeUsed ? "true" : "false") + + " phaseSafeFirstCycleCorr=" + juce::String (pitchOnlyEntryFirstCycleCorrelation, 3) + + " phaseSafeZeroCrossMs=" + juce::String (pitchOnlyEntryZeroCrossOffsetMs, 3) + + " downshiftCoreEnvelope=" + juce::String (pitchOnlyDownshiftCoreEnvelopePassUsed ? "true" : "false") + + " downshiftCoreRmsTrimDb=" + juce::String (pitchOnlyDownshiftCoreRmsTrimDb, 3) + + " entryContinuity=" + juce::String (pitchOnlyEntryRmsContinuityUsed ? "true" : "false") + + " entryContinuityGainDb=" + juce::String (pitchOnlyEntryRmsContinuityGainDb, 3) + + " entryTransientResidual=" + juce::String (pitchOnlyEntryTransientResidualUsed ? "true" : "false") + + " entryTransientResidualMix=" + juce::String (pitchOnlyEntryTransientResidualMix, 3) + + " exitDryRestore=" + juce::String (pitchOnlyExitDryRestoreUsed ? "true" : "false")); + } + else if (useContinuousWindowOutput) + { + renderedOutputBuffer.setSize (numChannels, windowNumSamples); + renderedOutputSamples = windowNumSamples; + candidateCoverageStartSec = requestedWindowStartSec; + candidateCoverageEndSec = requestedWindowEndSec; + for (int ch = 0; ch < numChannels; ++ch) { - float blend = 1.0f; - if (i < xfadeLen && windowStartSample > 0) - blend = 0.5f * (1.0f - std::cos (juce::MathConstants::pi - * static_cast (i) / static_cast (xfadeLen))); - int distFromEnd = copyLen - 1 - i; - if (distFromEnd < xfadeLen && (windowStartSample + copyLen) < clipNumSamples) + if (static_cast (ch) >= correctedWindow.size()) + continue; + const auto& corrected = correctedWindow[static_cast (ch)]; + const int copyLen = std::min (windowNumSamples, static_cast (corrected.size())); + for (int i = 0; i < copyLen; ++i) + renderedOutputBuffer.setSample (ch, i, corrected[static_cast (i)]); + } + } + else if (noteLocalSingleTrim) + { + const int trimStart = juce::jlimit (0, windowNumSamples, + static_cast ((requestedWindowStartSec - windowStartSec) * sr)); + const int segmentSamples = std::max (0, std::min ( + static_cast ((requestedWindowEndSec - requestedWindowStartSec) * sr), + windowNumSamples - trimStart)); + const int destStartSample = juce::jlimit (0, clipNumSamples, static_cast (requestedWindowStartSec * sr)); + const int copyLen = std::min (segmentSamples, clipNumSamples - destStartSample); + candidateCoverageStartSec = requestedWindowStartSec; + candidateCoverageEndSec = requestedWindowEndSec; + renderedOutputBuffer.setSize (numChannels, copyLen); + renderedOutputBuffer.clear(); + renderedOutputSamples = copyLen; + const int deClickLen = std::min ( + static_cast (std::round (0.008 * sr)), + copyLen / 6); + + for (int ch = 0; ch < numChannels; ++ch) + { + if (static_cast (ch) >= correctedWindow.size()) continue; + const auto& corrected = correctedWindow[static_cast (ch)]; + for (int i = 0; i < copyLen; ++i) { - float fo = 0.5f * (1.0f - std::cos (juce::MathConstants::pi - * static_cast (distFromEnd) / static_cast (xfadeLen))); - blend = std::min (blend, fo); + const int sourceIndex = trimStart + i; + const int destIndex = destStartSample + i; + if (sourceIndex < 0 || sourceIndex >= static_cast (corrected.size())) + continue; + + float corr = corrected[static_cast (sourceIndex)]; + if (deClickLen > 0 && i < deClickLen && destIndex < clipNumSamples) + { + const float t = static_cast (i) / static_cast (deClickLen); + const float edgeMatch = 0.5f * (1.0f + std::cos (juce::MathConstants::pi * t)); + const float originalAtBoundary = clipBuffer.getSample (ch, destIndex); + corr += (originalAtBoundary - corr) * edgeMatch; + } + const int distFromEnd = copyLen - 1 - i; + if (deClickLen > 0 && distFromEnd < deClickLen) + { + const int nextOriginalIndex = juce::jlimit (0, clipNumSamples - 1, destStartSample + copyLen); + const float t = static_cast (distFromEnd) / static_cast (deClickLen); + const float edgeMatch = 0.5f * (1.0f + std::cos (juce::MathConstants::pi * t)); + const float originalAtBoundary = clipBuffer.getSample (ch, nextOriginalIndex); + corr += (originalAtBoundary - corr) * edgeMatch; + } + renderedOutputBuffer.setSample (ch, i, corr); + } + } + } + else + { + candidateCoverageStartSec = windowStartSec; + candidateCoverageEndSec = windowEndSec; + // Blend corrected window into full clip buffer with cosine crossfade at splice points. + const int xfadeLen = std::min (2048, windowNumSamples / 4); + for (int ch = 0; ch < numChannels; ++ch) + { + if (static_cast (ch) >= correctedWindow.size()) continue; + const auto& corrected = correctedWindow[static_cast (ch)]; + int copyLen = std::min (windowNumSamples, static_cast (corrected.size())); + for (int i = 0; i < copyLen; ++i) + { + float blend = 1.0f; + if (i < xfadeLen && windowStartSample > 0) + blend = 0.5f * (1.0f - std::cos (juce::MathConstants::pi + * static_cast (i) / static_cast (xfadeLen))); + int distFromEnd = copyLen - 1 - i; + if (distFromEnd < xfadeLen && (windowStartSample + copyLen) < clipNumSamples) + { + float fo = 0.5f * (1.0f - std::cos (juce::MathConstants::pi + * static_cast (distFromEnd) / static_cast (xfadeLen))); + blend = std::min (blend, fo); + } + float orig = clipBuffer.getSample (ch, windowStartSample + i); + float corr = corrected[static_cast (i)]; + clipBuffer.setSample (ch, windowStartSample + i, orig * (1.0f - blend) + corr * blend); } - float orig = clipBuffer.getSample (ch, windowStartSample + i); - float corr = corrected[static_cast (i)]; - clipBuffer.setSample (ch, windowStartSample + i, orig * (1.0f - blend) + corr * blend); } } - renderedOutputBuffer = clipBuffer; - renderedOutputSamples = clipNumSamples; + if (! phraseHqRender && ! useContinuousWindowOutput && ! noteLocalSingleTrim) + { + renderedOutputBuffer = clipBuffer; + renderedOutputSamples = clipNumSamples; + } } } } @@ -8579,11 +12436,20 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j // truncated by a concurrent job — producing garbled audio. // A monotonic counter ensures each job writes to a different file slot. static std::atomic s_pitchCorrSeq { 0 }; - int seq = s_pitchCorrSeq.fetch_add(1) & 0x1F; // 32 rotating slots - juce::File outputFile = sourceFile.getSiblingFile( + const auto seq = s_pitchCorrSeq.fetch_add (1); + const auto modeSuffix = renderMode == "preview_segment" + ? juce::String ("_pcseg") + : (renderMode == "full_clip_hq" + ? juce::String ("_pcfinal") + : (renderMode == "note_hq" ? juce::String ("_pcnote") : juce::String ("_pc"))); + const auto uniqueToken = juce::String::toHexString ((juce::int64) juce::Time::getHighResolutionTicks()) + + "_" + juce::String (seq); + juce::File outputFile = sourceFile.getSiblingFile ( sourceFile.getFileNameWithoutExtension() - + (renderMode == "preview_segment" ? "_pcseg" : renderMode == "full_clip_hq" ? "_pcfinal" : "_pc") - + juce::String(seq) + ".wav"); + + modeSuffix + + "_" + + uniqueToken + + ".wav"); // Delete the old file first to prevent stale data if the write partially fails. // Without this, createOutputStream may fail to fully overwrite on some OS/filesystem combos. @@ -8615,7 +12481,22 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j if (previewSegmentRender) { - playbackEngine.setClipRenderedPreviewSegment(clipId, outputFile, previewCoverageStartSec, previewCoverageEndSec); + const bool installedPreviewSegment = playbackEngine.setClipRenderedPreviewSegment( + clipId, + outputFile, + previewCoverageStartSec, + previewCoverageEndSec, + 0.0, + previewRenderGenerationToken); + if (! installedPreviewSegment && previewSegmentRender) + return buildCancelledResult(); + } + else if (renderMode == "note_hq") + { + playbackEngine.replaceClipAudioFile(clipId, outputFile); + playbackEngine.clearPitchScrubPreview(clipId); + playbackEngine.clearClipPitchPreview(clipId); + playbackEngine.invalidateRenderedPreviewSegments(clipId); } else if (fullClipHQRender && isPlaying.load()) { @@ -8626,6 +12507,178 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j { playbackEngine.replaceClipAudioFile(clipId, outputFile); } + const auto postApplyRouteStatus = getPitchPreviewRoutingStatus(clipId); + juce::var appFinalCapture; + juce::var appFinalBakedCapture; + juce::var appFinalOfflineCapture; + juce::var appFinalParityReport; + juce::var appFinalOfflineParityReport; + juce::var appFinalLiveVsOfflineParityReport; + juce::String appFinalRouteReportPath; + juce::String appFinalBakedContextPath; + juce::String appFinalBakedResampledContextPath; + juce::String appFinalPlaybackContextPath; + juce::String appFinalOfflineContextPath; + juce::String appFinalParityReportPath; + juce::var appFinalBakedResampledCapture; + if (renderMode == "note_hq") + { + logPitchEditorFormant("note_hq post-apply route clip=" + clipId + + " monitorMode=" + postApplyRouteStatus.getProperty("monitorMode", {}).toString() + + " correctedSource=" + juce::String(static_cast(postApplyRouteStatus.getProperty("correctedSourceActive", false)) ? "true" : "false") + + " renderedSegment=" + juce::String(static_cast(postApplyRouteStatus.getProperty("renderedSegmentActive", false)) ? "true" : "false") + + " clipLivePreview=" + juce::String(static_cast(postApplyRouteStatus.getProperty("clipLivePreviewActive", false)) ? "true" : "false") + + " scrub=" + juce::String(static_cast(postApplyRouteStatus.getProperty("scrubPreviewActive", false)) ? "true" : "false")); + + if (! swapDeferred + && ! getPitchEnvFlag ("OPENSTUDIO_PITCH_APP_FINAL_CAPTURE_DISABLE", false)) + { + const double captureStartClipSec = juce::jlimit (0.0, clipDuration, noteHqAudibleCommitStartSec - 0.50); + const double captureEndClipSec = juce::jlimit (captureStartClipSec, clipDuration, noteHqAudibleCommitEndSec + 0.60); + const double captureDurationSec = captureEndClipSec - captureStartClipSec; + if (captureDurationSec >= 0.050) + { + const auto token = juce::String (juce::Time::currentTimeMillis()); + const auto bakedWav = outputFile.getSiblingFile ( + outputFile.getFileNameWithoutExtension() + "_appfinal_" + token + "_baked_context.wav"); + const auto bakedResampledWav = outputFile.getSiblingFile ( + outputFile.getFileNameWithoutExtension() + "_appfinal_" + token + "_baked_context_resampled.wav"); + const auto captureWav = outputFile.getSiblingFile ( + outputFile.getFileNameWithoutExtension() + "_appfinal_" + token + "_live_playback_context.wav"); + const auto offlineWav = outputFile.getSiblingFile ( + outputFile.getFileNameWithoutExtension() + "_appfinal_" + token + "_offline_render_context.wav"); + const auto routeJson = outputFile.getSiblingFile ( + outputFile.getFileNameWithoutExtension() + "_appfinal_" + token + "_route.json"); + const auto parityJson = outputFile.getSiblingFile ( + outputFile.getFileNameWithoutExtension() + "_appfinal_" + token + "_parity.json"); + + appFinalBakedCapture = writePitchBakedContextWav ( + outputFile, + captureStartClipSec, + captureDurationSec, + bakedWav); + appFinalBakedResampledCapture = writePitchBakedResampledContextWav ( + outputFile, + captureStartClipSec, + captureDurationSec, + sr, + bakedResampledWav); + appFinalCapture = capturePitchAuditionPlayback ( + trackId, + clipId, + foundClip->startTime + captureStartClipSec, + captureDurationSec, + captureWav.getFullPathName(), + sr, + false); + appFinalOfflineCapture = capturePitchAuditionPlayback ( + trackId, + clipId, + foundClip->startTime + captureStartClipSec, + captureDurationSec, + offlineWav.getFullPathName(), + sr, + true); + appFinalParityReport = comparePitchParityFiles ( + bakedWav, + captureWav, + captureStartClipSec, + noteHqAudibleCommitStartSec, + noteHqAudibleCommitEndSec); + appFinalOfflineParityReport = comparePitchParityFiles ( + bakedWav, + offlineWav, + captureStartClipSec, + noteHqAudibleCommitStartSec, + noteHqAudibleCommitEndSec); + appFinalLiveVsOfflineParityReport = comparePitchParityFiles ( + captureWav, + offlineWav, + captureStartClipSec, + noteHqAudibleCommitStartSec, + noteHqAudibleCommitEndSec); + appFinalRouteReportPath = routeJson.getFullPathName(); + appFinalBakedContextPath = bakedWav.getFullPathName(); + appFinalBakedResampledContextPath = bakedResampledWav.getFullPathName(); + appFinalPlaybackContextPath = captureWav.getFullPathName(); + appFinalOfflineContextPath = offlineWav.getFullPathName(); + appFinalParityReportPath = parityJson.getFullPathName(); + parityJson.getParentDirectory().createDirectory(); + parityJson.replaceWithText (juce::JSON::toString (appFinalParityReport, true)); + + juce::Array editedNoteDiagnostics; + for (const auto& note : editedNotes) + { + if (std::abs (note.correctedPitch - note.detectedPitch) <= editThreshold) + continue; + + auto* noteObj = new juce::DynamicObject(); + noteObj->setProperty ("id", note.id); + noteObj->setProperty ("startTime", static_cast (note.startTime)); + noteObj->setProperty ("endTime", static_cast (note.endTime)); + noteObj->setProperty ("effectiveStartTime", static_cast (note.effectiveStartTime)); + noteObj->setProperty ("effectiveEndTime", static_cast (note.effectiveEndTime)); + noteObj->setProperty ("detectedPitch", static_cast (note.detectedPitch)); + noteObj->setProperty ("correctedPitch", static_cast (note.correctedPitch)); + noteObj->setProperty ("requestedShiftSemitones", static_cast (note.correctedPitch - note.detectedPitch)); + editedNoteDiagnostics.add (juce::var (noteObj)); + } + + juce::DynamicObject::Ptr routeObj = new juce::DynamicObject(); + routeObj->setProperty ("clipId", clipId); + routeObj->setProperty ("trackId", trackId); + routeObj->setProperty ("renderMode", renderMode); + routeObj->setProperty ("outputFile", outputFile.getFullPathName()); + routeObj->setProperty ("bakedContextPath", bakedWav.getFullPathName()); + routeObj->setProperty ("bakedResampledContextPath", bakedResampledWav.getFullPathName()); + routeObj->setProperty ("appPlaybackContextPath", captureWav.getFullPathName()); + routeObj->setProperty ("offlineRenderContextPath", offlineWav.getFullPathName()); + routeObj->setProperty ("parityReportPath", parityJson.getFullPathName()); + routeObj->setProperty ("captureStartClipSec", captureStartClipSec); + routeObj->setProperty ("captureEndClipSec", captureEndClipSec); + routeObj->setProperty ("captureDurationSec", captureDurationSec); + routeObj->setProperty ("noteStartSec", noteHqAudibleCommitStartSec); + routeObj->setProperty ("noteEndSec", noteHqAudibleCommitEndSec); + routeObj->setProperty ("effectiveStartSec", noteHqEffectiveStartSec); + routeObj->setProperty ("effectiveEndSec", noteHqEffectiveEndSec); + routeObj->setProperty ("editedNotes", juce::var (editedNoteDiagnostics)); + routeObj->setProperty ("snapMode", "unknown_backend_capture"); + routeObj->setProperty ("capturedAt", juce::Time::getCurrentTime().toISO8601 (true)); + routeObj->setProperty ("buildDate", juce::String (__DATE__)); + routeObj->setProperty ("buildTime", juce::String (__TIME__)); + routeObj->setProperty ("actualRendererBranch", actualRendererBranch); + routeObj->setProperty ("pitchOnlyRecoveryPath", pitchOnlyRecoveryPath); + routeObj->setProperty ("formantCurveUsed", formantCurveUsed); + routeObj->setProperty ("pitchOnlyEntrySafeHandoffUsed", pitchOnlyEntrySafeHandoffUsed); + routeObj->setProperty ("pitchOnlyEntryDryHoldMs", pitchOnlyEntryDryHoldMs); + routeObj->setProperty ("pitchOnlyEntrySafeBridgeMs", pitchOnlyEntrySafeBridgeMs); + routeObj->setProperty ("pitchOnlyEntryWetAlignmentMs", pitchOnlyEntryWetAlignmentMs); + routeObj->setProperty ("pitchOnlyEntryEqualPowerBlendUsed", pitchOnlyEntryEqualPowerBlendUsed); + routeObj->setProperty ("pitchOnlyEntryRmsContinuityUsed", pitchOnlyEntryRmsContinuityUsed); + routeObj->setProperty ("pitchOnlyEntryTransientResidualUsed", pitchOnlyEntryTransientResidualUsed); + routeObj->setProperty ("pitchOnlyEntryTransientResidualMix", pitchOnlyEntryTransientResidualMix); + routeObj->setProperty ("pitchOnlyEntryTransientResidualMs", pitchOnlyEntryTransientResidualMs); + routeObj->setProperty ("postApplyRouteStatus", postApplyRouteStatus); + routeObj->setProperty ("bakedCapture", appFinalBakedCapture); + routeObj->setProperty ("bakedResampledCapture", appFinalBakedResampledCapture); + routeObj->setProperty ("appPlaybackCapture", appFinalCapture); + routeObj->setProperty ("offlineRenderCapture", appFinalOfflineCapture); + routeObj->setProperty ("parityReport", appFinalParityReport); + routeObj->setProperty ("offlineParityReport", appFinalOfflineParityReport); + routeObj->setProperty ("liveVsOfflineParityReport", appFinalLiveVsOfflineParityReport); + routeJson.getParentDirectory().createDirectory(); + routeJson.replaceWithText (juce::JSON::toString (juce::var (routeObj.get()), true)); + + logPitchEditorFormant ("note_hq app-final capture clip=" + clipId + + " baked=" + bakedWav.getFullPathName() + + " playback=" + captureWav.getFullPathName() + + " offline=" + offlineWav.getFullPathName() + + " route=" + routeJson.getFullPathName() + + " parity=" + parityJson.getFullPathName() + + " parityPassed=" + juce::String (static_cast (appFinalParityReport.getProperty ("parityPassed", false)) ? "true" : "false")); + } + } + } logPitchEditorFormant("wrote corrected file clip=" + clipId + " outputFile=" + outputFile.getFullPathName() + " swapDeferred=" + juce::String (swapDeferred ? "true" : "false")); @@ -8635,6 +12688,232 @@ juce::var AudioEngine::applyPitchCorrection(const juce::String& trackId, const j resultObj->setProperty("success", true); resultObj->setProperty("renderMode", renderMode); resultObj->setProperty("swapDeferred", swapDeferred); + resultObj->setProperty("previewCoverageStartSec", previewCoverageStartSec); + resultObj->setProperty("previewCoverageEndSec", previewCoverageEndSec); + resultObj->setProperty("candidateCoverageStartSec", candidateCoverageStartSec); + resultObj->setProperty("candidateCoverageEndSec", candidateCoverageEndSec); + resultObj->setProperty("postApplyRouteStatus", postApplyRouteStatus); + if (! appFinalCapture.isVoid()) + resultObj->setProperty("appFinalCapture", appFinalCapture); + if (! appFinalBakedCapture.isVoid()) + resultObj->setProperty("appFinalBakedCapture", appFinalBakedCapture); + if (! appFinalBakedResampledCapture.isVoid()) + resultObj->setProperty("appFinalBakedResampledCapture", appFinalBakedResampledCapture); + if (! appFinalOfflineCapture.isVoid()) + resultObj->setProperty("appFinalOfflineCapture", appFinalOfflineCapture); + if (! appFinalParityReport.isVoid()) + resultObj->setProperty("appFinalParityReport", appFinalParityReport); + if (! appFinalOfflineParityReport.isVoid()) + resultObj->setProperty("appFinalOfflineParityReport", appFinalOfflineParityReport); + if (! appFinalLiveVsOfflineParityReport.isVoid()) + resultObj->setProperty("appFinalLiveVsOfflineParityReport", appFinalLiveVsOfflineParityReport); + if (appFinalRouteReportPath.isNotEmpty()) + resultObj->setProperty("appFinalRouteReportPath", appFinalRouteReportPath); + if (appFinalBakedContextPath.isNotEmpty()) + resultObj->setProperty("appFinalBakedContextPath", appFinalBakedContextPath); + if (appFinalBakedResampledContextPath.isNotEmpty()) + resultObj->setProperty("appFinalBakedResampledContextPath", appFinalBakedResampledContextPath); + if (appFinalPlaybackContextPath.isNotEmpty()) + resultObj->setProperty("appFinalPlaybackContextPath", appFinalPlaybackContextPath); + if (appFinalOfflineContextPath.isNotEmpty()) + resultObj->setProperty("appFinalOfflineContextPath", appFinalOfflineContextPath); + if (appFinalParityReportPath.isNotEmpty()) + resultObj->setProperty("appFinalParityReportPath", appFinalParityReportPath); + resultObj->setProperty("requestedRendererBranch", requestedRendererBranch); + resultObj->setProperty("actualRendererBranch", actualRendererBranch); + resultObj->setProperty("pitchOnlyRecoveryPath", pitchOnlyRecoveryPath); + resultObj->setProperty("pitchOnlyNeutralFormantUsed", pitchOnlyNeutralFormantUsed); + resultObj->setProperty("processingMode", requestMode); + resultObj->setProperty("formantCurveUsed", formantCurveUsed); + resultObj->setProperty("explicitFormantRequested", explicitFormantRequested); + resultObj->setProperty("pitchOnlyFormantSuppressed", pitchOnlyFormantSuppressed); + resultObj->setProperty("usedFallback", usedRendererFallback); + resultObj->setProperty("fallbackReason", rendererFallbackReason); + resultObj->setProperty("hardFailReason", hardFailReason); + resultObj->setProperty("outputDurationSec", renderedOutputSamples > 0 ? static_cast (renderedOutputSamples) / sr : 0.0); + resultObj->setProperty("pitchRenderStrategy", + phraseHqRenderUsed ? "native_vsf_note_hq" : "legacy_window"); + resultObj->setProperty("pitchRenderProductPath", pitchRenderProductPath); + resultObj->setProperty("pitchRenderBackendId", pitchRenderBackendId); + resultObj->setProperty("pitchRenderBackendVersion", pitchRenderBackendVersion); + resultObj->setProperty("pitchRenderBackendFailureCode", pitchRenderBackendFailureCode); + resultObj->setProperty("pitchRenderBackendCapabilities", pitchRenderBackendCapabilities); + resultObj->setProperty("pitchRenderBackendDiagnostics", pitchRenderBackendDiagnostics); + resultObj->setProperty("phraseHqRenderUsed", phraseHqRenderUsed); + resultObj->setProperty("phraseHqExpandedToFullClip", phraseHqExpandedToFullClip); + resultObj->setProperty("phraseHqStartSec", phraseHqStartSec); + resultObj->setProperty("phraseHqEndSec", phraseHqEndSec); + resultObj->setProperty("pitchRenderCommitPolicy", pitchRenderCommitPolicy); + resultObj->setProperty("pitchRenderDryProtectedSamples", pitchRenderDryProtectedSamples); + resultObj->setProperty("pitchRenderContextDurationSec", pitchRenderContextDurationSec); + resultObj->setProperty("pitchRenderCommitDurationSec", pitchRenderCommitDurationSec); + resultObj->setProperty("pitchRenderJobStartDelayMs", pitchRenderJobStartDelayMs); + resultObj->setProperty("pitchRenderDirection", pitchRenderDirection); + resultObj->setProperty("downshiftFormantGuardUsed", downshiftFormantGuardUsed); + resultObj->setProperty("downshiftFormantGuardAlpha", downshiftFormantGuardAlpha); + resultObj->setProperty("noteHqEffectiveStartSec", noteHqEffectiveStartSec); + resultObj->setProperty("noteHqEffectiveEndSec", noteHqEffectiveEndSec); + resultObj->setProperty("noteHqContextStartSec", noteHqContextStartSec); + resultObj->setProperty("noteHqContextEndSec", noteHqContextEndSec); + resultObj->setProperty("noteHqAudibleCommitStartSec", noteHqAudibleCommitStartSec); + resultObj->setProperty("noteHqAudibleCommitEndSec", noteHqAudibleCommitEndSec); + resultObj->setProperty("noteHqPreBodyDryProtectedSamples", noteHqPreBodyDryProtectedSamples); + resultObj->setProperty("noteHqEntryInsideBodyFadeMs", noteHqEntryInsideBodyFadeMs); + resultObj->setProperty("noteHqExitLeadInMs", noteHqExitLeadInMs); + resultObj->setProperty("noteHqEntryBridgeStartSec", noteHqEntryBridgeStartSec); + resultObj->setProperty("noteHqEntryBridgeEndSec", noteHqEntryBridgeEndSec); + resultObj->setProperty("noteHqEntryBridgeWetLagMs", noteHqEntryBridgeWetLagMs); + resultObj->setProperty("noteHqEntryBridgeEnvelopeGainDb", noteHqEntryBridgeEnvelopeGainDb); + resultObj->setProperty("noteHqEntryBridgeUsed", noteHqEntryBridgeUsed); + resultObj->setProperty("noteHqEntryTransientDryPreservedMs", noteHqEntryTransientDryPreservedMs); + resultObj->setProperty("pitchOnlyEntrySimpleHandoffUsed", pitchOnlyEntrySimpleHandoffUsed); + resultObj->setProperty("pitchOnlyEntrySafeHandoffUsed", pitchOnlyEntrySafeHandoffUsed); + resultObj->setProperty("pitchOnlyEntryDryHoldMs", pitchOnlyEntryDryHoldMs); + resultObj->setProperty("pitchOnlyEntrySafeBridgeMs", pitchOnlyEntrySafeBridgeMs); + resultObj->setProperty("pitchOnlyEntryWetAlignmentMs", pitchOnlyEntryWetAlignmentMs); + resultObj->setProperty("pitchOnlyEntryWetGainDb", pitchOnlyEntryWetGainDb); + resultObj->setProperty("pitchOnlyEntryWetVsDryRmsDb", pitchOnlyEntryWetVsDryRmsDb); + resultObj->setProperty("pitchOnlyEntryEqualPowerBlendUsed", pitchOnlyEntryEqualPowerBlendUsed); + resultObj->setProperty("pitchOnlyEntryRmsContinuityUsed", pitchOnlyEntryRmsContinuityUsed); + resultObj->setProperty("pitchOnlyEntryRmsContinuityGainDb", pitchOnlyEntryRmsContinuityGainDb); + resultObj->setProperty("pitchOnlyEntryRmsContinuityMs", pitchOnlyEntryRmsContinuityMs); + resultObj->setProperty("pitchOnlyEntryTransientResidualUsed", pitchOnlyEntryTransientResidualUsed); + resultObj->setProperty("pitchOnlyEntryTransientResidualMix", pitchOnlyEntryTransientResidualMix); + resultObj->setProperty("pitchOnlyEntryTransientResidualMs", pitchOnlyEntryTransientResidualMs); + resultObj->setProperty("pitchOnlyEntryPhaseSafeUsed", pitchOnlyEntryPhaseSafeUsed); + resultObj->setProperty("pitchOnlyEntryWetAlignmentAccepted", pitchOnlyEntryWetAlignmentAccepted); + resultObj->setProperty("pitchOnlyEntryFirstCycleCorrelation", pitchOnlyEntryFirstCycleCorrelation); + resultObj->setProperty("pitchOnlyEntryZeroCrossOffsetMs", pitchOnlyEntryZeroCrossOffsetMs); + resultObj->setProperty("pitchOnlyEntryBridgeGainRampDb", pitchOnlyEntryBridgeGainRampDb); + resultObj->setProperty("pitchOnlyDownshiftCoreEnvelopePassUsed", pitchOnlyDownshiftCoreEnvelopePassUsed); + resultObj->setProperty("pitchOnlyDownshiftCoreRmsTrimDb", pitchOnlyDownshiftCoreRmsTrimDb); + resultObj->setProperty("pitchOnlyDownshiftCoreEnvelopeMaxDb", pitchOnlyDownshiftCoreEnvelopeMaxDb); + resultObj->setProperty("pitchOnlyDownshiftCoreEnvelopeFrames", pitchOnlyDownshiftCoreEnvelopeFrames); + resultObj->setProperty("pitchOnlyEntryWetLagMs", pitchOnlyEntryWetLagMs); + resultObj->setProperty("pitchOnlyEntryBridgeDurationMs", pitchOnlyEntryBridgeDurationMs); + resultObj->setProperty("pitchOnlyExitDryRestoreUsed", pitchOnlyExitDryRestoreUsed); + resultObj->setProperty("pitchOnlyExitDryRestoreStartSec", pitchOnlyExitDryRestoreStartSec); + resultObj->setProperty("pitchOnlyExitDryRestoreEndSec", pitchOnlyExitDryRestoreEndSec); + resultObj->setProperty("noteHqEntryBoundaryKind", noteHqEntryBoundaryKind); + resultObj->setProperty("noteHqExitBoundaryKind", noteHqExitBoundaryKind); + resultObj->setProperty("noteHqEntryBoundaryScore", noteHqEntryBoundaryScore); + resultObj->setProperty("noteHqExitBoundaryScore", noteHqExitBoundaryScore); + resultObj->setProperty("noteHqRendererEntryBoundaryKind", noteHqRendererEntryBoundaryKind); + resultObj->setProperty("noteHqRendererExitBoundaryKind", noteHqRendererExitBoundaryKind); + resultObj->setProperty("noteHqEditIslandCount", noteHqEditIslandCount); + resultObj->setProperty("noteHqEditedNoteCount", noteHqEditedNoteCount); + resultObj->setProperty("noteHqEntryPitchHandoffUsed", noteHqEntryPitchHandoffUsed); + resultObj->setProperty("noteHqEntryPitchHandoffStartSec", noteHqEntryPitchHandoffStartSec); + resultObj->setProperty("noteHqEntryPitchHandoffEndSec", noteHqEntryPitchHandoffEndSec); + resultObj->setProperty("noteHqEntryPitchHandoffPreMs", noteHqEntryPitchHandoffPreMs); + resultObj->setProperty("noteHqEntryPitchHandoffBodyMs", noteHqEntryPitchHandoffBodyMs); + resultObj->setProperty("noteHqEntryPitchSlopeJumpStPerSec", noteHqEntryPitchSlopeJumpStPerSec); + resultObj->setProperty("noteHqEntryPitchAccelerationLimited", noteHqEntryPitchAccelerationLimited); + resultObj->setProperty("bridgeUsed", bridgeUsed); + resultObj->setProperty("bridgeFallbackUsed", bridgeFallbackUsed); + resultObj->setProperty("bridgeStartSec", bridgeStartSec); + resultObj->setProperty("bridgeLengthMs", bridgeLengthMs); + resultObj->setProperty("bridgeAlignmentLagSamples", bridgeAlignmentLagSamples); + resultObj->setProperty("bridgeCorrelationScore", bridgeCorrelationScore); + resultObj->setProperty("bridgeGainDeltaDb", bridgeGainDeltaDb); + resultObj->setProperty("bodyReplacementUsed", bodyReplacementUsed); + resultObj->setProperty("bodyReplacementFallbackUsed", bodyReplacementFallbackUsed); + resultObj->setProperty("entryLockStartSec", entryLockStartSec); + resultObj->setProperty("entryLockLengthMs", entryLockLengthMs); + resultObj->setProperty("exitLockStartSec", exitLockStartSec); + resultObj->setProperty("renderedBodyStartSec", renderedBodyStartSec); + resultObj->setProperty("renderedBodyEndSec", renderedBodyEndSec); + resultObj->setProperty("islandNativeUsed", islandNativeUsed); + resultObj->setProperty("islandNativeFallbackUsed", islandNativeFallbackUsed); + resultObj->setProperty("islandRenderStartSec", islandRenderStartSec); + resultObj->setProperty("islandRenderEndSec", islandRenderEndSec); + resultObj->setProperty("transientMaskPeak", transientMaskPeak); + resultObj->setProperty("voicedCoreMaskPeak", voicedCoreMaskPeak); + resultObj->setProperty("hpssUsed", hpssUsed); + resultObj->setProperty("hpssFallbackUsed", hpssFallbackUsed); + resultObj->setProperty("harmonicMaskPeak", harmonicMaskPeak); + resultObj->setProperty("aperiodicMaskPeak", aperiodicMaskPeak); + resultObj->setProperty("spectralEnvelopeCorrectionUsed", spectralEnvelopeCorrectionUsed); + resultObj->setProperty("pitchOnlyCoreTimbreCorrectionUsed", pitchOnlyCoreTimbreCorrectionUsed); + resultObj->setProperty("pitchOnlyCoreEnvelopeMix", pitchOnlyCoreEnvelopeMix); + resultObj->setProperty("pitchOnlyCoreRmsTrimDb", pitchOnlyCoreRmsTrimDb); + resultObj->setProperty("pitchOnlyCoreEnvelopeLifter", pitchOnlyCoreEnvelopeLifter); + resultObj->setProperty("pitchOnlyEntryTimbreCorrectionUsed", pitchOnlyEntryTimbreCorrectionUsed); + resultObj->setProperty("pitchOnlyEntryRmsTrimDb", pitchOnlyEntryRmsTrimDb); + resultObj->setProperty("pitchOnlyEntryTiltDb", pitchOnlyEntryTiltDb); + resultObj->setProperty("pitchOnlyEntryHandoffUsed", pitchOnlyEntryHandoffUsed); + resultObj->setProperty("pitchOnlyExitHandoffUsed", pitchOnlyExitHandoffUsed); + resultObj->setProperty("vocalSourceFilterUsed", vocalSourceFilterUsed); + resultObj->setProperty("vocalSourceFilterVoicedCoverage", vocalSourceFilterVoicedCoverage); + resultObj->setProperty("vocalSourceFilterResidualMix", vocalSourceFilterResidualMix); + resultObj->setProperty("vocalSourceFilterFallbackUsed", vocalSourceFilterFallbackUsed); + resultObj->setProperty("vocalSourceFilterFallbackReason", vocalSourceFilterFallbackReason); + resultObj->setProperty("vocalSourceFilterEntryDryMs", vocalSourceFilterEntryDryMs); + resultObj->setProperty("vocalSourceFilterExitDryMs", vocalSourceFilterExitDryMs); + resultObj->setProperty("vocalSourceFilterResidualMixScale", vocalSourceFilterResidualMixScale); + resultObj->setProperty("vocalSourceFilterEpochInterpolationUsed", vocalSourceFilterEpochInterpolationUsed); + resultObj->setProperty("vocalSourceFilterEpochInterpolationStrength", vocalSourceFilterEpochInterpolationStrength); + resultObj->setProperty("vocalSourceFilterGrainRadiusScale", vocalSourceFilterGrainRadiusScale); + resultObj->setProperty("vocalSourceFilterUpPresenceTrimDb", vocalSourceFilterUpPresenceTrimDb); + resultObj->setProperty("vocalSourceFilterUpPresenceHz", vocalSourceFilterUpPresenceHz); + resultObj->setProperty("vocalSourceFilterDownNasalTrimDb", vocalSourceFilterDownNasalTrimDb); + resultObj->setProperty("vocalSourceFilterDownNasalHz", vocalSourceFilterDownNasalHz); + resultObj->setProperty("vocalSourceFilterDownBodyCompDb", vocalSourceFilterDownBodyCompDb); + resultObj->setProperty("vocalSourceFilterDownBodyCompHz", vocalSourceFilterDownBodyCompHz); + resultObj->setProperty("wsolaUsed", wsolaUsed); + resultObj->setProperty("wsolaFallbackUsed", wsolaFallbackUsed); + resultObj->setProperty("wsolaEntryLagSamples", wsolaEntryLagSamples); + resultObj->setProperty("wsolaExitLagSamples", wsolaExitLagSamples); + resultObj->setProperty("wsolaCorrelationScore", wsolaCorrelationScore); + resultObj->setProperty("phaseLockUsed", phaseLockUsed); + resultObj->setProperty("phaseLockFallbackUsed", phaseLockFallbackUsed); + resultObj->setProperty("phaseAlignedEntry", phaseAlignedEntry); + resultObj->setProperty("phaseAlignedExit", phaseAlignedExit); + resultObj->setProperty("phasePeakCount", phasePeakCount); + resultObj->setProperty("transitionHqUsed", transitionHqUsed); + resultObj->setProperty("transitionHqFallbackUsed", transitionHqFallbackUsed); + resultObj->setProperty("transitionStartSec", transitionStartSec); + resultObj->setProperty("transitionEndSec", transitionEndSec); + resultObj->setProperty("transitionTransientPeak", transitionTransientPeak); + resultObj->setProperty("transitionVoicedCorePeak", transitionVoicedCorePeak); + resultObj->setProperty("transitionResidualPeak", transitionResidualPeak); + resultObj->setProperty("transitionEnvelopeCorrectionUsed", transitionEnvelopeCorrectionUsed); + resultObj->setProperty("engineV2Used", engineV2Used); + resultObj->setProperty("engineV2FallbackUsed", engineV2FallbackUsed); + resultObj->setProperty("engineV2TransitionCount", engineV2TransitionCount); + resultObj->setProperty("engineV2TransitionStartSec", engineV2TransitionStartSec); + resultObj->setProperty("engineV2TransitionEndSec", engineV2TransitionEndSec); + resultObj->setProperty("engineV2HarmonicSupportPeak", engineV2HarmonicSupportPeak); + resultObj->setProperty("engineV2ResidualSupportPeak", engineV2ResidualSupportPeak); + resultObj->setProperty("engineV2EnvelopeSupportPeak", engineV2EnvelopeSupportPeak); + resultObj->setProperty("transientBypassUsed", transientBypassUsed); + resultObj->setProperty("residualCarryUsed", residualCarryUsed); + resultObj->setProperty("cepstralCutoffUsed", cepstralCutoffUsed); + resultObj->setProperty("fftSizeUsed", engineV2FftSize); + resultObj->setProperty("hopSizeUsed", engineV2HopSize); + resultObj->setProperty("immediateLeftNeighborUsed", immediateLeftNeighborUsed); + resultObj->setProperty("immediateRightNeighborUsed", immediateRightNeighborUsed); + resultObj->setProperty("leftNeighborSamplesRendered", leftNeighborSamplesRendered); + resultObj->setProperty("rightNeighborSamplesRendered", rightNeighborSamplesRendered); + resultObj->setProperty("leftNeighborSmoothMs", leftNeighborSmoothMs); + resultObj->setProperty("rightNeighborSmoothMs", rightNeighborSmoothMs); + resultObj->setProperty("nonImmediateNeighborTouched", nonImmediateNeighborTouched); + resultObj->setProperty("entryAlignmentOffsetMs", entryAlignmentOffsetMs); + resultObj->setProperty("exitAlignmentOffsetMs", exitAlignmentOffsetMs); + resultObj->setProperty("firstVoicedCyclesEntryUsed", firstVoicedCyclesEntryUsed); + resultObj->setProperty("firstVoicedCyclesExitUsed", firstVoicedCyclesExitUsed); + resultObj->setProperty("v3TransitionPairUsed", v3TransitionPairUsed); + resultObj->setProperty("v3ContinuousRenderUsed", v3ContinuousRenderUsed); + resultObj->setProperty("v3EntryAnchorMs", v3EntryAnchorMs); + resultObj->setProperty("v3ExitAnchorMs", v3ExitAnchorMs); + resultObj->setProperty("v3FirstCyclesEntryCount", v3FirstCyclesEntryCount); + resultObj->setProperty("v3FirstCyclesExitCount", v3FirstCyclesExitCount); + resultObj->setProperty("v3ShellDurationMs", v3ShellDurationMs); + resultObj->setProperty("v3BodyDurationMs", v3BodyDurationMs); + resultObj->setProperty("v3ResidualMix", v3ResidualMix); + resultObj->setProperty("v3FormantMode", v3FormantMode); + resultObj->setProperty("v3NeighborLeftOverlapMs", v3NeighborLeftOverlapMs); + resultObj->setProperty("v3NeighborRightOverlapMs", v3NeighborRightOverlapMs); return juce::var(resultObj.get()); } @@ -8644,6 +12923,313 @@ juce::var AudioEngine::previewPitchCorrection(const juce::String& trackId, const return applyPitchCorrection(trackId, clipId, notesJson, juce::var(), 0.0f); } +juce::var AudioEngine::startPitchScrubPreview(const juce::String& trackId, + const juce::String& clipId, + const juce::var& noteJson, + const juce::var& framesJson) +{ + const juce::var parsedNoteJson = noteJson.isString() ? juce::JSON::parse (noteJson.toString()) : noteJson; + const juce::var parsedFramesJson = framesJson.isString() ? juce::JSON::parse (framesJson.toString()) : framesJson; + + juce::Array noteArray; + noteArray.add (parsedNoteJson); + auto parsedNotes = PitchAnalyzer::notesFromJSON (juce::var (noteArray)); + if (parsedNotes.empty()) + return false; + const auto& parsedNote = parsedNotes.front(); + + auto clips = playbackEngine.getClipSnapshot(); + const PlaybackEngine::ClipInfo* foundClip = nullptr; + for (const auto& clip : clips) + { + if (clip.trackId == trackId && clip.clipId == clipId) + { + foundClip = &clip; + break; + } + } + if (foundClip == nullptr) + return false; + + juce::File sourceFile = foundClip->audioFile.existsAsFile() + ? foundClip->audioFile + : foundClip->originalAudioFile; + if (! sourceFile.existsAsFile()) + return false; + + juce::AudioFormatManager fmtMgr; + fmtMgr.registerBasicFormats(); + std::unique_ptr reader(fmtMgr.createReaderFor(sourceFile)); + if (! reader) + return false; + + auto analysisFrames = PitchAnalyzer::framesFromJSON (parsedFramesJson); + float noteStartTime = parsedNote.startTime; + float noteEndTime = parsedNote.endTime; + float detectedPitch = parsedNote.detectedPitch; + float correctedPitch = parsedNote.correctedPitch; + if (noteEndTime <= noteStartTime + 0.02f || detectedPitch <= 0.0f || correctedPitch <= 0.0f) + { + float derivedStartSec = std::numeric_limits::max(); + float derivedEndSec = 0.0f; + float derivedMidi = 0.0f; + int derivedCount = 0; + for (const auto& frame : analysisFrames) + { + if (! frame.voiced || frame.midiNote <= 0.0f || frame.confidence < 0.30f) + continue; + derivedStartSec = std::min (derivedStartSec, frame.time); + derivedEndSec = std::max (derivedEndSec, frame.time); + derivedMidi += frame.midiNote; + ++derivedCount; + } + if (derivedCount > 0 && derivedEndSec > derivedStartSec) + { + noteStartTime = derivedStartSec; + noteEndTime = derivedEndSec + 0.04f; + detectedPitch = derivedMidi / static_cast (derivedCount); + correctedPitch = detectedPitch; + } + } + const float noteDurationSec = std::max (0.02f, noteEndTime - noteStartTime); + + float basePitchHz = 0.0f; + int basePitchCount = 0; + const float innerStartSec = noteStartTime + 0.12f * noteDurationSec; + const float innerEndSec = noteEndTime - 0.12f * noteDurationSec; + for (const auto& frame : analysisFrames) + { + if (frame.time >= innerStartSec && frame.time <= innerEndSec && frame.frequency > 40.0f) + { + basePitchHz += frame.frequency; + ++basePitchCount; + } + } + if (basePitchCount > 0) + basePitchHz /= static_cast (basePitchCount); + else + basePitchHz = 440.0f * std::pow (2.0f, (detectedPitch - 69.0f) / 12.0f); + basePitchHz = juce::jlimit (55.0f, 1400.0f, basePitchHz); + + const float preferredPreviewSec = 0.240f; + const float maxPreviewSec = 0.320f; + const float minPreviewSec = 0.080f; + const float availableInteriorSec = std::max (0.02f, noteDurationSec * 0.80f); + const float clampedInteriorSec = std::max (0.04f, availableInteriorSec); + const float desiredLoopSec = juce::jlimit (std::min (minPreviewSec, clampedInteriorSec), + std::min (maxPreviewSec, clampedInteriorSec), + preferredPreviewSec); + + struct CandidateSpan + { + float startSec = 0.0f; + float endSec = 0.0f; + float score = -1.0f; + }; + + CandidateSpan bestSpan; + CandidateSpan currentSpan; + float currentScoreAccum = 0.0f; + int currentScoreCount = 0; + const auto flushCurrentSpan = [&]() + { + if (currentSpan.endSec > currentSpan.startSec && currentScoreCount > 0) + { + currentSpan.score = currentScoreAccum / static_cast (currentScoreCount); + const float currentLen = currentSpan.endSec - currentSpan.startSec; + const float bestLen = bestSpan.endSec - bestSpan.startSec; + if (currentLen > bestLen + 0.010f + || (std::abs (currentLen - bestLen) <= 0.010f && currentSpan.score > bestSpan.score)) + bestSpan = currentSpan; + } + currentSpan = {}; + currentScoreAccum = 0.0f; + currentScoreCount = 0; + }; + + for (const auto& frame : analysisFrames) + { + if (frame.time < innerStartSec || frame.time > innerEndSec) + continue; + + const float linearRms = std::pow (10.0f, frame.rmsDB / 20.0f); + const bool stableVoiced = frame.voiced && frame.frequency > 40.0f && frame.confidence >= 0.45f && linearRms >= 0.003f; + if (! stableVoiced) + { + flushCurrentSpan(); + continue; + } + + if (currentScoreCount == 0) + currentSpan.startSec = frame.time; + currentSpan.endSec = frame.time; + currentScoreAccum += frame.confidence * juce::jlimit (0.0f, 1.0f, linearRms * 8.0f); + ++currentScoreCount; + } + flushCurrentSpan(); + + const float noteMidSec = 0.5f * (noteStartTime + noteEndTime); + float anchorMidSec = noteMidSec; + if (bestSpan.endSec > bestSpan.startSec) + anchorMidSec = 0.5f * (bestSpan.startSec + bestSpan.endSec); + + float loopStartSec = anchorMidSec - 0.5f * desiredLoopSec; + float loopEndSec = loopStartSec + desiredLoopSec; + const float minAllowedStart = noteStartTime + 0.08f * noteDurationSec; + const float maxAllowedEnd = std::max (minAllowedStart + desiredLoopSec, noteEndTime - 0.08f * noteDurationSec); + if (loopStartSec < minAllowedStart) + { + loopStartSec = minAllowedStart; + loopEndSec = loopStartSec + desiredLoopSec; + } + if (loopEndSec > maxAllowedEnd) + { + loopEndSec = maxAllowedEnd; + loopStartSec = loopEndSec - desiredLoopSec; + } + loopStartSec = std::max (0.0f, loopStartSec); + loopEndSec = std::max (loopStartSec + std::max (0.04f, std::min (minPreviewSec, desiredLoopSec)), loopEndSec); + + const double clipStartTimeInSource = foundClip->offset; + const juce::int64 loopStartSample = static_cast (std::floor ((clipStartTimeInSource + loopStartSec) * reader->sampleRate)); + const int loopSamples = std::max (32, static_cast (std::round ((loopEndSec - loopStartSec) * reader->sampleRate))); + if (loopStartSample < 0 || loopSamples <= 0 || loopStartSample >= reader->lengthInSamples) + return false; + + const int availableSamples = static_cast (std::min (loopSamples, reader->lengthInSamples - loopStartSample)); + if (availableSamples <= 16) + return false; + + juce::AudioBuffer loopBuffer (static_cast (reader->numChannels), availableSamples); + reader->read (&loopBuffer, 0, availableSamples, loopStartSample, true, true); + + const int fadeSamples = juce::jlimit (16, std::max (24, availableSamples / 3), std::max (32, availableSamples / 6)); + float seamAbsDiff = 0.0f; + int seamSampleCount = 0; + float previewPeak = 0.0f; + double previewEnergy = 0.0; + int previewEnergySamples = 0; + for (int ch = 0; ch < loopBuffer.getNumChannels(); ++ch) + { + for (int i = 0; i < availableSamples; ++i) + { + const float sample = loopBuffer.getSample (ch, i); + previewPeak = juce::jmax (previewPeak, std::abs (sample)); + previewEnergy += static_cast (sample) * static_cast (sample); + ++previewEnergySamples; + } + } + const float previewRms = previewEnergySamples > 0 + ? std::sqrt (static_cast (previewEnergy / static_cast (previewEnergySamples))) + : 0.0f; + for (int ch = 0; ch < loopBuffer.getNumChannels(); ++ch) + { + for (int i = 0; i < fadeSamples; ++i) + { + const float t = static_cast (i) / static_cast (std::max (1, fadeSamples - 1)); + const int tailIndex = availableSamples - fadeSamples + i; + const float startSample = loopBuffer.getSample (ch, i); + const float tailSample = loopBuffer.getSample (ch, tailIndex); + seamAbsDiff += std::abs (startSample - tailSample); + ++seamSampleCount; + loopBuffer.setSample (ch, i, startSample * (1.0f - t) + tailSample * t); + loopBuffer.setSample (ch, tailIndex, tailSample * (1.0f - t) + startSample * t); + } + } + + const float seamAvgDiff = seamSampleCount > 0 ? seamAbsDiff / static_cast (seamSampleCount) : 1.0f; + const float seamDenominator = std::max (0.005f, previewPeak); + const float repeatStability = juce::jlimit (0.0f, 1.0f, 1.0f - (seamAvgDiff / seamDenominator)); + const float peakTarget = 0.18f; + const float rmsTarget = 0.055f; + float previewGain = 1.0f; + if (previewPeak > 1.0e-5f) + previewGain = std::max (previewGain, peakTarget / previewPeak); + if (previewRms > 1.0e-5f) + previewGain = std::max (previewGain, rmsTarget / previewRms); + previewGain = juce::jlimit (1.0f, 24.0f, previewGain); + + PlaybackEngine::PitchScrubPreviewData preview; + preview.trackId = trackId; + preview.clipId = clipId; + preview.loopBuffer = loopBuffer; + preview.sourceSampleRate = reader->sampleRate; + preview.loopStartSec = loopStartSec; + preview.loopEndSec = loopEndSec; + preview.basePitchHz = basePitchHz; + preview.pitchRatio = juce::jlimit (0.25f, 4.0f, + std::pow (2.0f, (correctedPitch - detectedPitch) / 12.0f)); + preview.active = true; + preview.readPosition = 0.0; + preview.loopCrossfadeSamples = fadeSamples; + preview.gain = previewGain; + preview.repeatStability = repeatStability; + playbackEngine.setPitchScrubPreview (preview); + + auto* resultObj = new juce::DynamicObject(); + resultObj->setProperty ("success", true); + resultObj->setProperty ("loopStartSec", loopStartSec); + resultObj->setProperty ("loopEndSec", loopEndSec); + resultObj->setProperty ("loopDurationMs", 1000.0 * (loopEndSec - loopStartSec)); + resultObj->setProperty ("basePitchHz", basePitchHz); + resultObj->setProperty ("pitchRatio", preview.pitchRatio); + resultObj->setProperty ("previewGain", previewGain); + resultObj->setProperty ("previewPeak", previewPeak); + resultObj->setProperty ("previewRms", previewRms); + resultObj->setProperty ("repeatStability", repeatStability); + return juce::var (resultObj); +} + +bool AudioEngine::updatePitchScrubPreview(const juce::String& clipId, float pitchRatio) +{ + return playbackEngine.updatePitchScrubPreview (clipId, pitchRatio); +} + +bool AudioEngine::stopPitchScrubPreview(const juce::String& clipId) +{ + const bool hadPreview = playbackEngine.hasPitchScrubPreview (clipId); + playbackEngine.clearPitchScrubPreview (clipId); + return hadPreview; +} + +juce::var AudioEngine::getPitchScrubPreviewStatus(const juce::String& clipId) +{ + const auto status = playbackEngine.getPitchScrubPreviewStatus (clipId); + auto* resultObj = new juce::DynamicObject(); + resultObj->setProperty ("active", status.active); + resultObj->setProperty ("releasePending", status.releasePending); + resultObj->setProperty ("audible", status.audible); + resultObj->setProperty ("previewArmed", status.previewArmed); + resultObj->setProperty ("firstCallbackServiced", status.firstCallbackServiced); + resultObj->setProperty ("firstDragAudible", status.firstDragAudible); + resultObj->setProperty ("trackId", status.trackId); + resultObj->setProperty ("clipId", status.clipId); + resultObj->setProperty ("pitchRatio", status.pitchRatio); + resultObj->setProperty ("basePitchHz", status.basePitchHz); + resultObj->setProperty ("currentGain", status.currentGain); + resultObj->setProperty ("targetGain", status.targetGain); + resultObj->setProperty ("repeatStability", status.repeatStability); + resultObj->setProperty ("lastPeak", status.lastPeak); + resultObj->setProperty ("loopDurationMs", status.loopDurationMs); + resultObj->setProperty ("lastRenderWallTimeMs", status.lastRenderWallTimeMs); + resultObj->setProperty ("mixedCallbackCount", status.mixedCallbackCount); + resultObj->setProperty ("mixedSampleCount", static_cast (status.mixedSampleCount)); + resultObj->setProperty ("renderMethod", status.renderMethod); + return juce::var (resultObj); +} + +juce::var AudioEngine::getPitchPreviewRoutingStatus(const juce::String& clipId) +{ + const auto status = playbackEngine.getPitchPreviewRoutingStatus (clipId); + auto* resultObj = new juce::DynamicObject(); + resultObj->setProperty ("scrubPreviewActive", status.scrubPreviewActive); + resultObj->setProperty ("clipLivePreviewActive", status.clipLivePreviewActive); + resultObj->setProperty ("renderedSegmentActive", status.renderedSegmentActive); + resultObj->setProperty ("correctedSourceActive", status.correctedSourceActive); + resultObj->setProperty ("monitorMode", status.monitorMode); + return juce::var (resultObj); +} + // ============================================================================= // Phase 6: Polyphonic Pitch Detection // ============================================================================= @@ -8935,9 +13521,14 @@ juce::var AudioEngine::refreshAiToolsStatus() return stemSeparator.refreshAiToolsStatus(); } -juce::var AudioEngine::installAiTools() +juce::var AudioEngine::installAiTools(bool userConfirmedDownload) +{ + return stemSeparator.installAiTools(userConfirmedDownload); +} + +juce::var AudioEngine::resetAiTools() { - return stemSeparator.installAiTools(); + return stemSeparator.resetAiTools(); } juce::var AudioEngine::separateStems(const juce::String& trackId, const juce::String& clipId) @@ -9114,6 +13705,171 @@ void AudioEngine::cancelAiToolsInstall() stemSeparator.cancelAiToolsInstall(); } +juce::var AudioEngine::startAIGeneration(const juce::String& trackId, + const juce::String& workflowId, + const juce::String& paramsJSON) +{ + juce::ignoreUnused(trackId); + + auto result = std::make_unique(); + + if (aiTrackEngine.isRunning()) + { + result->setProperty("started", false); + result->setProperty("error", "Another AI generation is already in progress."); + return juce::var(result.release()); + } + + auto aiToolsStatus = stemSeparator.getAiToolsStatus(); + if (auto* statusObject = aiToolsStatus.getDynamicObject()) + { + const auto musicGenerationReady = static_cast(statusObject->getProperty("musicGenerationReady")); + const auto layoutValid = static_cast(statusObject->getProperty("musicGenerationLayoutValid")); + const auto performanceReady = ! statusObject->hasProperty("musicGenerationPerformanceReady") + || static_cast(statusObject->getProperty("musicGenerationPerformanceReady")); + const auto availableProfilesVar = statusObject->getProperty("musicGenerationAvailableProfiles"); + auto hasNativeMusicProfile = true; + if (auto* availableProfilesArray = availableProfilesVar.getArray()) + { + if (! availableProfilesArray->isEmpty()) + { + hasNativeMusicProfile = false; + for (const auto& profile : *availableProfilesArray) + { + if (profile.toString() == "native-xl-turbo") + { + hasNativeMusicProfile = true; + break; + } + } + } + } + + if (! musicGenerationReady || ! layoutValid || ! performanceReady || ! hasNativeMusicProfile) + { + const auto musicGenerationStatusMessage = statusObject->getProperty("musicGenerationStatusMessage").toString(); + const auto musicGenerationPerformanceStatusMessage = statusObject->getProperty("musicGenerationPerformanceStatusMessage").toString(); + const auto statusMessage = statusObject->getProperty("message").toString(); + const auto errorMessage = statusObject->getProperty("error").toString(); + const auto modelId = statusObject->getProperty("musicGenerationModelId").toString(); + const auto checkpointRoot = statusObject->getProperty("musicGenerationCheckpointRoot").toString(); + result->setProperty("started", false); + result->setProperty( + "error", + errorMessage.isNotEmpty() + ? errorMessage + : (! hasNativeMusicProfile) + ? "The Native XL Turbo ACE-Step profile is still missing required music-generation assets. Retry AI Tools install to finish setup." + : musicGenerationPerformanceStatusMessage.isNotEmpty() + ? musicGenerationPerformanceStatusMessage + : musicGenerationStatusMessage.isNotEmpty() + ? musicGenerationStatusMessage + : (! layoutValid && modelId.isNotEmpty() && checkpointRoot.isNotEmpty()) + ? "Pinned ACE-Step model " + modelId + " is not ready in " + checkpointRoot + "." + : statusMessage); + return juce::var(result.release()); + } + } + + auto outputDir = juce::File::getSpecialLocation(juce::File::tempDirectory) + .getChildFile("OpenStudio") + .getChildFile("generated-music"); + + if (! aiTrackEngine.startGeneration(workflowId, paramsJSON, outputDir)) + { + auto progress = aiTrackEngine.pollProgress(); + result->setProperty("started", false); + result->setProperty("error", progress.error.isNotEmpty() + ? progress.error + : "Failed to start AI generation."); + return juce::var(result.release()); + } + + result->setProperty("started", true); + return juce::var(result.release()); +} + +juce::var AudioEngine::getAIGenerationProgress() +{ + auto obj = std::make_unique(); + auto progress = aiTrackEngine.pollProgress(); + + obj->setProperty("state", progress.state); + obj->setProperty("progress", static_cast(progress.progress)); + obj->setProperty("phase", progress.phase); + obj->setProperty("message", progress.message); + obj->setProperty("backend", progress.backend); + obj->setProperty("elapsedMs", progress.elapsedMs); + obj->setProperty("heartbeatTs", progress.heartbeatTs); + if (progress.phaseProgress >= 0.0) + obj->setProperty("phaseProgress", progress.phaseProgress); + if (progress.etaMs >= 0.0) + obj->setProperty("etaMs", progress.etaMs); + if (progress.runMode.isNotEmpty()) + obj->setProperty("runMode", progress.runMode); + if (progress.runtimeProfile.isNotEmpty()) + obj->setProperty("runtimeProfile", progress.runtimeProfile); + if (progress.lmModel.isNotEmpty()) + obj->setProperty("lmModel", progress.lmModel); + if (progress.statusNote.isNotEmpty()) + obj->setProperty("statusNote", progress.statusNote); + if (progress.failureKind.isNotEmpty()) + obj->setProperty("failureKind", progress.failureKind); + if (progress.sessionMode.isNotEmpty()) + obj->setProperty("sessionMode", progress.sessionMode); + if (progress.workerExitCode != 0) + obj->setProperty("workerExitCode", progress.workerExitCode); + if (progress.lastStdoutLine.isNotEmpty()) + obj->setProperty("lastStdoutLine", progress.lastStdoutLine); + if (progress.lastStderrLine.isNotEmpty()) + obj->setProperty("lastStderrLine", progress.lastStderrLine); + if (progress.attemptMode.isNotEmpty()) + obj->setProperty("attemptMode", progress.attemptMode); + if (progress.attemptIndex > 0) + obj->setProperty("attemptIndex", progress.attemptIndex); + if (progress.protocolVersion > 0) + obj->setProperty("protocolVersion", progress.protocolVersion); + if (progress.scriptVersion.isNotEmpty()) + obj->setProperty("scriptVersion", progress.scriptVersion); + if (progress.requestId.isNotEmpty()) + obj->setProperty("requestId", progress.requestId); + if (progress.priorFailure.isNotEmpty()) + obj->setProperty("priorFailure", progress.priorFailure); + if (progress.lastProgressAgeMs >= 0.0) + obj->setProperty("lastProgressAgeMs", progress.lastProgressAgeMs); + if (progress.tracePath.isNotEmpty()) + obj->setProperty("tracePath", progress.tracePath); + if (progress.failureDetail.isNotEmpty()) + obj->setProperty("failureDetail", progress.failureDetail); + if (progress.lmBackend.isNotEmpty()) + obj->setProperty("lmBackend", progress.lmBackend); + if (progress.lmStage.isNotEmpty()) + obj->setProperty("lmStage", progress.lmStage); + + if (progress.outputFile.isNotEmpty()) + { + obj->setProperty("outputFile", progress.outputFile); + + juce::File generatedFile(progress.outputFile); + if (generatedFile.existsAsFile()) + { + peakCache.generateAsync(generatedFile, [generatedFile]() { + juce::Logger::writeToLog("PeakCache: Generated peaks for AI clip " + generatedFile.getFileName()); + }); + } + } + + if (progress.error.isNotEmpty()) + obj->setProperty("error", progress.error); + + return juce::var(obj.release()); +} + +void AudioEngine::cancelAIGeneration() +{ + aiTrackEngine.cancel(); +} + // ============================================================================= // Phase 9: ARA Plugin Hosting // ============================================================================= diff --git a/Source/AudioEngine.h b/Source/AudioEngine.h index 02ccea8..2ef363b 100644 --- a/Source/AudioEngine.h +++ b/Source/AudioEngine.h @@ -22,8 +22,10 @@ #include "PolyPitchDetector.h" #include "PolyResynthesizer.h" #include "StemSeparator.h" +#include "AITrackEngine.h" #include #include +#include // ... (skip lines) ... class AudioEngine : public juce::AudioIODeviceCallback, @@ -61,7 +63,7 @@ class AudioEngine : public juce::AudioIODeviceCallback, float* const* outputChannelData, int numOutputChannels, int numSamples, bool hybrid64PostChainActive); void applyMasterGainPanMono (float* const* outputChannelData, int numOutputChannels, - int numSamples, double currentSamplePosition, + int numSamples, double currentTimeSeconds, bool hybrid64PostChainActive); juce::AudioDeviceManager& getDeviceManager() { return deviceManager; } @@ -263,14 +265,32 @@ class AudioEngine : public juce::AudioIODeviceCallback, bool renderProject(const juce::String& source, double startTime, double endTime, const juce::String& filePath, const juce::String& format, double renderSampleRate, int bitDepth, int numChannels, - bool normalize, bool addTail, double tailLengthMs); + bool normalize, bool addTail, double tailLengthMs, + bool includeMetronome = false); + juce::var capturePitchAuditionPlayback(const juce::String& trackId, + const juce::String& clipId, + double startTime, + double duration, + const juce::String& filePath, + double sampleRate = 44100.0, + bool offlineRenderMode = true); + juce::var capturePitchBakedContext(const juce::String& sourceFile, + double startSec, + double durationSec, + const juce::String& filePath); + juce::var comparePitchDebugAudioFiles(const juce::String& referenceFile, + const juce::String& candidateFile, + double captureStartClipSec, + double noteStartClipSec, + double noteEndClipSec); // Render with dither/noise shaping (Phase 9E) bool renderProjectWithDither(const juce::String& source, double startTime, double endTime, const juce::String& filePath, const juce::String& format, double renderSampleRate, int bitDepth, int numChannels, bool normalize, bool addTail, double tailLengthMs, - const juce::String& ditherType); + const juce::String& ditherType, + bool includeMetronome = false); // Plugin Delay Compensation (PDC) void recalculatePDC(); @@ -441,16 +461,27 @@ class AudioEngine : public juce::AudioIODeviceCallback, juce::var getPitchHistory(const juce::String& trackId, int fxIndex, int numFrames); // Pitch Corrector bridge methods (graphical mode) - juce::var analyzePitchContour(const juce::String& trackId, const juce::String& clipId); - juce::var analyzePitchContourDirect(const juce::String& filePath, double offset, double duration, const juce::String& clipId); + juce::var analyzePitchContour(const juce::String& trackId, const juce::String& clipId, + std::function shouldCancel = {}); + juce::var analyzePitchContourDirect(const juce::String& filePath, double offset, double duration, + const juce::String& clipId, + std::function shouldCancel = {}); juce::var applyPitchCorrection(const juce::String& trackId, const juce::String& clipId, const juce::var& notesJson, const juce::var& framesJson = juce::var(), float globalFormantSemitones = 0.0f, - std::optional windowStartSec = std::nullopt, - std::optional windowEndSec = std::nullopt, - const juce::String& renderMode = "single", - std::function shouldCancel = {}); + std::optional windowStartSec = std::nullopt, + std::optional windowEndSec = std::nullopt, + const juce::String& renderMode = "single", + std::function shouldCancel = {}, + double jobStartDelayMs = 0.0, + int previewRenderGenerationToken = 0); juce::var previewPitchCorrection(const juce::String& trackId, const juce::String& clipId, const juce::var& notesJson); + juce::var startPitchScrubPreview(const juce::String& trackId, const juce::String& clipId, + const juce::var& noteJson, const juce::var& framesJson = juce::var()); + bool updatePitchScrubPreview(const juce::String& clipId, float pitchRatio); + bool stopPitchScrubPreview(const juce::String& clipId); + juce::var getPitchScrubPreviewStatus(const juce::String& clipId = {}); + juce::var getPitchPreviewRoutingStatus(const juce::String& clipId = {}); // Polyphonic pitch detection (Phase 6) juce::var analyzePolyphonic(const juce::String& trackId, const juce::String& clipId); @@ -466,11 +497,17 @@ class AudioEngine : public juce::AudioIODeviceCallback, bool isStemSeparationAvailable() const; juce::var getAiToolsStatus(); juce::var refreshAiToolsStatus(); - juce::var installAiTools(); + juce::var installAiTools(bool userConfirmedDownload); + juce::var resetAiTools(); juce::var separateStemsAsync(const juce::String& trackId, const juce::String& clipId, const juce::String& optionsJSON); juce::var getStemSeparationProgress(); void cancelStemSeparation(); void cancelAiToolsInstall(); + juce::var startAIGeneration(const juce::String& trackId, + const juce::String& workflowId, + const juce::String& paramsJSON); + juce::var getAIGenerationProgress(); + void cancelAIGeneration(); // ARA Plugin Hosting (Phase 9) juce::var initializeARAForTrack(const juce::String& trackId, int fxIndex); @@ -589,6 +626,10 @@ class AudioEngine : public juce::AudioIODeviceCallback, // Device settings persistence void saveDeviceSettings(); void loadDeviceSettings(); + void loadDeviceSettingsWithChannelCounts(int inputChannels, int outputChannels); + bool isMicrophonePermissionGrantedForInput() const; + void requestMicrophonePermissionIfNeeded(std::function completion); + void applyAudioDeviceSetup(const juce::String& type, const juce::String& input, const juce::String& output, double sampleRate, int bufferSize); juce::File getDeviceSettingsFile() const; // MIDI message routing (Phase 2) @@ -622,7 +663,14 @@ class AudioEngine : public juce::AudioIODeviceCallback, int inputLatencySamples = 0; // Device input latency for recording compensation std::atomic lastAudioBlockWallTimeMs { 0.0 }; std::atomic lastAudioBlockDurationMs { 0.0 }; + std::atomic lastAudioCallbackProcessMs { 0.0 }; + std::atomic maxAudioCallbackProcessMs { 0.0 }; std::atomic audioCallbackCounter { 0 }; + std::atomic audioCallbackDeadlineMissCount { 0 }; + std::atomic audioCallbackScopedNoDenormalsCount { 0 }; + std::atomic audioCallbackTrackBufferResizeCount { 0 }; + std::atomic audioCallbackPitchScrubBufferResizeCount { 0 }; + std::atomic audioCallbackSidechainBufferResizeCount { 0 }; std::atomic firstCallbackAfterTransportStartPending { false }; std::atomic pendingRecordStartCapture { false }; // Audio thread captures start time double tempo = 120.0; // BPM (global default / fallback) @@ -737,6 +785,7 @@ class AudioEngine : public juce::AudioIODeviceCallback, juce::AudioBuffer reusableTrackBuffer; juce::AudioBuffer reusableMasterBuffer; juce::AudioBuffer reusableMasterBufferDouble; + juce::AudioBuffer reusablePitchScrubBuffer; juce::AudioBuffer masterFXFallbackBuffer; juce::AudioBuffer monitoringFXFallbackBuffer; std::atomic masterFXFallbackReuseCount { 0 }; @@ -793,6 +842,8 @@ class AudioEngine : public juce::AudioIODeviceCallback, int spectrumWritePos { 0 }; bool spectrumReady { false }; juce::CriticalSection spectrumLock; + std::atomic spectrumFftPublishCount { 0 }; + std::atomic spectrumFftLockMissCount { 0 }; // Polyphonic Pitch Detection (Phase 6) — lazy-loaded PolyPitchDetector polyPitchDetector; @@ -805,6 +856,7 @@ class AudioEngine : public juce::AudioIODeviceCallback, // Source Separation (Phase 8 + Phase 10) — Python subprocess StemSeparator stemSeparator; + AITrackEngine aiTrackEngine; // Stem file cache: hash(filePath+offset+duration) -> stem files (name -> path) std::map stemFileCache; diff --git a/Source/AutomationList.cpp b/Source/AutomationList.cpp index 0876c4e..6a5f2be 100644 --- a/Source/AutomationList.cpp +++ b/Source/AutomationList.cpp @@ -12,34 +12,34 @@ void AutomationList::setPoints(std::vector newPoints) // Sort by time before storing std::sort(newPoints.begin(), newPoints.end(), [](const AutomationPoint& a, const AutomationPoint& b) { - return a.timeSamples < b.timeSamples; + return a.timeSeconds < b.timeSeconds; }); const juce::ScopedLock sl(lock); points = std::move(newPoints); } -void AutomationList::addPoint(double timeSamples, float value) +void AutomationList::addPoint(double timeSeconds, float value) { const juce::ScopedLock sl(lock); - AutomationPoint pt { timeSamples, value }; + AutomationPoint pt { timeSeconds, value }; // Insert in sorted order auto it = std::lower_bound(points.begin(), points.end(), pt, [](const AutomationPoint& a, const AutomationPoint& b) { - return a.timeSamples < b.timeSamples; + return a.timeSeconds < b.timeSeconds; }); points.insert(it, pt); } -void AutomationList::removePointsInRange(double startSample, double endSample) +void AutomationList::removePointsInRange(double startTimeSeconds, double endTimeSeconds) { const juce::ScopedLock sl(lock); points.erase( std::remove_if(points.begin(), points.end(), - [startSample, endSample](const AutomationPoint& p) { - return p.timeSamples >= startSample && p.timeSamples <= endSample; + [startTimeSeconds, endTimeSeconds](const AutomationPoint& p) { + return p.timeSeconds >= startTimeSeconds && p.timeSeconds <= endTimeSeconds; }), points.end()); } @@ -95,10 +95,10 @@ bool AutomationList::shouldRecord() const } } -int AutomationList::findPointBefore(double timeSamples) const +int AutomationList::findPointBefore(double timeSeconds) const { - // Binary search: find last point at or before timeSamples - // points must be sorted by timeSamples (guaranteed by setPoints/addPoint) + // Binary search: find last point at or before timeSeconds + // points must be sorted by timeSeconds (guaranteed by setPoints/addPoint) if (points.empty()) return -1; @@ -109,7 +109,7 @@ int AutomationList::findPointBefore(double timeSamples) const while (lo <= hi) { int mid = lo + (hi - lo) / 2; - if (points[static_cast(mid)].timeSamples <= timeSamples) + if (points[static_cast(mid)].timeSeconds <= timeSeconds) { result = mid; lo = mid + 1; @@ -123,13 +123,13 @@ int AutomationList::findPointBefore(double timeSamples) const return result; } -float AutomationList::eval(double timeSamples) const +float AutomationList::eval(double timeSeconds) const { const juce::ScopedTryLock sl(lock); if (!sl.isLocked() || points.empty()) return defaultValue.load(std::memory_order_relaxed); - int idx = findPointBefore(timeSamples); + int idx = findPointBefore(timeSeconds); // Before first point — hold first point's value if (idx < 0) @@ -148,11 +148,11 @@ float AutomationList::eval(double timeSamples) const if (interpolation == AutomationInterpolation::Discrete) return p0.value; - double dt = p1.timeSamples - p0.timeSamples; + double dt = p1.timeSeconds - p0.timeSeconds; if (dt <= 0.0) return p0.value; - double t = (timeSamples - p0.timeSamples) / dt; // 0.0 to 1.0 + double t = (timeSeconds - p0.timeSeconds) / dt; // 0.0 to 1.0 if (interpolation == AutomationInterpolation::Exponential) { @@ -163,7 +163,7 @@ float AutomationList::eval(double timeSamples) const return static_cast(p0.value + (p1.value - p0.value) * t); } -void AutomationList::evalBlock(double startSample, double sampleRate, int numSamples, float* outputBuffer) const +void AutomationList::evalBlock(double startTimeSeconds, double sampleRate, int numSamples, float* outputBuffer) const { juce::ignoreUnused(sampleRate); @@ -179,14 +179,14 @@ void AutomationList::evalBlock(double startSample, double sampleRate, int numSam auto sz = static_cast(points.size()); // Start search from the point before the block start - int idx = findPointBefore(startSample); + int idx = findPointBefore(startTimeSeconds); for (int i = 0; i < numSamples; ++i) { - double t = startSample + static_cast(i); + double t = startTimeSeconds + static_cast(i) / sampleRate; // Advance idx to the correct segment for this sample - while (idx < sz - 1 && points[static_cast(idx + 1)].timeSamples <= t) + while (idx < sz - 1 && points[static_cast(idx + 1)].timeSeconds <= t) ++idx; if (idx < 0) @@ -213,14 +213,14 @@ void AutomationList::evalBlock(double startSample, double sampleRate, int numSam } else { - double dt = p1.timeSamples - p0.timeSamples; + double dt = p1.timeSeconds - p0.timeSeconds; if (dt <= 0.0) { outputBuffer[i] = p0.value; } else { - double frac = (t - p0.timeSamples) / dt; + double frac = (t - p0.timeSeconds) / dt; if (interpolation == AutomationInterpolation::Exponential) frac = frac * frac; outputBuffer[i] = static_cast(p0.value + (p1.value - p0.value) * frac); diff --git a/Source/AutomationList.h b/Source/AutomationList.h index cff5877..cc21b2d 100644 --- a/Source/AutomationList.h +++ b/Source/AutomationList.h @@ -22,10 +22,10 @@ enum class AutomationMode Latch // Record on touch, continue writing last value after release }; -// A single automation point (time + value) +// A single automation point (timeline time + value) struct AutomationPoint { - double timeSamples; // Position in samples (absolute timeline position) + double timeSeconds; // Absolute timeline position in seconds float value; // Normalised 0.0–1.0 for most params, or raw dB for volume }; @@ -45,10 +45,10 @@ class AutomationList void setPoints(std::vector newPoints); // Add a single point (for recording). Keeps list sorted. - void addPoint(double timeSamples, float value); + void addPoint(double timeSeconds, float value); // Remove points in a time range (for automation trim / delete) - void removePointsInRange(double startSample, double endSample); + void removePointsInRange(double startTimeSeconds, double endTimeSeconds); // Clear all points void clear(); @@ -78,12 +78,12 @@ class AutomationList // Evaluate automation value at a single sample position. // Returns the interpolated value, or defaultValue if no points / mode is Off. // Uses ScopedTryLock — returns defaultValue if lock is held (message thread writing). - float eval(double timeSamples) const; + float eval(double timeSeconds) const; // Batch evaluate: fill outputBuffer with per-sample values for a block. - // startSample = timeline position of first sample in block. + // startTimeSeconds = timeline position of first sample in block. // Much more efficient than calling eval() per sample — uses cached search position. - void evalBlock(double startSample, double sampleRate, int numSamples, float* outputBuffer) const; + void evalBlock(double startTimeSeconds, double sampleRate, int numSamples, float* outputBuffer) const; // Should the audio thread apply automation right now? // Read mode: always. Touch: only when NOT touching. Latch: when NOT touching. @@ -95,7 +95,7 @@ class AutomationList bool shouldRecord() const; private: - // Points — sorted by timeSamples, protected by lock + // Points — sorted by timeSeconds, protected by lock std::vector points; mutable juce::CriticalSection lock; @@ -106,8 +106,8 @@ class AutomationList AutomationInterpolation interpolation { AutomationInterpolation::Linear }; - // Binary search helper — find index of last point at or before timeSamples - int findPointBefore(double timeSamples) const; + // Binary search helper — find index of last point at or before timeSeconds + int findPointBefore(double timeSeconds) const; JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(AutomationList) }; diff --git a/Source/LpcEnvelopeTransfer.cpp b/Source/LpcEnvelopeTransfer.cpp new file mode 100644 index 0000000..fc687d9 --- /dev/null +++ b/Source/LpcEnvelopeTransfer.cpp @@ -0,0 +1,283 @@ +#include "LpcEnvelopeTransfer.h" + +#include + +#include +#include +#include + +namespace +{ +static bool computeLpc (const std::vector& frame, + int order, + std::vector& lpcOut, + float& errorOut) +{ + const int frameSize = static_cast (frame.size()); + if (frameSize <= order + 1 || order <= 0) + return false; + + std::vector autocorrelation (static_cast (order + 1), 0.0f); + for (int lag = 0; lag <= order; ++lag) + { + double sum = 0.0; + for (int i = 0; i < frameSize - lag; ++i) + sum += static_cast (frame[static_cast (i)]) + * static_cast (frame[static_cast (i + lag)]); + autocorrelation[static_cast (lag)] = static_cast (sum); + } + + if (autocorrelation[0] <= 1.0e-8f) + return false; + + std::vector a (static_cast (order + 1), 0.0f); + std::vector nextA (static_cast (order + 1), 0.0f); + a[0] = 1.0f; + float error = autocorrelation[0]; + + for (int i = 1; i <= order; ++i) + { + double reflectionNumerator = autocorrelation[static_cast (i)]; + for (int j = 1; j < i; ++j) + reflectionNumerator += static_cast (a[static_cast (j)]) + * autocorrelation[static_cast (i - j)]; + + const float reflection = juce::jlimit ( + -0.995f, + 0.995f, + static_cast (-reflectionNumerator / std::max (1.0e-8, static_cast (error)))); + if (! std::isfinite (reflection)) + return false; + nextA = a; + nextA[static_cast (i)] = reflection; + + for (int j = 1; j < i; ++j) + { + nextA[static_cast (j)] = a[static_cast (j)] + reflection * a[static_cast (i - j)]; + if (! std::isfinite (nextA[static_cast (j)])) + return false; + } + + error *= std::max (1.0e-5f, 1.0f - reflection * reflection); + if (! std::isfinite (error) || error <= 0.0f) + return false; + a.swap (nextA); + } + + lpcOut.assign (a.begin() + 1, a.end()); + for (const auto coeff : lpcOut) + { + if (! std::isfinite (coeff)) + return false; + } + errorOut = std::max (error, 1.0e-8f); + return true; +} + +static void lpcToEnvelope (const std::vector& lpc, + float errorPower, + int fftSize, + std::vector& envelope) +{ + const int halfBins = fftSize / 2 + 1; + envelope.assign (static_cast (halfBins), 1.0f); + const float gain = std::sqrt (std::max (errorPower, 1.0e-8f)); + + for (int bin = 0; bin < halfBins; ++bin) + { + const float omega = juce::MathConstants::twoPi + * static_cast (bin) / static_cast (fftSize); + std::complex denominator (1.0f, 0.0f); + + for (size_t k = 0; k < lpc.size(); ++k) + { + const float phase = -omega * static_cast (k + 1); + denominator += std::complex (std::cos (phase), std::sin (phase)) * lpc[k]; + } + + const float denomAbs = std::max (std::abs (denominator), 1.0e-5f); + const float value = gain / denomAbs; + envelope[static_cast (bin)] = std::isfinite (value) ? value : 1.0f; + } +} +} + +bool LpcEnvelopeTransfer::applyToBuffer (std::vector>& processed, + const float* const* original, + int numChannels, + int numSamples) +{ + return applyToBuffer (processed, original, numChannels, numSamples, Settings {}); +} + +bool LpcEnvelopeTransfer::applyToBuffer (std::vector>& processed, + const float* const* original, + int numChannels, + int numSamples, + const Settings& settings) +{ + if (numChannels <= 0 || numSamples <= 0 || original == nullptr || processed.size() != static_cast (numChannels)) + return false; + + const int fftOrder = juce::jlimit (8, 11, settings.fftOrder); + const int fftSize = 1 << fftOrder; + const int hopSize = juce::jlimit (std::max (1, fftSize / 8), fftSize, settings.hopSize); + const int lpcOrder = juce::jlimit (8, 24, settings.lpcOrder); + const float epsilonFloor = std::max (1.0e-5f, settings.epsilonFloor); + const float maxCorrectionGain = std::max (1.0f, settings.maxCorrectionGain); + + juce::dsp::FFT fft (fftOrder); + std::vector window (static_cast (fftSize), 0.0f); + for (int i = 0; i < fftSize; ++i) + window[static_cast (i)] = 0.5f * (1.0f - std::cos ( + juce::MathConstants::twoPi * static_cast (i) / static_cast (fftSize - 1))); + + bool used = false; + + for (int ch = 0; ch < numChannels; ++ch) + { + auto& channel = processed[static_cast (ch)]; + if (static_cast (channel.size()) != numSamples) + continue; + + std::vector overlapAdd (static_cast (numSamples), 0.0f); + std::vector windowSum (static_cast (numSamples), 0.0f); + std::vector inputFrame (static_cast (fftSize), 0.0f); + std::vector outputFrame (static_cast (fftSize), 0.0f); + std::vector inputLpc; + std::vector outputLpc; + std::vector inputEnvelope; + std::vector outputEnvelope; + std::vector> outputTime (static_cast (fftSize)); + std::vector> outputFreq (static_cast (fftSize)); + std::vector> correctedTime (static_cast (fftSize)); + + for (int pos = 0; pos < numSamples; pos += hopSize) + { + for (int i = 0; i < fftSize; ++i) + { + const int index = pos + i - fftSize / 2; + const float source = (index >= 0 && index < numSamples) ? original[ch][index] : 0.0f; + const float target = (index >= 0 && index < numSamples) ? channel[static_cast (index)] : 0.0f; + inputFrame[static_cast (i)] = source * window[static_cast (i)]; + outputFrame[static_cast (i)] = target * window[static_cast (i)]; + } + + float inputEnergy = 0.0f; + float outputEnergy = 0.0f; + for (int i = 0; i < fftSize; ++i) + { + inputEnergy += inputFrame[static_cast (i)] * inputFrame[static_cast (i)]; + outputEnergy += outputFrame[static_cast (i)] * outputFrame[static_cast (i)]; + } + + if (inputEnergy < 1.0e-6f || outputEnergy < 1.0e-6f) + { + for (int i = 0; i < fftSize; ++i) + { + const int index = pos + i - fftSize / 2; + if (index >= 0 && index < numSamples) + { + const float w = window[static_cast (i)]; + overlapAdd[static_cast (index)] += channel[static_cast (index)] * w; + windowSum[static_cast (index)] += w * w; + } + } + continue; + } + + float inputError = 0.0f; + float outputError = 0.0f; + if (! computeLpc (inputFrame, lpcOrder, inputLpc, inputError) + || ! computeLpc (outputFrame, lpcOrder, outputLpc, outputError)) + { + for (int i = 0; i < fftSize; ++i) + { + const int index = pos + i - fftSize / 2; + if (index >= 0 && index < numSamples) + { + const float w = window[static_cast (i)]; + overlapAdd[static_cast (index)] += channel[static_cast (index)] * w; + windowSum[static_cast (index)] += w * w; + } + } + continue; + } + + lpcToEnvelope (inputLpc, inputError, fftSize, inputEnvelope); + lpcToEnvelope (outputLpc, outputError, fftSize, outputEnvelope); + bool validEnvelope = true; + for (int bin = 0; bin < fftSize / 2 + 1; ++bin) + { + if (! std::isfinite (inputEnvelope[static_cast (bin)]) + || ! std::isfinite (outputEnvelope[static_cast (bin)])) + { + validEnvelope = false; + break; + } + } + if (! validEnvelope) + { + for (int i = 0; i < fftSize; ++i) + { + const int index = pos + i - fftSize / 2; + if (index >= 0 && index < numSamples) + { + const float w = window[static_cast (i)]; + overlapAdd[static_cast (index)] += channel[static_cast (index)] * w; + windowSum[static_cast (index)] += w * w; + } + } + continue; + } + + for (int i = 0; i < fftSize; ++i) + outputTime[static_cast (i)] = { outputFrame[static_cast (i)], 0.0f }; + fft.perform (outputTime.data(), outputFreq.data(), false); + + const int halfBins = fftSize / 2 + 1; + for (int bin = 0; bin < halfBins; ++bin) + { + const float correction = juce::jlimit ( + 1.0f / maxCorrectionGain, + maxCorrectionGain, + (inputEnvelope[static_cast (bin)] + epsilonFloor) + / (outputEnvelope[static_cast (bin)] + epsilonFloor)); + if (! std::isfinite (correction)) + continue; + outputFreq[static_cast (bin)] *= correction; + if (bin > 0 && bin < halfBins - 1) + outputFreq[static_cast (fftSize - bin)] = std::conj (outputFreq[static_cast (bin)]); + } + + fft.perform (outputFreq.data(), correctedTime.data(), true); + const float inverseScale = 1.0f / static_cast (fftSize); + for (int i = 0; i < fftSize; ++i) + { + const int index = pos + i - fftSize / 2; + if (index >= 0 && index < numSamples) + { + const float w = window[static_cast (i)]; + const float correctedSample = correctedTime[static_cast (i)].real() * inverseScale; + if (! std::isfinite (correctedSample)) + continue; + overlapAdd[static_cast (index)] += correctedSample * w; + windowSum[static_cast (index)] += w * w; + } + } + used = true; + } + + if (used) + { + for (int i = 0; i < numSamples; ++i) + { + const float normalizer = windowSum[static_cast (i)]; + if (normalizer > 1.0e-5f) + channel[static_cast (i)] = overlapAdd[static_cast (i)] / normalizer; + } + } + } + + return used; +} diff --git a/Source/LpcEnvelopeTransfer.h b/Source/LpcEnvelopeTransfer.h new file mode 100644 index 0000000..486ce9c --- /dev/null +++ b/Source/LpcEnvelopeTransfer.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +class LpcEnvelopeTransfer +{ +public: + struct Settings + { + int lpcOrder = 16; + int fftOrder = 9; // 512 samples + int hopSize = 256; + float epsilonFloor = 1.0e-3f; + float maxCorrectionGain = 2.2f; + }; + + static bool applyToBuffer (std::vector>& processed, + const float* const* original, + int numChannels, + int numSamples); + + static bool applyToBuffer (std::vector>& processed, + const float* const* original, + int numChannels, + int numSamples, + const Settings& settings); +}; diff --git a/Source/Main.cpp b/Source/Main.cpp index 2134349..e8281c2 100644 --- a/Source/Main.cpp +++ b/Source/Main.cpp @@ -5,6 +5,9 @@ #include "MainComponent.h" #include "MixerWindowManager.h" +#include +#include + #if JUCE_WINDOWS #include #endif @@ -13,9 +16,6 @@ namespace { bool commandLineHasFlag(const juce::String& commandLine, const juce::String& flag) { - if (commandLine.contains(flag)) - return true; - juce::StringArray tokens; tokens.addTokens(commandLine, " ", "\""); for (const auto& token : tokens) @@ -29,21 +29,23 @@ bool commandLineHasFlag(const juce::String& commandLine, const juce::String& fla juce::String getCommandLineOptionValue(const juce::String& commandLine, const juce::String& option) { - const auto optionIndex = commandLine.indexOf(option); - if (optionIndex < 0) - return {}; - - auto remainder = commandLine.substring(optionIndex + option.length()).trimStart(); - if (remainder.isEmpty()) - return {}; + juce::StringArray tokens; + tokens.addTokens(commandLine, " ", "\""); + tokens.trim(); + tokens.removeEmptyStrings(); - if (remainder.startsWithChar('"')) + for (int i = 0; i < tokens.size(); ++i) { - remainder = remainder.substring(1); - return remainder.upToFirstOccurrenceOf("\"", false, false); + const auto token = tokens[i].trim().unquoted(); + if (token == option) + return i + 1 < tokens.size() ? tokens[i + 1].trim().unquoted() : juce::String(); + + const auto equalsPrefix = option + "="; + if (token.startsWith(equalsPrefix)) + return token.fromFirstOccurrenceOf(equalsPrefix, false, false).trim().unquoted(); } - return remainder.upToFirstOccurrenceOf(" ", false, false); + return {}; } juce::File getWritableStartupLogFile() @@ -83,6 +85,385 @@ juce::var rectangleToVar(const juce::Rectangle& bounds) obj->setProperty("height", bounds.getHeight()); return juce::var(obj); } + +bool isFiniteNumericVar(const juce::var& value) +{ + if (! (value.isDouble() || value.isInt() || value.isInt64())) + return false; + + return std::isfinite(static_cast(value)); +} + +double getNumericProperty(const juce::var& object, const juce::Identifier& propertyName, double fallback) +{ + const auto value = object.getProperty(propertyName, juce::var()); + return isFiniteNumericVar(value) ? static_cast(value) : fallback; +} + +void addHarnessCheck(juce::Array& checks, + const juce::String& id, + const juce::String& status, + const juce::String& detail, + const juce::var& value = juce::var()) +{ + auto* obj = new juce::DynamicObject(); + obj->setProperty("id", id); + obj->setProperty("status", status); + obj->setProperty("detail", detail); + if (! value.isVoid()) + obj->setProperty("value", value); + checks.add(juce::var(obj)); +} + +void setProcessEnvironmentVariable(const juce::String& name, const juce::String& value) +{ + #if JUCE_WINDOWS + _putenv_s(name.toRawUTF8(), value.toRawUTF8()); + #else + setenv(name.toRawUTF8(), value.toRawUTF8(), 1); + #endif +} + +bool hasFailedHarnessCheck(const juce::Array& checks) +{ + for (const auto& check : checks) + if (check.getProperty("status", {}).toString() == "fail") + return true; + + return false; +} + +bool writeHeadlessResult(const juce::File& resultFile, const juce::var& result) +{ + if (resultFile == juce::File()) + return false; + + resultFile.getParentDirectory().createDirectory(); + return resultFile.replaceWithText(juce::JSON::toString(result, true)); +} + +int runHeadlessPitchRegressionJob(AudioEngine& audioEngine, const juce::String& jobPath) +{ + setProcessEnvironmentVariable("OPENSTUDIO_PITCH_HEADLESS", "1"); + setProcessEnvironmentVariable("OPENSTUDIO_PITCH_APP_FINAL_CAPTURE_DISABLE", "1"); + + const juce::File jobFile(jobPath.trim().unquoted()); + juce::File resultFile; + juce::Array checks; + + auto makeBaseResult = [&]() { + auto* obj = new juce::DynamicObject(); + obj->setProperty("harnessMode", "headless_lightweight"); + obj->setProperty("claimLevel", "objective_only"); + obj->setProperty("subjectiveQuality", "not_asserted"); + obj->setProperty("completionClaim", "objective gates may pass; subjective audio quality is not asserted; user audition required"); + obj->setProperty("jobPath", jobFile.getFullPathName()); + obj->setProperty("capturedAt", juce::Time::getCurrentTime().toISO8601(true)); + return juce::DynamicObject::Ptr(obj); + }; + + auto fail = [&](const juce::String& message) { + addHarnessCheck(checks, "headless_job", "fail", message); + auto resultObj = makeBaseResult(); + resultObj->setProperty("success", false); + resultObj->setProperty("objectiveGateStatus", "fail"); + resultObj->setProperty("error", message); + resultObj->setProperty("checks", juce::var(checks)); + if (resultFile != juce::File()) + writeHeadlessResult(resultFile, juce::var(resultObj.get())); + juce::Logger::writeToLog("[pitchRegression.headless] " + message); + return 2; + }; + + if (! jobFile.existsAsFile()) + return fail("Headless pitch regression job file not found: " + jobFile.getFullPathName()); + + auto job = juce::JSON::parse(jobFile); + if (! job.isObject()) + return fail("Headless pitch regression job JSON could not be parsed: " + jobFile.getFullPathName()); + + const auto resultPath = job.getProperty("resultJsonPath", {}).toString().trim().unquoted(); + if (resultPath.isNotEmpty()) + resultFile = juce::File(resultPath); + + const auto jobType = job.getProperty("jobType", "render").toString(); + if (jobType != "render") + return fail("Headless lightweight harness only supports jobType='render'; got '" + jobType + "'"); + + const auto sourceAudioPath = job.getProperty("sourceAudioPath", {}).toString().trim().unquoted(); + const auto trackId = job.getProperty("trackId", "pitch-regression-track-1").toString(); + const auto clipId = job.getProperty("clipId", "pitch-regression-clip-1").toString(); + const auto renderMode = job.getProperty("renderMode", "note_hq").toString(); + if (sourceAudioPath.isEmpty()) + return fail("Headless job is missing sourceAudioPath"); + if (trackId.isEmpty() || clipId.isEmpty()) + return fail("Headless job is missing trackId or clipId"); + + const juce::File sourceFile(sourceAudioPath); + if (! sourceFile.existsAsFile()) + return fail("Source audio file not found: " + sourceFile.getFullPathName()); + + juce::AudioFormatManager formatManager; + formatManager.registerBasicFormats(); + std::unique_ptr sourceReader(formatManager.createReaderFor(sourceFile)); + if (sourceReader == nullptr || sourceReader->sampleRate <= 0.0 || sourceReader->lengthInSamples <= 0) + return fail("Could not read source audio metadata: " + sourceFile.getFullPathName()); + + const double sourceDurationSec = static_cast(sourceReader->lengthInSamples) / sourceReader->sampleRate; + const int sourceChannels = static_cast(sourceReader->numChannels); + sourceReader.reset(); + + auto notes = juce::JSON::parse(juce::JSON::toString(job.getProperty("notes", juce::var()), false)); + auto* noteArray = notes.getArray(); + if (noteArray == nullptr && notes.isObject()) + { + juce::Array wrappedNotes; + wrappedNotes.add(notes); + notes = juce::var(wrappedNotes); + noteArray = notes.getArray(); + } + if (noteArray == nullptr || noteArray->isEmpty()) + return fail("Headless render job requires a non-empty notes array"); + + const auto targetShiftVar = job.getProperty("targetShiftSemitones", juce::var()); + const bool hasTargetShift = isFiniteNumericVar(targetShiftVar); + double actualRequestedShift = 0.0; + double maxShiftErrorSemitones = 0.0; + if (hasTargetShift) + { + const double targetShift = static_cast(targetShiftVar); + for (auto& note : *noteArray) + { + auto* noteObj = note.getDynamicObject(); + if (noteObj == nullptr) + return fail("Each note must be a JSON object"); + + const auto detectedPitchVar = note.getProperty("detectedPitch", juce::var()); + if (! isFiniteNumericVar(detectedPitchVar)) + return fail("Cannot apply targetShiftSemitones because a note is missing numeric detectedPitch"); + + const double detectedPitch = static_cast(detectedPitchVar); + const double correctedPitch = detectedPitch + targetShift; + noteObj->setProperty("detectedPitch", detectedPitch); + noteObj->setProperty("correctedPitch", correctedPitch); + actualRequestedShift += correctedPitch - detectedPitch; + maxShiftErrorSemitones = juce::jmax(maxShiftErrorSemitones, + std::abs((correctedPitch - detectedPitch) - targetShift)); + } + + actualRequestedShift /= static_cast(noteArray->size()); + const double maxErrorCents = maxShiftErrorSemitones * 100.0; + addHarnessCheck(checks, + "exact_relative_pitch_shift", + maxErrorCents <= 1.0 ? "pass" : "fail", + "Requested shift is computed as detectedPitch + targetShiftSemitones, without chromatic snapping.", + maxErrorCents); + } + else + { + addHarnessCheck(checks, + "exact_relative_pitch_shift", + "not_asserted", + "Job did not provide targetShiftSemitones; exact relative pitch shift cannot be asserted."); + } + + audioEngine.addTrack(trackId); + audioEngine.setMasterVolume(1.0f); + audioEngine.setMasterPan(0.0f); + audioEngine.setTrackVolume(trackId, 0.0f); + audioEngine.setTrackPan(trackId, 0.0f); + audioEngine.clearPlaybackClips(); + audioEngine.addPlaybackClip(trackId, sourceFile.getFullPathName(), 0.0, sourceDurationSec, 0.0, 0.0, 0.0, 0.0, clipId); + + std::optional windowStartSec; + std::optional windowEndSec; + const auto windowStartVar = job.getProperty("windowStartSec", juce::var()); + const auto windowEndVar = job.getProperty("windowEndSec", juce::var()); + if (isFiniteNumericVar(windowStartVar) && isFiniteNumericVar(windowEndVar)) + { + windowStartSec = static_cast(windowStartVar); + windowEndSec = static_cast(windowEndVar); + } + + const auto frames = job.getProperty("frames", juce::var()); + const float globalFormantSemitones = static_cast( + getNumericProperty(job, "globalFormantSemitones", 0.0)); + + juce::Logger::writeToLog("[pitchRegression.headless] Running render job clip=" + clipId + + " renderMode=" + renderMode + + " source=" + sourceFile.getFullPathName()); + + auto nativeResult = audioEngine.applyPitchCorrection(trackId, + clipId, + notes, + frames, + globalFormantSemitones, + windowStartSec, + windowEndSec, + renderMode); + + const bool nativeSuccess = nativeResult.isObject() + && static_cast(nativeResult.getProperty("success", false)); + if (! nativeSuccess) + addHarnessCheck(checks, "native_render_success", "fail", "AudioEngine::applyPitchCorrection did not return success."); + else + addHarnessCheck(checks, "native_render_success", "pass", "AudioEngine::applyPitchCorrection returned success."); + + const auto outputPath = nativeResult.getProperty("outputFile", {}).toString(); + const juce::File outputFile(outputPath); + const bool outputExists = outputPath.isNotEmpty() && outputFile.existsAsFile(); + addHarnessCheck(checks, + "output_file_exists", + outputExists ? "pass" : "fail", + outputExists ? "Corrected output file exists." : "Corrected output file is missing.", + outputPath); + + if (outputExists) + { + std::unique_ptr outputReader(formatManager.createReaderFor(outputFile)); + if (outputReader != nullptr && outputReader->sampleRate > 0.0) + { + const double outputDurationSec = static_cast(outputReader->lengthInSamples) / outputReader->sampleRate; + const double durationDeltaMs = std::abs(outputDurationSec - sourceDurationSec) * 1000.0; + addHarnessCheck(checks, + "output_duration_sane", + durationDeltaMs <= 5.0 ? "pass" : "fail", + "Corrected full-clip output duration should match source duration within 5 ms.", + durationDeltaMs); + addHarnessCheck(checks, + "output_channels_sane", + static_cast(outputReader->numChannels) == sourceChannels ? "pass" : "fail", + "Corrected output channel count should match source channel count.", + static_cast(outputReader->numChannels)); + } + else + { + addHarnessCheck(checks, "output_duration_sane", "fail", "Corrected output file could not be read."); + addHarnessCheck(checks, "output_channels_sane", "fail", "Corrected output file could not be read."); + } + } + + const auto actualRendererBranch = nativeResult.getProperty("actualRendererBranch", {}).toString(); + addHarnessCheck(checks, + "renderer_branch_recorded", + actualRendererBranch.isNotEmpty() ? "pass" : "fail", + actualRendererBranch.isNotEmpty() ? "Renderer branch was reported." : "Renderer branch was not reported.", + actualRendererBranch); + + const auto formantCurveUsedVar = nativeResult.getProperty("formantCurveUsed", juce::var()); + const bool formantCurveRecorded = formantCurveUsedVar.isBool(); + const bool formantCurveUsed = formantCurveRecorded && static_cast(formantCurveUsedVar); + addHarnessCheck(checks, + "pitch_only_formant_curve_disabled", + formantCurveRecorded && ! formantCurveUsed ? "pass" : "fail", + formantCurveRecorded + ? "Pitch-only render reported formantCurveUsed=false." + : "Pitch-only render did not report formantCurveUsed.", + formantCurveRecorded ? juce::var(formantCurveUsed) : juce::var()); + + const auto routeStatus = nativeResult.getProperty("postApplyRouteStatus", juce::var()); + if (routeStatus.isObject()) + { + const bool routeClean = routeStatus.getProperty("monitorMode", {}).toString() == "corrected_source" + && ! static_cast(routeStatus.getProperty("renderedSegmentActive", false)) + && ! static_cast(routeStatus.getProperty("clipLivePreviewActive", false)) + && ! static_cast(routeStatus.getProperty("scrubPreviewActive", false)); + addHarnessCheck(checks, + "corrected_source_route_clean", + routeClean ? "pass" : "fail", + "After note-HQ render, corrected source should be active with preview/scrub/rendered-segment routes inactive.", + routeStatus); + } + else + { + addHarnessCheck(checks, + "corrected_source_route_clean", + "not_asserted", + "Native render did not report postApplyRouteStatus."); + } + + addHarnessCheck(checks, + "subjective_audio_quality", + "not_asserted", + "Harness cannot assert naturalness, robotic tone, doubled voice, stutter feel, or target-sample closeness. User audition is required."); + addHarnessCheck(checks, + "spectral_similarity", + "diagnostic_only", + "Mel/formant/spectrogram similarity is intentionally not a pass/fail gate in the lightweight harness."); + + const bool failed = hasFailedHarnessCheck(checks); + auto resultObj = makeBaseResult(); + resultObj->setProperty("success", nativeSuccess && ! failed); + resultObj->setProperty("objectiveGateStatus", failed ? "fail" : "pass"); + resultObj->setProperty("done", false); + resultObj->setProperty("targetShiftSemitones", hasTargetShift ? targetShiftVar : juce::var()); + resultObj->setProperty("actualRequestedShiftSemitones", hasTargetShift ? juce::var(actualRequestedShift) : juce::var()); + resultObj->setProperty("requestedShiftErrorCents", hasTargetShift ? juce::var(maxShiftErrorSemitones * 100.0) : juce::var()); + resultObj->setProperty("chromaticSnapBypassed", hasTargetShift); + resultObj->setProperty("outputFile", outputPath); + resultObj->setProperty("actualRendererBranch", actualRendererBranch); + resultObj->setProperty("formantCurveUsed", formantCurveRecorded ? juce::var(formantCurveUsed) : juce::var()); + const char* const vsfDiagnosticKeys[] = { + "vocalSourceFilterResidualMix", + "vocalSourceFilterResidualMixScale", + "vocalSourceFilterEpochInterpolationUsed", + "vocalSourceFilterEpochInterpolationStrength", + "vocalSourceFilterGrainRadiusScale", + "vocalSourceFilterUpPresenceTrimDb", + "vocalSourceFilterUpPresenceHz", + "vocalSourceFilterDownNasalTrimDb", + "vocalSourceFilterDownNasalHz", + "vocalSourceFilterDownBodyCompDb", + "vocalSourceFilterDownBodyCompHz" + }; + for (const auto* key : vsfDiagnosticKeys) + { + const auto value = nativeResult.getProperty(key, juce::var()); + if (! value.isVoid()) + resultObj->setProperty(key, value); + } + resultObj->setProperty("nativeResult", nativeResult); + resultObj->setProperty("checks", juce::var(checks)); + + if (resultFile != juce::File()) + { + const auto routeReportFile = resultFile.getSiblingFile( + resultFile.getFileNameWithoutExtension() + "_route.json"); + auto* routeObj = new juce::DynamicObject(); + routeObj->setProperty("purpose", "headless_pitch_route_report"); + routeObj->setProperty("harnessMode", "headless_lightweight"); + routeObj->setProperty("trackId", trackId); + routeObj->setProperty("clipId", clipId); + routeObj->setProperty("sourceAudioPath", sourceFile.getFullPathName()); + routeObj->setProperty("outputFile", outputPath); + routeObj->setProperty("renderMode", renderMode); + routeObj->setProperty("targetShiftSemitones", hasTargetShift ? targetShiftVar : juce::var()); + routeObj->setProperty("actualRequestedShiftSemitones", hasTargetShift ? juce::var(actualRequestedShift) : juce::var()); + routeObj->setProperty("requestedShiftErrorCents", hasTargetShift ? juce::var(maxShiftErrorSemitones * 100.0) : juce::var()); + routeObj->setProperty("chromaticSnapBypassed", hasTargetShift); + routeObj->setProperty("actualRendererBranch", actualRendererBranch); + routeObj->setProperty("formantCurveUsed", formantCurveRecorded ? juce::var(formantCurveUsed) : juce::var()); + for (const auto* key : vsfDiagnosticKeys) + { + const auto value = nativeResult.getProperty(key, juce::var()); + if (! value.isVoid()) + routeObj->setProperty(key, value); + } + routeObj->setProperty("postApplyRouteStatus", routeStatus); + routeObj->setProperty("objectiveGateStatus", failed ? "fail" : "pass"); + routeObj->setProperty("subjectiveQuality", "not_asserted"); + routeObj->setProperty("checks", juce::var(checks)); + routeReportFile.replaceWithText(juce::JSON::toString(juce::var(routeObj), true)); + resultObj->setProperty("routeReportPath", routeReportFile.getFullPathName()); + } + + if (! writeHeadlessResult(resultFile, juce::var(resultObj.get()))) + return fail("Could not write headless result JSON: " + resultFile.getFullPathName()); + + juce::Logger::writeToLog("[pitchRegression.headless] Wrote result to: " + resultFile.getFullPathName() + + " objectiveGateStatus=" + juce::String(failed ? "fail" : "pass")); + return failed ? 2 : 0; +} } //============================================================================== @@ -97,9 +478,19 @@ class OpenStudioApplication : public juce::JUCEApplication void initialise (const juce::String& commandLine) override { + // Raise process priority so the audio thread is less likely to be preempted + // by competing background processes. ASIO drivers handle thread priority + // themselves (via MMCSS), but HIGH_PRIORITY_CLASS reduces scheduler jitter + // from other apps at 32-sample buffer sizes. + #if JUCE_WINDOWS + ::SetPriorityClass(::GetCurrentProcess(), HIGH_PRIORITY_CLASS); + #endif + OpenStudioLaunchState::setPendingProjectPath(commandLine); const auto startupSelfTestMode = commandLineHasFlag(commandLine, "--startup-self-test"); const auto startupSelfTestReportPath = getCommandLineOptionValue(commandLine, "--report"); + const auto pitchRegressionHeadlessJobPath = getCommandLineOptionValue(commandLine, "--pitch-regression-headless"); + const auto pitchRegressionJobPath = getCommandLineOptionValue(commandLine, "--pitch-regression"); startupMode = commandLineHasFlag(commandLine, "--ui-safe-mode") ? MainComponent::StartupMode::safe : MainComponent::StartupMode::normal; @@ -109,6 +500,16 @@ class OpenStudioApplication : public juce::JUCEApplication juce::Logger::writeToLog("Application Initialising..."); juce::Logger::writeToLog("Startup log path: " + logFile.getFullPathName()); juce::Logger::writeToLog("Startup mode: " + juce::String(startupMode == MainComponent::StartupMode::safe ? "safe" : "normal")); + if (pitchRegressionJobPath.isNotEmpty()) + { + juce::Logger::writeToLog("Pitch regression job path: " + pitchRegressionJobPath); + juce::Logger::writeToLog("OPENSTUDIO_PITCH_DEBUG=" + juce::SystemStats::getEnvironmentVariable("OPENSTUDIO_PITCH_DEBUG", "")); + } + if (pitchRegressionHeadlessJobPath.isNotEmpty()) + { + juce::Logger::writeToLog("Pitch regression headless job path: " + pitchRegressionHeadlessJobPath); + juce::Logger::writeToLog("OPENSTUDIO_PITCH_DEBUG=" + juce::SystemStats::getEnvironmentVariable("OPENSTUDIO_PITCH_DEBUG", "")); + } if (startupSelfTestMode) { @@ -123,6 +524,14 @@ class OpenStudioApplication : public juce::JUCEApplication return; } + if (pitchRegressionHeadlessJobPath.isNotEmpty()) + { + const auto exitCode = runHeadlessPitchRegressionJob(audioEngine, pitchRegressionHeadlessJobPath); + setApplicationReturnValue(exitCode); + quit(); + return; + } + mixerWindowManager = std::make_unique( [this]() { @@ -141,7 +550,8 @@ class OpenStudioApplication : public juce::JUCEApplication audioEngine, appUpdater, startupMode, - createWindowCallbacks()); + createWindowCallbacks(), + pitchRegressionJobPath); if (auto* component = mainWindow->getMainComponent()) audioEngine.setPluginWindowOwnerComponent(component); @@ -197,7 +607,8 @@ class OpenStudioApplication : public juce::JUCEApplication AudioEngine& audioEngine, AppUpdater& appUpdater, MainComponent::StartupMode startupMode, - MainComponent::WindowCallbacks callbacks) + MainComponent::WindowCallbacks callbacks, + const juce::String& pitchRegressionJobPath = {}) : DocumentWindow (name, juce::Colours::black, #if JUCE_MAC @@ -216,7 +627,8 @@ class OpenStudioApplication : public juce::JUCEApplication appUpdater, startupMode, MainComponent::WindowRole::main, - std::move(callbacks)), + std::move(callbacks), + pitchRegressionJobPath), true); #if JUCE_IOS || JUCE_ANDROID diff --git a/Source/MainComponent.cpp b/Source/MainComponent.cpp index 7710858..1389e1b 100644 --- a/Source/MainComponent.cpp +++ b/Source/MainComponent.cpp @@ -18,15 +18,17 @@ namespace { constexpr int kFrontendStartupTimeoutMs = 8000; +static bool shouldEnablePitchEditorFormantDebugLogs() +{ #if JUCE_DEBUG -constexpr bool kPitchEditorFormantDebugLogs = true; + return true; #else -constexpr bool kPitchEditorFormantDebugLogs = false; + return juce::SystemStats::getEnvironmentVariable ("OPENSTUDIO_PITCH_DEBUG", {}).trim() == "1"; #endif - +} static void logPitchEditorFormant(const juce::String& message) { - if (kPitchEditorFormantDebugLogs) + if (shouldEnablePitchEditorFormantDebugLogs()) juce::Logger::writeToLog ("[pitchEditor.formant] " + message); } @@ -540,8 +542,43 @@ juce::String getWindowChromeQueryValue(MainComponent::WindowRole role) bool isLocalFrontendDevServerReachable() { - juce::StreamingSocket socket; - return socket.connect("127.0.0.1", 5173, 750); + if (juce::SystemStats::getEnvironmentVariable ("OPENSTUDIO_FORCE_PACKAGED_FRONTEND", {}).trim() == "1") + { + juce::Logger::writeToLog("OPENSTUDIO_FORCE_PACKAGED_FRONTEND=1; loading the packaged frontend."); + return false; + } + + int statusCode = 0; + auto input = juce::URL("http://127.0.0.1:5173/").createInputStream( + juce::URL::InputStreamOptions(juce::URL::ParameterHandling::inAddress) + .withConnectionTimeoutMs(750) + .withNumRedirectsToFollow(1) + .withStatusCode(&statusCode)); + + if (input == nullptr) + return false; + + if (statusCode >= 400) + { + juce::Logger::writeToLog("localhost:5173 responded with HTTP " + juce::String(statusCode) + + "; falling back to the packaged frontend."); + return false; + } + + const auto indexHtml = input->readEntireStreamAsString(); + const bool looksLikeOpenStudioVite = + indexHtml.contains("OpenStudio") + && indexHtml.contains("id=\"root\"") + && (indexHtml.contains("/src/main.tsx") || indexHtml.contains("./src/main.tsx")); + + if (! looksLikeOpenStudioVite) + { + juce::Logger::writeToLog("localhost:5173 is reachable, but it did not return the OpenStudio Vite index; " + "falling back to the packaged frontend."); + return false; + } + + return true; } juce::String appendFrontendStartupQuery(const juce::String& baseUrl, @@ -559,6 +596,7 @@ juce::String appendFrontendStartupQuery(const juce::String& baseUrl, appendParameter("startup", getStartupModeQueryValue(startupMode)); appendParameter("platform", getHostPlatformQueryValue()); appendParameter("windowChrome", getWindowChromeQueryValue(role)); + appendParameter("cacheBust", juce::String(juce::Time::getCurrentTime().toMilliseconds())); return url; } @@ -708,7 +746,8 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, AppUpdater& appUpdaterIn, StartupMode startupModeIn, WindowRole roleIn, - WindowCallbacks callbacksIn) + WindowCallbacks callbacksIn, + const juce::String& pitchRegressionJobPathIn) : audioEngine(audioEngineIn), appUpdater(appUpdaterIn), startupMode(startupModeIn), @@ -853,12 +892,17 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, return function(...args) { return new Promise((resolve, reject) => { const resultId = Date.now() * 1000 + Math.floor(Math.random() * 1000); - - // Timeout after 15 seconds (audio device enumeration can take time) + + // Dialog functions are interactive — user may spend several minutes navigating. + // Worker-backed startup calls may also legitimately take longer than the default bridge timeout. + // Use a 5-minute timeout for file choosers and AI generation startup, 15 seconds for everything else. + const DIALOG_FUNCTIONS = ['showRenderSaveDialog', 'showSaveDialog', 'showOpenDialog', 'showOpenFileDialog', 'showDirectoryDialog']; + const LONG_RUNNING_FUNCTIONS = ['startAIGeneration']; + const timeoutMs = (DIALOG_FUNCTIONS.indexOf(name) >= 0 || LONG_RUNNING_FUNCTIONS.indexOf(name) >= 0) ? 300000 : 15000; const timeout = setTimeout(() => { window.__JUCE__.backend.removeEventListener(listener); reject(new Error("Native function call timeout: " + name)); - }, 15000); + }, timeoutMs); const listener = window.__JUCE__.backend.addEventListener('__juce__complete', (data) => { if (data.promiseId === resultId) { @@ -2029,7 +2073,7 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, } }); }) - .withNativeFunction ("saveProjectToFile", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + .withNativeFunction ("saveProjectToFile", [] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { // Save project JSON to file // Args: [filePath, jsonContent] if (args.size() == 2 && args[0].isString() && args[1].isString()) { @@ -2050,7 +2094,7 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion(false); } }) - .withNativeFunction ("loadProjectFromFile", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + .withNativeFunction ("loadProjectFromFile", [] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { // Load project JSON from file // Args: [filePath] if (args.size() == 1 && args[0].isString()) { @@ -2069,10 +2113,29 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion(""); } }) - .withNativeFunction ("consumePendingLaunchProjectPath", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + .withNativeFunction ("consumePendingLaunchProjectPath", [] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { juce::ignoreUnused(args); completion(OpenStudioLaunchState::consumePendingProjectPath()); }) + .withNativeFunction ("getPitchRegressionJob", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + juce::ignoreUnused(args); + + if (! isMainWindow() + || pitchRegressionJob.isVoid() + || pitchRegressionJobConsumed + || pitchRegressionJobCompleted) + { + completion(juce::String()); + return; + } + + pitchRegressionJobConsumed = true; + completion(juce::JSON::toString(pitchRegressionJob, false)); + }) + .withNativeFunction ("completePitchRegressionJob", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + const auto result = args.size() > 0 ? args[0] : juce::var(); + completion(completePitchRegressionJob(result)); + }) .withNativeFunction ("getPluginState", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { // Get plugin state as base64 string // Args: [trackId, fxIndex, isInputFX] @@ -2123,7 +2186,7 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion(false); } }) - .withNativeFunction ("importMediaFile", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + .withNativeFunction ("importMediaFile", [] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { // Read audio file metadata (duration, sample rate, channels, format). // For video files that JUCE can't read directly, attempts FFmpeg extraction. // Args: [filePath] @@ -2211,7 +2274,7 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion(juce::var()); } }) - .withNativeFunction ("saveDroppedFile", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + .withNativeFunction ("saveDroppedFile", [] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { // Save a base64-encoded file dropped from the OS to a temp directory. // Args: [fileName, base64Data] // Returns: the full path to the saved file, or empty string on failure. @@ -2261,11 +2324,20 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, }) .withNativeFunction ("showRenderSaveDialog", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { // Show save dialog for render/export with audio format filter - // Args: [defaultFileName, formatExtension] + // Args: [defaultFileName, formatExtension, initialDirectory?] juce::String defaultFileName = args.size() > 0 ? args[0].toString() : "untitled"; juce::String formatExt = args.size() > 1 ? args[1].toString() : "wav"; + juce::String initialDirectoryPath = args.size() > 2 ? args[2].toString() : juce::String(); - juce::File initialDir = juce::File::getSpecialLocation(juce::File::userDocumentsDirectory); + juce::File initialDir; + if (initialDirectoryPath.isNotEmpty()) + { + const auto requestedDir = juce::File(initialDirectoryPath); + if (requestedDir.isDirectory()) + initialDir = requestedDir; + } + if (! initialDir.isDirectory()) + initialDir = juce::File::getSpecialLocation(juce::File::userDocumentsDirectory); juce::String filter = "*." + formatExt; juce::String fullFileName = defaultFileName + "." + formatExt; @@ -2294,8 +2366,8 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, }) .withNativeFunction ("renderProject", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { // Render/Export project to audio file - // Args: [source, startTime, endTime, filePath, format, sampleRate, bitDepth, channels, normalize, addTail, tailLength] - if (args.size() == 11) { + // Args: [source, startTime, endTime, filePath, format, sampleRate, bitDepth, channels, normalize, addTail, tailLength, includeMetronome?] + if (args.size() >= 11) { juce::String source = args[0].toString(); double startTime = (double)args[1]; double endTime = (double)args[2]; @@ -2307,14 +2379,17 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, bool normalizeArg = (bool)args[8]; bool addTail = (bool)args[9]; double tailLength = (double)args[10]; + bool includeMetronome = args.size() >= 12 ? (bool)args[11] : false; // Run on background thread to avoid blocking message thread std::thread([this, source, startTime, endTime, filePathArg, format, sampleRate, bitDepth, channels, normalizeArg, addTail, tailLength, + includeMetronome, completion = std::make_shared<juce::WebBrowserComponent::NativeFunctionCompletion>(std::move(completion))]() { bool success = audioEngine.renderProject( source, startTime, endTime, filePathArg, format, - sampleRate, bitDepth, channels, normalizeArg, addTail, tailLength); + sampleRate, bitDepth, channels, normalizeArg, addTail, tailLength, + includeMetronome); // Call completion on the message thread to avoid crash // (WebView callbacks must not be invoked from background threads) juce::MessageManager::callAsync([completion, success]() { @@ -2326,9 +2401,153 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion(false); } }) + .withNativeFunction ("capturePitchAuditionPlayback", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + // Capture the current live playback-engine source for a clip after pitch apply. + // Args: [trackId, clipId, startTime, duration, filePath, sampleRate, offlineRenderMode?] + if (args.size() >= 5) { + juce::String trackId = args[0].toString(); + juce::String clipId = args[1].toString(); + double startTime = static_cast<double>(args[2]); + double duration = static_cast<double>(args[3]); + juce::String filePathArg = args[4].toString(); + double sampleRate = args.size() >= 6 ? static_cast<double>(args[5]) : 44100.0; + bool offlineRenderMode = args.size() >= 7 ? static_cast<bool>(args[6]) : true; + + std::thread([this, trackId, clipId, startTime, duration, filePathArg, sampleRate, + offlineRenderMode, + completion = std::make_shared<juce::WebBrowserComponent::NativeFunctionCompletion>(std::move(completion))]() { + auto result = audioEngine.capturePitchAuditionPlayback(trackId, clipId, startTime, duration, filePathArg, sampleRate, offlineRenderMode); + juce::MessageManager::callAsync([completion, result]() { + (*completion)(result); + }); + }).detach(); + } else { + juce::Logger::writeToLog("capturePitchAuditionPlayback: Invalid args count: " + juce::String(args.size())); + completion(juce::var()); + } + }) + .withNativeFunction ("capturePitchAppFinalContext", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + // Capture the actual playback-engine route after final note-HQ apply. + // Args: [trackId, clipId, startTime, duration, wavPath, routeJsonPath, sampleRate, metadata?] + if (args.size() >= 6) { + juce::String trackId = args[0].toString(); + juce::String clipId = args[1].toString(); + double startTime = static_cast<double>(args[2]); + double duration = static_cast<double>(args[3]); + juce::String wavPath = args[4].toString(); + juce::String routeJsonPath = args[5].toString(); + double sampleRate = args.size() >= 7 ? static_cast<double>(args[6]) : 44100.0; + juce::var metadata = args.size() >= 8 ? args[7] : juce::var(); + + std::thread([this, trackId, clipId, startTime, duration, wavPath, routeJsonPath, sampleRate, metadata, + completion = std::make_shared<juce::WebBrowserComponent::NativeFunctionCompletion>(std::move(completion))]() { + const juce::File liveWav(wavPath); + const auto liveStem = liveWav.getFileNameWithoutExtension(); + const auto bakedWav = liveWav.getSiblingFile(liveStem + "_baked_corrected.wav"); + const auto offlineWav = liveWav.getSiblingFile(liveStem + "_offline_render.wav"); + const auto compareJson = liveWav.getSiblingFile(liveStem + "_comparison.json"); + const auto routeBefore = audioEngine.getPitchPreviewRoutingStatus(clipId); + auto capture = audioEngine.capturePitchAuditionPlayback(trackId, clipId, startTime, duration, wavPath, sampleRate, false); + auto offlineCapture = audioEngine.capturePitchAuditionPlayback(trackId, clipId, startTime, duration, offlineWav.getFullPathName(), sampleRate, true); + const auto routeAfter = audioEngine.getPitchPreviewRoutingStatus(clipId); + juce::var bakedCapture; + juce::var bakedVsLive; + juce::var bakedVsOffline; + juce::var liveVsOffline; + + const auto outputFile = metadata.getProperty("outputFile", {}).toString(); + const double captureStartClipSec = static_cast<double>(metadata.getProperty("clipContextStartSec", 0.0)); + const double noteStartSec = static_cast<double>(metadata.getProperty("noteStartSec", captureStartClipSec)); + const double noteEndSec = static_cast<double>(metadata.getProperty("noteEndSec", captureStartClipSec + duration)); + if (outputFile.isNotEmpty()) + { + bakedCapture = audioEngine.capturePitchBakedContext(outputFile, + captureStartClipSec, + duration, + bakedWav.getFullPathName()); + if (static_cast<bool>(bakedCapture.getProperty("success", false))) + { + bakedVsLive = audioEngine.comparePitchDebugAudioFiles(bakedWav.getFullPathName(), + liveWav.getFullPathName(), + captureStartClipSec, + noteStartSec, + noteEndSec); + bakedVsOffline = audioEngine.comparePitchDebugAudioFiles(bakedWav.getFullPathName(), + offlineWav.getFullPathName(), + captureStartClipSec, + noteStartSec, + noteEndSec); + } + } + liveVsOffline = audioEngine.comparePitchDebugAudioFiles(liveWav.getFullPathName(), + offlineWav.getFullPathName(), + captureStartClipSec, + noteStartSec, + noteEndSec); + + auto* resultObj = new juce::DynamicObject(); + resultObj->setProperty("success", capture.isObject() && static_cast<bool>(capture.getProperty("success", false))); + resultObj->setProperty("trackId", trackId); + resultObj->setProperty("clipId", clipId); + resultObj->setProperty("capture", capture); + resultObj->setProperty("livePlaybackCapture", capture); + resultObj->setProperty("offlineRenderCapture", offlineCapture); + if (! bakedCapture.isVoid()) + resultObj->setProperty("bakedCorrectedCapture", bakedCapture); + if (! bakedVsLive.isVoid()) + resultObj->setProperty("bakedVsLiveParityReport", bakedVsLive); + if (! bakedVsOffline.isVoid()) + resultObj->setProperty("bakedVsOfflineParityReport", bakedVsOffline); + if (! liveVsOffline.isVoid()) + resultObj->setProperty("liveVsOfflineParityReport", liveVsOffline); + resultObj->setProperty("routeBefore", routeBefore); + resultObj->setProperty("routeAfter", routeAfter); + resultObj->setProperty("routeReportPath", routeJsonPath); + resultObj->setProperty("bakedCorrectedPath", bakedWav.getFullPathName()); + resultObj->setProperty("livePlaybackPath", liveWav.getFullPathName()); + resultObj->setProperty("offlineRenderPath", offlineWav.getFullPathName()); + resultObj->setProperty("comparisonReportPath", compareJson.getFullPathName()); + resultObj->setProperty("capturedAt", juce::Time::getCurrentTime().toISO8601(true)); + if (! metadata.isVoid()) + resultObj->setProperty("metadata", metadata); + + juce::var result(resultObj); + compareJson.getParentDirectory().createDirectory(); + auto* compareObj = new juce::DynamicObject(); + compareObj->setProperty("bakedCorrectedPath", bakedWav.getFullPathName()); + compareObj->setProperty("livePlaybackPath", liveWav.getFullPathName()); + compareObj->setProperty("offlineRenderPath", offlineWav.getFullPathName()); + compareObj->setProperty("bakedVsLiveParityReport", bakedVsLive); + compareObj->setProperty("bakedVsOfflineParityReport", bakedVsOffline); + compareObj->setProperty("liveVsOfflineParityReport", liveVsOffline); + compareJson.replaceWithText(juce::JSON::toString(juce::var(compareObj), true)); + if (routeJsonPath.isNotEmpty()) + { + const juce::File routeFile(routeJsonPath); + routeFile.getParentDirectory().createDirectory(); + const bool wrote = routeFile.replaceWithText(juce::JSON::toString(result, true)); + resultObj->setProperty("routeReportWritten", wrote); + if (! wrote) + juce::Logger::writeToLog("capturePitchAppFinalContext: failed to write route report " + routeFile.getFullPathName()); + } + + juce::Logger::writeToLog("capturePitchAppFinalContext clip=" + clipId + + " wav=" + wavPath + + " routeReport=" + routeJsonPath + + " success=" + juce::String(static_cast<bool>(resultObj->getProperty("success")) ? "true" : "false")); + + juce::MessageManager::callAsync([completion, result]() { + (*completion)(result); + }); + }).detach(); + } else { + juce::Logger::writeToLog("capturePitchAppFinalContext: Invalid args count: " + juce::String(args.size())); + completion(juce::var()); + } + }) .withNativeFunction ("renderProjectWithDither", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { - // Args: [source, startTime, endTime, filePath, format, sampleRate, bitDepth, channels, normalize, addTail, tailLength, ditherType] - if (args.size() == 12) { + // Args: [source, startTime, endTime, filePath, format, sampleRate, bitDepth, channels, normalize, addTail, tailLength, ditherType, includeMetronome?] + if (args.size() >= 12) { juce::String source = args[0].toString(); double startTime = (double)args[1]; double endTime = (double)args[2]; @@ -2341,13 +2560,16 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, bool addTail = (bool)args[9]; double tailLength = (double)args[10]; juce::String ditherType = args[11].toString(); + bool includeMetronome = args.size() >= 13 ? (bool)args[12] : false; std::thread([this, source, startTime, endTime, filePathArg, format, sampleRate, bitDepth, channels, normalizeArg, addTail, tailLength, ditherType, + includeMetronome, completion = std::make_shared<juce::WebBrowserComponent::NativeFunctionCompletion>(std::move(completion))]() { bool success = audioEngine.renderProjectWithDither( source, startTime, endTime, filePathArg, format, - sampleRate, bitDepth, channels, normalizeArg, addTail, tailLength, ditherType); + sampleRate, bitDepth, channels, normalizeArg, addTail, tailLength, ditherType, + includeMetronome); juce::MessageManager::callAsync([completion, success]() { (*completion)(success); }); @@ -2587,7 +2809,7 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, } }) // ===== Phase 12: Media & File Management ===== - .withNativeFunction ("browseDirectory", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + .withNativeFunction ("browseDirectory", [] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { // Args: [directoryPath] // Returns: Array of {name, path, size, isDirectory, format, duration, sampleRate, numChannels} if (args.size() >= 1 && args[0].isString()) { @@ -2644,7 +2866,7 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion(juce::Array<juce::var>()); } }) - .withNativeFunction ("previewAudioFile", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + .withNativeFunction ("previewAudioFile", [] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { // Preview an audio file through the output device (not through the track graph) if (args.size() >= 1 && args[0].isString()) { juce::String filePath = args[0].toString(); @@ -2655,12 +2877,12 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion(false); } }) - .withNativeFunction ("stopPreview", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + .withNativeFunction ("stopPreview", [] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { juce::ignoreUnused(args); juce::Logger::writeToLog("stopPreview called"); completion(true); }) - .withNativeFunction ("cleanProjectDirectory", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + .withNativeFunction ("cleanProjectDirectory", [] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { // Args: [projectDir, referencedFilesArray] // Returns: { orphanedFiles: Array<{path, size}>, totalSize } if (args.size() >= 2 && args[0].isString() && args[1].isArray()) { @@ -2889,7 +3111,7 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion(false); } }) - .withNativeFunction ("getHomeDirectory", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + .withNativeFunction ("getHomeDirectory", [] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { juce::ignoreUnused(args); completion(juce::File::getSpecialLocation(juce::File::userHomeDirectory).getFullPathName()); }) @@ -3169,7 +3391,7 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, audioEngine.getControlSurfaceManager().getOSCControl().disconnect(); completion(true); }) - .withNativeFunction ("getControlSurfaceMIDIDevices", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + .withNativeFunction ("getControlSurfaceMIDIDevices", [] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { juce::ignoreUnused(args); juce::DynamicObject::Ptr result = new juce::DynamicObject(); auto inputs = ControlSurfaceManager::getAvailableMIDIInputs(); @@ -3530,7 +3752,7 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, } }) // ========== Phase 3.7: Surround / Spatial Audio ========== - .withNativeFunction ("getSurroundLayouts", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + .withNativeFunction ("getSurroundLayouts", [] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { juce::ignoreUnused(args); juce::Array<juce::var> layouts; auto addLayout = [&](const juce::String& name, int channels) { @@ -3729,6 +3951,7 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, // ==================== Window Management ==================== .withNativeFunction ("minimizeWindow", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + juce::ignoreUnused(this); juce::ignoreUnused(args); #if JUCE_WINDOWS if (auto* peer = getTopLevelComponent()->getPeer()) @@ -4097,6 +4320,15 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, auto trackId = args[0].toString(); auto clipId = args[1].toString(); // Fire-and-forget: start analysis, emit event when done + if (pitchNoteHqPriorityActive.load()) + { + auto obj = std::make_unique<juce::DynamicObject>(); + obj->setProperty ("started", false); + obj->setProperty ("deferred", true); + obj->setProperty ("error", "Pitch HQ apply in progress"); + completion (juce::var (obj.release())); + return; + } if (pitchAnalysisRunning.load()) { auto obj = std::make_unique<juce::DynamicObject>(); @@ -4105,19 +4337,27 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion (juce::var (obj.release())); return; } + const int analysisGeneration = ++pitchAnalysisGeneration; pitchAnalysisRunning.store (true); auto obj = std::make_unique<juce::DynamicObject>(); obj->setProperty ("started", true); completion (juce::var (obj.release())); - std::thread([this, trackId, clipId]() { + pitchAnalysisPool.addJob ([this, trackId, clipId, analysisGeneration]() { juce::Logger::writeToLog ("PitchAnalysis: Starting for track=" + trackId + " clip=" + clipId); - auto result = audioEngine.analyzePitchContour(trackId, clipId); - pitchAnalysisRunning.store (false); + auto shouldCancelAnalysis = [this, analysisGeneration]() + { + return pitchAnalysisGeneration.load() != analysisGeneration + || pitchNoteHqPriorityActive.load(); + }; + auto result = audioEngine.analyzePitchContour(trackId, clipId, shouldCancelAnalysis); + const bool cancelled = shouldCancelAnalysis(); + if (pitchAnalysisGeneration.load() == analysisGeneration) + pitchAnalysisRunning.store (false); int noteCount = 0; bool hasResult = false; - if (auto* obj = result.getDynamicObject()) + if (auto* obj = (! cancelled ? result.getDynamicObject() : nullptr)) { auto notesVar = obj->getProperty ("notes"); noteCount = notesVar.isArray() ? notesVar.getArray()->size() : 0; @@ -4130,22 +4370,24 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, juce::Logger::writeToLog ("PitchAnalysis: Result is VOID/empty!"); } + if (! cancelled) { const juce::ScopedLock sl (pitchResultLock); lastPitchAnalysisResult = result; } - juce::MessageManager::callAsync ([this, clipId, noteCount, hasResult]() { + juce::MessageManager::callAsync ([this, clipId, noteCount, hasResult, cancelled]() { auto notification = std::make_unique<juce::DynamicObject>(); notification->setProperty ("clipId", clipId); notification->setProperty ("noteCount", noteCount); - notification->setProperty ("ready", hasResult); + notification->setProperty ("ready", hasResult && ! cancelled); + notification->setProperty ("cancelled", cancelled); juce::Logger::writeToLog ("PitchAnalysis: Emitting lightweight event (noteCount=" + juce::String(noteCount) + ")"); webView.emitEventIfBrowserIsVisible ("pitchAnalysisComplete", juce::var (notification.release())); }); - }).detach(); + }); } else completion(juce::var()); @@ -4158,6 +4400,15 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, auto duration = static_cast<double>(args[2]); auto clipId = args[3].toString(); // Fire-and-forget: start analysis, emit event when done + if (pitchNoteHqPriorityActive.load()) + { + auto obj = std::make_unique<juce::DynamicObject>(); + obj->setProperty ("started", false); + obj->setProperty ("deferred", true); + obj->setProperty ("error", "Pitch HQ apply in progress"); + completion (juce::var (obj.release())); + return; + } if (pitchAnalysisRunning.load()) { auto obj = std::make_unique<juce::DynamicObject>(); @@ -4166,20 +4417,28 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion (juce::var (obj.release())); return; } + const int analysisGeneration = ++pitchAnalysisGeneration; pitchAnalysisRunning.store (true); auto obj = std::make_unique<juce::DynamicObject>(); obj->setProperty ("started", true); completion (juce::var (obj.release())); - std::thread([this, filePath, offset, duration, clipId]() { + pitchAnalysisPool.addJob ([this, filePath, offset, duration, clipId, analysisGeneration]() { juce::Logger::writeToLog ("PitchAnalysis: Starting for " + filePath + " offset=" + juce::String(offset) + " dur=" + juce::String(duration)); - auto result = audioEngine.analyzePitchContourDirect(filePath, offset, duration, clipId); - pitchAnalysisRunning.store (false); + auto shouldCancelAnalysis = [this, analysisGeneration]() + { + return pitchAnalysisGeneration.load() != analysisGeneration + || pitchNoteHqPriorityActive.load(); + }; + auto result = audioEngine.analyzePitchContourDirect(filePath, offset, duration, clipId, shouldCancelAnalysis); + const bool cancelled = shouldCancelAnalysis(); + if (pitchAnalysisGeneration.load() == analysisGeneration) + pitchAnalysisRunning.store (false); int noteCount = 0; bool hasResult = false; - if (auto* obj = result.getDynamicObject()) + if (auto* obj = (! cancelled ? result.getDynamicObject() : nullptr)) { auto notesVar = obj->getProperty ("notes"); noteCount = notesVar.isArray() ? notesVar.getArray()->size() : 0; @@ -4193,23 +4452,25 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, } // Store result for fetch-after-event pattern (avoids large event payload) + if (! cancelled) { const juce::ScopedLock sl (pitchResultLock); lastPitchAnalysisResult = result; } - juce::MessageManager::callAsync ([this, clipId, noteCount, hasResult]() { + juce::MessageManager::callAsync ([this, clipId, noteCount, hasResult, cancelled]() { // Send lightweight notification with metadata only auto notification = std::make_unique<juce::DynamicObject>(); notification->setProperty ("clipId", clipId); notification->setProperty ("noteCount", noteCount); - notification->setProperty ("ready", hasResult); + notification->setProperty ("ready", hasResult && ! cancelled); + notification->setProperty ("cancelled", cancelled); juce::Logger::writeToLog ("PitchAnalysis: Emitting lightweight event (noteCount=" + juce::String(noteCount) + ")"); webView.emitEventIfBrowserIsVisible ("pitchAnalysisComplete", juce::var (notification.release())); }); - }).detach(); + }); } else completion(juce::var()); @@ -4257,7 +4518,17 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, // pile up and corrupt the output file with stale note data. // The currently-running job (if any) is allowed to finish safely. const bool isPreviewSegment = renderMode == "preview_segment"; - juce::ThreadPool* targetPool = isPreviewSegment ? &previewSegmentPool : &fullClipHQPool; + const bool isNoteRender = renderMode == "note_hq"; + if (isNoteRender) + { + pitchNoteHqPriorityActive.store (true); + ++pitchAnalysisGeneration; + pitchAnalysisRunning.store (false); + pitchAnalysisPool.removeAllJobs (false, 0); + } + juce::ThreadPool* targetPool = isPreviewSegment + ? &previewSegmentPool + : (isNoteRender ? ¬eRenderPool : &fullClipHQPool); int renderGeneration = 0; { const juce::ScopedLock sl (pitchCorrectionJobLock); @@ -4266,15 +4537,35 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, if (activePreviewRequestGroup != requestGroupId) { previewSegmentPool.removeAllJobs (false, 0); - audioEngine.getPlaybackEngine().clearClipRenderedPreviewSegments (clipId); activePreviewRequestGroup = requestGroupId; renderGeneration = ++previewRenderGeneration; + audioEngine.getPlaybackEngine().beginRenderedPreviewSegmentGeneration (clipId, renderGeneration); } else { renderGeneration = previewRenderGeneration.load(); } } + else if (isNoteRender) + { + previewSegmentPool.removeAllJobs (false, 0); + activePreviewRequestGroup = {}; + ++previewRenderGeneration; + audioEngine.getPlaybackEngine().clearAllPitchPreviewRoutes (clipId); + logPitchEditorFormant ("note_hq invalidated preview routes clip=" + clipId + + " requestId=" + requestId + + " requestGroupId=" + requestGroupId); + if (activeNoteRenderRequestGroup != requestGroupId) + { + noteRenderPool.removeAllJobs (false, 0); + activeNoteRenderRequestGroup = requestGroupId; + renderGeneration = ++noteRenderGeneration; + } + else + { + renderGeneration = noteRenderGeneration.load(); + } + } else { if (activeFullClipRequestGroup != requestGroupId) @@ -4289,12 +4580,17 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, } } } + if (isNoteRender) + pitchNoteHqPriorityGeneration.store (renderGeneration); + const auto queuedAtMs = juce::Time::currentTimeMillis(); completion(true); - targetPool->addJob ([this, trackId, clipId, notes, frames, requestId, requestGroupId, globalFormantSemitones, windowStartSec, windowEndSec, renderMode, renderGeneration, isPreviewSegment]() mutable { - auto shouldCancel = [this, renderGeneration, requestGroupId, isPreviewSegment]() { + targetPool->addJob ([this, trackId, clipId, notes, frames, requestId, requestGroupId, globalFormantSemitones, windowStartSec, windowEndSec, renderMode, renderGeneration, isPreviewSegment, isNoteRender, queuedAtMs]() mutable { + auto shouldCancel = [this, renderGeneration, requestGroupId, isPreviewSegment, isNoteRender]() { const juce::ScopedLock sl (pitchCorrectionJobLock); if (isPreviewSegment) return previewRenderGeneration.load() != renderGeneration || activePreviewRequestGroup != requestGroupId; + if (isNoteRender) + return noteRenderGeneration.load() != renderGeneration || activeNoteRenderRequestGroup != requestGroupId; return fullClipRenderGeneration.load() != renderGeneration || activeFullClipRequestGroup != requestGroupId; }; logPitchEditorFormant ("job starting clip=" + clipId @@ -4303,9 +4599,12 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, + " globalFormantSt=" + juce::String (globalFormantSemitones, 3) + " renderMode=" + renderMode + " generation=" + juce::String (renderGeneration)); + const double jobStartDelayMs = static_cast<double> (juce::Time::currentTimeMillis() - queuedAtMs); auto result = shouldCancel() ? juce::var() - : audioEngine.applyPitchCorrection(trackId, clipId, notes, frames, globalFormantSemitones, windowStartSec, windowEndSec, renderMode, shouldCancel); + : audioEngine.applyPitchCorrection(trackId, clipId, notes, frames, globalFormantSemitones, windowStartSec, windowEndSec, renderMode, shouldCancel, jobStartDelayMs, isPreviewSegment ? renderGeneration : 0); + if (isNoteRender && pitchNoteHqPriorityGeneration.load() == renderGeneration) + pitchNoteHqPriorityActive.store (false); bool success = result.isObject() && static_cast<bool> (result.getProperty ("success", false)); juce::String outputFile = (success && result["outputFile"].isString()) @@ -4313,6 +4612,400 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, bool restored = result.isObject() && static_cast<bool> (result.getProperty ("restored", false)); bool cancelled = result.isObject() && static_cast<bool> (result.getProperty ("cancelled", false)); bool swapDeferred = result.isObject() && static_cast<bool> (result.getProperty ("swapDeferred", false)); + double previewCoverageStartSec = result.isObject() ? static_cast<double> (result.getProperty ("previewCoverageStartSec", 0.0)) : 0.0; + double previewCoverageEndSec = result.isObject() ? static_cast<double> (result.getProperty ("previewCoverageEndSec", 0.0)) : 0.0; + double candidateCoverageStartSec = result.isObject() ? static_cast<double> (result.getProperty ("candidateCoverageStartSec", 0.0)) : 0.0; + double candidateCoverageEndSec = result.isObject() ? static_cast<double> (result.getProperty ("candidateCoverageEndSec", 0.0)) : 0.0; + juce::String requestedRendererBranch = result.isObject() ? result.getProperty ("requestedRendererBranch", {}).toString() : juce::String(); + juce::String actualRendererBranch = result.isObject() ? result.getProperty ("actualRendererBranch", {}).toString() : juce::String(); + juce::String pitchOnlyRecoveryPath = result.isObject() ? result.getProperty ("pitchOnlyRecoveryPath", {}).toString() : juce::String(); + bool pitchOnlyNeutralFormantUsed = result.isObject() && static_cast<bool> (result.getProperty ("pitchOnlyNeutralFormantUsed", false)); + juce::String processingMode = result.isObject() ? result.getProperty ("processingMode", {}).toString() : juce::String(); + bool formantCurveUsed = result.isObject() && static_cast<bool> (result.getProperty ("formantCurveUsed", false)); + bool explicitFormantRequested = result.isObject() && static_cast<bool> (result.getProperty ("explicitFormantRequested", false)); + bool pitchOnlyFormantSuppressed = result.isObject() && static_cast<bool> (result.getProperty ("pitchOnlyFormantSuppressed", false)); + bool usedFallback = result.isObject() && static_cast<bool> (result.getProperty ("usedFallback", false)); + juce::String fallbackReason = result.isObject() ? result.getProperty ("fallbackReason", {}).toString() : juce::String(); + juce::String hardFailReason = result.isObject() ? result.getProperty ("hardFailReason", {}).toString() : juce::String(); + juce::String pitchRenderStrategy = result.isObject() ? result.getProperty ("pitchRenderStrategy", {}).toString() : juce::String(); + bool phraseHqRenderUsed = result.isObject() && static_cast<bool> (result.getProperty ("phraseHqRenderUsed", false)); + bool phraseHqExpandedToFullClip = result.isObject() && static_cast<bool> (result.getProperty ("phraseHqExpandedToFullClip", false)); + double phraseHqStartSec = result.isObject() ? static_cast<double> (result.getProperty ("phraseHqStartSec", 0.0)) : 0.0; + double phraseHqEndSec = result.isObject() ? static_cast<double> (result.getProperty ("phraseHqEndSec", 0.0)) : 0.0; + juce::String pitchRenderProductPath = result.isObject() ? result.getProperty ("pitchRenderProductPath", {}).toString() : juce::String(); + juce::String pitchRenderBackendId = result.isObject() ? result.getProperty ("pitchRenderBackendId", {}).toString() : juce::String(); + juce::String pitchRenderBackendVersion = result.isObject() ? result.getProperty ("pitchRenderBackendVersion", {}).toString() : juce::String(); + juce::String pitchRenderBackendFailureCode = result.isObject() ? result.getProperty ("pitchRenderBackendFailureCode", {}).toString() : juce::String(); + juce::var pitchRenderBackendCapabilities = result.isObject() ? result.getProperty ("pitchRenderBackendCapabilities", juce::var()) : juce::var(); + juce::var pitchRenderBackendDiagnostics = result.isObject() ? result.getProperty ("pitchRenderBackendDiagnostics", juce::var()) : juce::var(); + juce::String pitchRenderCommitPolicy = result.isObject() ? result.getProperty ("pitchRenderCommitPolicy", {}).toString() : juce::String(); + int pitchRenderDryProtectedSamples = result.isObject() ? static_cast<int> (result.getProperty ("pitchRenderDryProtectedSamples", 0)) : 0; + double pitchRenderContextDurationSec = result.isObject() ? static_cast<double> (result.getProperty ("pitchRenderContextDurationSec", 0.0)) : 0.0; + double pitchRenderCommitDurationSec = result.isObject() ? static_cast<double> (result.getProperty ("pitchRenderCommitDurationSec", 0.0)) : 0.0; + double pitchRenderJobStartDelayMs = result.isObject() ? static_cast<double> (result.getProperty ("pitchRenderJobStartDelayMs", 0.0)) : 0.0; + juce::String pitchRenderDirection = result.isObject() ? result.getProperty ("pitchRenderDirection", {}).toString() : juce::String(); + bool downshiftFormantGuardUsed = result.isObject() && static_cast<bool> (result.getProperty ("downshiftFormantGuardUsed", false)); + double downshiftFormantGuardAlpha = result.isObject() ? static_cast<double> (result.getProperty ("downshiftFormantGuardAlpha", 0.0)) : 0.0; + double noteHqEffectiveStartSec = result.isObject() ? static_cast<double> (result.getProperty ("noteHqEffectiveStartSec", 0.0)) : 0.0; + double noteHqEffectiveEndSec = result.isObject() ? static_cast<double> (result.getProperty ("noteHqEffectiveEndSec", 0.0)) : 0.0; + double noteHqContextStartSec = result.isObject() ? static_cast<double> (result.getProperty ("noteHqContextStartSec", 0.0)) : 0.0; + double noteHqContextEndSec = result.isObject() ? static_cast<double> (result.getProperty ("noteHqContextEndSec", 0.0)) : 0.0; + double noteHqAudibleCommitStartSec = result.isObject() ? static_cast<double> (result.getProperty ("noteHqAudibleCommitStartSec", 0.0)) : 0.0; + double noteHqAudibleCommitEndSec = result.isObject() ? static_cast<double> (result.getProperty ("noteHqAudibleCommitEndSec", 0.0)) : 0.0; + int noteHqPreBodyDryProtectedSamples = result.isObject() ? static_cast<int> (result.getProperty ("noteHqPreBodyDryProtectedSamples", 0)) : 0; + double noteHqEntryInsideBodyFadeMs = result.isObject() ? static_cast<double> (result.getProperty ("noteHqEntryInsideBodyFadeMs", 0.0)) : 0.0; + double noteHqExitLeadInMs = result.isObject() ? static_cast<double> (result.getProperty ("noteHqExitLeadInMs", 0.0)) : 0.0; + double noteHqEntryBridgeStartSec = result.isObject() ? static_cast<double> (result.getProperty ("noteHqEntryBridgeStartSec", 0.0)) : 0.0; + double noteHqEntryBridgeEndSec = result.isObject() ? static_cast<double> (result.getProperty ("noteHqEntryBridgeEndSec", 0.0)) : 0.0; + double noteHqEntryBridgeWetLagMs = result.isObject() ? static_cast<double> (result.getProperty ("noteHqEntryBridgeWetLagMs", 0.0)) : 0.0; + double noteHqEntryBridgeEnvelopeGainDb = result.isObject() ? static_cast<double> (result.getProperty ("noteHqEntryBridgeEnvelopeGainDb", 0.0)) : 0.0; + bool noteHqEntryBridgeUsed = result.isObject() && static_cast<bool> (result.getProperty ("noteHqEntryBridgeUsed", false)); + double noteHqEntryTransientDryPreservedMs = result.isObject() ? static_cast<double> (result.getProperty ("noteHqEntryTransientDryPreservedMs", 0.0)) : 0.0; + bool pitchOnlyEntrySimpleHandoffUsed = result.isObject() && static_cast<bool> (result.getProperty ("pitchOnlyEntrySimpleHandoffUsed", false)); + bool pitchOnlyEntrySafeHandoffUsed = result.isObject() && static_cast<bool> (result.getProperty ("pitchOnlyEntrySafeHandoffUsed", false)); + double pitchOnlyEntryDryHoldMs = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyEntryDryHoldMs", 0.0)) : 0.0; + double pitchOnlyEntrySafeBridgeMs = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyEntrySafeBridgeMs", 0.0)) : 0.0; + double pitchOnlyEntryWetAlignmentMs = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyEntryWetAlignmentMs", 0.0)) : 0.0; + double pitchOnlyEntryWetGainDb = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyEntryWetGainDb", 0.0)) : 0.0; + double pitchOnlyEntryWetVsDryRmsDb = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyEntryWetVsDryRmsDb", 0.0)) : 0.0; + bool pitchOnlyEntryEqualPowerBlendUsed = result.isObject() && static_cast<bool> (result.getProperty ("pitchOnlyEntryEqualPowerBlendUsed", false)); + bool pitchOnlyEntryRmsContinuityUsed = result.isObject() && static_cast<bool> (result.getProperty ("pitchOnlyEntryRmsContinuityUsed", false)); + double pitchOnlyEntryRmsContinuityGainDb = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyEntryRmsContinuityGainDb", 0.0)) : 0.0; + double pitchOnlyEntryRmsContinuityMs = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyEntryRmsContinuityMs", 0.0)) : 0.0; + bool pitchOnlyEntryPhaseSafeUsed = result.isObject() && static_cast<bool> (result.getProperty ("pitchOnlyEntryPhaseSafeUsed", false)); + bool pitchOnlyEntryWetAlignmentAccepted = result.isObject() && static_cast<bool> (result.getProperty ("pitchOnlyEntryWetAlignmentAccepted", false)); + double pitchOnlyEntryFirstCycleCorrelation = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyEntryFirstCycleCorrelation", 0.0)) : 0.0; + double pitchOnlyEntryZeroCrossOffsetMs = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyEntryZeroCrossOffsetMs", 0.0)) : 0.0; + double pitchOnlyEntryBridgeGainRampDb = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyEntryBridgeGainRampDb", 0.0)) : 0.0; + bool pitchOnlyDownshiftCoreEnvelopePassUsed = result.isObject() && static_cast<bool> (result.getProperty ("pitchOnlyDownshiftCoreEnvelopePassUsed", false)); + double pitchOnlyDownshiftCoreRmsTrimDb = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyDownshiftCoreRmsTrimDb", 0.0)) : 0.0; + double pitchOnlyDownshiftCoreEnvelopeMaxDb = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyDownshiftCoreEnvelopeMaxDb", 0.0)) : 0.0; + int pitchOnlyDownshiftCoreEnvelopeFrames = result.isObject() ? static_cast<int> (result.getProperty ("pitchOnlyDownshiftCoreEnvelopeFrames", 0)) : 0; + double pitchOnlyEntryWetLagMs = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyEntryWetLagMs", 0.0)) : 0.0; + double pitchOnlyEntryBridgeDurationMs = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyEntryBridgeDurationMs", 0.0)) : 0.0; + bool pitchOnlyExitDryRestoreUsed = result.isObject() && static_cast<bool> (result.getProperty ("pitchOnlyExitDryRestoreUsed", false)); + double pitchOnlyExitDryRestoreStartSec = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyExitDryRestoreStartSec", 0.0)) : 0.0; + double pitchOnlyExitDryRestoreEndSec = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyExitDryRestoreEndSec", 0.0)) : 0.0; + int noteHqEditIslandCount = result.isObject() ? static_cast<int> (result.getProperty ("noteHqEditIslandCount", 0)) : 0; + int noteHqEditedNoteCount = result.isObject() ? static_cast<int> (result.getProperty ("noteHqEditedNoteCount", 0)) : 0; + bool noteHqEntryPitchHandoffUsed = result.isObject() && static_cast<bool> (result.getProperty ("noteHqEntryPitchHandoffUsed", false)); + double noteHqEntryPitchHandoffStartSec = result.isObject() ? static_cast<double> (result.getProperty ("noteHqEntryPitchHandoffStartSec", 0.0)) : 0.0; + double noteHqEntryPitchHandoffEndSec = result.isObject() ? static_cast<double> (result.getProperty ("noteHqEntryPitchHandoffEndSec", 0.0)) : 0.0; + double noteHqEntryPitchHandoffPreMs = result.isObject() ? static_cast<double> (result.getProperty ("noteHqEntryPitchHandoffPreMs", 0.0)) : 0.0; + double noteHqEntryPitchHandoffBodyMs = result.isObject() ? static_cast<double> (result.getProperty ("noteHqEntryPitchHandoffBodyMs", 0.0)) : 0.0; + double noteHqEntryPitchSlopeJumpStPerSec = result.isObject() ? static_cast<double> (result.getProperty ("noteHqEntryPitchSlopeJumpStPerSec", 0.0)) : 0.0; + bool noteHqEntryPitchAccelerationLimited = result.isObject() && static_cast<bool> (result.getProperty ("noteHqEntryPitchAccelerationLimited", false)); + double outputDurationSec = result.isObject() ? static_cast<double> (result.getProperty ("outputDurationSec", 0.0)) : 0.0; + juce::var postApplyRouteStatus = result.isObject() ? result.getProperty ("postApplyRouteStatus", juce::var()) : juce::var(); + juce::var appFinalCapture = result.isObject() ? result.getProperty ("appFinalCapture", juce::var()) : juce::var(); + juce::var appFinalBakedCapture = result.isObject() ? result.getProperty ("appFinalBakedCapture", juce::var()) : juce::var(); + juce::var appFinalParityReport = result.isObject() ? result.getProperty ("appFinalParityReport", juce::var()) : juce::var(); + juce::String appFinalRouteReportPath = result.isObject() ? result.getProperty ("appFinalRouteReportPath", {}).toString() : juce::String(); + juce::String appFinalBakedContextPath = result.isObject() ? result.getProperty ("appFinalBakedContextPath", {}).toString() : juce::String(); + juce::String appFinalPlaybackContextPath = result.isObject() ? result.getProperty ("appFinalPlaybackContextPath", {}).toString() : juce::String(); + juce::String appFinalParityReportPath = result.isObject() ? result.getProperty ("appFinalParityReportPath", {}).toString() : juce::String(); + bool bridgeUsed = result.isObject() && static_cast<bool> (result.getProperty ("bridgeUsed", false)); + bool bridgeFallbackUsed = result.isObject() && static_cast<bool> (result.getProperty ("bridgeFallbackUsed", false)); + double bridgeStartSec = result.isObject() ? static_cast<double> (result.getProperty ("bridgeStartSec", 0.0)) : 0.0; + double bridgeLengthMs = result.isObject() ? static_cast<double> (result.getProperty ("bridgeLengthMs", 0.0)) : 0.0; + int bridgeAlignmentLagSamples = result.isObject() ? static_cast<int> (result.getProperty ("bridgeAlignmentLagSamples", 0)) : 0; + double bridgeCorrelationScore = result.isObject() ? static_cast<double> (result.getProperty ("bridgeCorrelationScore", 0.0)) : 0.0; + double bridgeGainDeltaDb = result.isObject() ? static_cast<double> (result.getProperty ("bridgeGainDeltaDb", 0.0)) : 0.0; + bool bodyReplacementUsed = result.isObject() && static_cast<bool> (result.getProperty ("bodyReplacementUsed", false)); + bool bodyReplacementFallbackUsed = result.isObject() && static_cast<bool> (result.getProperty ("bodyReplacementFallbackUsed", false)); + double entryLockStartSec = result.isObject() ? static_cast<double> (result.getProperty ("entryLockStartSec", 0.0)) : 0.0; + double entryLockLengthMs = result.isObject() ? static_cast<double> (result.getProperty ("entryLockLengthMs", 0.0)) : 0.0; + double exitLockStartSec = result.isObject() ? static_cast<double> (result.getProperty ("exitLockStartSec", 0.0)) : 0.0; + double renderedBodyStartSec = result.isObject() ? static_cast<double> (result.getProperty ("renderedBodyStartSec", 0.0)) : 0.0; + double renderedBodyEndSec = result.isObject() ? static_cast<double> (result.getProperty ("renderedBodyEndSec", 0.0)) : 0.0; + bool islandNativeUsed = result.isObject() && static_cast<bool> (result.getProperty ("islandNativeUsed", false)); + bool islandNativeFallbackUsed = result.isObject() && static_cast<bool> (result.getProperty ("islandNativeFallbackUsed", false)); + double islandRenderStartSec = result.isObject() ? static_cast<double> (result.getProperty ("islandRenderStartSec", 0.0)) : 0.0; + double islandRenderEndSec = result.isObject() ? static_cast<double> (result.getProperty ("islandRenderEndSec", 0.0)) : 0.0; + double transientMaskPeak = result.isObject() ? static_cast<double> (result.getProperty ("transientMaskPeak", 0.0)) : 0.0; + double voicedCoreMaskPeak = result.isObject() ? static_cast<double> (result.getProperty ("voicedCoreMaskPeak", 0.0)) : 0.0; + bool hpssUsed = result.isObject() && static_cast<bool> (result.getProperty ("hpssUsed", false)); + bool hpssFallbackUsed = result.isObject() && static_cast<bool> (result.getProperty ("hpssFallbackUsed", false)); + double harmonicMaskPeak = result.isObject() ? static_cast<double> (result.getProperty ("harmonicMaskPeak", 0.0)) : 0.0; + double aperiodicMaskPeak = result.isObject() ? static_cast<double> (result.getProperty ("aperiodicMaskPeak", 0.0)) : 0.0; + bool spectralEnvelopeCorrectionUsed = result.isObject() && static_cast<bool> (result.getProperty ("spectralEnvelopeCorrectionUsed", false)); + bool pitchOnlyCoreTimbreCorrectionUsed = result.isObject() && static_cast<bool> (result.getProperty ("pitchOnlyCoreTimbreCorrectionUsed", false)); + double pitchOnlyCoreEnvelopeMix = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyCoreEnvelopeMix", 0.0)) : 0.0; + double pitchOnlyCoreRmsTrimDb = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyCoreRmsTrimDb", 0.0)) : 0.0; + int pitchOnlyCoreEnvelopeLifter = result.isObject() ? static_cast<int> (result.getProperty ("pitchOnlyCoreEnvelopeLifter", 0)) : 0; + bool pitchOnlyEntryTimbreCorrectionUsed = result.isObject() && static_cast<bool> (result.getProperty ("pitchOnlyEntryTimbreCorrectionUsed", false)); + double pitchOnlyEntryRmsTrimDb = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyEntryRmsTrimDb", 0.0)) : 0.0; + double pitchOnlyEntryTiltDb = result.isObject() ? static_cast<double> (result.getProperty ("pitchOnlyEntryTiltDb", 0.0)) : 0.0; + bool pitchOnlyEntryHandoffUsed = result.isObject() && static_cast<bool> (result.getProperty ("pitchOnlyEntryHandoffUsed", false)); + bool pitchOnlyExitHandoffUsed = result.isObject() && static_cast<bool> (result.getProperty ("pitchOnlyExitHandoffUsed", false)); + bool vocalSourceFilterUsed = result.isObject() && static_cast<bool> (result.getProperty ("vocalSourceFilterUsed", false)); + double vocalSourceFilterVoicedCoverage = result.isObject() ? static_cast<double> (result.getProperty ("vocalSourceFilterVoicedCoverage", 0.0)) : 0.0; + double vocalSourceFilterResidualMix = result.isObject() ? static_cast<double> (result.getProperty ("vocalSourceFilterResidualMix", 0.0)) : 0.0; + bool vocalSourceFilterFallbackUsed = result.isObject() && static_cast<bool> (result.getProperty ("vocalSourceFilterFallbackUsed", false)); + juce::String vocalSourceFilterFallbackReason = result.isObject() ? result.getProperty ("vocalSourceFilterFallbackReason", {}).toString() : juce::String(); + double vocalSourceFilterEntryDryMs = result.isObject() ? static_cast<double> (result.getProperty ("vocalSourceFilterEntryDryMs", 0.0)) : 0.0; + double vocalSourceFilterExitDryMs = result.isObject() ? static_cast<double> (result.getProperty ("vocalSourceFilterExitDryMs", 0.0)) : 0.0; + bool wsolaUsed = result.isObject() && static_cast<bool> (result.getProperty ("wsolaUsed", false)); + bool wsolaFallbackUsed = result.isObject() && static_cast<bool> (result.getProperty ("wsolaFallbackUsed", false)); + int wsolaEntryLagSamples = result.isObject() ? static_cast<int> (result.getProperty ("wsolaEntryLagSamples", 0)) : 0; + int wsolaExitLagSamples = result.isObject() ? static_cast<int> (result.getProperty ("wsolaExitLagSamples", 0)) : 0; + double wsolaCorrelationScore = result.isObject() ? static_cast<double> (result.getProperty ("wsolaCorrelationScore", 0.0)) : 0.0; + bool phaseLockUsed = result.isObject() && static_cast<bool> (result.getProperty ("phaseLockUsed", false)); + bool phaseLockFallbackUsed = result.isObject() && static_cast<bool> (result.getProperty ("phaseLockFallbackUsed", false)); + bool phaseAlignedEntry = result.isObject() && static_cast<bool> (result.getProperty ("phaseAlignedEntry", false)); + bool phaseAlignedExit = result.isObject() && static_cast<bool> (result.getProperty ("phaseAlignedExit", false)); + int phasePeakCount = result.isObject() ? static_cast<int> (result.getProperty ("phasePeakCount", 0)) : 0; + bool transitionHqUsed = result.isObject() && static_cast<bool> (result.getProperty ("transitionHqUsed", false)); + bool transitionHqFallbackUsed = result.isObject() && static_cast<bool> (result.getProperty ("transitionHqFallbackUsed", false)); + double transitionStartSec = result.isObject() ? static_cast<double> (result.getProperty ("transitionStartSec", 0.0)) : 0.0; + double transitionEndSec = result.isObject() ? static_cast<double> (result.getProperty ("transitionEndSec", 0.0)) : 0.0; + double transitionTransientPeak = result.isObject() ? static_cast<double> (result.getProperty ("transitionTransientPeak", 0.0)) : 0.0; + double transitionVoicedCorePeak = result.isObject() ? static_cast<double> (result.getProperty ("transitionVoicedCorePeak", 0.0)) : 0.0; + double transitionResidualPeak = result.isObject() ? static_cast<double> (result.getProperty ("transitionResidualPeak", 0.0)) : 0.0; + bool transitionEnvelopeCorrectionUsed = result.isObject() && static_cast<bool> (result.getProperty ("transitionEnvelopeCorrectionUsed", false)); + bool engineV2Used = result.isObject() && static_cast<bool> (result.getProperty ("engineV2Used", false)); + bool engineV2FallbackUsed = result.isObject() && static_cast<bool> (result.getProperty ("engineV2FallbackUsed", false)); + int engineV2TransitionCount = result.isObject() ? static_cast<int> (result.getProperty ("engineV2TransitionCount", 0)) : 0; + double engineV2TransitionStartSec = result.isObject() ? static_cast<double> (result.getProperty ("engineV2TransitionStartSec", 0.0)) : 0.0; + double engineV2TransitionEndSec = result.isObject() ? static_cast<double> (result.getProperty ("engineV2TransitionEndSec", 0.0)) : 0.0; + double engineV2HarmonicSupportPeak = result.isObject() ? static_cast<double> (result.getProperty ("engineV2HarmonicSupportPeak", 0.0)) : 0.0; + double engineV2ResidualSupportPeak = result.isObject() ? static_cast<double> (result.getProperty ("engineV2ResidualSupportPeak", 0.0)) : 0.0; + double engineV2EnvelopeSupportPeak = result.isObject() ? static_cast<double> (result.getProperty ("engineV2EnvelopeSupportPeak", 0.0)) : 0.0; + bool transientBypassUsed = result.isObject() && static_cast<bool> (result.getProperty ("transientBypassUsed", false)); + bool residualCarryUsed = result.isObject() && static_cast<bool> (result.getProperty ("residualCarryUsed", false)); + double cepstralCutoffUsed = result.isObject() ? static_cast<double> (result.getProperty ("cepstralCutoffUsed", 0.0)) : 0.0; + int engineV2FftSize = result.isObject() ? static_cast<int> (result.getProperty ("fftSizeUsed", 0)) : 0; + int engineV2HopSize = result.isObject() ? static_cast<int> (result.getProperty ("hopSizeUsed", 0)) : 0; + bool immediateLeftNeighborUsed = result.isObject() && static_cast<bool> (result.getProperty ("immediateLeftNeighborUsed", false)); + bool immediateRightNeighborUsed = result.isObject() && static_cast<bool> (result.getProperty ("immediateRightNeighborUsed", false)); + int leftNeighborSamplesRendered = result.isObject() ? static_cast<int> (result.getProperty ("leftNeighborSamplesRendered", 0)) : 0; + int rightNeighborSamplesRendered = result.isObject() ? static_cast<int> (result.getProperty ("rightNeighborSamplesRendered", 0)) : 0; + double leftNeighborSmoothMs = result.isObject() ? static_cast<double> (result.getProperty ("leftNeighborSmoothMs", 0.0)) : 0.0; + double rightNeighborSmoothMs = result.isObject() ? static_cast<double> (result.getProperty ("rightNeighborSmoothMs", 0.0)) : 0.0; + bool nonImmediateNeighborTouched = result.isObject() && static_cast<bool> (result.getProperty ("nonImmediateNeighborTouched", false)); + double entryAlignmentOffsetMs = result.isObject() ? static_cast<double> (result.getProperty ("entryAlignmentOffsetMs", 0.0)) : 0.0; + double exitAlignmentOffsetMs = result.isObject() ? static_cast<double> (result.getProperty ("exitAlignmentOffsetMs", 0.0)) : 0.0; + bool firstVoicedCyclesEntryUsed = result.isObject() && static_cast<bool> (result.getProperty ("firstVoicedCyclesEntryUsed", false)); + bool firstVoicedCyclesExitUsed = result.isObject() && static_cast<bool> (result.getProperty ("firstVoicedCyclesExitUsed", false)); + bool v3TransitionPairUsed = result.isObject() && static_cast<bool> (result.getProperty ("v3TransitionPairUsed", false)); + bool v3ContinuousRenderUsed = result.isObject() && static_cast<bool> (result.getProperty ("v3ContinuousRenderUsed", false)); + double v3EntryAnchorMs = result.isObject() ? static_cast<double> (result.getProperty ("v3EntryAnchorMs", 0.0)) : 0.0; + double v3ExitAnchorMs = result.isObject() ? static_cast<double> (result.getProperty ("v3ExitAnchorMs", 0.0)) : 0.0; + int v3FirstCyclesEntryCount = result.isObject() ? static_cast<int> (result.getProperty ("v3FirstCyclesEntryCount", 0)) : 0; + int v3FirstCyclesExitCount = result.isObject() ? static_cast<int> (result.getProperty ("v3FirstCyclesExitCount", 0)) : 0; + double v3ShellDurationMs = result.isObject() ? static_cast<double> (result.getProperty ("v3ShellDurationMs", 0.0)) : 0.0; + double v3BodyDurationMs = result.isObject() ? static_cast<double> (result.getProperty ("v3BodyDurationMs", 0.0)) : 0.0; + double v3ResidualMix = result.isObject() ? static_cast<double> (result.getProperty ("v3ResidualMix", 0.0)) : 0.0; + juce::String v3FormantMode = result.isObject() ? result.getProperty ("v3FormantMode", {}).toString() : juce::String(); + double v3NeighborLeftOverlapMs = result.isObject() ? static_cast<double> (result.getProperty ("v3NeighborLeftOverlapMs", 0.0)) : 0.0; + double v3NeighborRightOverlapMs = result.isObject() ? static_cast<double> (result.getProperty ("v3NeighborRightOverlapMs", 0.0)) : 0.0; + if (! pitchRegressionJob.isVoid()) + { + auto nativeResult = juce::var (new juce::DynamicObject()); + if (auto* nativeResultObject = nativeResult.getDynamicObject()) + { + nativeResultObject->setProperty ("clipId", clipId); + nativeResultObject->setProperty ("requestId", requestId); + nativeResultObject->setProperty ("renderMode", renderMode); + nativeResultObject->setProperty ("requestedRendererBranch", requestedRendererBranch); + nativeResultObject->setProperty ("actualRendererBranch", actualRendererBranch); + nativeResultObject->setProperty ("pitchOnlyRecoveryPath", pitchOnlyRecoveryPath); + nativeResultObject->setProperty ("pitchOnlyNeutralFormantUsed", pitchOnlyNeutralFormantUsed); + nativeResultObject->setProperty ("processingMode", processingMode); + nativeResultObject->setProperty ("formantCurveUsed", formantCurveUsed); + nativeResultObject->setProperty ("explicitFormantRequested", explicitFormantRequested); + nativeResultObject->setProperty ("pitchOnlyFormantSuppressed", pitchOnlyFormantSuppressed); + nativeResultObject->setProperty ("usedFallback", usedFallback); + nativeResultObject->setProperty ("fallbackReason", fallbackReason); + nativeResultObject->setProperty ("hardFailReason", hardFailReason); + nativeResultObject->setProperty ("pitchRenderStrategy", pitchRenderStrategy); + nativeResultObject->setProperty ("phraseHqRenderUsed", phraseHqRenderUsed); + nativeResultObject->setProperty ("phraseHqExpandedToFullClip", phraseHqExpandedToFullClip); + nativeResultObject->setProperty ("phraseHqStartSec", phraseHqStartSec); + nativeResultObject->setProperty ("phraseHqEndSec", phraseHqEndSec); + nativeResultObject->setProperty ("pitchRenderProductPath", pitchRenderProductPath); + nativeResultObject->setProperty ("pitchRenderBackendId", pitchRenderBackendId); + nativeResultObject->setProperty ("pitchRenderBackendVersion", pitchRenderBackendVersion); + nativeResultObject->setProperty ("pitchRenderBackendFailureCode", pitchRenderBackendFailureCode); + nativeResultObject->setProperty ("pitchRenderBackendCapabilities", pitchRenderBackendCapabilities); + nativeResultObject->setProperty ("pitchRenderBackendDiagnostics", pitchRenderBackendDiagnostics); + nativeResultObject->setProperty ("pitchRenderCommitPolicy", pitchRenderCommitPolicy); + nativeResultObject->setProperty ("pitchRenderDryProtectedSamples", pitchRenderDryProtectedSamples); + nativeResultObject->setProperty ("pitchRenderContextDurationSec", pitchRenderContextDurationSec); + nativeResultObject->setProperty ("pitchRenderCommitDurationSec", pitchRenderCommitDurationSec); + nativeResultObject->setProperty ("pitchRenderJobStartDelayMs", pitchRenderJobStartDelayMs); + nativeResultObject->setProperty ("pitchRenderDirection", pitchRenderDirection); + nativeResultObject->setProperty ("downshiftFormantGuardUsed", downshiftFormantGuardUsed); + nativeResultObject->setProperty ("downshiftFormantGuardAlpha", downshiftFormantGuardAlpha); + nativeResultObject->setProperty ("noteHqEffectiveStartSec", noteHqEffectiveStartSec); + nativeResultObject->setProperty ("noteHqEffectiveEndSec", noteHqEffectiveEndSec); + nativeResultObject->setProperty ("noteHqContextStartSec", noteHqContextStartSec); + nativeResultObject->setProperty ("noteHqContextEndSec", noteHqContextEndSec); + nativeResultObject->setProperty ("noteHqAudibleCommitStartSec", noteHqAudibleCommitStartSec); + nativeResultObject->setProperty ("noteHqAudibleCommitEndSec", noteHqAudibleCommitEndSec); + nativeResultObject->setProperty ("noteHqPreBodyDryProtectedSamples", noteHqPreBodyDryProtectedSamples); + nativeResultObject->setProperty ("noteHqEntryInsideBodyFadeMs", noteHqEntryInsideBodyFadeMs); + nativeResultObject->setProperty ("noteHqExitLeadInMs", noteHqExitLeadInMs); + nativeResultObject->setProperty ("noteHqEntryBridgeStartSec", noteHqEntryBridgeStartSec); + nativeResultObject->setProperty ("noteHqEntryBridgeEndSec", noteHqEntryBridgeEndSec); + nativeResultObject->setProperty ("noteHqEntryBridgeWetLagMs", noteHqEntryBridgeWetLagMs); + nativeResultObject->setProperty ("noteHqEntryBridgeEnvelopeGainDb", noteHqEntryBridgeEnvelopeGainDb); + nativeResultObject->setProperty ("noteHqEntryBridgeUsed", noteHqEntryBridgeUsed); + nativeResultObject->setProperty ("noteHqEntryTransientDryPreservedMs", noteHqEntryTransientDryPreservedMs); + nativeResultObject->setProperty ("pitchOnlyEntrySimpleHandoffUsed", pitchOnlyEntrySimpleHandoffUsed); + nativeResultObject->setProperty ("pitchOnlyEntrySafeHandoffUsed", pitchOnlyEntrySafeHandoffUsed); + nativeResultObject->setProperty ("pitchOnlyEntryDryHoldMs", pitchOnlyEntryDryHoldMs); + nativeResultObject->setProperty ("pitchOnlyEntrySafeBridgeMs", pitchOnlyEntrySafeBridgeMs); + nativeResultObject->setProperty ("pitchOnlyEntryWetAlignmentMs", pitchOnlyEntryWetAlignmentMs); + nativeResultObject->setProperty ("pitchOnlyEntryWetGainDb", pitchOnlyEntryWetGainDb); + nativeResultObject->setProperty ("pitchOnlyEntryWetVsDryRmsDb", pitchOnlyEntryWetVsDryRmsDb); + nativeResultObject->setProperty ("pitchOnlyEntryEqualPowerBlendUsed", pitchOnlyEntryEqualPowerBlendUsed); + nativeResultObject->setProperty ("pitchOnlyEntryRmsContinuityUsed", pitchOnlyEntryRmsContinuityUsed); + nativeResultObject->setProperty ("pitchOnlyEntryRmsContinuityGainDb", pitchOnlyEntryRmsContinuityGainDb); + nativeResultObject->setProperty ("pitchOnlyEntryRmsContinuityMs", pitchOnlyEntryRmsContinuityMs); + nativeResultObject->setProperty ("pitchOnlyEntryPhaseSafeUsed", pitchOnlyEntryPhaseSafeUsed); + nativeResultObject->setProperty ("pitchOnlyEntryWetAlignmentAccepted", pitchOnlyEntryWetAlignmentAccepted); + nativeResultObject->setProperty ("pitchOnlyEntryFirstCycleCorrelation", pitchOnlyEntryFirstCycleCorrelation); + nativeResultObject->setProperty ("pitchOnlyEntryZeroCrossOffsetMs", pitchOnlyEntryZeroCrossOffsetMs); + nativeResultObject->setProperty ("pitchOnlyEntryBridgeGainRampDb", pitchOnlyEntryBridgeGainRampDb); + nativeResultObject->setProperty ("pitchOnlyDownshiftCoreEnvelopePassUsed", pitchOnlyDownshiftCoreEnvelopePassUsed); + nativeResultObject->setProperty ("pitchOnlyDownshiftCoreRmsTrimDb", pitchOnlyDownshiftCoreRmsTrimDb); + nativeResultObject->setProperty ("pitchOnlyDownshiftCoreEnvelopeMaxDb", pitchOnlyDownshiftCoreEnvelopeMaxDb); + nativeResultObject->setProperty ("pitchOnlyDownshiftCoreEnvelopeFrames", pitchOnlyDownshiftCoreEnvelopeFrames); + nativeResultObject->setProperty ("pitchOnlyEntryWetLagMs", pitchOnlyEntryWetLagMs); + nativeResultObject->setProperty ("pitchOnlyEntryBridgeDurationMs", pitchOnlyEntryBridgeDurationMs); + nativeResultObject->setProperty ("pitchOnlyExitDryRestoreUsed", pitchOnlyExitDryRestoreUsed); + nativeResultObject->setProperty ("pitchOnlyExitDryRestoreStartSec", pitchOnlyExitDryRestoreStartSec); + nativeResultObject->setProperty ("pitchOnlyExitDryRestoreEndSec", pitchOnlyExitDryRestoreEndSec); + nativeResultObject->setProperty ("noteHqEditIslandCount", noteHqEditIslandCount); + nativeResultObject->setProperty ("noteHqEditedNoteCount", noteHqEditedNoteCount); + nativeResultObject->setProperty ("noteHqEntryPitchHandoffUsed", noteHqEntryPitchHandoffUsed); + nativeResultObject->setProperty ("noteHqEntryPitchHandoffStartSec", noteHqEntryPitchHandoffStartSec); + nativeResultObject->setProperty ("noteHqEntryPitchHandoffEndSec", noteHqEntryPitchHandoffEndSec); + nativeResultObject->setProperty ("noteHqEntryPitchHandoffPreMs", noteHqEntryPitchHandoffPreMs); + nativeResultObject->setProperty ("noteHqEntryPitchHandoffBodyMs", noteHqEntryPitchHandoffBodyMs); + nativeResultObject->setProperty ("noteHqEntryPitchSlopeJumpStPerSec", noteHqEntryPitchSlopeJumpStPerSec); + nativeResultObject->setProperty ("noteHqEntryPitchAccelerationLimited", noteHqEntryPitchAccelerationLimited); + nativeResultObject->setProperty ("outputDurationSec", outputDurationSec); + nativeResultObject->setProperty ("postApplyRouteStatus", postApplyRouteStatus); + if (! appFinalCapture.isVoid()) + nativeResultObject->setProperty ("appFinalCapture", appFinalCapture); + if (! appFinalBakedCapture.isVoid()) + nativeResultObject->setProperty ("appFinalBakedCapture", appFinalBakedCapture); + if (! appFinalParityReport.isVoid()) + nativeResultObject->setProperty ("appFinalParityReport", appFinalParityReport); + if (appFinalRouteReportPath.isNotEmpty()) + nativeResultObject->setProperty ("appFinalRouteReportPath", appFinalRouteReportPath); + if (appFinalBakedContextPath.isNotEmpty()) + nativeResultObject->setProperty ("appFinalBakedContextPath", appFinalBakedContextPath); + if (appFinalPlaybackContextPath.isNotEmpty()) + nativeResultObject->setProperty ("appFinalPlaybackContextPath", appFinalPlaybackContextPath); + if (appFinalParityReportPath.isNotEmpty()) + nativeResultObject->setProperty ("appFinalParityReportPath", appFinalParityReportPath); + nativeResultObject->setProperty ("bridgeUsed", bridgeUsed); + nativeResultObject->setProperty ("bridgeFallbackUsed", bridgeFallbackUsed); + nativeResultObject->setProperty ("bridgeStartSec", bridgeStartSec); + nativeResultObject->setProperty ("bridgeLengthMs", bridgeLengthMs); + nativeResultObject->setProperty ("bridgeAlignmentLagSamples", bridgeAlignmentLagSamples); + nativeResultObject->setProperty ("bridgeCorrelationScore", bridgeCorrelationScore); + nativeResultObject->setProperty ("bridgeGainDeltaDb", bridgeGainDeltaDb); + nativeResultObject->setProperty ("bodyReplacementUsed", bodyReplacementUsed); + nativeResultObject->setProperty ("bodyReplacementFallbackUsed", bodyReplacementFallbackUsed); + nativeResultObject->setProperty ("entryLockStartSec", entryLockStartSec); + nativeResultObject->setProperty ("entryLockLengthMs", entryLockLengthMs); + nativeResultObject->setProperty ("exitLockStartSec", exitLockStartSec); + nativeResultObject->setProperty ("renderedBodyStartSec", renderedBodyStartSec); + nativeResultObject->setProperty ("renderedBodyEndSec", renderedBodyEndSec); + nativeResultObject->setProperty ("islandNativeUsed", islandNativeUsed); + nativeResultObject->setProperty ("islandNativeFallbackUsed", islandNativeFallbackUsed); + nativeResultObject->setProperty ("islandRenderStartSec", islandRenderStartSec); + nativeResultObject->setProperty ("islandRenderEndSec", islandRenderEndSec); + nativeResultObject->setProperty ("transientMaskPeak", transientMaskPeak); + nativeResultObject->setProperty ("voicedCoreMaskPeak", voicedCoreMaskPeak); + nativeResultObject->setProperty ("hpssUsed", hpssUsed); + nativeResultObject->setProperty ("hpssFallbackUsed", hpssFallbackUsed); + nativeResultObject->setProperty ("harmonicMaskPeak", harmonicMaskPeak); + nativeResultObject->setProperty ("aperiodicMaskPeak", aperiodicMaskPeak); + nativeResultObject->setProperty ("spectralEnvelopeCorrectionUsed", spectralEnvelopeCorrectionUsed); + nativeResultObject->setProperty ("pitchOnlyCoreTimbreCorrectionUsed", pitchOnlyCoreTimbreCorrectionUsed); + nativeResultObject->setProperty ("pitchOnlyCoreEnvelopeMix", pitchOnlyCoreEnvelopeMix); + nativeResultObject->setProperty ("pitchOnlyCoreRmsTrimDb", pitchOnlyCoreRmsTrimDb); + nativeResultObject->setProperty ("pitchOnlyCoreEnvelopeLifter", pitchOnlyCoreEnvelopeLifter); + nativeResultObject->setProperty ("pitchOnlyEntryTimbreCorrectionUsed", pitchOnlyEntryTimbreCorrectionUsed); + nativeResultObject->setProperty ("pitchOnlyEntryRmsTrimDb", pitchOnlyEntryRmsTrimDb); + nativeResultObject->setProperty ("pitchOnlyEntryTiltDb", pitchOnlyEntryTiltDb); + nativeResultObject->setProperty ("pitchOnlyEntryHandoffUsed", pitchOnlyEntryHandoffUsed); + nativeResultObject->setProperty ("pitchOnlyExitHandoffUsed", pitchOnlyExitHandoffUsed); + nativeResultObject->setProperty ("vocalSourceFilterUsed", vocalSourceFilterUsed); + nativeResultObject->setProperty ("vocalSourceFilterVoicedCoverage", vocalSourceFilterVoicedCoverage); + nativeResultObject->setProperty ("vocalSourceFilterResidualMix", vocalSourceFilterResidualMix); + nativeResultObject->setProperty ("vocalSourceFilterFallbackUsed", vocalSourceFilterFallbackUsed); + nativeResultObject->setProperty ("vocalSourceFilterFallbackReason", vocalSourceFilterFallbackReason); + nativeResultObject->setProperty ("vocalSourceFilterEntryDryMs", vocalSourceFilterEntryDryMs); + nativeResultObject->setProperty ("vocalSourceFilterExitDryMs", vocalSourceFilterExitDryMs); + nativeResultObject->setProperty ("wsolaUsed", wsolaUsed); + nativeResultObject->setProperty ("wsolaFallbackUsed", wsolaFallbackUsed); + nativeResultObject->setProperty ("wsolaEntryLagSamples", wsolaEntryLagSamples); + nativeResultObject->setProperty ("wsolaExitLagSamples", wsolaExitLagSamples); + nativeResultObject->setProperty ("wsolaCorrelationScore", wsolaCorrelationScore); + nativeResultObject->setProperty ("phaseLockUsed", phaseLockUsed); + nativeResultObject->setProperty ("phaseLockFallbackUsed", phaseLockFallbackUsed); + nativeResultObject->setProperty ("phaseAlignedEntry", phaseAlignedEntry); + nativeResultObject->setProperty ("phaseAlignedExit", phaseAlignedExit); + nativeResultObject->setProperty ("phasePeakCount", phasePeakCount); + nativeResultObject->setProperty ("transitionHqUsed", transitionHqUsed); + nativeResultObject->setProperty ("transitionHqFallbackUsed", transitionHqFallbackUsed); + nativeResultObject->setProperty ("transitionStartSec", transitionStartSec); + nativeResultObject->setProperty ("transitionEndSec", transitionEndSec); + nativeResultObject->setProperty ("transitionTransientPeak", transitionTransientPeak); + nativeResultObject->setProperty ("transitionVoicedCorePeak", transitionVoicedCorePeak); + nativeResultObject->setProperty ("transitionResidualPeak", transitionResidualPeak); + nativeResultObject->setProperty ("transitionEnvelopeCorrectionUsed", transitionEnvelopeCorrectionUsed); + nativeResultObject->setProperty ("engineV2Used", engineV2Used); + nativeResultObject->setProperty ("engineV2FallbackUsed", engineV2FallbackUsed); + nativeResultObject->setProperty ("engineV2TransitionCount", engineV2TransitionCount); + nativeResultObject->setProperty ("engineV2TransitionStartSec", engineV2TransitionStartSec); + nativeResultObject->setProperty ("engineV2TransitionEndSec", engineV2TransitionEndSec); + nativeResultObject->setProperty ("engineV2HarmonicSupportPeak", engineV2HarmonicSupportPeak); + nativeResultObject->setProperty ("engineV2ResidualSupportPeak", engineV2ResidualSupportPeak); + nativeResultObject->setProperty ("engineV2EnvelopeSupportPeak", engineV2EnvelopeSupportPeak); + nativeResultObject->setProperty ("transientBypassUsed", transientBypassUsed); + nativeResultObject->setProperty ("residualCarryUsed", residualCarryUsed); + nativeResultObject->setProperty ("cepstralCutoffUsed", cepstralCutoffUsed); + nativeResultObject->setProperty ("fftSizeUsed", engineV2FftSize); + nativeResultObject->setProperty ("hopSizeUsed", engineV2HopSize); + nativeResultObject->setProperty ("immediateLeftNeighborUsed", immediateLeftNeighborUsed); + nativeResultObject->setProperty ("immediateRightNeighborUsed", immediateRightNeighborUsed); + nativeResultObject->setProperty ("leftNeighborSamplesRendered", leftNeighborSamplesRendered); + nativeResultObject->setProperty ("rightNeighborSamplesRendered", rightNeighborSamplesRendered); + nativeResultObject->setProperty ("leftNeighborSmoothMs", leftNeighborSmoothMs); + nativeResultObject->setProperty ("rightNeighborSmoothMs", rightNeighborSmoothMs); + nativeResultObject->setProperty ("nonImmediateNeighborTouched", nonImmediateNeighborTouched); + nativeResultObject->setProperty ("entryAlignmentOffsetMs", entryAlignmentOffsetMs); + nativeResultObject->setProperty ("exitAlignmentOffsetMs", exitAlignmentOffsetMs); + nativeResultObject->setProperty ("firstVoicedCyclesEntryUsed", firstVoicedCyclesEntryUsed); + nativeResultObject->setProperty ("firstVoicedCyclesExitUsed", firstVoicedCyclesExitUsed); + nativeResultObject->setProperty ("v3TransitionPairUsed", v3TransitionPairUsed); + nativeResultObject->setProperty ("v3ContinuousRenderUsed", v3ContinuousRenderUsed); + nativeResultObject->setProperty ("v3EntryAnchorMs", v3EntryAnchorMs); + nativeResultObject->setProperty ("v3ExitAnchorMs", v3ExitAnchorMs); + nativeResultObject->setProperty ("v3FirstCyclesEntryCount", v3FirstCyclesEntryCount); + nativeResultObject->setProperty ("v3FirstCyclesExitCount", v3FirstCyclesExitCount); + nativeResultObject->setProperty ("v3ShellDurationMs", v3ShellDurationMs); + nativeResultObject->setProperty ("v3BodyDurationMs", v3BodyDurationMs); + nativeResultObject->setProperty ("v3ResidualMix", v3ResidualMix); + nativeResultObject->setProperty ("v3FormantMode", v3FormantMode); + nativeResultObject->setProperty ("v3NeighborLeftOverlapMs", v3NeighborLeftOverlapMs); + nativeResultObject->setProperty ("v3NeighborRightOverlapMs", v3NeighborRightOverlapMs); + nativeResultObject->setProperty ("previewCoverageStartSec", previewCoverageStartSec); + nativeResultObject->setProperty ("previewCoverageEndSec", previewCoverageEndSec); + nativeResultObject->setProperty ("candidateCoverageStartSec", candidateCoverageStartSec); + nativeResultObject->setProperty ("candidateCoverageEndSec", candidateCoverageEndSec); + const juce::ScopedLock resultLock (pitchRegressionNativeResultLock); + lastPitchRegressionNativeResult = nativeResult; + } + } + logPitchEditorFormant ("job finished clip=" + clipId + " requestId=" + requestId + " requestGroupId=" + requestGroupId @@ -4320,14 +5013,25 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, + " success=" + juce::String(success ? "true" : "false") + " cancelled=" + juce::String(cancelled ? "true" : "false") + " swapDeferred=" + juce::String(swapDeferred ? "true" : "false") + + " requestedBranch=" + requestedRendererBranch + + " actualBranch=" + actualRendererBranch + + " usedFallback=" + juce::String (usedFallback ? "true" : "false") + + " fallbackReason=" + fallbackReason + + " previewCoverageStart=" + juce::String(previewCoverageStartSec, 3) + + " previewCoverageEnd=" + juce::String(previewCoverageEndSec, 3) + " restored=" + juce::String(restored ? "true" : "false") + " outputFile=" + outputFile); - juce::MessageManager::callAsync ([this, clipId, success, outputFile, requestId, restored, renderMode, cancelled, swapDeferred]() { + juce::MessageManager::callAsync ([this, clipId, success, outputFile, requestId, restored, renderMode, cancelled, swapDeferred, previewCoverageStartSec, previewCoverageEndSec, candidateCoverageStartSec, candidateCoverageEndSec, requestedRendererBranch, actualRendererBranch, pitchOnlyRecoveryPath, pitchOnlyNeutralFormantUsed, processingMode, formantCurveUsed, explicitFormantRequested, pitchOnlyFormantSuppressed, usedFallback, fallbackReason, hardFailReason, pitchRenderStrategy, phraseHqRenderUsed, phraseHqExpandedToFullClip, phraseHqStartSec, phraseHqEndSec, pitchRenderProductPath, pitchRenderBackendId, pitchRenderBackendVersion, pitchRenderBackendFailureCode, pitchRenderBackendCapabilities, pitchRenderBackendDiagnostics, pitchRenderCommitPolicy, pitchRenderDryProtectedSamples, pitchRenderContextDurationSec, pitchRenderCommitDurationSec, pitchRenderJobStartDelayMs, pitchRenderDirection, downshiftFormantGuardUsed, downshiftFormantGuardAlpha, noteHqEffectiveStartSec, noteHqEffectiveEndSec, noteHqContextStartSec, noteHqContextEndSec, noteHqAudibleCommitStartSec, noteHqAudibleCommitEndSec, noteHqPreBodyDryProtectedSamples, noteHqEntryInsideBodyFadeMs, noteHqExitLeadInMs, noteHqEntryBridgeStartSec, noteHqEntryBridgeEndSec, noteHqEntryBridgeWetLagMs, noteHqEntryBridgeEnvelopeGainDb, noteHqEntryBridgeUsed, noteHqEntryTransientDryPreservedMs, pitchOnlyEntrySimpleHandoffUsed, pitchOnlyEntrySafeHandoffUsed, pitchOnlyEntryDryHoldMs, pitchOnlyEntrySafeBridgeMs, pitchOnlyEntryWetAlignmentMs, pitchOnlyEntryWetGainDb, pitchOnlyEntryWetVsDryRmsDb, pitchOnlyEntryEqualPowerBlendUsed, pitchOnlyEntryRmsContinuityUsed, pitchOnlyEntryRmsContinuityGainDb, pitchOnlyEntryRmsContinuityMs, pitchOnlyEntryPhaseSafeUsed, pitchOnlyEntryWetAlignmentAccepted, pitchOnlyEntryFirstCycleCorrelation, pitchOnlyEntryZeroCrossOffsetMs, pitchOnlyEntryBridgeGainRampDb, pitchOnlyDownshiftCoreEnvelopePassUsed, pitchOnlyDownshiftCoreRmsTrimDb, pitchOnlyDownshiftCoreEnvelopeMaxDb, pitchOnlyDownshiftCoreEnvelopeFrames, pitchOnlyEntryWetLagMs, pitchOnlyEntryBridgeDurationMs, pitchOnlyExitDryRestoreUsed, pitchOnlyExitDryRestoreStartSec, pitchOnlyExitDryRestoreEndSec, noteHqEditIslandCount, noteHqEditedNoteCount, noteHqEntryPitchHandoffUsed, noteHqEntryPitchHandoffStartSec, noteHqEntryPitchHandoffEndSec, noteHqEntryPitchHandoffPreMs, noteHqEntryPitchHandoffBodyMs, noteHqEntryPitchSlopeJumpStPerSec, noteHqEntryPitchAccelerationLimited, outputDurationSec, postApplyRouteStatus, appFinalCapture, appFinalBakedCapture, appFinalParityReport, appFinalRouteReportPath, appFinalBakedContextPath, appFinalPlaybackContextPath, appFinalParityReportPath, bridgeUsed, bridgeFallbackUsed, bridgeStartSec, bridgeLengthMs, bridgeAlignmentLagSamples, bridgeCorrelationScore, bridgeGainDeltaDb, bodyReplacementUsed, bodyReplacementFallbackUsed, entryLockStartSec, entryLockLengthMs, exitLockStartSec, renderedBodyStartSec, renderedBodyEndSec, islandNativeUsed, islandNativeFallbackUsed, islandRenderStartSec, islandRenderEndSec, transientMaskPeak, voicedCoreMaskPeak, hpssUsed, hpssFallbackUsed, harmonicMaskPeak, aperiodicMaskPeak, spectralEnvelopeCorrectionUsed, pitchOnlyCoreTimbreCorrectionUsed, pitchOnlyCoreEnvelopeMix, pitchOnlyCoreRmsTrimDb, pitchOnlyCoreEnvelopeLifter, pitchOnlyEntryTimbreCorrectionUsed, pitchOnlyEntryRmsTrimDb, pitchOnlyEntryTiltDb, pitchOnlyEntryHandoffUsed, pitchOnlyExitHandoffUsed, vocalSourceFilterUsed, vocalSourceFilterVoicedCoverage, vocalSourceFilterResidualMix, vocalSourceFilterFallbackUsed, vocalSourceFilterFallbackReason, vocalSourceFilterEntryDryMs, vocalSourceFilterExitDryMs, wsolaUsed, wsolaFallbackUsed, wsolaEntryLagSamples, wsolaExitLagSamples, wsolaCorrelationScore, phaseLockUsed, phaseLockFallbackUsed, phaseAlignedEntry, phaseAlignedExit, phasePeakCount, transitionHqUsed, transitionHqFallbackUsed, transitionStartSec, transitionEndSec, transitionTransientPeak, transitionVoicedCorePeak, transitionResidualPeak, transitionEnvelopeCorrectionUsed, engineV2Used, engineV2FallbackUsed, engineV2TransitionCount, engineV2TransitionStartSec, engineV2TransitionEndSec, engineV2HarmonicSupportPeak, engineV2ResidualSupportPeak, engineV2EnvelopeSupportPeak, transientBypassUsed, residualCarryUsed, cepstralCutoffUsed, engineV2FftSize, engineV2HopSize, immediateLeftNeighborUsed, immediateRightNeighborUsed, leftNeighborSamplesRendered, rightNeighborSamplesRendered, leftNeighborSmoothMs, rightNeighborSmoothMs, nonImmediateNeighborTouched, entryAlignmentOffsetMs, exitAlignmentOffsetMs, firstVoicedCyclesEntryUsed, firstVoicedCyclesExitUsed, v3TransitionPairUsed, v3ContinuousRenderUsed, v3EntryAnchorMs, v3ExitAnchorMs, v3FirstCyclesEntryCount, v3FirstCyclesExitCount, v3ShellDurationMs, v3BodyDurationMs, v3ResidualMix, v3FormantMode, v3NeighborLeftOverlapMs, v3NeighborRightOverlapMs]() { logPitchEditorFormant ("emitting pitchCorrectionComplete clip=" + clipId + " requestId=" + requestId + " renderMode=" + renderMode + " cancelled=" + juce::String (cancelled ? "true" : "false") + " swapDeferred=" + juce::String (swapDeferred ? "true" : "false") + + " requestedBranch=" + requestedRendererBranch + + " actualBranch=" + actualRendererBranch + + " usedFallback=" + juce::String (usedFallback ? "true" : "false") + + " previewCoverageStart=" + juce::String(previewCoverageStartSec, 3) + + " previewCoverageEnd=" + juce::String(previewCoverageEndSec, 3) + " success=" + juce::String (success ? "true" : "false") + " outputFile=" + outputFile); auto obj = std::make_unique<juce::DynamicObject>(); @@ -4339,6 +5043,200 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, obj->setProperty ("renderMode", renderMode); obj->setProperty ("cancelled", cancelled); obj->setProperty ("swapDeferred", swapDeferred); + obj->setProperty ("previewCoverageStartSec", previewCoverageStartSec); + obj->setProperty ("previewCoverageEndSec", previewCoverageEndSec); + obj->setProperty ("candidateCoverageStartSec", candidateCoverageStartSec); + obj->setProperty ("candidateCoverageEndSec", candidateCoverageEndSec); + obj->setProperty ("requestedRendererBranch", requestedRendererBranch); + obj->setProperty ("actualRendererBranch", actualRendererBranch); + obj->setProperty ("pitchOnlyRecoveryPath", pitchOnlyRecoveryPath); + obj->setProperty ("pitchOnlyNeutralFormantUsed", pitchOnlyNeutralFormantUsed); + obj->setProperty ("processingMode", processingMode); + obj->setProperty ("formantCurveUsed", formantCurveUsed); + obj->setProperty ("explicitFormantRequested", explicitFormantRequested); + obj->setProperty ("pitchOnlyFormantSuppressed", pitchOnlyFormantSuppressed); + obj->setProperty ("usedFallback", usedFallback); + obj->setProperty ("fallbackReason", fallbackReason); + obj->setProperty ("hardFailReason", hardFailReason); + obj->setProperty ("pitchRenderStrategy", pitchRenderStrategy); + obj->setProperty ("phraseHqRenderUsed", phraseHqRenderUsed); + obj->setProperty ("phraseHqExpandedToFullClip", phraseHqExpandedToFullClip); + obj->setProperty ("phraseHqStartSec", phraseHqStartSec); + obj->setProperty ("phraseHqEndSec", phraseHqEndSec); + obj->setProperty ("pitchRenderProductPath", pitchRenderProductPath); + obj->setProperty ("pitchRenderBackendId", pitchRenderBackendId); + obj->setProperty ("pitchRenderBackendVersion", pitchRenderBackendVersion); + obj->setProperty ("pitchRenderBackendFailureCode", pitchRenderBackendFailureCode); + obj->setProperty ("pitchRenderBackendCapabilities", pitchRenderBackendCapabilities); + obj->setProperty ("pitchRenderBackendDiagnostics", pitchRenderBackendDiagnostics); + obj->setProperty ("pitchRenderCommitPolicy", pitchRenderCommitPolicy); + obj->setProperty ("pitchRenderDryProtectedSamples", pitchRenderDryProtectedSamples); + obj->setProperty ("pitchRenderContextDurationSec", pitchRenderContextDurationSec); + obj->setProperty ("pitchRenderCommitDurationSec", pitchRenderCommitDurationSec); + obj->setProperty ("pitchRenderJobStartDelayMs", pitchRenderJobStartDelayMs); + obj->setProperty ("pitchRenderDirection", pitchRenderDirection); + obj->setProperty ("downshiftFormantGuardUsed", downshiftFormantGuardUsed); + obj->setProperty ("downshiftFormantGuardAlpha", downshiftFormantGuardAlpha); + obj->setProperty ("noteHqEffectiveStartSec", noteHqEffectiveStartSec); + obj->setProperty ("noteHqEffectiveEndSec", noteHqEffectiveEndSec); + obj->setProperty ("noteHqContextStartSec", noteHqContextStartSec); + obj->setProperty ("noteHqContextEndSec", noteHqContextEndSec); + obj->setProperty ("noteHqAudibleCommitStartSec", noteHqAudibleCommitStartSec); + obj->setProperty ("noteHqAudibleCommitEndSec", noteHqAudibleCommitEndSec); + obj->setProperty ("noteHqPreBodyDryProtectedSamples", noteHqPreBodyDryProtectedSamples); + obj->setProperty ("noteHqEntryInsideBodyFadeMs", noteHqEntryInsideBodyFadeMs); + obj->setProperty ("noteHqExitLeadInMs", noteHqExitLeadInMs); + obj->setProperty ("noteHqEntryBridgeStartSec", noteHqEntryBridgeStartSec); + obj->setProperty ("noteHqEntryBridgeEndSec", noteHqEntryBridgeEndSec); + obj->setProperty ("noteHqEntryBridgeWetLagMs", noteHqEntryBridgeWetLagMs); + obj->setProperty ("noteHqEntryBridgeEnvelopeGainDb", noteHqEntryBridgeEnvelopeGainDb); + obj->setProperty ("noteHqEntryBridgeUsed", noteHqEntryBridgeUsed); + obj->setProperty ("noteHqEntryTransientDryPreservedMs", noteHqEntryTransientDryPreservedMs); + obj->setProperty ("pitchOnlyEntrySimpleHandoffUsed", pitchOnlyEntrySimpleHandoffUsed); + obj->setProperty ("pitchOnlyEntrySafeHandoffUsed", pitchOnlyEntrySafeHandoffUsed); + obj->setProperty ("pitchOnlyEntryDryHoldMs", pitchOnlyEntryDryHoldMs); + obj->setProperty ("pitchOnlyEntrySafeBridgeMs", pitchOnlyEntrySafeBridgeMs); + obj->setProperty ("pitchOnlyEntryWetAlignmentMs", pitchOnlyEntryWetAlignmentMs); + obj->setProperty ("pitchOnlyEntryWetGainDb", pitchOnlyEntryWetGainDb); + obj->setProperty ("pitchOnlyEntryWetVsDryRmsDb", pitchOnlyEntryWetVsDryRmsDb); + obj->setProperty ("pitchOnlyEntryEqualPowerBlendUsed", pitchOnlyEntryEqualPowerBlendUsed); + obj->setProperty ("pitchOnlyEntryRmsContinuityUsed", pitchOnlyEntryRmsContinuityUsed); + obj->setProperty ("pitchOnlyEntryRmsContinuityGainDb", pitchOnlyEntryRmsContinuityGainDb); + obj->setProperty ("pitchOnlyEntryRmsContinuityMs", pitchOnlyEntryRmsContinuityMs); + obj->setProperty ("pitchOnlyEntryPhaseSafeUsed", pitchOnlyEntryPhaseSafeUsed); + obj->setProperty ("pitchOnlyEntryWetAlignmentAccepted", pitchOnlyEntryWetAlignmentAccepted); + obj->setProperty ("pitchOnlyEntryFirstCycleCorrelation", pitchOnlyEntryFirstCycleCorrelation); + obj->setProperty ("pitchOnlyEntryZeroCrossOffsetMs", pitchOnlyEntryZeroCrossOffsetMs); + obj->setProperty ("pitchOnlyEntryBridgeGainRampDb", pitchOnlyEntryBridgeGainRampDb); + obj->setProperty ("pitchOnlyDownshiftCoreEnvelopePassUsed", pitchOnlyDownshiftCoreEnvelopePassUsed); + obj->setProperty ("pitchOnlyDownshiftCoreRmsTrimDb", pitchOnlyDownshiftCoreRmsTrimDb); + obj->setProperty ("pitchOnlyDownshiftCoreEnvelopeMaxDb", pitchOnlyDownshiftCoreEnvelopeMaxDb); + obj->setProperty ("pitchOnlyDownshiftCoreEnvelopeFrames", pitchOnlyDownshiftCoreEnvelopeFrames); + obj->setProperty ("pitchOnlyEntryWetLagMs", pitchOnlyEntryWetLagMs); + obj->setProperty ("pitchOnlyEntryBridgeDurationMs", pitchOnlyEntryBridgeDurationMs); + obj->setProperty ("pitchOnlyExitDryRestoreUsed", pitchOnlyExitDryRestoreUsed); + obj->setProperty ("pitchOnlyExitDryRestoreStartSec", pitchOnlyExitDryRestoreStartSec); + obj->setProperty ("pitchOnlyExitDryRestoreEndSec", pitchOnlyExitDryRestoreEndSec); + obj->setProperty ("noteHqEditIslandCount", noteHqEditIslandCount); + obj->setProperty ("noteHqEditedNoteCount", noteHqEditedNoteCount); + obj->setProperty ("noteHqEntryPitchHandoffUsed", noteHqEntryPitchHandoffUsed); + obj->setProperty ("noteHqEntryPitchHandoffStartSec", noteHqEntryPitchHandoffStartSec); + obj->setProperty ("noteHqEntryPitchHandoffEndSec", noteHqEntryPitchHandoffEndSec); + obj->setProperty ("noteHqEntryPitchHandoffPreMs", noteHqEntryPitchHandoffPreMs); + obj->setProperty ("noteHqEntryPitchHandoffBodyMs", noteHqEntryPitchHandoffBodyMs); + obj->setProperty ("noteHqEntryPitchSlopeJumpStPerSec", noteHqEntryPitchSlopeJumpStPerSec); + obj->setProperty ("noteHqEntryPitchAccelerationLimited", noteHqEntryPitchAccelerationLimited); + obj->setProperty ("outputDurationSec", outputDurationSec); + obj->setProperty ("postApplyRouteStatus", postApplyRouteStatus); + if (! appFinalCapture.isVoid()) + obj->setProperty ("appFinalCapture", appFinalCapture); + if (! appFinalBakedCapture.isVoid()) + obj->setProperty ("appFinalBakedCapture", appFinalBakedCapture); + if (! appFinalParityReport.isVoid()) + obj->setProperty ("appFinalParityReport", appFinalParityReport); + if (appFinalRouteReportPath.isNotEmpty()) + obj->setProperty ("appFinalRouteReportPath", appFinalRouteReportPath); + if (appFinalBakedContextPath.isNotEmpty()) + obj->setProperty ("appFinalBakedContextPath", appFinalBakedContextPath); + if (appFinalPlaybackContextPath.isNotEmpty()) + obj->setProperty ("appFinalPlaybackContextPath", appFinalPlaybackContextPath); + if (appFinalParityReportPath.isNotEmpty()) + obj->setProperty ("appFinalParityReportPath", appFinalParityReportPath); + obj->setProperty ("bridgeUsed", bridgeUsed); + obj->setProperty ("bridgeFallbackUsed", bridgeFallbackUsed); + obj->setProperty ("bridgeStartSec", bridgeStartSec); + obj->setProperty ("bridgeLengthMs", bridgeLengthMs); + obj->setProperty ("bridgeAlignmentLagSamples", bridgeAlignmentLagSamples); + obj->setProperty ("bridgeCorrelationScore", bridgeCorrelationScore); + obj->setProperty ("bridgeGainDeltaDb", bridgeGainDeltaDb); + obj->setProperty ("bodyReplacementUsed", bodyReplacementUsed); + obj->setProperty ("bodyReplacementFallbackUsed", bodyReplacementFallbackUsed); + obj->setProperty ("entryLockStartSec", entryLockStartSec); + obj->setProperty ("entryLockLengthMs", entryLockLengthMs); + obj->setProperty ("exitLockStartSec", exitLockStartSec); + obj->setProperty ("renderedBodyStartSec", renderedBodyStartSec); + obj->setProperty ("renderedBodyEndSec", renderedBodyEndSec); + obj->setProperty ("islandNativeUsed", islandNativeUsed); + obj->setProperty ("islandNativeFallbackUsed", islandNativeFallbackUsed); + obj->setProperty ("islandRenderStartSec", islandRenderStartSec); + obj->setProperty ("islandRenderEndSec", islandRenderEndSec); + obj->setProperty ("transientMaskPeak", transientMaskPeak); + obj->setProperty ("voicedCoreMaskPeak", voicedCoreMaskPeak); + obj->setProperty ("hpssUsed", hpssUsed); + obj->setProperty ("hpssFallbackUsed", hpssFallbackUsed); + obj->setProperty ("harmonicMaskPeak", harmonicMaskPeak); + obj->setProperty ("aperiodicMaskPeak", aperiodicMaskPeak); + obj->setProperty ("spectralEnvelopeCorrectionUsed", spectralEnvelopeCorrectionUsed); + obj->setProperty ("pitchOnlyCoreTimbreCorrectionUsed", pitchOnlyCoreTimbreCorrectionUsed); + obj->setProperty ("pitchOnlyCoreEnvelopeMix", pitchOnlyCoreEnvelopeMix); + obj->setProperty ("pitchOnlyCoreRmsTrimDb", pitchOnlyCoreRmsTrimDb); + obj->setProperty ("pitchOnlyCoreEnvelopeLifter", pitchOnlyCoreEnvelopeLifter); + obj->setProperty ("pitchOnlyEntryTimbreCorrectionUsed", pitchOnlyEntryTimbreCorrectionUsed); + obj->setProperty ("pitchOnlyEntryRmsTrimDb", pitchOnlyEntryRmsTrimDb); + obj->setProperty ("pitchOnlyEntryTiltDb", pitchOnlyEntryTiltDb); + obj->setProperty ("pitchOnlyEntryHandoffUsed", pitchOnlyEntryHandoffUsed); + obj->setProperty ("pitchOnlyExitHandoffUsed", pitchOnlyExitHandoffUsed); + obj->setProperty ("vocalSourceFilterUsed", vocalSourceFilterUsed); + obj->setProperty ("vocalSourceFilterVoicedCoverage", vocalSourceFilterVoicedCoverage); + obj->setProperty ("vocalSourceFilterResidualMix", vocalSourceFilterResidualMix); + obj->setProperty ("vocalSourceFilterFallbackUsed", vocalSourceFilterFallbackUsed); + obj->setProperty ("vocalSourceFilterFallbackReason", vocalSourceFilterFallbackReason); + obj->setProperty ("vocalSourceFilterEntryDryMs", vocalSourceFilterEntryDryMs); + obj->setProperty ("vocalSourceFilterExitDryMs", vocalSourceFilterExitDryMs); + obj->setProperty ("wsolaUsed", wsolaUsed); + obj->setProperty ("wsolaFallbackUsed", wsolaFallbackUsed); + obj->setProperty ("wsolaEntryLagSamples", wsolaEntryLagSamples); + obj->setProperty ("wsolaExitLagSamples", wsolaExitLagSamples); + obj->setProperty ("wsolaCorrelationScore", wsolaCorrelationScore); + obj->setProperty ("phaseLockUsed", phaseLockUsed); + obj->setProperty ("phaseLockFallbackUsed", phaseLockFallbackUsed); + obj->setProperty ("phaseAlignedEntry", phaseAlignedEntry); + obj->setProperty ("phaseAlignedExit", phaseAlignedExit); + obj->setProperty ("phasePeakCount", phasePeakCount); + obj->setProperty ("transitionHqUsed", transitionHqUsed); + obj->setProperty ("transitionHqFallbackUsed", transitionHqFallbackUsed); + obj->setProperty ("transitionStartSec", transitionStartSec); + obj->setProperty ("transitionEndSec", transitionEndSec); + obj->setProperty ("transitionTransientPeak", transitionTransientPeak); + obj->setProperty ("transitionVoicedCorePeak", transitionVoicedCorePeak); + obj->setProperty ("transitionResidualPeak", transitionResidualPeak); + obj->setProperty ("transitionEnvelopeCorrectionUsed", transitionEnvelopeCorrectionUsed); + obj->setProperty ("engineV2Used", engineV2Used); + obj->setProperty ("engineV2FallbackUsed", engineV2FallbackUsed); + obj->setProperty ("engineV2TransitionCount", engineV2TransitionCount); + obj->setProperty ("engineV2TransitionStartSec", engineV2TransitionStartSec); + obj->setProperty ("engineV2TransitionEndSec", engineV2TransitionEndSec); + obj->setProperty ("engineV2HarmonicSupportPeak", engineV2HarmonicSupportPeak); + obj->setProperty ("engineV2ResidualSupportPeak", engineV2ResidualSupportPeak); + obj->setProperty ("engineV2EnvelopeSupportPeak", engineV2EnvelopeSupportPeak); + obj->setProperty ("transientBypassUsed", transientBypassUsed); + obj->setProperty ("residualCarryUsed", residualCarryUsed); + obj->setProperty ("cepstralCutoffUsed", cepstralCutoffUsed); + obj->setProperty ("fftSizeUsed", engineV2FftSize); + obj->setProperty ("hopSizeUsed", engineV2HopSize); + obj->setProperty ("immediateLeftNeighborUsed", immediateLeftNeighborUsed); + obj->setProperty ("immediateRightNeighborUsed", immediateRightNeighborUsed); + obj->setProperty ("leftNeighborSamplesRendered", leftNeighborSamplesRendered); + obj->setProperty ("rightNeighborSamplesRendered", rightNeighborSamplesRendered); + obj->setProperty ("leftNeighborSmoothMs", leftNeighborSmoothMs); + obj->setProperty ("rightNeighborSmoothMs", rightNeighborSmoothMs); + obj->setProperty ("nonImmediateNeighborTouched", nonImmediateNeighborTouched); + obj->setProperty ("entryAlignmentOffsetMs", entryAlignmentOffsetMs); + obj->setProperty ("exitAlignmentOffsetMs", exitAlignmentOffsetMs); + obj->setProperty ("firstVoicedCyclesEntryUsed", firstVoicedCyclesEntryUsed); + obj->setProperty ("firstVoicedCyclesExitUsed", firstVoicedCyclesExitUsed); + obj->setProperty ("v3TransitionPairUsed", v3TransitionPairUsed); + obj->setProperty ("v3ContinuousRenderUsed", v3ContinuousRenderUsed); + obj->setProperty ("v3EntryAnchorMs", v3EntryAnchorMs); + obj->setProperty ("v3ExitAnchorMs", v3ExitAnchorMs); + obj->setProperty ("v3FirstCyclesEntryCount", v3FirstCyclesEntryCount); + obj->setProperty ("v3FirstCyclesExitCount", v3FirstCyclesExitCount); + obj->setProperty ("v3ShellDurationMs", v3ShellDurationMs); + obj->setProperty ("v3BodyDurationMs", v3BodyDurationMs); + obj->setProperty ("v3ResidualMix", v3ResidualMix); + obj->setProperty ("v3FormantMode", v3FormantMode); + obj->setProperty ("v3NeighborLeftOverlapMs", v3NeighborLeftOverlapMs); + obj->setProperty ("v3NeighborRightOverlapMs", v3NeighborRightOverlapMs); webView.emitEventIfBrowserIsVisible ("pitchCorrectionComplete", juce::var (obj.release())); }); @@ -4388,6 +5286,37 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, else completion(juce::var()); }) + .withNativeFunction ("startPitchScrubPreview", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + if (args.size() >= 4) + completion(audioEngine.startPitchScrubPreview(args[0].toString(), args[1].toString(), args[2], args[3])); + else + completion(false); + }) + .withNativeFunction ("updatePitchScrubPreview", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + if (args.size() >= 2) + completion(audioEngine.updatePitchScrubPreview(args[0].toString(), + static_cast<float> (static_cast<double> (args[1])))); + else + completion(false); + }) + .withNativeFunction ("stopPitchScrubPreview", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + if (args.size() >= 1) + completion(audioEngine.stopPitchScrubPreview(args[0].toString())); + else + completion(false); + }) + .withNativeFunction ("getPitchScrubPreviewStatus", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + if (args.size() >= 1) + completion(audioEngine.getPitchScrubPreviewStatus(args[0].toString())); + else + completion(audioEngine.getPitchScrubPreviewStatus()); + }) + .withNativeFunction ("getPitchPreviewRoutingStatus", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + if (args.size() >= 1) + completion(audioEngine.getPitchPreviewRoutingStatus(args[0].toString())); + else + completion(audioEngine.getPitchPreviewRoutingStatus()); + }) .withNativeFunction ("setClipPitchPreview", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { if (args.size() >= 2) { @@ -4418,11 +5347,14 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, previewData.previewStartSec = static_cast<double> (previewObj->getProperty ("previewStartSec")); if (previewObj->hasProperty ("previewEndSec")) previewData.previewEndSec = static_cast<double> (previewObj->getProperty ("previewEndSec")); + if (previewObj->hasProperty ("allowReplacingCorrectedSource")) + previewData.allowReplacingCorrectedSource = static_cast<bool> (previewObj->getProperty ("allowReplacingCorrectedSource")); } logPitchEditorFormant ("setClipPitchPreview clip=" + clipId + " pitchSegments=" + juce::String (static_cast<int> (previewData.pitchSegments.size())) + " globalFormantSt=" + juce::String (previewData.globalFormantSemitones, 3) + + " allowReplacingCorrectedSource=" + juce::String (previewData.allowReplacingCorrectedSource ? "true" : "false") + " window=[" + juce::String (previewData.previewStartSec, 3) + "," + juce::String (previewData.previewEndSec, 3) + "]"); @@ -4450,6 +5382,13 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, else completion (false); }) + .withNativeFunction ("clearAllPitchPreviewRoutes", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + audioEngine.getPlaybackEngine().clearAllPitchPreviewRoutes (args.size() >= 1 ? args[0].toString() : juce::String()); + completion (true); + }) + .withNativeFunction ("clearPitchPreviewRoutesForCorrectedSources", [this] (const juce::Array<juce::var>&, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + completion (audioEngine.getPlaybackEngine().clearPitchPreviewRoutesForCorrectedSources()); + }) .withNativeFunction ("separateStems", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { if (args.size() >= 2) completion(audioEngine.separateStems(args[0].toString(), args[1].toString())); @@ -4465,8 +5404,12 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, .withNativeFunction ("refreshAiToolsStatus", [this] (const juce::Array<juce::var>&, juce::WebBrowserComponent::NativeFunctionCompletion completion) { completion(audioEngine.refreshAiToolsStatus()); }) - .withNativeFunction ("installAiTools", [this] (const juce::Array<juce::var>&, juce::WebBrowserComponent::NativeFunctionCompletion completion) { - completion(audioEngine.installAiTools()); + .withNativeFunction ("installAiTools", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + const bool userConfirmedDownload = args.size() > 0 && static_cast<bool>(args[0]); + completion(audioEngine.installAiTools(userConfirmedDownload)); + }) + .withNativeFunction ("resetAiTools", [this] (const juce::Array<juce::var>&, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + completion(audioEngine.resetAiTools()); }) .withNativeFunction ("separateStemsAsync", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { if (args.size() >= 3) @@ -4485,6 +5428,19 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, audioEngine.cancelAiToolsInstall(); completion(juce::var()); }) + .withNativeFunction ("startAIGeneration", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + if (args.size() >= 3) + completion(audioEngine.startAIGeneration(args[0].toString(), args[1].toString(), args[2].toString())); + else + completion(juce::var()); + }) + .withNativeFunction ("getAIGenerationProgress", [this] (const juce::Array<juce::var>&, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + completion(audioEngine.getAIGenerationProgress()); + }) + .withNativeFunction ("cancelAIGeneration", [this] (const juce::Array<juce::var>&, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + audioEngine.cancelAIGeneration(); + completion(juce::var()); + }) .withNativeFunction ("initializeARA", [this] (const juce::Array<juce::var>& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { if (args.size() >= 2) completion(audioEngine.initializeARAForTrack(args[0].toString(), static_cast<int>(args[1]))); @@ -4566,14 +5522,14 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, fallbackMessage.setJustificationType(juce::Justification::centred); fallbackMessage.setColour(juce::Label::textColourId, juce::Colours::white); fallbackMessage.setColour(juce::Label::backgroundColourId, juce::Colour(0xff111827)); - fallbackMessage.setFont(juce::Font(16.0f)); + fallbackMessage.setFont(juce::Font(juce::FontOptions(16.0f))); fallbackMessage.setText({}, juce::dontSendNotification); fallbackMessage.setVisible(false); startupStatusMessage.setJustificationType(juce::Justification::centred); startupStatusMessage.setColour(juce::Label::textColourId, juce::Colours::white); startupStatusMessage.setColour(juce::Label::backgroundColourId, juce::Colour(0xf0111827)); - startupStatusMessage.setFont(juce::Font(16.0f)); + startupStatusMessage.setFont(juce::Font(juce::FontOptions(16.0f))); startupStatusMessage.setText({}, juce::dontSendNotification); startupStatusMessage.setVisible(false); startupStatusMessage.setInterceptsMouseClicks(false, false); @@ -4661,13 +5617,8 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, if (! loadedFrontend) { - if (packagedFrontend.existsAsFile()) + if (loadPackagedFrontend()) { - webuiDir = packagedFrontend.getParentDirectory(); - juce::Logger::writeToLog("Loading packaged frontend from: " + packagedFrontend.getFullPathName()); - const auto frontendUrl = appendFrontendStartupQuery(juce::WebBrowserComponent::getResourceProviderRoot() + "index.html", windowRole, startupMode); - beginFrontendStartupWatchdog(frontendUrl); - webView.goToURL(frontendUrl); loadedFrontend = true; } else @@ -4695,7 +5646,9 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, activeInstances.add(this); } - if (isMainWindow()) + initializePitchRegressionJob(pitchRegressionJobPathIn); + + if (isMainWindow() && pitchRegressionJob.isVoid()) juce::Timer::callAfterDelay(2000, [this]() { appUpdater.checkForUpdates(false, [this](const juce::var& status) @@ -4733,6 +5686,270 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, juce::Logger::writeToLog("MainComponent initialized successfully"); } +void MainComponent::initializePitchRegressionJob(const juce::String& pitchRegressionJobPathIn) +{ + if (! isMainWindow()) + return; + + const auto trimmedPath = pitchRegressionJobPathIn.trim().unquoted(); + if (trimmedPath.isEmpty()) + return; + + pitchRegressionJobFile = juce::File(trimmedPath); + if (! pitchRegressionJobFile.existsAsFile()) + { + juce::Logger::writeToLog("[pitchRegression] Job file not found: " + pitchRegressionJobFile.getFullPathName()); + return; + } + + const auto parsedJob = juce::JSON::parse(pitchRegressionJobFile); + if (parsedJob.isVoid()) + { + juce::Logger::writeToLog("[pitchRegression] Failed to parse job JSON: " + pitchRegressionJobFile.getFullPathName()); + pitchRegressionJobFile = juce::File(); + return; + } + + pitchRegressionJob = parsedJob; + lastPitchRegressionNativeResult = juce::var(); + juce::Logger::writeToLog("[pitchRegression] Loaded job file: " + pitchRegressionJobFile.getFullPathName()); +} + +bool MainComponent::completePitchRegressionJob(const juce::var& result) +{ + if (! isMainWindow() || pitchRegressionJob.isVoid() || pitchRegressionJobCompleted) + return false; + + const auto resultPath = getStringProperty(pitchRegressionJob, "resultJsonPath").trim(); + if (resultPath.isEmpty()) + { + juce::Logger::writeToLog("[pitchRegression] Missing resultJsonPath in job payload."); + return false; + } + + auto payload = result; + if (! payload.isObject()) + { + auto* fallback = new juce::DynamicObject(); + fallback->setProperty("success", false); + fallback->setProperty("error", "Regression frontend returned a non-object result."); + payload = juce::var(fallback); + } + + auto* payloadObject = payload.getDynamicObject(); + if (payloadObject == nullptr) + return false; + + juce::var nativeResultSnapshot; + { + const juce::ScopedLock resultLock (pitchRegressionNativeResultLock); + nativeResultSnapshot = lastPitchRegressionNativeResult; + } + + if (nativeResultSnapshot.isObject()) + { + auto mergeFromNative = [payloadObject, &nativeResultSnapshot] (const juce::Identifier& propertyName, bool overwriteExisting) + { + if (! overwriteExisting) + { + const auto currentValue = payloadObject->getProperty (propertyName); + const bool missingValue = currentValue.isVoid() + || (currentValue.isString() && currentValue.toString().isEmpty()); + if (! missingValue) + return; + } + + const auto mergedValue = nativeResultSnapshot.getProperty (propertyName, juce::var()); + if (mergedValue.isVoid()) + return; + payloadObject->setProperty (propertyName, mergedValue); + }; + + mergeFromNative ("requestedRendererBranch", true); + mergeFromNative ("actualRendererBranch", true); + mergeFromNative ("pitchOnlyRecoveryPath", true); + mergeFromNative ("pitchOnlyNeutralFormantUsed", true); + mergeFromNative ("processingMode", true); + mergeFromNative ("formantCurveUsed", true); + mergeFromNative ("explicitFormantRequested", true); + mergeFromNative ("pitchOnlyFormantSuppressed", true); + mergeFromNative ("usedFallback", true); + mergeFromNative ("fallbackReason", true); + mergeFromNative ("hardFailReason", true); + mergeFromNative ("pitchRenderStrategy", true); + mergeFromNative ("phraseHqRenderUsed", true); + mergeFromNative ("phraseHqExpandedToFullClip", true); + mergeFromNative ("phraseHqStartSec", true); + mergeFromNative ("phraseHqEndSec", true); + mergeFromNative ("pitchRenderProductPath", true); + mergeFromNative ("pitchRenderBackendId", true); + mergeFromNative ("pitchRenderBackendVersion", true); + mergeFromNative ("pitchRenderBackendFailureCode", true); + mergeFromNative ("pitchRenderBackendCapabilities", true); + mergeFromNative ("pitchRenderBackendDiagnostics", true); + mergeFromNative ("pitchRenderCommitPolicy", true); + mergeFromNative ("pitchRenderDryProtectedSamples", true); + mergeFromNative ("pitchRenderContextDurationSec", true); + mergeFromNative ("pitchRenderCommitDurationSec", true); + mergeFromNative ("pitchRenderJobStartDelayMs", true); + mergeFromNative ("pitchRenderDirection", true); + mergeFromNative ("downshiftFormantGuardUsed", true); + mergeFromNative ("downshiftFormantGuardAlpha", true); + mergeFromNative ("noteHqEffectiveStartSec", true); + mergeFromNative ("noteHqEffectiveEndSec", true); + mergeFromNative ("noteHqContextStartSec", true); + mergeFromNative ("noteHqContextEndSec", true); + mergeFromNative ("noteHqAudibleCommitStartSec", true); + mergeFromNative ("noteHqAudibleCommitEndSec", true); + mergeFromNative ("noteHqPreBodyDryProtectedSamples", true); + mergeFromNative ("noteHqEntryInsideBodyFadeMs", true); + mergeFromNative ("noteHqExitLeadInMs", true); + mergeFromNative ("noteHqEntryBridgeStartSec", true); + mergeFromNative ("noteHqEntryBridgeEndSec", true); + mergeFromNative ("noteHqEntryBridgeWetLagMs", true); + mergeFromNative ("noteHqEntryBridgeEnvelopeGainDb", true); + mergeFromNative ("noteHqEntryBridgeUsed", true); + mergeFromNative ("noteHqEntryTransientDryPreservedMs", true); + mergeFromNative ("pitchOnlyEntrySimpleHandoffUsed", true); + mergeFromNative ("pitchOnlyEntrySafeHandoffUsed", true); + mergeFromNative ("pitchOnlyEntryDryHoldMs", true); + mergeFromNative ("pitchOnlyEntrySafeBridgeMs", true); + mergeFromNative ("pitchOnlyEntryWetAlignmentMs", true); + mergeFromNative ("pitchOnlyEntryWetGainDb", true); + mergeFromNative ("pitchOnlyEntryWetVsDryRmsDb", true); + mergeFromNative ("pitchOnlyEntryEqualPowerBlendUsed", true); + mergeFromNative ("pitchOnlyEntryRmsContinuityUsed", true); + mergeFromNative ("pitchOnlyEntryRmsContinuityGainDb", true); + mergeFromNative ("pitchOnlyEntryRmsContinuityMs", true); + mergeFromNative ("pitchOnlyEntryPhaseSafeUsed", true); + mergeFromNative ("pitchOnlyEntryWetAlignmentAccepted", true); + mergeFromNative ("pitchOnlyEntryFirstCycleCorrelation", true); + mergeFromNative ("pitchOnlyEntryZeroCrossOffsetMs", true); + mergeFromNative ("pitchOnlyEntryBridgeGainRampDb", true); + mergeFromNative ("pitchOnlyDownshiftCoreEnvelopePassUsed", true); + mergeFromNative ("pitchOnlyDownshiftCoreRmsTrimDb", true); + mergeFromNative ("pitchOnlyDownshiftCoreEnvelopeMaxDb", true); + mergeFromNative ("pitchOnlyDownshiftCoreEnvelopeFrames", true); + mergeFromNative ("pitchOnlyEntryWetLagMs", true); + mergeFromNative ("pitchOnlyEntryBridgeDurationMs", true); + mergeFromNative ("pitchOnlyExitDryRestoreUsed", true); + mergeFromNative ("pitchOnlyExitDryRestoreStartSec", true); + mergeFromNative ("pitchOnlyExitDryRestoreEndSec", true); + mergeFromNative ("noteHqEditIslandCount", true); + mergeFromNative ("noteHqEditedNoteCount", true); + mergeFromNative ("noteHqEntryPitchHandoffUsed", true); + mergeFromNative ("noteHqEntryPitchHandoffStartSec", true); + mergeFromNative ("noteHqEntryPitchHandoffEndSec", true); + mergeFromNative ("noteHqEntryPitchHandoffPreMs", true); + mergeFromNative ("noteHqEntryPitchHandoffBodyMs", true); + mergeFromNative ("noteHqEntryPitchSlopeJumpStPerSec", true); + mergeFromNative ("noteHqEntryPitchAccelerationLimited", true); + mergeFromNative ("outputDurationSec", true); + mergeFromNative ("postApplyRouteStatus", true); + mergeFromNative ("appFinalCapture", true); + mergeFromNative ("appFinalBakedCapture", true); + mergeFromNative ("appFinalParityReport", true); + mergeFromNative ("appFinalRouteReportPath", true); + mergeFromNative ("appFinalBakedContextPath", true); + mergeFromNative ("appFinalPlaybackContextPath", true); + mergeFromNative ("appFinalParityReportPath", true); + mergeFromNative ("bridgeUsed", true); + mergeFromNative ("bridgeFallbackUsed", true); + mergeFromNative ("bridgeStartSec", true); + mergeFromNative ("bridgeLengthMs", true); + mergeFromNative ("bridgeAlignmentLagSamples", true); + mergeFromNative ("bridgeCorrelationScore", true); + mergeFromNative ("bridgeGainDeltaDb", true); + mergeFromNative ("bodyReplacementUsed", true); + mergeFromNative ("bodyReplacementFallbackUsed", true); + mergeFromNative ("entryLockStartSec", true); + mergeFromNative ("entryLockLengthMs", true); + mergeFromNative ("exitLockStartSec", true); + mergeFromNative ("renderedBodyStartSec", true); + mergeFromNative ("renderedBodyEndSec", true); + mergeFromNative ("islandNativeUsed", true); + mergeFromNative ("islandNativeFallbackUsed", true); + mergeFromNative ("islandRenderStartSec", true); + mergeFromNative ("islandRenderEndSec", true); + mergeFromNative ("transientMaskPeak", true); + mergeFromNative ("voicedCoreMaskPeak", true); + mergeFromNative ("hpssUsed", true); + mergeFromNative ("hpssFallbackUsed", true); + mergeFromNative ("harmonicMaskPeak", true); + mergeFromNative ("aperiodicMaskPeak", true); + mergeFromNative ("spectralEnvelopeCorrectionUsed", true); + mergeFromNative ("pitchOnlyCoreTimbreCorrectionUsed", true); + mergeFromNative ("pitchOnlyCoreEnvelopeMix", true); + mergeFromNative ("pitchOnlyCoreRmsTrimDb", true); + mergeFromNative ("pitchOnlyCoreEnvelopeLifter", true); + mergeFromNative ("pitchOnlyEntryTimbreCorrectionUsed", true); + mergeFromNative ("pitchOnlyEntryRmsTrimDb", true); + mergeFromNative ("pitchOnlyEntryTiltDb", true); + mergeFromNative ("pitchOnlyEntryHandoffUsed", true); + mergeFromNative ("pitchOnlyExitHandoffUsed", true); + mergeFromNative ("vocalSourceFilterUsed", true); + mergeFromNative ("vocalSourceFilterVoicedCoverage", true); + mergeFromNative ("vocalSourceFilterResidualMix", true); + mergeFromNative ("vocalSourceFilterFallbackUsed", true); + mergeFromNative ("vocalSourceFilterFallbackReason", true); + mergeFromNative ("vocalSourceFilterEntryDryMs", true); + mergeFromNative ("vocalSourceFilterExitDryMs", true); + mergeFromNative ("wsolaUsed", true); + mergeFromNative ("wsolaFallbackUsed", true); + mergeFromNative ("wsolaEntryLagSamples", true); + mergeFromNative ("wsolaExitLagSamples", true); + mergeFromNative ("wsolaCorrelationScore", true); + mergeFromNative ("phaseLockUsed", true); + mergeFromNative ("phaseLockFallbackUsed", true); + mergeFromNative ("phaseAlignedEntry", true); + mergeFromNative ("phaseAlignedExit", true); + mergeFromNative ("phasePeakCount", true); + mergeFromNative ("transitionHqUsed", true); + mergeFromNative ("transitionHqFallbackUsed", true); + mergeFromNative ("transitionStartSec", true); + mergeFromNative ("transitionEndSec", true); + mergeFromNative ("transitionTransientPeak", true); + mergeFromNative ("transitionVoicedCorePeak", true); + mergeFromNative ("transitionResidualPeak", true); + mergeFromNative ("transitionEnvelopeCorrectionUsed", true); + mergeFromNative ("engineV2Used", true); + mergeFromNative ("engineV2FallbackUsed", true); + mergeFromNative ("engineV2TransitionCount", true); + mergeFromNative ("engineV2TransitionStartSec", true); + mergeFromNative ("engineV2TransitionEndSec", true); + mergeFromNative ("engineV2HarmonicSupportPeak", true); + mergeFromNative ("engineV2ResidualSupportPeak", true); + mergeFromNative ("engineV2EnvelopeSupportPeak", true); + mergeFromNative ("previewCoverageStartSec", true); + mergeFromNative ("previewCoverageEndSec", true); + mergeFromNative ("candidateCoverageStartSec", true); + mergeFromNative ("candidateCoverageEndSec", true); + } + + payloadObject->setProperty("jobPath", pitchRegressionJobFile.getFullPathName()); + payloadObject->setProperty("completedAt", juce::Time::getCurrentTime().toISO8601(true)); + + const auto outputFile = juce::File(resultPath); + outputFile.getParentDirectory().createDirectory(); + const auto writeOk = outputFile.replaceWithText(juce::JSON::toString(payload, true)); + pitchRegressionJobCompleted = writeOk; + juce::Logger::writeToLog("[pitchRegression] Wrote result to: " + outputFile.getFullPathName() + + " success=" + juce::String(writeOk ? "true" : "false")); + + if (auto* app = juce::JUCEApplication::getInstance()) + { + const auto success = writeOk && static_cast<bool>(payloadObject->getProperty("success")); + app->setApplicationReturnValue(success ? 0 : 1); + juce::Timer::callAfterDelay(100, []() + { + if (auto* currentApp = juce::JUCEApplication::getInstance()) + currentApp->systemRequestedQuit(); + }); + } + + return writeOk; +} + MainComponent::~MainComponent() { stopTimer(); @@ -4802,6 +6019,40 @@ bool MainComponent::isMainWindow() const return windowRole == WindowRole::main; } +bool MainComponent::loadPackagedFrontend() +{ + const auto packagedFrontend = getPackagedFrontendEntryPoint(); + if (! packagedFrontend.existsAsFile()) + return false; + + webuiDir = packagedFrontend.getParentDirectory(); + juce::Logger::writeToLog("Loading packaged frontend from: " + packagedFrontend.getFullPathName()); + + const auto frontendUrl = appendFrontendStartupQuery( + juce::WebBrowserComponent::getResourceProviderRoot() + "index.html", + windowRole, + startupMode); + beginFrontendStartupWatchdog(frontendUrl); + webView.goToURL(frontendUrl); + return true; +} + +bool MainComponent::tryFallbackToPackagedFrontendAfterLocalTimeout() +{ + if (attemptedPackagedFrontendFallbackAfterLocalTimeout) + return false; + + if (! frontendStartupTargetUrl.startsWithIgnoreCase("http://localhost:5173") + && ! frontendStartupTargetUrl.startsWithIgnoreCase("http://127.0.0.1:5173")) + return false; + + attemptedPackagedFrontendFallbackAfterLocalTimeout = true; + + juce::Logger::writeToLog("Frontend startup timed out while using localhost:5173; " + "retrying with the packaged frontend."); + return loadPackagedFrontend(); +} + void MainComponent::beginFrontendStartupWatchdog(const juce::String& targetUrl) { frontendStartupTargetUrl = targetUrl; @@ -5175,6 +6426,9 @@ void MainComponent::timerCallback() const auto elapsedMs = static_cast<int>(frontendStartupNavigationTicks * 100); if (elapsedMs >= kFrontendStartupTimeoutMs) { + if (tryFallbackToPackagedFrontendAfterLocalTimeout()) + return; + frontendStartupState = FrontendStartupState::timedOut; frontendStartupDetail = "No boot-ready signal was received from the embedded frontend within " + juce::String(kFrontendStartupTimeoutMs / 1000.0, 1) diff --git a/Source/MainComponent.h b/Source/MainComponent.h index 41d5102..80135a9 100644 --- a/Source/MainComponent.h +++ b/Source/MainComponent.h @@ -57,7 +57,8 @@ class MainComponent : public juce::Component, AppUpdater& appUpdaterIn, StartupMode startupModeIn, WindowRole roleIn, - WindowCallbacks callbacksIn = {}); + WindowCallbacks callbacksIn = {}, + const juce::String& pitchRegressionJobPathIn = {}); ~MainComponent() override; //============================================================================== @@ -80,6 +81,8 @@ class MainComponent : public juce::Component, void startDesktopWindowDrag(); void emitFrontendEvent(const juce::String& eventId, const juce::var& payload = {}); bool isMainWindow() const; + bool loadPackagedFrontend(); + bool tryFallbackToPackagedFrontendAfterLocalTimeout(); void beginFrontendStartupWatchdog(const juce::String& targetUrl); void showStartupOverlay(const juce::String& title, const juce::String& detail); void hideStartupOverlay(); @@ -93,6 +96,8 @@ class MainComponent : public juce::Component, void repairInstalledApplication(); void repairWindowsPrerequisites(); juce::var buildStartupDiagnostics() const; + void initializePitchRegressionJob(const juce::String& pitchRegressionJobPathIn); + bool completePitchRegressionJob(const juce::var& result); //============================================================================== // Your private member variables go here... @@ -115,16 +120,23 @@ class MainComponent : public juce::Component, // Async pitch analysis state std::atomic<bool> pitchAnalysisRunning { false }; + std::atomic<bool> pitchNoteHqPriorityActive { false }; + std::atomic<int> pitchAnalysisGeneration { 0 }; + std::atomic<int> pitchNoteHqPriorityGeneration { 0 }; + juce::ThreadPool pitchAnalysisPool { 1 }; juce::var lastPitchAnalysisResult; // Cached result for fetch-after-event pattern juce::CriticalSection pitchResultLock; // Background thread for pitch correction (1 slot — serialises apply calls) juce::ThreadPool previewSegmentPool { 2 }; + juce::ThreadPool noteRenderPool { 1 }; juce::ThreadPool fullClipHQPool { 1 }; juce::CriticalSection pitchCorrectionJobLock; juce::String activePreviewRequestGroup; + juce::String activeNoteRenderRequestGroup; juce::String activeFullClipRequestGroup; std::atomic<int> previewRenderGeneration { 0 }; + std::atomic<int> noteRenderGeneration { 0 }; std::atomic<int> fullClipRenderGeneration { 0 }; FrontendStartupState frontendStartupState = FrontendStartupState::idle; juce::String frontendStartupTargetUrl; @@ -132,9 +144,16 @@ class MainComponent : public juce::Component, juce::uint32 frontendStartupNavigationTicks = 0; bool startupFallbackVisible = false; bool startupWatchdogActive = false; + bool attemptedPackagedFrontendFallbackAfterLocalTimeout = false; StartupRepairAction startupRepairAction = StartupRepairAction::none; juce::String lastAiToolsStatusDigest; double lastAiToolsStatusEmitMs = 0.0; + juce::File pitchRegressionJobFile; + juce::var pitchRegressionJob; + bool pitchRegressionJobConsumed = false; + bool pitchRegressionJobCompleted = false; + juce::CriticalSection pitchRegressionNativeResultLock; + juce::var lastPitchRegressionNativeResult; static juce::CriticalSection instanceListLock; static juce::Array<MainComponent*> activeInstances; diff --git a/Source/OwnPitchEngine.cpp b/Source/OwnPitchEngine.cpp new file mode 100644 index 0000000..4a0dc3c --- /dev/null +++ b/Source/OwnPitchEngine.cpp @@ -0,0 +1,2453 @@ +#include "OwnPitchEngine.h" + +#include <algorithm> +#include <cmath> +#include <cstdint> +#include <limits> +#include <numeric> +#include <optional> + +namespace +{ +static bool shouldEnableOwnPitchEngineLogs() +{ +#if JUCE_DEBUG + return true; +#else + return juce::SystemStats::getEnvironmentVariable ("OPENSTUDIO_PITCH_DEBUG", {}).trim() == "1"; +#endif +} + +static void logOwnPitchEngine (const juce::String& message) +{ + if (shouldEnableOwnPitchEngineLogs()) + juce::Logger::writeToLog ("[pitchEditor.ownEngine] " + message); +} + +static bool isHybridStructuralBranchRequested(); + +static float getEnvFloat (const char* name, float fallback) +{ + const auto value = juce::SystemStats::getEnvironmentVariable (name, {}).trim(); + if (value.isEmpty()) + return fallback; + const float parsed = value.getFloatValue(); + return std::isfinite (parsed) ? parsed : fallback; +} + +static bool getEnvBool (const char* name, bool fallback) +{ + const auto value = juce::SystemStats::getEnvironmentVariable (name, {}).trim().toLowerCase(); + if (value.isEmpty()) + return fallback; + if (value == "1" || value == "true" || value == "yes" || value == "on") + return true; + if (value == "0" || value == "false" || value == "no" || value == "off") + return false; + return fallback; +} + +static juce::File getVsfLayerDumpDirectory() +{ + auto path = juce::SystemStats::getEnvironmentVariable ("OPENSTUDIO_VSF_LAYER_DUMP_DIR", {}).trim(); + if (path.isEmpty()) + path = juce::SystemStats::getEnvironmentVariable ("OPENSTUDIO_PITCH_LAYER_DUMP_DIR", {}).trim(); + return path.isEmpty() ? juce::File() : juce::File (path); +} + +static bool writeOwnPitchLayerDumpWav ( + const juce::File& directory, + const juce::String& name, + const std::vector<std::vector<float>>& channels, + double sampleRate) +{ + if (directory == juce::File() || channels.empty() || sampleRate <= 0.0) + return false; + + const int numChannels = static_cast<int> (channels.size()); + const int numSamples = static_cast<int> (channels.front().size()); + if (numChannels <= 0 || numSamples <= 0) + return false; + + directory.createDirectory(); + const auto file = directory.getChildFile (name + ".wav"); + file.deleteFile(); + + juce::AudioBuffer<float> buffer (numChannels, numSamples); + buffer.clear(); + for (int ch = 0; ch < numChannels; ++ch) + { + if (channels[static_cast<size_t> (ch)].empty()) + continue; + const int copyCount = std::min (numSamples, static_cast<int> (channels[static_cast<size_t> (ch)].size())); + buffer.copyFrom (ch, 0, channels[static_cast<size_t> (ch)].data(), copyCount); + } + + juce::WavAudioFormat format; + std::unique_ptr<juce::FileOutputStream> stream (file.createOutputStream()); + if (stream == nullptr) + return false; + + std::unique_ptr<juce::AudioFormatWriter> writer ( + format.createWriterFor (stream.get(), sampleRate, static_cast<unsigned int> (numChannels), 24, {}, 0)); + if (writer == nullptr) + return false; + + stream.release(); + return writer->writeFromAudioSampleBuffer (buffer, 0, numSamples); +} + +static float smoothstep01 (float x) +{ + x = juce::jlimit (0.0f, 1.0f, x); + return x * x * (3.0f - 2.0f * x); +} + +static float equalPowerFadeIn (float x) +{ + x = juce::jlimit (0.0f, 1.0f, x); + return std::sin (0.5f * juce::MathConstants<float>::pi * x); +} + +static float equalPowerFadeOut (float x) +{ + x = juce::jlimit (0.0f, 1.0f, x); + return std::cos (0.5f * juce::MathConstants<float>::pi * x); +} + +static void applyPeakingEqToRange ( + std::vector<float>& signal, + int startSample, + int endSample, + double sampleRate, + float centreHz, + float q, + float gainDb, + int fadeSamples) +{ + if (signal.empty() || sampleRate <= 0.0 || centreHz <= 0.0f || q <= 0.0f) + return; + + startSample = juce::jlimit (0, static_cast<int> (signal.size()), startSample); + endSample = juce::jlimit (startSample, static_cast<int> (signal.size()), endSample); + const int count = endSample - startSample; + if (count <= 2) + return; + + const double a = std::pow (10.0, static_cast<double> (gainDb) / 40.0); + const double omega = juce::MathConstants<double>::twoPi * static_cast<double> (centreHz) / sampleRate; + const double alpha = std::sin (omega) / (2.0 * static_cast<double> (q)); + const double cosine = std::cos (omega); + + double b0 = 1.0 + alpha * a; + double b1 = -2.0 * cosine; + double b2 = 1.0 - alpha * a; + const double a0 = 1.0 + alpha / a; + double a1 = -2.0 * cosine; + double a2 = 1.0 - alpha / a; + + b0 /= a0; + b1 /= a0; + b2 /= a0; + a1 /= a0; + a2 /= a0; + + fadeSamples = juce::jlimit (1, std::max (1, count / 2), fadeSamples); + double z1 = 0.0; + double z2 = 0.0; + for (int i = 0; i < count; ++i) + { + const int sampleIndex = startSample + i; + const double inputSample = static_cast<double> (signal[static_cast<size_t> (sampleIndex)]); + const double filtered = b0 * inputSample + z1; + z1 = b1 * inputSample - a1 * filtered + z2; + z2 = b2 * inputSample - a2 * filtered; + + float fade = 1.0f; + if (i < fadeSamples) + fade *= equalPowerFadeIn (static_cast<float> (i) / static_cast<float> (fadeSamples)); + if (i >= count - fadeSamples) + fade *= equalPowerFadeOut (static_cast<float> (i - (count - fadeSamples)) / static_cast<float> (fadeSamples)); + + const float original = signal[static_cast<size_t> (sampleIndex)]; + signal[static_cast<size_t> (sampleIndex)] = original * (1.0f - fade) + static_cast<float> (filtered) * fade; + } +} + +static float getEffectiveNoteStartTime (const PitchAnalyzer::PitchNote& note) +{ + return std::min (note.startTime, note.effectiveStartTime); +} + +static float getEffectiveNoteEndTime (const PitchAnalyzer::PitchNote& note) +{ + return std::max (note.endTime, note.effectiveEndTime); +} + +static bool hasPitchStyleEdit (const PitchAnalyzer::PitchNote& note) +{ + if (std::abs (note.correctedPitch - note.detectedPitch) > 0.01f) + return true; + if (note.driftCorrectionAmount > 0.01f) + return true; + if (std::abs (note.vibratoDepth - 1.0f) > 0.01f) + return true; + for (const auto drift : note.pitchDrift) + if (std::abs (drift) > 0.01f) + return true; + return false; +} + +static float computeRms (const std::vector<float>& signal, int startSample, int endSample) +{ + if (signal.empty()) + return 0.0f; + + startSample = juce::jlimit (0, static_cast<int> (signal.size()), startSample); + endSample = juce::jlimit (startSample, static_cast<int> (signal.size()), endSample); + if (endSample <= startSample) + return 0.0f; + + double sum = 0.0; + int count = 0; + for (int i = startSample; i < endSample; ++i) + { + const float s = signal[static_cast<size_t> (i)]; + sum += static_cast<double> (s) * s; + ++count; + } + + return count > 0 ? std::sqrt (static_cast<float> (sum / static_cast<double> (count))) : 0.0f; +} + +static std::vector<float> buildMonoSignal ( + const float* const* input, + int numChannels, + int startSample, + int endSample) +{ + const int numSamples = std::max (0, endSample - startSample); + std::vector<float> mono (static_cast<size_t> (numSamples), 0.0f); + if (numChannels <= 0 || numSamples <= 0) + return mono; + + const float channelScale = 1.0f / static_cast<float> (numChannels); + for (int ch = 0; ch < numChannels; ++ch) + { + const auto* src = input[ch] + startSample; + for (int i = 0; i < numSamples; ++i) + mono[static_cast<size_t> (i)] += src[i] * channelScale; + } + + return mono; +} + +static std::vector<float> buildF0TrackHz ( + int numSamples, + int contextStartSample, + double sampleRate, + const std::vector<PitchAnalyzer::PitchFrame>& frames) +{ + std::vector<float> f0 (static_cast<size_t> (numSamples), 0.0f); + if (frames.empty()) + return f0; + + int hopSize = 256; + if (frames.size() >= 2) + hopSize = std::max (1, static_cast<int> (std::round ((frames[1].time - frames[0].time) * sampleRate))); + + for (const auto& frame : frames) + { + if (frame.frequency <= 0.0f) + continue; + + const int sampleStart = juce::jlimit (0, std::max (0, numSamples - 1), + static_cast<int> (std::floor (frame.time * sampleRate)) - contextStartSample); + const int sampleEnd = juce::jlimit (0, numSamples, sampleStart + hopSize); + for (int s = sampleStart; s < sampleEnd; ++s) + f0[static_cast<size_t> (s)] = frame.frequency; + } + + int lastVoiced = -1; + for (int i = 0; i < numSamples; ++i) + { + if (f0[static_cast<size_t> (i)] > 0.0f) + { + if (lastVoiced >= 0 && i > lastVoiced + 1) + { + const float left = f0[static_cast<size_t> (lastVoiced)]; + const float right = f0[static_cast<size_t> (i)]; + const int gap = i - lastVoiced - 1; + for (int g = 1; g <= gap; ++g) + { + const float t = static_cast<float> (g) / static_cast<float> (gap + 1); + f0[static_cast<size_t> (lastVoiced + g)] = left + (right - left) * t; + } + } + lastVoiced = i; + } + } + + return f0; +} + +static std::vector<float> buildVoicedMask ( + int numSamples, + int contextStartSample, + double sampleRate, + const std::vector<PitchAnalyzer::PitchFrame>& frames) +{ + std::vector<float> mask (static_cast<size_t> (numSamples), 0.0f); + if (frames.empty()) + return mask; + + int hopSize = 256; + if (frames.size() >= 2) + hopSize = std::max (1, static_cast<int> (std::round ((frames[1].time - frames[0].time) * sampleRate))); + + for (const auto& frame : frames) + { + const float voicedScore = frame.voiced && frame.frequency > 0.0f + ? juce::jlimit (0.0f, 1.0f, frame.confidence) + : 0.0f; + if (voicedScore <= 0.0f) + continue; + + const int sampleStart = juce::jlimit (0, std::max (0, numSamples - 1), + static_cast<int> (std::floor (frame.time * sampleRate)) - contextStartSample); + const int sampleEnd = juce::jlimit (0, numSamples, sampleStart + hopSize); + for (int s = sampleStart; s < sampleEnd; ++s) + mask[static_cast<size_t> (s)] = std::max (mask[static_cast<size_t> (s)], voicedScore); + } + + std::vector<float> smoothed = mask; + for (int i = 1; i < numSamples - 1; ++i) + smoothed[static_cast<size_t> (i)] = (mask[static_cast<size_t> (i - 1)] + mask[static_cast<size_t> (i)] + mask[static_cast<size_t> (i + 1)]) / 3.0f; + + return smoothed; +} + +static std::vector<float> buildAmplitudeEnvelope ( + const std::vector<float>& monoSignal, + double sampleRate) +{ + std::vector<float> env (monoSignal.size(), 0.0f); + if (monoSignal.empty()) + return env; + + const int radius = std::max (4, static_cast<int> (std::round (0.008 * sampleRate))); + for (int i = 0; i < static_cast<int> (monoSignal.size()); ++i) + { + double sum = 0.0; + int count = 0; + for (int j = std::max (0, i - radius); j <= std::min (static_cast<int> (monoSignal.size()) - 1, i + radius); ++j) + { + const float sample = monoSignal[static_cast<size_t> (j)]; + sum += static_cast<double> (sample) * sample; + ++count; + } + env[static_cast<size_t> (i)] = count > 0 ? std::sqrt (static_cast<float> (sum / static_cast<double> (count))) : 0.0f; + } + + return env; +} + +static std::vector<int> buildEpochs ( + const std::vector<float>& f0TrackHz, + int coreStartSample, + int coreEndSample, + double sampleRate) +{ + std::vector<int> epochs; + if (coreEndSample <= coreStartSample || f0TrackHz.empty()) + return epochs; + + int s = coreStartSample; + while (s < coreEndSample) + { + const int idx = juce::jlimit (0, static_cast<int> (f0TrackHz.size()) - 1, s); + const float f0 = juce::jlimit (55.0f, 1200.0f, std::max (f0TrackHz[static_cast<size_t> (idx)], 55.0f)); + const int period = std::max (1, static_cast<int> (std::round (sampleRate / f0))); + epochs.push_back (s); + s += period; + } + + return epochs; +} + +static float computeMedianPositive (const std::vector<float>& values, int startSample, int endSample) +{ + std::vector<float> positive; + positive.reserve (static_cast<size_t> (std::max (0, endSample - startSample))); + + for (int i = startSample; i < endSample && i < static_cast<int> (values.size()); ++i) + { + const float value = values[static_cast<size_t> (i)]; + if (value > 0.0f) + positive.push_back (value); + } + + if (positive.empty()) + return 0.0f; + + const auto mid = positive.begin() + static_cast<ptrdiff_t> (positive.size() / 2); + std::nth_element (positive.begin(), mid, positive.end()); + return *mid; +} + +static std::vector<float> buildHannWindow (int fftSize) +{ + std::vector<float> window (static_cast<size_t> (fftSize), 0.0f); + const float twoPi = juce::MathConstants<float>::twoPi; + for (int i = 0; i < fftSize; ++i) + window[static_cast<size_t> (i)] = 0.5f * (1.0f - std::cos (twoPi * static_cast<float> (i) / static_cast<float> (fftSize - 1))); + return window; +} + +static void smoothLogSpectrum ( + const std::vector<float>& logMagnitude, + int smoothHalfW, + std::vector<float>& smoothedMagnitude) +{ + const int halfBins = static_cast<int> (logMagnitude.size()); + smoothedMagnitude.assign (logMagnitude.size(), 0.0f); + for (int i = 0; i < halfBins; ++i) + { + const int lo = std::max (0, i - smoothHalfW); + const int hi = std::min (halfBins - 1, i + smoothHalfW); + float sum = 0.0f; + for (int j = lo; j <= hi; ++j) + sum += logMagnitude[static_cast<size_t> (j)]; + smoothedMagnitude[static_cast<size_t> (i)] = std::exp (sum / static_cast<float> (hi - lo + 1)); + } +} + +template <typename FrameType> +static int advanceFrameCursor ( + const std::vector<FrameType>& frames, + int sampleIndex, + int cursor) +{ + if (frames.empty()) + return 0; + + cursor = juce::jlimit (0, static_cast<int> (frames.size()) - 1, cursor); + while (cursor + 1 < static_cast<int> (frames.size()) + && frames[static_cast<size_t> (cursor + 1)].sampleIndex <= sampleIndex) + { + ++cursor; + } + while (cursor > 0 + && frames[static_cast<size_t> (cursor)].sampleIndex > sampleIndex) + { + --cursor; + } + return cursor; +} + +static float interpolateEnvelopeMagnitudeAt ( + const OwnPitchEngine::SpectralEnvelopeModel& model, + int sampleIndex, + int bin, + int cursor) +{ + if (model.frames.empty()) + { + if (model.averageMagnitude.empty()) + return 1.0e-8f; + + const int clampedBin = juce::jlimit (0, static_cast<int> (model.averageMagnitude.size()) - 1, bin); + return model.averageMagnitude[static_cast<size_t> (clampedBin)] + 1.0e-8f; + } + + const int clampedCursor = juce::jlimit (0, static_cast<int> (model.frames.size()) - 1, cursor); + const auto& loFrame = model.frames[static_cast<size_t> (clampedCursor)]; + const int clampedBin = juce::jlimit (0, static_cast<int> (loFrame.smoothedMagnitude.size()) - 1, bin); + + if (clampedCursor + 1 >= static_cast<int> (model.frames.size())) + return loFrame.smoothedMagnitude[static_cast<size_t> (clampedBin)] + 1.0e-8f; + + const auto& hiFrame = model.frames[static_cast<size_t> (clampedCursor + 1)]; + const int span = std::max (1, hiFrame.sampleIndex - loFrame.sampleIndex); + const float frac = juce::jlimit (0.0f, 1.0f, static_cast<float> (sampleIndex - loFrame.sampleIndex) / static_cast<float> (span)); + const float lo = loFrame.smoothedMagnitude[static_cast<size_t> (clampedBin)]; + const float hi = hiFrame.smoothedMagnitude[static_cast<size_t> (juce::jlimit (0, static_cast<int> (hiFrame.smoothedMagnitude.size()) - 1, bin))]; + return lo + (hi - lo) * frac + 1.0e-8f; +} + +static float interpolateEnvelopeMagnitudeAtHz ( + const OwnPitchEngine::SpectralEnvelopeModel& model, + int sampleIndex, + float frequencyHz, + int cursor) +{ + if (model.binHz <= 0.0f) + return 1.0e-8f; + + const float binPosition = frequencyHz / model.binHz; + const int loBin = static_cast<int> (std::floor (binPosition)); + const int hiBin = loBin + 1; + const float frac = juce::jlimit (0.0f, 1.0f, binPosition - static_cast<float> (loBin)); + const float lo = interpolateEnvelopeMagnitudeAt (model, sampleIndex, loBin, cursor); + const float hi = interpolateEnvelopeMagnitudeAt (model, sampleIndex, hiBin, cursor); + return lo + (hi - lo) * frac; +} + +static OwnPitchEngine::SpectralEnvelopeModel buildSpectralEnvelope ( + const std::vector<float>& signal, + const std::vector<float>& f0TrackHz, + const std::vector<int>& epochs, + int coreStartSample, + int coreEndSample, + double sampleRate) +{ + OwnPitchEngine::SpectralEnvelopeModel model; + model.fftOrder = 10; + model.fftSize = 1 << model.fftOrder; + model.binHz = static_cast<float> (sampleRate / static_cast<double> (model.fftSize)); + + const int halfBins = model.fftSize / 2 + 1; + model.averageMagnitude.assign (static_cast<size_t> (halfBins), 0.0f); + + if (signal.empty() || coreEndSample <= coreStartSample) + return model; + + juce::dsp::FFT fft (model.fftOrder); + const auto hannWindow = buildHannWindow (model.fftSize); + std::vector<float> fftBuffer (static_cast<size_t> (model.fftSize * 2), 0.0f); + std::vector<float> logMagnitude (static_cast<size_t> (halfBins), 0.0f); + std::vector<float> smoothedMagnitude (static_cast<size_t> (halfBins), 0.0f); + + std::vector<int> frameCenters; + if (! epochs.empty()) + { + for (const int epoch : epochs) + if (epoch >= coreStartSample && epoch < coreEndSample) + frameCenters.push_back (epoch); + } + else + { + const int hop = std::max (32, model.fftSize / 8); + for (int pos = coreStartSample; pos < coreEndSample; pos += hop) + frameCenters.push_back (pos); + } + + if (frameCenters.empty()) + frameCenters.push_back ((coreStartSample + coreEndSample) / 2); + + for (const int center : frameCenters) + { + const int f0Index = juce::jlimit (0, static_cast<int> (f0TrackHz.size()) - 1, center); + const float localF0 = (f0TrackHz.empty() ? 0.0f : f0TrackHz[static_cast<size_t> (f0Index)]); + const float safeF0 = juce::jlimit (70.0f, 1200.0f, localF0 > 0.0f ? localF0 : computeMedianPositive (f0TrackHz, coreStartSample, coreEndSample)); + const int pitchWindowSamples = juce::jlimit (256, model.fftSize, static_cast<int> (std::round ((3.0 * sampleRate) / std::max (safeF0, 1.0f)))); + const int frameStart = center - pitchWindowSamples / 2; + + std::fill (fftBuffer.begin(), fftBuffer.end(), 0.0f); + for (int i = 0; i < pitchWindowSamples; ++i) + { + const int srcIndex = frameStart + i; + if (srcIndex < 0 || srcIndex >= static_cast<int> (signal.size())) + continue; + + const float x = static_cast<float> (i) / static_cast<float> (std::max (1, pitchWindowSamples - 1)); + const float w = 0.5f * (1.0f - std::cos (juce::MathConstants<float>::twoPi * x)); + const int dstIndex = juce::jlimit (0, model.fftSize - 1, i); + fftBuffer[static_cast<size_t> (dstIndex)] = signal[static_cast<size_t> (srcIndex)] * w; + } + + fft.performRealOnlyForwardTransform (fftBuffer.data(), true); + for (int bin = 0; bin < halfBins; ++bin) + { + const float re = fftBuffer[static_cast<size_t> (bin * 2)]; + const float im = fftBuffer[static_cast<size_t> (bin * 2 + 1)]; + logMagnitude[static_cast<size_t> (bin)] = std::log (std::sqrt (re * re + im * im) + 1.0e-10f); + } + + const int smoothHalfW = std::min (halfBins / 3, std::max (4, static_cast<int> (std::round ((1.6f * safeF0) / std::max (1.0f, model.binHz))))); + smoothLogSpectrum (logMagnitude, smoothHalfW, smoothedMagnitude); + + OwnPitchEngine::SpectralEnvelopeFrame frame; + frame.sampleIndex = center; + frame.f0Hz = safeF0; + frame.rawMagnitude.resize (static_cast<size_t> (halfBins), 0.0f); + frame.smoothedMagnitude = smoothedMagnitude; + for (int bin = 0; bin < halfBins; ++bin) + { + frame.rawMagnitude[static_cast<size_t> (bin)] = std::exp (logMagnitude[static_cast<size_t> (bin)]); + model.averageMagnitude[static_cast<size_t> (bin)] += smoothedMagnitude[static_cast<size_t> (bin)]; + } + model.frames.push_back (std::move (frame)); + } + + if (! model.frames.empty()) + { + const float invCount = 1.0f / static_cast<float> (model.frames.size()); + for (auto& value : model.averageMagnitude) + value *= invCount; + } + + return model; +} + +static OwnPitchEngine::HarmonicModel buildHarmonicModel ( + float meanF0Hz, + float coreRms, + const OwnPitchEngine::SpectralEnvelopeModel& spectralEnvelopeModel, + double sampleRate) +{ + OwnPitchEngine::HarmonicModel model; + model.fftSize = spectralEnvelopeModel.fftSize > 0 ? spectralEnvelopeModel.fftSize : 2048; + model.binHz = spectralEnvelopeModel.binHz > 0.0f + ? spectralEnvelopeModel.binHz + : static_cast<float> (sampleRate / static_cast<double> (model.fftSize)); + + const float clampedF0 = juce::jlimit (70.0f, 900.0f, meanF0Hz > 0.0f ? meanF0Hz : 180.0f); + model.partialCount = juce::jlimit (6, 28, static_cast<int> ((sampleRate * 0.45) / clampedF0)); + model.averageMagnitude.resize (static_cast<size_t> (model.partialCount), 0.0f); + + const float base = std::max (coreRms, 0.01f); + for (int p = 0; p < model.partialCount; ++p) + { + const float harmonicHz = clampedF0 * static_cast<float> (p + 1); + const float envelope = spectralEnvelopeModel.averageMagnitude.empty() + ? 1.0f / std::pow (static_cast<float> (p + 1), 0.88f) + : spectralEnvelopeModel.averageMagnitude[static_cast<size_t> (juce::jlimit ( + 0, + static_cast<int> (spectralEnvelopeModel.averageMagnitude.size()) - 1, + static_cast<int> (std::round (harmonicHz / std::max (1.0f, model.binHz)))) )]; + model.averageMagnitude[static_cast<size_t> (p)] = base * envelope; + } + + return model; +} + +static OwnPitchEngine::ResidualModel buildResidualModel ( + const std::vector<float>& monoSignal, + const std::vector<float>& voicedMask, + const OwnPitchEngine::SpectralEnvelopeModel& spectralEnvelopeModel, + double sampleRate) +{ + OwnPitchEngine::ResidualModel model; + model.monoResidual.resize (monoSignal.size(), 0.0f); + model.voicedHighBandResidual.resize (monoSignal.size(), 0.0f); + model.bandAperiodicity.assign (6, 0.0f); + + float residualEnergy = 0.0f; + const float cutoffHz = 1800.0f; + const float alpha = juce::jlimit (0.02f, 0.45f, + static_cast<float> ((2.0 * juce::MathConstants<double>::pi * cutoffHz) / (sampleRate + 2.0 * juce::MathConstants<double>::pi * cutoffHz))); + float lowpass = monoSignal.empty() ? 0.0f : monoSignal.front(); + for (size_t i = 0; i < monoSignal.size(); ++i) + { + lowpass += alpha * (monoSignal[i] - lowpass); + const float highBand = monoSignal[i] - lowpass; + const float unvoiced = 1.0f - juce::jlimit (0.0f, 1.0f, voicedMask.empty() ? 0.0f : voicedMask[i]); + model.monoResidual[i] = monoSignal[i] * unvoiced; + model.voicedHighBandResidual[i] = highBand * (voicedMask.empty() ? 0.0f : voicedMask[i]); + residualEnergy += std::abs (model.monoResidual[i]); + } + + if (! spectralEnvelopeModel.frames.empty()) + { + const int halfBins = spectralEnvelopeModel.fftSize / 2 + 1; + const int numBands = static_cast<int> (model.bandAperiodicity.size()); + std::vector<float> bandSum (static_cast<size_t> (numBands), 0.0f); + std::vector<int> bandCount (static_cast<size_t> (numBands), 0); + for (const auto& frame : spectralEnvelopeModel.frames) + { + for (int band = 0; band < numBands; ++band) + { + const int lo = (band * halfBins) / numBands; + const int hi = ((band + 1) * halfBins) / numBands; + float rawSum = 0.0f; + float envSum = 0.0f; + for (int bin = lo; bin < hi; ++bin) + { + rawSum += frame.rawMagnitude[static_cast<size_t> (bin)]; + envSum += frame.smoothedMagnitude[static_cast<size_t> (bin)]; + } + if (rawSum > 1.0e-6f) + { + const float aperiodicity = juce::jlimit (0.0f, 1.0f, std::max (0.0f, rawSum - envSum) / rawSum); + bandSum[static_cast<size_t> (band)] += aperiodicity; + ++bandCount[static_cast<size_t> (band)]; + } + } + } + + for (int band = 0; band < numBands; ++band) + { + model.bandAperiodicity[static_cast<size_t> (band)] = bandCount[static_cast<size_t> (band)] > 0 + ? bandSum[static_cast<size_t> (band)] / static_cast<float> (bandCount[static_cast<size_t> (band)]) + : 0.0f; + } + } + + const float avgResidual = monoSignal.empty() ? 0.0f : residualEnergy / static_cast<float> (monoSignal.size()); + const float avgAperiodicity = model.bandAperiodicity.empty() + ? 0.0f + : std::accumulate (model.bandAperiodicity.begin(), model.bandAperiodicity.end(), 0.0f) + / static_cast<float> (model.bandAperiodicity.size()); + model.suggestedMix = juce::jlimit (0.02f, 0.16f, 0.04f + avgResidual * 0.5f + avgAperiodicity * 0.10f); + model.voicedMix = juce::jlimit (0.02f, 0.14f, 0.03f + avgAperiodicity * 0.16f); + model.highBandMix = juce::jlimit (0.02f, 0.18f, + 0.03f + (model.bandAperiodicity.size() >= 2 + ? 0.5f * (model.bandAperiodicity[model.bandAperiodicity.size() - 1] + + model.bandAperiodicity[model.bandAperiodicity.size() - 2]) + : avgAperiodicity) * 0.22f); + return model; +} + +static void applySourceFilterCorrection ( + std::vector<float>& synthSignal, + const std::vector<float>& f0TrackHz, + const std::vector<int>& epochs, + int coreStartSample, + int coreEndSample, + const OwnPitchEngine::SpectralEnvelopeModel& sourceEnvelopeModel, + const OwnPitchEngine::ResidualModel& residualModel, + double sampleRate, + bool downwardShift, + bool longBody) +{ + if (synthSignal.empty() + || coreEndSample <= coreStartSample + || sourceEnvelopeModel.averageMagnitude.empty()) + return; + + const auto synthEnvelopeModel = buildSpectralEnvelope ( + synthSignal, + f0TrackHz, + epochs, + coreStartSample, + coreEndSample, + sampleRate); + + if (synthEnvelopeModel.averageMagnitude.empty()) + return; + + const int fftOrder = sourceEnvelopeModel.fftOrder > 0 ? sourceEnvelopeModel.fftOrder : 10; + const int fftSize = 1 << fftOrder; + const int halfBins = fftSize / 2 + 1; + const int hopLen = fftSize / 4; + const auto hannWindow = buildHannWindow (fftSize); + juce::dsp::FFT fft (fftOrder); + std::vector<float> fftBuffer (static_cast<size_t> (fftSize * 2), 0.0f); + std::vector<float> corrected (synthSignal.size(), 0.0f); + std::vector<float> windowSum (synthSignal.size(), 0.0f); + int sourceFrameCursor = 0; + int synthFrameCursor = 0; + const bool hybridStructural = isHybridStructuralBranchRequested(); + + const float correctionStrength = downwardShift + ? (hybridStructural ? (longBody ? 0.76f : 0.68f) : 0.58f) + : (longBody ? 0.74f : 0.68f); + const float minGain = downwardShift + ? (hybridStructural ? 0.60f : 0.72f) + : (longBody ? 0.72f : 0.68f); + const float maxGain = downwardShift + ? (hybridStructural ? (longBody ? 1.52f : 1.44f) : 1.36f) + : (longBody ? 1.34f : 1.44f); + const float highBandLimit = downwardShift + ? (hybridStructural ? 1.08f : 1.18f) + : (longBody ? 1.16f : 1.28f); + const float localEnvelopeBlend = downwardShift + ? (hybridStructural ? (longBody ? 0.70f : 0.60f) : 0.48f) + : (longBody ? 0.72f : 0.52f); + + for (int pos = std::max (0, coreStartSample - fftSize / 2); + pos < std::min (static_cast<int> (synthSignal.size()), coreEndSample + fftSize / 2); + pos += hopLen) + { + const int frameSample = juce::jlimit (coreStartSample, std::max (coreStartSample, coreEndSample - 1), pos + fftSize / 2); + sourceFrameCursor = advanceFrameCursor (sourceEnvelopeModel.frames, frameSample, sourceFrameCursor); + synthFrameCursor = advanceFrameCursor (synthEnvelopeModel.frames, frameSample, synthFrameCursor); + + std::fill (fftBuffer.begin(), fftBuffer.end(), 0.0f); + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + if (idx >= 0 && idx < static_cast<int> (synthSignal.size())) + fftBuffer[static_cast<size_t> (i)] = synthSignal[static_cast<size_t> (idx)] * hannWindow[static_cast<size_t> (i)]; + } + + fft.performRealOnlyForwardTransform (fftBuffer.data(), true); + float energyBefore = 0.0f; + float energyAfter = 0.0f; + for (int bin = 0; bin < halfBins; ++bin) + { + const float re = fftBuffer[static_cast<size_t> (bin * 2)]; + const float im = fftBuffer[static_cast<size_t> (bin * 2 + 1)]; + energyBefore += re * re + im * im; + + const int srcAvgBin = juce::jlimit (0, static_cast<int> (sourceEnvelopeModel.averageMagnitude.size()) - 1, bin); + const int synthAvgBin = juce::jlimit (0, static_cast<int> (synthEnvelopeModel.averageMagnitude.size()) - 1, bin); + const float srcLocal = interpolateEnvelopeMagnitudeAt (sourceEnvelopeModel, frameSample, bin, sourceFrameCursor); + const float synthLocal = interpolateEnvelopeMagnitudeAt (synthEnvelopeModel, frameSample, bin, synthFrameCursor); + const float srcAverage = sourceEnvelopeModel.averageMagnitude[static_cast<size_t> (srcAvgBin)] + 1.0e-8f; + const float synthAverage = synthEnvelopeModel.averageMagnitude[static_cast<size_t> (synthAvgBin)] + 1.0e-8f; + const float srcEnv = srcAverage + (srcLocal - srcAverage) * localEnvelopeBlend; + const float synthEnv = synthAverage + (synthLocal - synthAverage) * localEnvelopeBlend; + const float rawGain = std::pow (srcEnv / synthEnv, correctionStrength); + const float freqNorm = static_cast<float> (bin) / static_cast<float> (std::max (1, halfBins - 1)); + const float bandWeight = downwardShift + ? (hybridStructural + ? (0.92f + 0.08f * smoothstep01 ((freqNorm - 0.05f) / 0.42f)) + : (0.78f + 0.22f * smoothstep01 ((freqNorm - 0.08f) / 0.55f))) + : (0.70f + 0.30f * smoothstep01 ((freqNorm - 0.06f) / 0.48f)); + float gain = juce::jlimit (minGain, maxGain, 1.0f + (rawGain - 1.0f) * bandWeight); + if (freqNorm > 0.60f) + gain = std::min (gain, highBandLimit); + + fftBuffer[static_cast<size_t> (bin * 2)] *= gain; + fftBuffer[static_cast<size_t> (bin * 2 + 1)] *= gain; + } + + for (int bin = 0; bin < halfBins; ++bin) + { + const float re = fftBuffer[static_cast<size_t> (bin * 2)]; + const float im = fftBuffer[static_cast<size_t> (bin * 2 + 1)]; + energyAfter += re * re + im * im; + } + + if (energyAfter > 1.0e-10f) + { + const float scale = std::pow (energyBefore / energyAfter, 0.18f); + for (int bin = 0; bin < halfBins; ++bin) + { + fftBuffer[static_cast<size_t> (bin * 2)] *= scale; + fftBuffer[static_cast<size_t> (bin * 2 + 1)] *= scale; + } + } + + fft.performRealOnlyInverseTransform (fftBuffer.data()); + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + if (idx >= 0 && idx < static_cast<int> (synthSignal.size())) + { + const float w = hannWindow[static_cast<size_t> (i)]; + corrected[static_cast<size_t> (idx)] += fftBuffer[static_cast<size_t> (i)] * w; + windowSum[static_cast<size_t> (idx)] += w * w; + } + } + } + + for (int i = coreStartSample; i < coreEndSample && i < static_cast<int> (synthSignal.size()); ++i) + { + if (windowSum[static_cast<size_t> (i)] > 1.0e-4f) + synthSignal[static_cast<size_t> (i)] = corrected[static_cast<size_t> (i)] / windowSum[static_cast<size_t> (i)]; + } + + const float residualMix = downwardShift + ? residualModel.voicedMix * (hybridStructural ? (longBody ? 0.28f : 0.36f) : 0.55f) + : residualModel.highBandMix * (longBody ? 0.75f : 0.70f); + if (! residualModel.voicedHighBandResidual.empty() && residualMix > 0.0f) + { + for (int i = coreStartSample; i < coreEndSample && i < static_cast<int> (synthSignal.size()); ++i) + synthSignal[static_cast<size_t> (i)] += residualModel.voicedHighBandResidual[static_cast<size_t> (i)] * residualMix; + } +} + +struct AnalysisCacheKey +{ + int mode = 0; + int quality = 0; + int numChannels = 0; + int numSamples = 0; + int sampleRateRounded = 0; + std::uint64_t noteSignature = 0; + std::uint64_t frameSignature = 0; + std::uint64_t audioSignature = 0; + + bool operator== (const AnalysisCacheKey& other) const noexcept + { + return mode == other.mode + && quality == other.quality + && numChannels == other.numChannels + && numSamples == other.numSamples + && sampleRateRounded == other.sampleRateRounded + && noteSignature == other.noteSignature + && frameSignature == other.frameSignature + && audioSignature == other.audioSignature; + } +}; + +static void hashCombine (std::uint64_t& seed, std::uint64_t value) +{ + seed ^= value + 0x9e3779b97f4a7c15ULL + (seed << 6U) + (seed >> 2U); +} + +static std::uint64_t buildNoteSignature (const std::vector<PitchAnalyzer::PitchNote>& notes) +{ + std::uint64_t seed = 1469598103934665603ULL; + for (const auto& note : notes) + { + hashCombine (seed, static_cast<std::uint64_t> (std::llround (note.startTime * 1000.0f))); + hashCombine (seed, static_cast<std::uint64_t> (std::llround (note.endTime * 1000.0f))); + hashCombine (seed, static_cast<std::uint64_t> (std::llround (note.detectedPitch * 100.0f))); + hashCombine (seed, static_cast<std::uint64_t> (std::llround (note.correctedPitch * 100.0f))); + hashCombine (seed, static_cast<std::uint64_t> (std::llround (note.formantShift * 100.0f))); + } + return seed; +} + +static std::uint64_t buildFrameSignature (const std::vector<PitchAnalyzer::PitchFrame>& frames) +{ + std::uint64_t seed = 1099511628211ULL; + if (frames.empty()) + return seed; + + const int stride = std::max (1, static_cast<int> (frames.size() / 32)); + for (size_t i = 0; i < frames.size(); i += static_cast<size_t> (stride)) + { + const auto& frame = frames[i]; + hashCombine (seed, static_cast<std::uint64_t> (std::llround (frame.time * 1000.0f))); + hashCombine (seed, static_cast<std::uint64_t> (std::llround (frame.frequency * 10.0f))); + hashCombine (seed, static_cast<std::uint64_t> (std::llround (frame.confidence * 1000.0f))); + hashCombine (seed, static_cast<std::uint64_t> (frame.voiced ? 1 : 0)); + } + return seed; +} + +static std::uint64_t buildAudioSignature ( + const float* const* input, + int numChannels, + int numSamples) +{ + std::uint64_t seed = 0xcbf29ce484222325ULL; + if (numChannels <= 0 || numSamples <= 0) + return seed; + + const int probeCount = std::min (numSamples, 256); + for (int ch = 0; ch < numChannels; ++ch) + { + const auto* src = input[ch]; + for (int i = 0; i < probeCount; ++i) + hashCombine (seed, static_cast<std::uint64_t> (std::llround (src[i] * 100000.0f))); + for (int i = std::max (0, numSamples - probeCount); i < numSamples; ++i) + hashCombine (seed, static_cast<std::uint64_t> (std::llround (src[i] * 100000.0f))); + } + + return seed; +} + +static std::optional<std::pair<AnalysisCacheKey, OwnPitchEngine::SharedAnalysis>> gLastSharedAnalysisCache; + +static float getTargetPitchRatioAtSample ( + int absoluteSample, + const std::vector<float>& pitchRatios) +{ + if (pitchRatios.empty()) + return 1.0f; + const int clamped = juce::jlimit (0, static_cast<int> (pitchRatios.size()) - 1, absoluteSample); + return juce::jlimit (0.25f, 4.0f, pitchRatios[static_cast<size_t> (clamped)]); +} + +static bool isHybridStructuralBranchRequested() +{ + return false; +} + +static int findSustainedVoicedSample ( + const std::vector<float>& voicedMask, + const std::vector<float>& amplitudeEnvelope, + int searchStartSample, + int searchEndSample, + int sustainSamples, + float voicedThreshold, + float envThreshold) +{ + if (voicedMask.empty() || amplitudeEnvelope.empty() || searchEndSample <= searchStartSample) + return searchStartSample; + + const int endLimit = std::max (searchStartSample, searchEndSample - sustainSamples); + for (int sample = searchStartSample; sample <= endLimit; ++sample) + { + bool sustained = true; + for (int offset = 0; offset < sustainSamples; ++offset) + { + const int idx = sample + offset; + if (idx < 0 + || idx >= static_cast<int> (voicedMask.size()) + || idx >= static_cast<int> (amplitudeEnvelope.size()) + || voicedMask[static_cast<size_t> (idx)] < voicedThreshold + || amplitudeEnvelope[static_cast<size_t> (idx)] < envThreshold) + { + sustained = false; + break; + } + } + + if (sustained) + return sample; + } + + return searchStartSample; +} + +static int findLastSustainedVoicedSample ( + const std::vector<float>& voicedMask, + const std::vector<float>& amplitudeEnvelope, + int searchStartSample, + int searchEndSample, + int sustainSamples, + float voicedThreshold, + float envThreshold) +{ + if (voicedMask.empty() || amplitudeEnvelope.empty() || searchEndSample <= searchStartSample) + return searchEndSample; + + const int startLimit = std::min (searchEndSample - 1, std::max (searchStartSample, searchStartSample + sustainSamples - 1)); + for (int sample = searchEndSample - 1; sample >= startLimit; --sample) + { + bool sustained = true; + for (int offset = 0; offset < sustainSamples; ++offset) + { + const int idx = sample - offset; + if (idx < 0 + || idx >= static_cast<int> (voicedMask.size()) + || idx >= static_cast<int> (amplitudeEnvelope.size()) + || voicedMask[static_cast<size_t> (idx)] < voicedThreshold + || amplitudeEnvelope[static_cast<size_t> (idx)] < envThreshold) + { + sustained = false; + break; + } + } + + if (sustained) + return sample; + } + + return searchEndSample; +} + +static float computeIslandWet ( + int absoluteSample, + int renderStartSample, + int renderEndSample, + int bodyStartSample, + int bodyEndSample, + int coreStartSample, + int coreEndSample, + int voicedEntrySample, + int voicedExitSample, + double sampleRate, + float voiced, + bool downwardShift, + bool longBody, + float coreMaxWet, + float bodyEntryWet, + float bodyExitWet, + float outsideCoreScale) +{ + const bool hybridStructural = isHybridStructuralBranchRequested(); + const int entryProtect = std::max (1, static_cast<int> (std::round (0.034 * sampleRate))); + const int exitProtect = std::max (1, static_cast<int> (std::round (0.040 * sampleRate))); + const int entryPreRoll = hybridStructural + ? std::max (1, static_cast<int> (std::round ((downwardShift ? 0.008 : 0.010) * sampleRate))) + : 0; + const int exitPostRoll = hybridStructural + ? std::max (1, static_cast<int> (std::round ((downwardShift ? 0.010 : 0.012) * sampleRate))) + : 0; + const int maxEntryAnchorShift = hybridStructural + ? std::max (1, static_cast<int> (std::round ((downwardShift ? 0.016 : (longBody ? 0.024 : 0.018)) * sampleRate))) + : 0; + const int maxExitAnchorShift = hybridStructural + ? std::max (1, static_cast<int> (std::round ((downwardShift ? 0.018 : (longBody ? 0.028 : 0.020)) * sampleRate))) + : 0; + const int desiredEntryStart = hybridStructural + ? std::max (bodyStartSample, voicedEntrySample - entryPreRoll) + : bodyStartSample; + const int desiredExitEnd = hybridStructural + ? std::min (bodyEndSample, voicedExitSample + exitPostRoll) + : bodyEndSample; + const int anchoredBodyStart = juce::jlimit ( + bodyStartSample, + std::max (bodyStartSample, bodyEndSample - 1), + bodyStartSample + std::min (maxEntryAnchorShift, std::max (0, desiredEntryStart - bodyStartSample))); + const int anchoredBodyEnd = juce::jlimit ( + anchoredBodyStart + 1, + bodyEndSample, + bodyEndSample - std::min (maxExitAnchorShift, std::max (0, bodyEndSample - desiredExitEnd))); + + if (absoluteSample < renderStartSample || absoluteSample >= renderEndSample) + return 0.0f; + + float wet = 0.0f; + if (absoluteSample < anchoredBodyStart) + { + const float t = static_cast<float> (absoluteSample - renderStartSample) / static_cast<float> (std::max (1, anchoredBodyStart - renderStartSample)); + wet = hybridStructural + ? (downwardShift ? 0.66f * smoothstep01 (t) : 0.58f * smoothstep01 (t)) + : 0.68f * smoothstep01 (t); + } + else if (absoluteSample > anchoredBodyEnd) + { + const float t = static_cast<float> (renderEndSample - absoluteSample) / static_cast<float> (std::max (1, renderEndSample - anchoredBodyEnd)); + wet = hybridStructural + ? (downwardShift ? 0.70f * smoothstep01 (t) : 0.60f * smoothstep01 (t)) + : 0.74f * smoothstep01 (t); + } + else + { + wet = coreMaxWet; + + const int effectiveEntryProtect = hybridStructural + ? (downwardShift + ? std::max (entryProtect, longBody ? static_cast<int> (std::round (0.050 * sampleRate)) + : static_cast<int> (std::round (0.042 * sampleRate))) + : std::max (entryProtect, longBody ? static_cast<int> (std::round (0.056 * sampleRate)) + : static_cast<int> (std::round (0.046 * sampleRate)))) + : (downwardShift + ? entryProtect + : std::max (entryProtect, longBody ? static_cast<int> (std::round (0.060 * sampleRate)) + : static_cast<int> (std::round (0.052 * sampleRate)))); + const int effectiveExitProtect = hybridStructural + ? (downwardShift + ? std::max (exitProtect, longBody ? static_cast<int> (std::round (0.058 * sampleRate)) + : static_cast<int> (std::round (0.050 * sampleRate))) + : std::max (exitProtect, longBody ? static_cast<int> (std::round (0.062 * sampleRate)) + : static_cast<int> (std::round (0.048 * sampleRate)))) + : (downwardShift + ? exitProtect + : std::max (exitProtect, longBody ? static_cast<int> (std::round (0.068 * sampleRate)) + : exitProtect)); + + if (! downwardShift) + { + const int upwardDryHold = std::max (1, static_cast<int> (std::round ((hybridStructural ? (longBody ? 0.024 : 0.018) + : (longBody ? 0.024 : 0.018)) * sampleRate))); + if (absoluteSample < anchoredBodyStart + upwardDryHold) + { + const float t = static_cast<float> (absoluteSample - anchoredBodyStart) / static_cast<float> (upwardDryHold); + wet = std::min (wet, juce::jmap ( + smoothstep01 (t), + hybridStructural ? 0.16f : 0.10f, + hybridStructural ? (longBody ? 0.36f : 0.40f) : 0.32f)); + } + } + else if (hybridStructural) + { + const int downwardDryHold = std::max (1, static_cast<int> (std::round ((longBody ? 0.020 : 0.014) * sampleRate))); + if (absoluteSample < anchoredBodyStart + downwardDryHold) + { + const float t = static_cast<float> (absoluteSample - anchoredBodyStart) / static_cast<float> (downwardDryHold); + wet = std::min (wet, juce::jmap (smoothstep01 (t), 0.24f, longBody ? 0.46f : 0.52f)); + } + } + + const float effectiveBodyEntryWet = downwardShift + ? std::min (bodyEntryWet, hybridStructural ? (longBody ? 0.54f : 0.60f) : bodyEntryWet) + : std::min (bodyEntryWet, hybridStructural ? (longBody ? 0.42f : 0.50f) : (longBody ? 0.34f : 0.42f)); + const float effectiveBodyExitWet = downwardShift + ? std::min (bodyExitWet, hybridStructural ? (longBody ? 0.56f : 0.62f) : bodyExitWet) + : std::min (bodyExitWet, hybridStructural ? (longBody ? 0.46f : 0.56f) : (longBody ? 0.44f : 0.56f)); + + if (absoluteSample < anchoredBodyStart + effectiveEntryProtect) + { + const float t = static_cast<float> (absoluteSample - anchoredBodyStart) / static_cast<float> (effectiveEntryProtect); + wet = std::min (wet, effectiveBodyEntryWet + (coreMaxWet - effectiveBodyEntryWet) * smoothstep01 (t)); + } + + if (absoluteSample > anchoredBodyEnd - effectiveExitProtect) + { + const float t = static_cast<float> (anchoredBodyEnd - absoluteSample) / static_cast<float> (effectiveExitProtect); + wet = std::min (wet, effectiveBodyExitWet + (coreMaxWet - effectiveBodyExitWet) * smoothstep01 (t)); + } + } + + if (absoluteSample < coreStartSample || absoluteSample >= coreEndSample) + { + if (hybridStructural) + wet *= (!downwardShift && longBody) ? outsideCoreScale * 0.88f : outsideCoreScale * (downwardShift ? 0.92f : 0.90f); + else + wet *= (!downwardShift && longBody) ? outsideCoreScale * 0.82f : outsideCoreScale; + } + + wet *= juce::jlimit (0.0f, 1.0f, 0.20f + 0.80f * voiced); + return juce::jlimit (0.0f, 1.0f, wet); +} +} + +OwnPitchEngine::SharedAnalysis OwnPitchEngine::analyze ( + const float* const* input, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchFrame>& frames, + const std::vector<PitchAnalyzer::PitchNote>& notes, + Mode mode, + Quality quality) +{ + SharedAnalysis analysis; + analysis.mode = mode; + analysis.quality = quality; + analysis.sampleRate = sampleRate; + analysis.numChannels = numChannels; + analysis.numSamples = numSamples; + + AnalysisCacheKey key; + key.mode = static_cast<int> (mode); + key.quality = static_cast<int> (quality); + key.numChannels = numChannels; + key.numSamples = numSamples; + key.sampleRateRounded = static_cast<int> (std::round (sampleRate)); + key.noteSignature = buildNoteSignature (notes); + key.frameSignature = buildFrameSignature (frames); + key.audioSignature = buildAudioSignature (input, numChannels, numSamples); + + if (gLastSharedAnalysisCache.has_value() && gLastSharedAnalysisCache->first == key) + { + analysis = gLastSharedAnalysisCache->second; + analysis.cacheHit = true; + return analysis; + } + + std::vector<PitchAnalyzer::PitchNote> editedNotes; + editedNotes.reserve (notes.size()); + for (const auto& note : notes) + { + const bool includeForMode = mode == Mode::PitchOnly + ? hasPitchStyleEdit (note) + : (std::abs (note.formantShift) > 0.01f || hasPitchStyleEdit (note)); + if (includeForMode) + editedNotes.push_back (note); + } + + std::sort (editedNotes.begin(), editedNotes.end(), [] (const auto& a, const auto& b) + { + return getEffectiveNoteStartTime (a) < getEffectiveNoteStartTime (b); + }); + + const float mergeGapSec = 0.020f; + const float contextPadSec = quality == Quality::PreviewFast ? 0.120f : 0.180f; + + std::vector<NoteIslandAnalysis> islands; + for (const auto& note : editedNotes) + { + if (islands.empty() + || getEffectiveNoteStartTime (note) > (static_cast<float> (islands.back().contextEndSample / sampleRate) + mergeGapSec)) + { + NoteIslandAnalysis island; + island.notes.push_back (note); + islands.push_back (std::move (island)); + } + else + { + islands.back().notes.push_back (note); + } + } + + for (auto& island : islands) + { + float islandRenderStartSec = std::numeric_limits<float>::max(); + float islandRenderEndSec = 0.0f; + float islandBodyStartSec = std::numeric_limits<float>::max(); + float islandBodyEndSec = 0.0f; + double islandRatioSum = 0.0; + int islandRatioCount = 0; + + for (const auto& note : island.notes) + { + islandRenderStartSec = std::min (islandRenderStartSec, getEffectiveNoteStartTime (note)); + islandRenderEndSec = std::max (islandRenderEndSec, getEffectiveNoteEndTime (note)); + islandBodyStartSec = std::min (islandBodyStartSec, note.startTime); + islandBodyEndSec = std::max (islandBodyEndSec, note.endTime); + islandRatioSum += std::pow (2.0, static_cast<double> (note.correctedPitch - note.detectedPitch) / 12.0); + ++islandRatioCount; + } + + island.renderStartSample = juce::jlimit (0, numSamples, static_cast<int> (std::floor (islandRenderStartSec * sampleRate))); + island.renderEndSample = juce::jlimit (island.renderStartSample, numSamples, static_cast<int> (std::ceil (islandRenderEndSec * sampleRate))); + island.contextStartSample = juce::jlimit (0, numSamples, island.renderStartSample - static_cast<int> (std::round (contextPadSec * sampleRate))); + island.contextEndSample = juce::jlimit (island.renderEndSample, numSamples, island.renderEndSample + static_cast<int> (std::round (contextPadSec * sampleRate))); + + island.monoSignal = buildMonoSignal (input, numChannels, island.contextStartSample, island.contextEndSample); + island.f0TrackHz = buildF0TrackHz (static_cast<int> (island.monoSignal.size()), island.contextStartSample, sampleRate, frames); + island.voicedMask = buildVoicedMask (static_cast<int> (island.monoSignal.size()), island.contextStartSample, sampleRate, frames); + island.consonantMask.resize (island.voicedMask.size(), 0.0f); + for (size_t i = 0; i < island.voicedMask.size(); ++i) + island.consonantMask[i] = 1.0f - juce::jlimit (0.0f, 1.0f, island.voicedMask[i]); + island.amplitudeEnvelope = buildAmplitudeEnvelope (island.monoSignal, sampleRate); + + const int bodyStartSample = juce::jlimit (0, static_cast<int> (island.monoSignal.size()), + static_cast<int> (std::floor (islandBodyStartSec * sampleRate)) - island.contextStartSample); + const int bodyEndSample = juce::jlimit (bodyStartSample, static_cast<int> (island.monoSignal.size()), + static_cast<int> (std::ceil (islandBodyEndSec * sampleRate)) - island.contextStartSample); + island.bodyStartSample = bodyStartSample; + island.bodyEndSample = bodyEndSample; + + const float averagePitchRatio = islandRatioCount > 0 + ? static_cast<float> (islandRatioSum / static_cast<double> (islandRatioCount)) + : 1.0f; + const bool downwardShift = averagePitchRatio < 0.999f; + const float bodyDurationSec = static_cast<float> (std::max (0, bodyEndSample - bodyStartSample)) / static_cast<float> (sampleRate); + const bool longBody = bodyDurationSec >= 0.90f; + + const int entryProtect = std::max (1, static_cast<int> (std::round ( + (!downwardShift && longBody ? 0.052 : 0.032) * sampleRate))); + const int exitProtect = std::max (1, static_cast<int> (std::round ( + (!downwardShift && longBody ? 0.070 : 0.038) * sampleRate))); + const int minCoreLen = std::max (1, static_cast<int> (std::round (0.055 * sampleRate))); + + island.core.startSample = juce::jlimit (bodyStartSample, bodyEndSample, bodyStartSample + entryProtect); + island.core.endSample = juce::jlimit (island.core.startSample, bodyEndSample, bodyEndSample - exitProtect); + if (island.core.endSample - island.core.startSample < minCoreLen) + { + island.core.startSample = bodyStartSample; + island.core.endSample = bodyEndSample; + } + + float voicedSum = 0.0f; + int voicedCount = 0; + for (int i = island.core.startSample; i < island.core.endSample && i < static_cast<int> (island.voicedMask.size()); ++i) + { + voicedSum += island.voicedMask[static_cast<size_t> (i)]; + ++voicedCount; + } + island.core.voicedRatio = voicedCount > 0 ? voicedSum / static_cast<float> (voicedCount) : 0.0f; + island.core.meanF0Hz = computeMedianPositive (island.f0TrackHz, island.core.startSample, island.core.endSample); + island.core.rms = computeRms (island.monoSignal, island.core.startSample, island.core.endSample); + island.epochs = buildEpochs (island.f0TrackHz, island.core.startSample, island.core.endSample, sampleRate); + island.core.epochCount = static_cast<int> (island.epochs.size()); + + island.spectralEnvelopeModel = buildSpectralEnvelope ( + island.monoSignal, + island.f0TrackHz, + island.epochs, + island.core.startSample, + island.core.endSample, + sampleRate); + island.harmonicModel = buildHarmonicModel ( + island.core.meanF0Hz, + island.core.rms, + island.spectralEnvelopeModel, + sampleRate); + island.residualModel = buildResidualModel ( + island.monoSignal, + island.voicedMask, + island.spectralEnvelopeModel, + sampleRate); + + analysis.totalEpochCount += island.core.epochCount; + analysis.maxPartialCount = std::max (analysis.maxPartialCount, island.harmonicModel.partialCount); + } + + analysis.islands = std::move (islands); + gLastSharedAnalysisCache = std::make_pair (key, analysis); + return analysis; +} + +OwnPitchEngine::RenderResult OwnPitchEngine::renderPitchOnly ( + const float* const* input, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchFrame>& frames, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& pitchRatios, + Quality quality, + std::function<bool()> shouldCancel) +{ + RenderResult result; + result.output.resize (static_cast<size_t> (numChannels)); + for (int ch = 0; ch < numChannels; ++ch) + result.output[static_cast<size_t> (ch)].assign (input[ch], input[ch] + numSamples); + + const double analysisStart = juce::Time::getMillisecondCounterHiRes(); + result.analysis = analyze (input, numChannels, numSamples, sampleRate, frames, notes, Mode::PitchOnly, quality); + result.analysisMs = juce::Time::getMillisecondCounterHiRes() - analysisStart; + + const double renderStart = juce::Time::getMillisecondCounterHiRes(); + const double twoPi = juce::MathConstants<double>::twoPi; + + for (const auto& island : result.analysis.islands) + { + if (shouldCancel && shouldCancel()) + { + result.usedFallback = true; + result.fallbackReason = "cancelled"; + break; + } + + const int localRenderStart = juce::jlimit (0, static_cast<int> (island.monoSignal.size()), island.renderStartSample - island.contextStartSample); + const int localRenderEnd = juce::jlimit (localRenderStart, static_cast<int> (island.monoSignal.size()), island.renderEndSample - island.contextStartSample); + const int renderSamples = localRenderEnd - localRenderStart; + if (renderSamples <= 0) + continue; + + float averagePitchRatio = 1.0f; + if (! island.notes.empty()) + { + double ratioSum = 0.0; + for (const auto& note : island.notes) + ratioSum += std::pow (2.0, static_cast<double> (note.correctedPitch - note.detectedPitch) / 12.0); + averagePitchRatio = static_cast<float> (ratioSum / static_cast<double> (island.notes.size())); + } + const bool downwardShift = averagePitchRatio < 0.999f; + const float bodyDurationSec = static_cast<float> (std::max (0, island.bodyEndSample - island.bodyStartSample)) / static_cast<float> (sampleRate); + const bool longBody = bodyDurationSec >= 0.90f; + + float coreMaxWet = 0.88f; + float bodyEntryWet = 0.66f; + float bodyExitWet = 0.70f; + float outsideCoreScale = 0.64f; + float cutoffHz = quality == Quality::PreviewFast ? 2600.0f : 3200.0f; + float filteredMix = 0.34f; + if (downwardShift) + { + coreMaxWet = 0.96f; + bodyEntryWet = 0.76f; + bodyExitWet = 0.80f; + outsideCoreScale = 0.78f; + cutoffHz = quality == Quality::PreviewFast ? 3400.0f : 4200.0f; + filteredMix = 0.26f; + if (! longBody) + { + coreMaxWet = 1.0f; + outsideCoreScale = 0.84f; + } + } + else if (longBody) + { + coreMaxWet = 0.92f; + bodyEntryWet = 0.78f; + bodyExitWet = 0.82f; + outsideCoreScale = 0.82f; + cutoffHz = quality == Quality::PreviewFast ? 3200.0f : 3800.0f; + filteredMix = 0.24f; + } + + const float maxBodyEnv = island.amplitudeEnvelope.empty() || island.bodyEndSample <= island.bodyStartSample + ? 0.0f + : *std::max_element (island.amplitudeEnvelope.begin() + island.bodyStartSample, + island.amplitudeEnvelope.begin() + island.bodyEndSample); + const float entryEnvThreshold = maxBodyEnv * (downwardShift ? 0.12f : (longBody ? 0.18f : 0.15f)); + const float exitEnvThreshold = maxBodyEnv * (downwardShift ? 0.10f : (longBody ? 0.14f : 0.12f)); + const int entrySearchEnd = std::min (island.bodyEndSample, + island.bodyStartSample + static_cast<int> (std::round ((longBody ? 0.120 : 0.080) * sampleRate))); + const int exitSearchStart = std::max (island.bodyStartSample, + island.bodyEndSample - static_cast<int> (std::round ((longBody ? 0.140 : 0.095) * sampleRate))); + const int sustainSamples = std::max (1, static_cast<int> (std::round (0.010 * sampleRate))); + const int voicedEntrySample = findSustainedVoicedSample ( + island.voicedMask, + island.amplitudeEnvelope, + island.bodyStartSample, + entrySearchEnd, + sustainSamples, + downwardShift ? 0.42f : 0.48f, + entryEnvThreshold); + const int voicedExitSample = findLastSustainedVoicedSample ( + island.voicedMask, + island.amplitudeEnvelope, + exitSearchStart, + island.bodyEndSample, + sustainSamples, + downwardShift ? 0.34f : 0.40f, + exitEnvThreshold); + + std::vector<float> synthMono (static_cast<size_t> (renderSamples), 0.0f); + const bool useLongUpwardHarmonicCarrier = ! downwardShift && longBody; + const bool canUseEpochCarrier = island.epochs.size() >= 4 && ! island.monoSignal.empty() && ! useLongUpwardHarmonicCarrier; + if (canUseEpochCarrier) + { + std::vector<int> targetEpochs; + int targetSample = island.core.startSample; + while (targetSample < island.core.endSample) + { + targetEpochs.push_back (targetSample); + const int absoluteSample = island.contextStartSample + targetSample; + const float pitchRatio = getTargetPitchRatioAtSample (absoluteSample, pitchRatios); + const float sourceHz = island.f0TrackHz.empty() + ? island.core.meanF0Hz + : std::max (island.f0TrackHz[static_cast<size_t> (juce::jlimit (0, static_cast<int> (island.f0TrackHz.size()) - 1, targetSample))], + island.core.meanF0Hz * 0.75f); + const float targetHz = juce::jlimit (55.0f, 1400.0f, std::max (55.0f, sourceHz) * pitchRatio); + const int targetPeriod = std::max (16, static_cast<int> (std::round (sampleRate / targetHz))); + targetSample += targetPeriod; + } + + const int sourceEpochCount = static_cast<int> (island.epochs.size()); + const int targetEpochCount = static_cast<int> (targetEpochs.size()); + const bool interpolateEpochSources = ! downwardShift && longBody && sourceEpochCount >= 6; + for (int k = 0; k < targetEpochCount; ++k) + { + const float norm = targetEpochCount > 1 + ? static_cast<float> (k) / static_cast<float> (targetEpochCount - 1) + : 0.0f; + const float sourceEpochPos = norm * static_cast<float> (std::max (0, sourceEpochCount - 1)); + const int sourceEpochIndexLo = juce::jlimit (0, sourceEpochCount - 1, + static_cast<int> (std::floor (sourceEpochPos))); + const int sourceEpochIndexHi = juce::jlimit (0, sourceEpochCount - 1, sourceEpochIndexLo + 1); + const float sourceEpochFrac = interpolateEpochSources + ? juce::jlimit (0.0f, 1.0f, sourceEpochPos - static_cast<float> (sourceEpochIndexLo)) + : 0.0f; + const int sourceEpochIndex = interpolateEpochSources + ? sourceEpochIndexLo + : juce::jlimit (0, sourceEpochCount - 1, static_cast<int> (std::round (sourceEpochPos))); + const int sourceEpoch = island.epochs[static_cast<size_t> (sourceEpochIndex)]; + const int targetEpoch = targetEpochs[static_cast<size_t> (k)]; + + const int prevSource = sourceEpochIndex > 0 ? island.epochs[static_cast<size_t> (sourceEpochIndex - 1)] : sourceEpoch; + const int nextSource = sourceEpochIndex + 1 < sourceEpochCount ? island.epochs[static_cast<size_t> (sourceEpochIndex + 1)] : sourceEpoch; + const int sourcePeriod = std::max (16, std::max (nextSource - sourceEpoch, sourceEpoch - prevSource)); + const int sourceEpochHi = island.epochs[static_cast<size_t> (sourceEpochIndexHi)]; + const int prevSourceHi = sourceEpochIndexHi > 0 ? island.epochs[static_cast<size_t> (sourceEpochIndexHi - 1)] : sourceEpochHi; + const int nextSourceHi = sourceEpochIndexHi + 1 < sourceEpochCount ? island.epochs[static_cast<size_t> (sourceEpochIndexHi + 1)] : sourceEpochHi; + const int sourcePeriodHi = std::max (16, std::max (nextSourceHi - sourceEpochHi, sourceEpochHi - prevSourceHi)); + + const int prevTarget = k > 0 ? targetEpochs[static_cast<size_t> (k - 1)] : targetEpoch; + const int nextTarget = k + 1 < targetEpochCount ? targetEpochs[static_cast<size_t> (k + 1)] : targetEpoch; + const int targetPeriod = std::max (16, std::max (nextTarget - targetEpoch, targetEpoch - prevTarget)); + + const int blendedSourcePeriod = interpolateEpochSources + ? std::max (16, static_cast<int> (std::round ( + sourcePeriod + (sourcePeriodHi - sourcePeriod) * sourceEpochFrac))) + : sourcePeriod; + const int grainRadius = std::max (16, std::min (blendedSourcePeriod, targetPeriod)); + for (int offset = -grainRadius; offset <= grainRadius; ++offset) + { + const int sourcePos = sourceEpoch + offset; + const int targetPos = targetEpoch + offset; + if (sourcePos < 0 || sourcePos >= static_cast<int> (island.monoSignal.size())) + continue; + if (targetPos < localRenderStart || targetPos >= localRenderEnd) + continue; + + float sourceSample = island.monoSignal[static_cast<size_t> (sourcePos)]; + if (interpolateEpochSources) + { + const int sourcePosHi = sourceEpochHi + offset; + if (sourcePosHi >= 0 && sourcePosHi < static_cast<int> (island.monoSignal.size())) + { + sourceSample += (island.monoSignal[static_cast<size_t> (sourcePosHi)] - sourceSample) * sourceEpochFrac; + } + } + + const float x = static_cast<float> (offset + grainRadius) / static_cast<float> (std::max (1, grainRadius * 2)); + const float window = 0.5f - 0.5f * std::cos (static_cast<float> (twoPi) * x); + const int destIndex = targetPos - localRenderStart; + synthMono[static_cast<size_t> (destIndex)] += sourceSample * window; + } + } + } + else + { + std::vector<double> phases (static_cast<size_t> (std::max (1, island.harmonicModel.partialCount)), 0.0); + for (int i = 0; i < renderSamples; ++i) + { + const int localSample = localRenderStart + i; + const int absoluteSample = island.contextStartSample + localSample; + const float voiced = island.voicedMask.empty() ? 0.0f : island.voicedMask[static_cast<size_t> (localSample)]; + const float envelope = island.amplitudeEnvelope.empty() ? 0.0f : island.amplitudeEnvelope[static_cast<size_t> (localSample)]; + + float sampleValue = 0.0f; + if (localSample >= island.core.startSample + && localSample < island.core.endSample + && voiced > 0.05f + && island.core.meanF0Hz > 0.0f + && island.harmonicModel.partialCount > 0) + { + const float pitchRatio = getTargetPitchRatioAtSample (absoluteSample, pitchRatios); + const float sourceHz = island.f0TrackHz.empty() + ? island.core.meanF0Hz + : std::max (island.f0TrackHz[static_cast<size_t> (localSample)], island.core.meanF0Hz * 0.75f); + const float targetHz = juce::jlimit (55.0f, 1400.0f, sourceHz * pitchRatio); + const float voicedScale = 0.55f + 0.45f * voiced; + + for (int p = 0; p < island.harmonicModel.partialCount; ++p) + { + const double omega = twoPi * static_cast<double> (targetHz * static_cast<float> (p + 1)) / sampleRate; + phases[static_cast<size_t> (p)] += omega; + if (phases[static_cast<size_t> (p)] > twoPi) + phases[static_cast<size_t> (p)] -= twoPi; + + const float amp = island.harmonicModel.averageMagnitude[static_cast<size_t> (p)]; + sampleValue += amp * static_cast<float> (std::sin (phases[static_cast<size_t> (p)])); + } + + sampleValue *= envelope * voicedScale; + } + else if (! island.monoSignal.empty()) + { + sampleValue = island.monoSignal[static_cast<size_t> (localSample)]; + } + + synthMono[static_cast<size_t> (i)] = sampleValue; + } + } + + const float synthCoreRms = computeRms ( + synthMono, + std::max (0, island.core.startSample - localRenderStart), + std::min (renderSamples, island.core.endSample - localRenderStart)); + const float gain = synthCoreRms > 1.0e-5f ? island.core.rms / synthCoreRms : 1.0f; + if (std::isfinite (gain)) + { + for (auto& sample : synthMono) + sample *= gain; + } + + std::vector<float> localF0Track (static_cast<size_t> (renderSamples), 0.0f); + for (int i = 0; i < renderSamples; ++i) + { + const int localSample = localRenderStart + i; + if (localSample >= 0 && localSample < static_cast<int> (island.f0TrackHz.size())) + localF0Track[static_cast<size_t> (i)] = island.f0TrackHz[static_cast<size_t> (localSample)]; + } + + std::vector<int> localEpochs; + localEpochs.reserve (island.epochs.size()); + for (const int epoch : island.epochs) + { + if (epoch >= localRenderStart && epoch < localRenderEnd) + localEpochs.push_back (epoch - localRenderStart); + } + + applySourceFilterCorrection ( + synthMono, + localF0Track, + localEpochs, + std::max (0, island.core.startSample - localRenderStart), + std::min (renderSamples, island.core.endSample - localRenderStart), + island.spectralEnvelopeModel, + island.residualModel, + sampleRate, + downwardShift, + longBody); + + // Keep a light protection low-pass after envelope correction so the + // preview path stays stable while the source-filter model matures. + const float alpha = juce::jlimit (0.02f, 0.45f, + static_cast<float> ((2.0 * juce::MathConstants<double>::pi * cutoffHz) / (sampleRate + 2.0 * juce::MathConstants<double>::pi * cutoffHz))); + float lp = synthMono.empty() ? 0.0f : synthMono.front(); + for (auto& sample : synthMono) + { + lp += alpha * (sample - lp); + sample = filteredMix * lp + (1.0f - filteredMix) * sample; + } + + for (int ch = 0; ch < numChannels; ++ch) + { + for (int i = 0; i < renderSamples; ++i) + { + const int absoluteSample = island.renderStartSample + i; + if (absoluteSample < 0 || absoluteSample >= numSamples) + continue; + + const int localSample = localRenderStart + i; + const float dry = input[ch][absoluteSample]; + const float voiced = island.voicedMask.empty() ? 0.0f : island.voicedMask[static_cast<size_t> (localSample)]; + const float wet = computeIslandWet ( + absoluteSample, + island.renderStartSample, + island.renderEndSample, + island.contextStartSample + island.bodyStartSample, + island.contextStartSample + island.bodyEndSample, + island.contextStartSample + island.core.startSample, + island.contextStartSample + island.core.endSample, + island.contextStartSample + voicedEntrySample, + island.contextStartSample + voicedExitSample, + sampleRate, + voiced, + downwardShift, + longBody, + coreMaxWet, + bodyEntryWet, + bodyExitWet, + outsideCoreScale); + + result.output[static_cast<size_t> (ch)][static_cast<size_t> (absoluteSample)] + = dry * (1.0f - wet) + synthMono[static_cast<size_t> (i)] * wet; + } + } + } + + result.renderMs = juce::Time::getMillisecondCounterHiRes() - renderStart; + + logOwnPitchEngine ("pitchOnly islands=" + juce::String (static_cast<int> (result.analysis.islands.size())) + + " cacheHit=" + juce::String (result.analysis.cacheHit ? "true" : "false") + + " epochs=" + juce::String (result.analysis.totalEpochCount) + + " maxPartials=" + juce::String (result.analysis.maxPartialCount) + + " analysisMs=" + juce::String (result.analysisMs, 2) + + " renderMs=" + juce::String (result.renderMs, 2)); + + return result; +} + +OwnPitchEngine::RenderResult OwnPitchEngine::renderVocalSourceFilterHq ( + const float* const* input, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchFrame>& frames, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& pitchRatios, + Quality quality, + std::function<bool()> shouldCancel) +{ + RenderResult result; + result.vocalSourceFilterUsed = true; + result.output.resize (static_cast<size_t> (numChannels)); + for (int ch = 0; ch < numChannels; ++ch) + result.output[static_cast<size_t> (ch)].assign (input[ch], input[ch] + numSamples); + + const auto layerDumpDir = getVsfLayerDumpDirectory(); + const bool dumpLayers = layerDumpDir != juce::File(); + std::vector<std::vector<float>> dryLayer; + std::vector<std::vector<float>> coreLayer; + std::vector<std::vector<float>> residualLayer; + std::vector<std::vector<float>> wetEnvelopeLayer; + if (dumpLayers) + { + dryLayer = result.output; + coreLayer.assign (static_cast<size_t> (numChannels), std::vector<float> (static_cast<size_t> (numSamples), 0.0f)); + residualLayer.assign (static_cast<size_t> (numChannels), std::vector<float> (static_cast<size_t> (numSamples), 0.0f)); + wetEnvelopeLayer.assign (static_cast<size_t> (numChannels), std::vector<float> (static_cast<size_t> (numSamples), 0.0f)); + } + + const double analysisStart = juce::Time::getMillisecondCounterHiRes(); + result.analysis = analyze (input, numChannels, numSamples, sampleRate, frames, notes, Mode::PitchOnly, quality); + result.analysisMs = juce::Time::getMillisecondCounterHiRes() - analysisStart; + + const double renderStart = juce::Time::getMillisecondCounterHiRes(); + const double twoPi = juce::MathConstants<double>::twoPi; + const float nyquistLimit = static_cast<float> (sampleRate * 0.46); + + int renderedIslandCount = 0; + int fallbackIslandCount = 0; + int totalBodySamples = 0; + int totalVoicedSamples = 0; + double residualMixSum = 0.0; + int residualMixCount = 0; + double residualMixScaleSum = 0.0; + int residualMixScaleCount = 0; + bool anyEpochInterpolationUsed = false; + double epochInterpolationStrengthSum = 0.0; + int epochInterpolationStrengthCount = 0; + double grainRadiusScaleSum = 0.0; + int grainRadiusScaleCount = 0; + double upPresenceTrimDbSum = 0.0; + double upPresenceHzSum = 0.0; + int upPresenceCount = 0; + double downNasalTrimDbSum = 0.0; + double downNasalHzSum = 0.0; + int downNasalCount = 0; + double downBodyCompDbSum = 0.0; + double downBodyCompHzSum = 0.0; + int downBodyCompCount = 0; + int maxEntryDrySamples = 0; + int maxExitDrySamples = 0; + + for (const auto& island : result.analysis.islands) + { + if (shouldCancel && shouldCancel()) + { + result.usedFallback = true; + result.fallbackReason = "cancelled"; + result.vocalSourceFilterFallbackUsed = true; + result.vocalSourceFilterFallbackReason = "cancelled"; + break; + } + + const int localRenderStart = juce::jlimit (0, static_cast<int> (island.monoSignal.size()), island.renderStartSample - island.contextStartSample); + const int localRenderEnd = juce::jlimit (localRenderStart, static_cast<int> (island.monoSignal.size()), island.renderEndSample - island.contextStartSample); + const int renderSamples = localRenderEnd - localRenderStart; + if (renderSamples <= 0) + { + ++fallbackIslandCount; + continue; + } + + const int bodyStart = juce::jlimit (localRenderStart, localRenderEnd, island.bodyStartSample); + const int bodyEnd = juce::jlimit (bodyStart, localRenderEnd, island.bodyEndSample); + int coreStart = juce::jlimit (bodyStart, bodyEnd, island.core.startSample); + int coreEnd = juce::jlimit (coreStart, bodyEnd, island.core.endSample); + if (coreEnd - coreStart < static_cast<int> (std::round (0.045 * sampleRate))) + { + coreStart = bodyStart; + coreEnd = bodyEnd; + } + + if (coreEnd <= coreStart + || island.core.meanF0Hz <= 0.0f + || island.harmonicModel.partialCount < 3 + || island.spectralEnvelopeModel.averageMagnitude.empty()) + { + ++fallbackIslandCount; + continue; + } + + float averagePitchRatio = 1.0f; + if (! island.notes.empty()) + { + double ratioSum = 0.0; + for (const auto& note : island.notes) + ratioSum += std::pow (2.0, static_cast<double> (note.correctedPitch - note.detectedPitch) / 12.0); + averagePitchRatio = static_cast<float> (ratioSum / static_cast<double> (island.notes.size())); + } + const bool downwardShift = averagePitchRatio < 0.999f; + const bool upwardShift = averagePitchRatio > 1.001f; + const float bodyDurationSec = static_cast<float> (std::max (0, bodyEnd - bodyStart)) / static_cast<float> (sampleRate); + const bool longBody = bodyDurationSec >= 0.90f; + const bool interpolateEpochSourcesRequested = upwardShift + && getEnvBool ("OPENSTUDIO_VSF_UP_SOURCE_EPOCH_INTERP_ENABLE", + getEnvBool ("OPENSTUDIO_VSF_SOURCE_EPOCH_INTERP_ENABLE", true)); + const float effectiveEpochInterpolationStrength = interpolateEpochSourcesRequested + ? juce::jlimit ( + 0.0f, + 1.0f, + getEnvFloat ("OPENSTUDIO_VSF_UP_SOURCE_EPOCH_INTERP_STRENGTH", + getEnvFloat ("OPENSTUDIO_VSF_SOURCE_EPOCH_INTERP_STRENGTH", 0.75f))) + : 0.0f; + epochInterpolationStrengthSum += effectiveEpochInterpolationStrength; + ++epochInterpolationStrengthCount; + const float effectiveGrainRadiusScale = upwardShift + ? juce::jlimit ( + 0.45f, + 1.25f, + getEnvFloat ("OPENSTUDIO_VSF_UP_GRAIN_RADIUS_SCALE", + getEnvFloat ("OPENSTUDIO_VSF_GRAIN_RADIUS_SCALE", 0.70f))) + : 1.0f; + grainRadiusScaleSum += effectiveGrainRadiusScale; + ++grainRadiusScaleCount; + const float upBodyPresenceTrimDb = upwardShift + ? juce::jlimit (-6.0f, 3.0f, getEnvFloat ("OPENSTUDIO_VSF_UP_BODY_PRESENCE_TRIM_DB", 0.0f)) + : 0.0f; + const float upBodyPresenceHz = upwardShift && std::abs (upBodyPresenceTrimDb) > 1.0e-4f + ? juce::jlimit (1200.0f, 4000.0f, getEnvFloat ("OPENSTUDIO_VSF_UP_BODY_PRESENCE_HZ", 2400.0f)) + : 0.0f; + const float upBodyPresenceQ = juce::jlimit (0.35f, 3.0f, getEnvFloat ("OPENSTUDIO_VSF_UP_BODY_PRESENCE_Q", 0.90f)); + upPresenceTrimDbSum += upBodyPresenceTrimDb; + upPresenceHzSum += upBodyPresenceHz; + ++upPresenceCount; + const float downBodyNasalTrimDb = downwardShift + ? juce::jlimit (-6.0f, 3.0f, getEnvFloat ("OPENSTUDIO_VSF_DOWN_BODY_NASAL_TRIM_DB", -1.5f)) + : 0.0f; + const float downBodyNasalHz = downwardShift && std::abs (downBodyNasalTrimDb) > 1.0e-4f + ? juce::jlimit (750.0f, 1700.0f, getEnvFloat ("OPENSTUDIO_VSF_DOWN_BODY_NASAL_HZ", 1100.0f)) + : 0.0f; + const float downBodyNasalQ = juce::jlimit (0.35f, 3.0f, getEnvFloat ("OPENSTUDIO_VSF_DOWN_BODY_NASAL_Q", 1.00f)); + downNasalTrimDbSum += downBodyNasalTrimDb; + downNasalHzSum += downBodyNasalHz; + ++downNasalCount; + const float downBodyCompDb = downwardShift + ? juce::jlimit (-2.0f, 4.0f, getEnvFloat ("OPENSTUDIO_VSF_DOWN_BODY_COMP_DB", 0.0f)) + : 0.0f; + const float downBodyCompHz = downwardShift && std::abs (downBodyCompDb) > 1.0e-4f + ? juce::jlimit (220.0f, 850.0f, getEnvFloat ("OPENSTUDIO_VSF_DOWN_BODY_COMP_HZ", 430.0f)) + : 0.0f; + const float downBodyCompQ = juce::jlimit (0.35f, 3.0f, getEnvFloat ("OPENSTUDIO_VSF_DOWN_BODY_COMP_Q", 0.75f)); + downBodyCompDbSum += downBodyCompDb; + downBodyCompHzSum += downBodyCompHz; + ++downBodyCompCount; + const float envelopeLookupRatio = downwardShift + ? juce::jlimit (0.82f, 1.0f, 1.0f + (averagePitchRatio - 1.0f) * 0.55f) + : 1.0f; + + const float maxBodyEnv = island.amplitudeEnvelope.empty() || bodyEnd <= bodyStart + ? 0.0f + : *std::max_element (island.amplitudeEnvelope.begin() + bodyStart, + island.amplitudeEnvelope.begin() + bodyEnd); + const int sustainSamples = std::max (1, static_cast<int> (std::round (0.010 * sampleRate))); + const int entrySearchEnd = std::min (bodyEnd, bodyStart + static_cast<int> (std::round ((longBody ? 0.135 : 0.095) * sampleRate))); + const int voicedEntrySample = findSustainedVoicedSample ( + island.voicedMask, + island.amplitudeEnvelope, + bodyStart, + entrySearchEnd, + sustainSamples, + downwardShift ? 0.36f : 0.42f, + maxBodyEnv * (downwardShift ? 0.09f : 0.12f)); + + const int maxEntryDryByBody = std::max (1, (bodyEnd - bodyStart) / 4); + const double requestedEntryDrySec = (! longBody && ! downwardShift) ? 0.024 : 0.032; + const int requestedEntryDrySamples = std::max (1, static_cast<int> (std::round (requestedEntryDrySec * sampleRate))); + const int baseEntryDrySamples = std::min (requestedEntryDrySamples, maxEntryDryByBody); + const int maxEntryDryAllowed = std::max ( + baseEntryDrySamples, + std::min ( + maxEntryDryByBody, + static_cast<int> (std::round ((longBody ? 0.044 : 0.034) * sampleRate)))); + const int entryDrySamples = juce::jlimit ( + baseEntryDrySamples, + maxEntryDryAllowed, + std::max (baseEntryDrySamples, voicedEntrySample - bodyStart)); + const int exitDrySamples = std::max (1, static_cast<int> (std::round ((downwardShift ? 0.012 : 0.014) * sampleRate))); + const int wetStart = juce::jlimit (bodyStart, bodyEnd, bodyStart + entryDrySamples); + const int wetEnd = juce::jlimit (wetStart, bodyEnd, bodyEnd - exitDrySamples); + if (wetEnd <= wetStart) + { + ++fallbackIslandCount; + continue; + } + + maxEntryDrySamples = std::max (maxEntryDrySamples, wetStart - bodyStart); + maxExitDrySamples = std::max (maxExitDrySamples, bodyEnd - wetEnd); + + std::vector<float> synthMono (static_cast<size_t> (renderSamples), 0.0f); + for (int i = 0; i < renderSamples; ++i) + { + const int localSample = localRenderStart + i; + if (localSample >= 0 && localSample < static_cast<int> (island.monoSignal.size())) + synthMono[static_cast<size_t> (i)] = island.monoSignal[static_cast<size_t> (localSample)]; + } + + const bool useEpochCarrier = island.epochs.size() >= 4; + if (useEpochCarrier) + { + std::vector<float> olaWeight (static_cast<size_t> (renderSamples), 0.0f); + for (int localSample = wetStart; localSample < wetEnd; ++localSample) + { + const int dest = localSample - localRenderStart; + if (dest >= 0 && dest < renderSamples) + synthMono[static_cast<size_t> (dest)] = 0.0f; + } + + std::vector<int> targetEpochs; + int targetSample = wetStart; + while (targetSample < wetEnd) + { + targetEpochs.push_back (targetSample); + const int absoluteSample = island.contextStartSample + targetSample; + const float pitchRatio = getTargetPitchRatioAtSample (absoluteSample, pitchRatios); + const float sourceHz = island.f0TrackHz.empty() + ? island.core.meanF0Hz + : std::max (island.f0TrackHz[static_cast<size_t> (juce::jlimit (0, static_cast<int> (island.f0TrackHz.size()) - 1, targetSample))], + island.core.meanF0Hz * 0.70f); + const float targetHz = juce::jlimit (55.0f, 1400.0f, std::max (55.0f, sourceHz) * pitchRatio); + const int targetPeriod = std::max (16, static_cast<int> (std::round (sampleRate / targetHz))); + targetSample += targetPeriod; + } + + const int sourceEpochCount = static_cast<int> (island.epochs.size()); + const int targetEpochCount = static_cast<int> (targetEpochs.size()); + const bool interpolateEpochSources = interpolateEpochSourcesRequested + && effectiveEpochInterpolationStrength > 1.0e-4f + && sourceEpochCount > 1 + && targetEpochCount > 1; + anyEpochInterpolationUsed = anyEpochInterpolationUsed || interpolateEpochSources; + for (int k = 0; k < targetEpochCount; ++k) + { + const float norm = targetEpochCount > 1 + ? static_cast<float> (k) / static_cast<float> (targetEpochCount - 1) + : 0.0f; + const float sourceEpochPos = norm * static_cast<float> (std::max (0, sourceEpochCount - 1)); + const int sourceEpochIndexLo = juce::jlimit (0, sourceEpochCount - 1, + static_cast<int> (std::floor (sourceEpochPos))); + const float sourceEpochFrac = interpolateEpochSources + ? juce::jlimit (0.0f, 1.0f, sourceEpochPos - static_cast<float> (sourceEpochIndexLo)) + : 0.0f; + const int nearestSourceEpochIndex = juce::jlimit ( + 0, + sourceEpochCount - 1, + static_cast<int> (std::round (sourceEpochPos))); + const int sourceEpochIndexHi = interpolateEpochSources + ? juce::jlimit (0, sourceEpochCount - 1, sourceEpochIndexLo + 1) + : nearestSourceEpochIndex; + const int sourceEpoch = island.epochs[static_cast<size_t> (sourceEpochIndexLo)]; + const int sourceEpochHi = island.epochs[static_cast<size_t> (sourceEpochIndexHi)]; + const int nearestSourceEpoch = island.epochs[static_cast<size_t> (nearestSourceEpochIndex)]; + const int targetEpoch = targetEpochs[static_cast<size_t> (k)]; + + const int prevNearestSource = nearestSourceEpochIndex > 0 ? island.epochs[static_cast<size_t> (nearestSourceEpochIndex - 1)] : nearestSourceEpoch; + const int nextNearestSource = nearestSourceEpochIndex + 1 < sourceEpochCount ? island.epochs[static_cast<size_t> (nearestSourceEpochIndex + 1)] : nearestSourceEpoch; + const int nearestSourcePeriod = std::max (16, std::max (nextNearestSource - nearestSourceEpoch, nearestSourceEpoch - prevNearestSource)); + const int prevSource = sourceEpochIndexLo > 0 ? island.epochs[static_cast<size_t> (sourceEpochIndexLo - 1)] : sourceEpoch; + const int nextSource = sourceEpochIndexLo + 1 < sourceEpochCount ? island.epochs[static_cast<size_t> (sourceEpochIndexLo + 1)] : sourceEpoch; + const int sourcePeriod = std::max (16, std::max (nextSource - sourceEpoch, sourceEpoch - prevSource)); + const int prevSourceHi = sourceEpochIndexHi > 0 ? island.epochs[static_cast<size_t> (sourceEpochIndexHi - 1)] : sourceEpochHi; + const int nextSourceHi = sourceEpochIndexHi + 1 < sourceEpochCount ? island.epochs[static_cast<size_t> (sourceEpochIndexHi + 1)] : sourceEpochHi; + const int sourcePeriodHi = std::max (16, std::max (nextSourceHi - sourceEpochHi, sourceEpochHi - prevSourceHi)); + const int interpolatedSourcePeriod = interpolateEpochSources + ? std::max ( + 16, + static_cast<int> (std::round ( + static_cast<float> (sourcePeriod) + + static_cast<float> (sourcePeriodHi - sourcePeriod) * sourceEpochFrac))) + : sourcePeriod; + const int blendedSourcePeriod = interpolateEpochSources + ? std::max ( + 16, + static_cast<int> (std::round ( + static_cast<float> (nearestSourcePeriod) + + static_cast<float> (interpolatedSourcePeriod - nearestSourcePeriod) + * effectiveEpochInterpolationStrength))) + : nearestSourcePeriod; + const int prevTarget = k > 0 ? targetEpochs[static_cast<size_t> (k - 1)] : targetEpoch; + const int nextTarget = k + 1 < targetEpochCount ? targetEpochs[static_cast<size_t> (k + 1)] : targetEpoch; + const int targetPeriod = std::max (16, std::max (nextTarget - targetEpoch, targetEpoch - prevTarget)); + const int maxGrainRadius = std::max (16, static_cast<int> (std::round (0.032 * sampleRate))); + const int grainRadius = juce::jlimit ( + 16, + maxGrainRadius, + static_cast<int> (std::round ( + static_cast<float> (std::max (blendedSourcePeriod, targetPeriod)) + * effectiveGrainRadiusScale))); + + for (int offset = -grainRadius; offset <= grainRadius; ++offset) + { + const int sourcePos = sourceEpoch + offset; + const int sourcePosHi = sourceEpochHi + offset; + const int nearestSourcePos = nearestSourceEpoch + offset; + const int targetPos = targetEpoch + offset; + if (nearestSourcePos < 0 || nearestSourcePos >= static_cast<int> (island.monoSignal.size())) + continue; + if (targetPos < wetStart || targetPos >= wetEnd) + continue; + const int destIndex = targetPos - localRenderStart; + if (destIndex < 0 || destIndex >= renderSamples) + continue; + + const float x = static_cast<float> (offset + grainRadius) / static_cast<float> (std::max (1, grainRadius * 2)); + const float window = 0.5f - 0.5f * std::cos (static_cast<float> (twoPi) * x); + float sourceSample = island.monoSignal[static_cast<size_t> (nearestSourcePos)]; + if (interpolateEpochSources + && sourcePos >= 0 + && sourcePos < static_cast<int> (island.monoSignal.size()) + && sourcePosHi >= 0 + && sourcePosHi < static_cast<int> (island.monoSignal.size())) + { + float interpolatedSourceSample = island.monoSignal[static_cast<size_t> (sourcePos)]; + interpolatedSourceSample += (island.monoSignal[static_cast<size_t> (sourcePosHi)] - interpolatedSourceSample) + * sourceEpochFrac; + sourceSample += (interpolatedSourceSample - sourceSample) + * effectiveEpochInterpolationStrength; + } + synthMono[static_cast<size_t> (destIndex)] += sourceSample * window; + olaWeight[static_cast<size_t> (destIndex)] += window; + } + } + + for (int localSample = wetStart; localSample < wetEnd; ++localSample) + { + const int dest = localSample - localRenderStart; + if (dest < 0 || dest >= renderSamples) + continue; + + const float weight = olaWeight[static_cast<size_t> (dest)]; + if (weight > 1.0e-4f) + synthMono[static_cast<size_t> (dest)] /= weight; + } + } + else + { + std::vector<double> phases (static_cast<size_t> (96), 0.0); + int envelopeFrameCursor = 0; + + for (int i = 0; i < renderSamples; ++i) + { + const int localSample = localRenderStart + i; + if (localSample < wetStart || localSample >= wetEnd) + continue; + + const float voiced = island.voicedMask.empty() ? 0.0f : island.voicedMask[static_cast<size_t> (localSample)]; + const float envelope = island.amplitudeEnvelope.empty() ? 0.0f : island.amplitudeEnvelope[static_cast<size_t> (localSample)]; + if (voiced <= 0.04f || envelope <= 1.0e-6f) + continue; + + envelopeFrameCursor = advanceFrameCursor (island.spectralEnvelopeModel.frames, localSample, envelopeFrameCursor); + const int absoluteSample = island.contextStartSample + localSample; + const float pitchRatio = getTargetPitchRatioAtSample (absoluteSample, pitchRatios); + const float sourceHz = island.f0TrackHz.empty() + ? island.core.meanF0Hz + : std::max (island.f0TrackHz[static_cast<size_t> (juce::jlimit (0, static_cast<int> (island.f0TrackHz.size()) - 1, localSample))], + island.core.meanF0Hz * 0.70f); + const float targetHz = juce::jlimit (55.0f, 1400.0f, std::max (55.0f, sourceHz) * pitchRatio); + const int partialCount = juce::jlimit ( + 3, + static_cast<int> (phases.size()), + std::min (36, static_cast<int> (nyquistLimit / std::max (targetHz, 1.0f)))); + + float sampleValue = 0.0f; + float envelopeReference = interpolateEnvelopeMagnitudeAtHz ( + island.spectralEnvelopeModel, + localSample, + targetHz / envelopeLookupRatio, + envelopeFrameCursor); + envelopeReference = std::max (envelopeReference, 1.0e-6f); + + for (int p = 0; p < partialCount; ++p) + { + const float harmonicHz = targetHz * static_cast<float> (p + 1); + if (harmonicHz >= nyquistLimit) + break; + + const double omega = twoPi * static_cast<double> (harmonicHz) / sampleRate; + phases[static_cast<size_t> (p)] += omega; + if (phases[static_cast<size_t> (p)] > twoPi) + phases[static_cast<size_t> (p)] = std::fmod (phases[static_cast<size_t> (p)], twoPi); + + const float sourceEnvelope = interpolateEnvelopeMagnitudeAtHz ( + island.spectralEnvelopeModel, + localSample, + harmonicHz / envelopeLookupRatio, + envelopeFrameCursor); + const float harmonicTilt = 1.0f / std::pow (static_cast<float> (p + 1), 0.68f); + const float highBandTame = 1.0f - smoothstep01 ((harmonicHz - 2200.0f) / 6200.0f) + * (downwardShift ? 0.84f : 0.68f); + const float downshiftMidLift = downwardShift + ? 1.0f + 0.18f + * smoothstep01 ((harmonicHz - 520.0f) / 560.0f) + * (1.0f - smoothstep01 ((harmonicHz - 1850.0f) / 1100.0f)) + : 1.0f; + const float amp = std::pow (std::max (sourceEnvelope, 1.0e-8f) / envelopeReference, 0.58f) + * harmonicTilt + * highBandTame + * downshiftMidLift; + sampleValue += amp * static_cast<float> (std::sin (phases[static_cast<size_t> (p)])); + } + + const float voicedScale = 0.42f + 0.58f * juce::jlimit (0.0f, 1.0f, voiced); + synthMono[static_cast<size_t> (i)] = sampleValue * envelope * voicedScale; + } + } + + const int localCoreStart = wetStart - localRenderStart; + const int localCoreEnd = wetEnd - localRenderStart; + const float synthCoreRms = computeRms (synthMono, localCoreStart, localCoreEnd); + const float sourceCoreRms = computeRms (island.monoSignal, wetStart, wetEnd); + if (synthCoreRms > 1.0e-6f && sourceCoreRms > 1.0e-6f) + { + const float defaultTargetRmsScale = longBody + ? (downwardShift ? 0.96f : 0.78f) + : (downwardShift ? 0.86f : 0.92f); + const float targetRmsScale = juce::jlimit ( + 0.45f, + 1.15f, + getEnvFloat (downwardShift + ? "OPENSTUDIO_VSF_TARGET_RMS_SCALE_DOWN" + : "OPENSTUDIO_VSF_TARGET_RMS_SCALE_UP", + getEnvFloat ("OPENSTUDIO_VSF_TARGET_RMS_SCALE", defaultTargetRmsScale))); + const float gain = juce::jlimit (0.20f, 3.5f, (sourceCoreRms * targetRmsScale) / synthCoreRms); + for (int i = localCoreStart; i < localCoreEnd && i < static_cast<int> (synthMono.size()); ++i) + synthMono[static_cast<size_t> (i)] *= gain; + } + + std::vector<float> coreBeforeResidual; + if (dumpLayers) + coreBeforeResidual = synthMono; + + const float baseResidualMix = downwardShift + ? juce::jlimit (0.08f, 0.20f, 0.12f + island.residualModel.voicedMix * 0.35f) + : juce::jlimit (0.03f, 0.085f, 0.04f + island.residualModel.highBandMix * 0.20f); + const float defaultResidualMixScale = downwardShift ? 0.55f : 0.0f; + const float residualMixScale = juce::jlimit ( + 0.0f, + 2.2f, + getEnvFloat (downwardShift + ? "OPENSTUDIO_VSF_RESIDUAL_MIX_SCALE_DOWN" + : "OPENSTUDIO_VSF_RESIDUAL_MIX_SCALE_UP", + getEnvFloat ("OPENSTUDIO_VSF_RESIDUAL_MIX_SCALE", defaultResidualMixScale))); + residualMixScaleSum += residualMixScale; + ++residualMixScaleCount; + const float residualMix = juce::jlimit (0.0f, downwardShift ? 0.32f : 0.18f, + baseResidualMix * residualMixScale); + for (int i = 0; i < renderSamples; ++i) + { + const int localSample = localRenderStart + i; + if (localSample < wetStart || localSample >= wetEnd) + continue; + const float voiced = juce::jlimit (0.0f, 1.0f, island.voicedMask.empty() ? 0.0f : island.voicedMask[static_cast<size_t> (localSample)]); + if (! island.residualModel.voicedHighBandResidual.empty()) + { + const float stableVoiced = smoothstep01 ((voiced - 0.46f) / 0.32f); + const float voicedResidualGate = (1.0f - stableVoiced) * (downwardShift ? 0.42f : 0.28f); + synthMono[static_cast<size_t> (i)] += island.residualModel.voicedHighBandResidual[static_cast<size_t> (localSample)] + * residualMix + * voicedResidualGate; + } + if (! island.residualModel.monoResidual.empty()) + { + const float unvoiced = 1.0f - voiced; + synthMono[static_cast<size_t> (i)] += island.residualModel.monoResidual[static_cast<size_t> (localSample)] * residualMix * 0.35f * unvoiced; + } + } + + std::vector<float> channelGain (static_cast<size_t> (numChannels), 1.0f); + const float monoRms = std::max (1.0e-6f, computeRms (island.monoSignal, wetStart, wetEnd)); + for (int ch = 0; ch < numChannels; ++ch) + { + double sum = 0.0; + int count = 0; + for (int localSample = wetStart; localSample < wetEnd; ++localSample) + { + const int absoluteSample = island.contextStartSample + localSample; + if (absoluteSample < 0 || absoluteSample >= numSamples) + continue; + const float value = input[ch][absoluteSample]; + sum += static_cast<double> (value) * value; + ++count; + } + const float channelRms = count > 0 ? std::sqrt (static_cast<float> (sum / static_cast<double> (count))) : monoRms; + channelGain[static_cast<size_t> (ch)] = juce::jlimit (0.35f, 2.30f, channelRms / monoRms); + } + + const float coreMaxWet = juce::jlimit ( + 0.55f, + 1.0f, + getEnvFloat (downwardShift + ? "OPENSTUDIO_VSF_CORE_MAX_WET_DOWN" + : "OPENSTUDIO_VSF_CORE_MAX_WET_UP", + getEnvFloat ("OPENSTUDIO_VSF_CORE_MAX_WET", 1.0f))); + const int fadeInSamples = std::max (1, static_cast<int> (std::round (0.016 * sampleRate))); + const int fadeOutSamples = std::max (1, static_cast<int> (std::round ((downwardShift ? 0.030 : 0.036) * sampleRate))); + + std::vector<float> wetEnvelope (static_cast<size_t> (renderSamples), 0.0f); + const float attackCoeff = static_cast<float> (std::exp (-1.0 / std::max (1.0, 0.010 * sampleRate))); + const float releaseCoeff = static_cast<float> (std::exp (-1.0 / std::max (1.0, 0.050 * sampleRate))); + float smoothedVoicedGate = 0.0f; + for (int i = 0; i < renderSamples; ++i) + { + const int localSample = localRenderStart + i; + if (localSample < wetStart || localSample >= wetEnd) + continue; + + float wet = coreMaxWet; + if (localSample < wetStart + fadeInSamples) + { + const float t = static_cast<float> (localSample - wetStart) / static_cast<float> (fadeInSamples); + wet *= equalPowerFadeIn (t); + } + if (localSample >= wetEnd - fadeOutSamples) + { + const float t = static_cast<float> (wetEnd - localSample) / static_cast<float> (fadeOutSamples); + wet *= equalPowerFadeOut (1.0f - t); + } + + const float voiced = island.voicedMask.empty() ? 0.0f : island.voicedMask[static_cast<size_t> (localSample)]; + const float targetGate = smoothstep01 ((voiced - 0.12f) / 0.52f); + const float coeff = targetGate > smoothedVoicedGate ? attackCoeff : releaseCoeff; + smoothedVoicedGate = targetGate + (smoothedVoicedGate - targetGate) * coeff; + const float voicedFloor = downwardShift ? 0.46f : 0.40f; + wet *= voicedFloor + (1.0f - voicedFloor) * juce::jlimit (0.0f, 1.0f, smoothedVoicedGate); + wetEnvelope[static_cast<size_t> (i)] = juce::jlimit (0.0f, coreMaxWet, wet); + } + + for (int ch = 0; ch < numChannels; ++ch) + { + for (int i = 0; i < renderSamples; ++i) + { + const int localSample = localRenderStart + i; + const int absoluteSample = island.contextStartSample + localSample; + if (absoluteSample < 0 || absoluteSample >= numSamples || localSample < wetStart || localSample >= wetEnd) + continue; + + const float wet = wetEnvelope[static_cast<size_t> (i)]; + if (wet <= 1.0e-4f) + continue; + + const float dry = input[ch][absoluteSample]; + const float wetSample = synthMono[static_cast<size_t> (i)] * channelGain[static_cast<size_t> (ch)]; + if (dumpLayers) + { + const float coreSample = coreBeforeResidual.empty() + ? 0.0f + : coreBeforeResidual[static_cast<size_t> (i)] * channelGain[static_cast<size_t> (ch)]; + const float residualSample = wetSample - coreSample; + coreLayer[static_cast<size_t> (ch)][static_cast<size_t> (absoluteSample)] += coreSample * wet; + residualLayer[static_cast<size_t> (ch)][static_cast<size_t> (absoluteSample)] += residualSample * wet; + wetEnvelopeLayer[static_cast<size_t> (ch)][static_cast<size_t> (absoluteSample)] = + std::max (wetEnvelopeLayer[static_cast<size_t> (ch)][static_cast<size_t> (absoluteSample)], wet); + } + result.output[static_cast<size_t> (ch)][static_cast<size_t> (absoluteSample)] = + dry * (1.0f - wet) + wetSample * wet; + } + } + + const int absoluteBodyShapeStart = juce::jlimit (0, numSamples, island.contextStartSample + wetStart); + const int absoluteBodyShapeEnd = juce::jlimit (absoluteBodyShapeStart, numSamples, island.contextStartSample + wetEnd); + if (absoluteBodyShapeEnd > absoluteBodyShapeStart) + { + const int bodyEqFadeSamples = std::max (1, static_cast<int> (std::round (0.020 * sampleRate))); + for (int ch = 0; ch < numChannels; ++ch) + { + auto& channel = result.output[static_cast<size_t> (ch)]; + if (std::abs (upBodyPresenceTrimDb) > 1.0e-4f && upBodyPresenceHz > 0.0f) + { + applyPeakingEqToRange ( + channel, + absoluteBodyShapeStart, + absoluteBodyShapeEnd, + sampleRate, + upBodyPresenceHz, + upBodyPresenceQ, + upBodyPresenceTrimDb, + bodyEqFadeSamples); + } + if (std::abs (downBodyNasalTrimDb) > 1.0e-4f && downBodyNasalHz > 0.0f) + { + applyPeakingEqToRange ( + channel, + absoluteBodyShapeStart, + absoluteBodyShapeEnd, + sampleRate, + downBodyNasalHz, + downBodyNasalQ, + downBodyNasalTrimDb, + bodyEqFadeSamples); + } + if (std::abs (downBodyCompDb) > 1.0e-4f && downBodyCompHz > 0.0f) + { + applyPeakingEqToRange ( + channel, + absoluteBodyShapeStart, + absoluteBodyShapeEnd, + sampleRate, + downBodyCompHz, + downBodyCompQ, + downBodyCompDb, + bodyEqFadeSamples); + } + } + } + + const int entryShapeStart = juce::jlimit ( + bodyStart, + bodyEnd, + bodyStart + static_cast<int> (std::round (0.012 * sampleRate))); + const int entryShapeEnd = std::min (bodyEnd, bodyStart + static_cast<int> (std::round (0.140 * sampleRate))); + if (entryShapeEnd > entryShapeStart) + { + const float entryGainDb = -1.5f; + const float entryGain = std::pow (10.0f, entryGainDb / 20.0f); + const int gainAttackSamples = std::max (1, static_cast<int> (std::round (0.010 * sampleRate))); + const int gainHoldSamples = std::max (1, static_cast<int> (std::round (0.080 * sampleRate))); + const int gainReleaseSamples = std::max (1, static_cast<int> (std::round (0.060 * sampleRate))); + const float entryMidCutDb = downwardShift + ? -12.0f + : (longBody ? -18.0f : getEnvFloat ("OPENSTUDIO_VSF_SHORT_UP_ENTRY_MID_CUT_DB", -3.0f)); + const float entryMidHz = downwardShift ? 1000.0f : 1300.0f; + const int eqFadeSamples = std::max (1, static_cast<int> (std::round (0.006 * sampleRate))); + const int absoluteEntryStart = juce::jlimit (0, numSamples, island.contextStartSample + entryShapeStart); + const int absoluteEntryEnd = juce::jlimit (absoluteEntryStart, numSamples, island.contextStartSample + entryShapeEnd); + + for (int ch = 0; ch < numChannels; ++ch) + { + auto& channel = result.output[static_cast<size_t> (ch)]; + for (int sampleIndex = absoluteEntryStart; sampleIndex < absoluteEntryEnd; ++sampleIndex) + { + const int age = sampleIndex - absoluteEntryStart; + const float attackT = juce::jlimit ( + 0.0f, + 1.0f, + static_cast<float> (age) / static_cast<float> (gainAttackSamples)); + const float releaseT = juce::jlimit ( + 0.0f, + 1.0f, + static_cast<float> (age - gainHoldSamples) / static_cast<float> (gainReleaseSamples)); + const float trimAmount = attackT * (1.0f - releaseT); + const float gain = 1.0f + (entryGain - 1.0f) * trimAmount; + channel[static_cast<size_t> (sampleIndex)] *= gain; + } + + applyPeakingEqToRange ( + channel, + absoluteEntryStart, + absoluteEntryEnd, + sampleRate, + entryMidHz, + 0.75f, + entryMidCutDb, + eqFadeSamples); + } + } + + for (int localSample = bodyStart; localSample < bodyEnd; ++localSample) + { + ++totalBodySamples; + const float voiced = island.voicedMask.empty() ? 0.0f : island.voicedMask[static_cast<size_t> (localSample)]; + if (voiced > 0.35f) + ++totalVoicedSamples; + } + residualMixSum += residualMix; + ++residualMixCount; + ++renderedIslandCount; + } + + if (renderedIslandCount == 0) + { + result.usedFallback = true; + result.fallbackReason = result.fallbackReason.isEmpty() ? "no_usable_voiced_island" : result.fallbackReason; + result.vocalSourceFilterFallbackUsed = true; + result.vocalSourceFilterFallbackReason = result.fallbackReason; + } + else if (fallbackIslandCount > 0) + { + result.vocalSourceFilterFallbackUsed = true; + result.vocalSourceFilterFallbackReason = "partial_island_fallback"; + } + + result.vocalSourceFilterVoicedCoverage = totalBodySamples > 0 + ? static_cast<double> (totalVoicedSamples) / static_cast<double> (totalBodySamples) + : 0.0; + result.vocalSourceFilterResidualMix = residualMixCount > 0 + ? residualMixSum / static_cast<double> (residualMixCount) + : 0.0; + result.vocalSourceFilterResidualMixScale = residualMixScaleCount > 0 + ? residualMixScaleSum / static_cast<double> (residualMixScaleCount) + : 1.0; + result.vocalSourceFilterEpochInterpolationUsed = anyEpochInterpolationUsed; + result.vocalSourceFilterEpochInterpolationStrength = epochInterpolationStrengthCount > 0 + ? epochInterpolationStrengthSum / static_cast<double> (epochInterpolationStrengthCount) + : 0.0; + result.vocalSourceFilterGrainRadiusScale = grainRadiusScaleCount > 0 + ? grainRadiusScaleSum / static_cast<double> (grainRadiusScaleCount) + : 1.0; + result.vocalSourceFilterUpPresenceTrimDb = upPresenceCount > 0 + ? upPresenceTrimDbSum / static_cast<double> (upPresenceCount) + : 0.0; + result.vocalSourceFilterUpPresenceHz = upPresenceCount > 0 + ? upPresenceHzSum / static_cast<double> (upPresenceCount) + : 0.0; + result.vocalSourceFilterDownNasalTrimDb = downNasalCount > 0 + ? downNasalTrimDbSum / static_cast<double> (downNasalCount) + : 0.0; + result.vocalSourceFilterDownNasalHz = downNasalCount > 0 + ? downNasalHzSum / static_cast<double> (downNasalCount) + : 0.0; + result.vocalSourceFilterDownBodyCompDb = downBodyCompCount > 0 + ? downBodyCompDbSum / static_cast<double> (downBodyCompCount) + : 0.0; + result.vocalSourceFilterDownBodyCompHz = downBodyCompCount > 0 + ? downBodyCompHzSum / static_cast<double> (downBodyCompCount) + : 0.0; + result.vocalSourceFilterEntryDryMs = sampleRate > 0.0 + ? 1000.0 * static_cast<double> (maxEntryDrySamples) / sampleRate + : 0.0; + result.vocalSourceFilterExitDryMs = sampleRate > 0.0 + ? 1000.0 * static_cast<double> (maxExitDrySamples) / sampleRate + : 0.0; + result.renderMs = juce::Time::getMillisecondCounterHiRes() - renderStart; + + logOwnPitchEngine ("vocalSourceFilter islands=" + juce::String (renderedIslandCount) + + " fallbackIslands=" + juce::String (fallbackIslandCount) + + " cacheHit=" + juce::String (result.analysis.cacheHit ? "true" : "false") + + " voicedCoverage=" + juce::String (result.vocalSourceFilterVoicedCoverage, 3) + + " residualMix=" + juce::String (result.vocalSourceFilterResidualMix, 3) + + " residualScale=" + juce::String (result.vocalSourceFilterResidualMixScale, 3) + + " epochInterp=" + juce::String (result.vocalSourceFilterEpochInterpolationUsed ? "true" : "false") + + " epochInterpStrength=" + juce::String (result.vocalSourceFilterEpochInterpolationStrength, 3) + + " grainScale=" + juce::String (result.vocalSourceFilterGrainRadiusScale, 3) + + " upPresenceTrimDb=" + juce::String (result.vocalSourceFilterUpPresenceTrimDb, 2) + + " downNasalTrimDb=" + juce::String (result.vocalSourceFilterDownNasalTrimDb, 2) + + " downBodyCompDb=" + juce::String (result.vocalSourceFilterDownBodyCompDb, 2) + + " entryDryMs=" + juce::String (result.vocalSourceFilterEntryDryMs, 1) + + " exitDryMs=" + juce::String (result.vocalSourceFilterExitDryMs, 1) + + " analysisMs=" + juce::String (result.analysisMs, 2) + + " renderMs=" + juce::String (result.renderMs, 2)); + + if (dumpLayers) + { + writeOwnPitchLayerDumpWav (layerDumpDir, "vsf_dry_input", dryLayer, sampleRate); + writeOwnPitchLayerDumpWav (layerDumpDir, "vsf_core_pre_mix", coreLayer, sampleRate); + writeOwnPitchLayerDumpWav (layerDumpDir, "vsf_residual_noise_layer", residualLayer, sampleRate); + writeOwnPitchLayerDumpWav (layerDumpDir, "vsf_wet_envelope", wetEnvelopeLayer, sampleRate); + writeOwnPitchLayerDumpWav (layerDumpDir, "vsf_final_output", result.output, sampleRate); + } + + return result; +} diff --git a/Source/OwnPitchEngine.h b/Source/OwnPitchEngine.h new file mode 100644 index 0000000..fc0038f --- /dev/null +++ b/Source/OwnPitchEngine.h @@ -0,0 +1,173 @@ +#pragma once + +#include <JuceHeader.h> +#include "PitchAnalyzer.h" +#include <functional> +#include <vector> + +class OwnPitchEngine +{ +public: + enum class Mode + { + PitchOnly, + FormantOnly, + PitchPlusFormant + }; + + enum class Quality + { + PreviewFast, + FinalHQ + }; + + struct VoicedCoreAnalysis + { + int startSample = 0; + int endSample = 0; + float voicedRatio = 0.0f; + float meanF0Hz = 0.0f; + float rms = 0.0f; + int epochCount = 0; + }; + + struct HarmonicFrame + { + int sampleIndex = 0; + float f0Hz = 0.0f; + std::vector<float> amplitudes; + }; + + struct SpectralEnvelopeFrame + { + int sampleIndex = 0; + float f0Hz = 0.0f; + std::vector<float> rawMagnitude; + std::vector<float> smoothedMagnitude; + }; + + struct HarmonicModel + { + int fftSize = 0; + float binHz = 0.0f; + int partialCount = 0; + std::vector<float> averageMagnitude; + std::vector<HarmonicFrame> frames; + }; + + struct ResidualModel + { + std::vector<float> monoResidual; + std::vector<float> voicedHighBandResidual; + std::vector<float> bandAperiodicity; + float suggestedMix = 0.0f; + float voicedMix = 0.0f; + float highBandMix = 0.0f; + }; + + struct SpectralEnvelopeModel + { + int fftOrder = 0; + int fftSize = 0; + float binHz = 0.0f; + std::vector<float> averageMagnitude; + std::vector<SpectralEnvelopeFrame> frames; + }; + + struct NoteIslandAnalysis + { + int renderStartSample = 0; + int renderEndSample = 0; + int contextStartSample = 0; + int contextEndSample = 0; + int bodyStartSample = 0; + int bodyEndSample = 0; + std::vector<PitchAnalyzer::PitchNote> notes; + + std::vector<float> monoSignal; + std::vector<float> voicedMask; + std::vector<float> consonantMask; + std::vector<float> f0TrackHz; + std::vector<float> amplitudeEnvelope; + std::vector<int> epochs; + + VoicedCoreAnalysis core; + HarmonicModel harmonicModel; + ResidualModel residualModel; + SpectralEnvelopeModel spectralEnvelopeModel; + }; + + struct SharedAnalysis + { + Mode mode = Mode::PitchOnly; + Quality quality = Quality::FinalHQ; + double sampleRate = 0.0; + int numChannels = 0; + int numSamples = 0; + bool cacheHit = false; + std::vector<NoteIslandAnalysis> islands; + int totalEpochCount = 0; + int maxPartialCount = 0; + }; + + struct RenderResult + { + std::vector<std::vector<float>> output; + SharedAnalysis analysis; + bool usedFallback = false; + juce::String fallbackReason; + double analysisMs = 0.0; + double renderMs = 0.0; + bool vocalSourceFilterUsed = false; + double vocalSourceFilterVoicedCoverage = 0.0; + double vocalSourceFilterResidualMix = 0.0; + bool vocalSourceFilterFallbackUsed = false; + juce::String vocalSourceFilterFallbackReason; + double vocalSourceFilterEntryDryMs = 0.0; + double vocalSourceFilterExitDryMs = 0.0; + double vocalSourceFilterResidualMixScale = 1.0; + bool vocalSourceFilterEpochInterpolationUsed = false; + double vocalSourceFilterEpochInterpolationStrength = 0.0; + double vocalSourceFilterGrainRadiusScale = 1.0; + double vocalSourceFilterUpPresenceTrimDb = 0.0; + double vocalSourceFilterUpPresenceHz = 0.0; + double vocalSourceFilterDownNasalTrimDb = 0.0; + double vocalSourceFilterDownNasalHz = 0.0; + double vocalSourceFilterDownBodyCompDb = 0.0; + double vocalSourceFilterDownBodyCompHz = 0.0; + }; + + OwnPitchEngine() = default; + + SharedAnalysis analyze ( + const float* const* input, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchFrame>& frames, + const std::vector<PitchAnalyzer::PitchNote>& notes, + Mode mode, + Quality quality); + + RenderResult renderPitchOnly ( + const float* const* input, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchFrame>& frames, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& pitchRatios, + Quality quality, + std::function<bool()> shouldCancel = {}); + + RenderResult renderVocalSourceFilterHq ( + const float* const* input, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchFrame>& frames, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& pitchRatios, + Quality quality, + std::function<bool()> shouldCancel = {}); +}; diff --git a/Source/PitchAnalyzer.cpp b/Source/PitchAnalyzer.cpp index 9b19b2a..9e617fe 100644 --- a/Source/PitchAnalyzer.cpp +++ b/Source/PitchAnalyzer.cpp @@ -1,219 +1,664 @@ #include "PitchAnalyzer.h" -#include <cmath> #include <algorithm> +#include <array> +#include <cmath> +#include <limits> #include <numeric> -PitchAnalyzer::PitchAnalyzer() = default; +namespace +{ +constexpr float kPi = 3.14159265358979323846f; -static float hzToMidi(float hz) +float hzToMidi(float hz) { if (hz <= 0.0f) return 0.0f; return 69.0f + 12.0f * std::log2(hz / 440.0f); } -// midiToHz used by PitchResynthesizer, not needed here +float clamp01(float value) +{ + return juce::jlimit(0.0f, 1.0f, value); +} -float PitchAnalyzer::analyzeFrame(const float *frame, int frameSize, double sr, - float &outConfidence) +float safeLogProbability(float probability) { - const int halfSize = frameSize / 2; - const int tauMin = static_cast<int>(sr / maxFreq); - const int tauMax = std::min(halfSize - 1, static_cast<int>(sr / minFreq)); + return std::log(std::max(probability, 1.0e-6f)); +} + +float parabolicInterpolation(const std::vector<float>& values, int index) +{ + if (index <= 0 || index >= static_cast<int>(values.size()) - 1) + return static_cast<float>(index); + + const float left = values[static_cast<size_t>(index - 1)]; + const float center = values[static_cast<size_t>(index)]; + const float right = values[static_cast<size_t>(index + 1)]; + const float denominator = 2.0f * (2.0f * center - left - right); + + if (std::abs(denominator) < 1.0e-8f) + return static_cast<float>(index); + + return static_cast<float>(index) + (right - left) / denominator; +} + +bool shouldCompareDirectYin() +{ + return juce::SystemStats::getEnvironmentVariable("OPENSTUDIO_ANALYZER_COMPARE_DIRECT_YIN", {}).trim() == "1"; +} + +bool shouldUseFftYin() +{ + return juce::SystemStats::getEnvironmentVariable("OPENSTUDIO_ANALYZER_USE_FFT_YIN", {}).trim() == "1"; +} + +struct PitchCandidate +{ + float frequency = 0.0f; + float midi = 0.0f; + float probability = 0.0f; + float confidence = 0.0f; + int tau = 0; + float cmndf = 1.0f; +}; + +struct FrameObservation +{ + std::vector<PitchCandidate> candidates; + float voicedProbability = 0.0f; + float unvoicedProbability = 1.0f; + float bestCmndf = 1.0f; + float rmsDB = -100.0f; +}; + +struct DecodedFrame +{ + float frequency = 0.0f; + float midi = 0.0f; + float confidence = 0.0f; + bool voiced = false; +}; + +struct DecoderCell +{ + float score = -std::numeric_limits<float>::infinity(); + int previousState = -1; +}; +} - if (tauMax <= tauMin || tauMax >= halfSize) +PitchAnalyzer::PitchAnalyzer() + : fft(std::make_unique<juce::dsp::FFT>(analysisFftOrder)) +{ + analysisWindow.resize(static_cast<size_t>(analysisFrameSize)); + for (int i = 0; i < analysisFrameSize; ++i) { - outConfidence = 0.0f; - return 0.0f; + analysisWindow[static_cast<size_t>(i)] = + 0.5f * (1.0f - std::cos((2.0f * kPi * static_cast<float>(i)) / static_cast<float>(analysisFrameSize - 1))); } - yinBuffer.resize(static_cast<size_t>(halfSize)); + yinBuffer.assign(static_cast<size_t>(analysisFrameSize / 2), 0.0f); + differenceBuffer.assign(static_cast<size_t>(analysisFrameSize / 2), 0.0f); + directDifferenceBuffer.assign(static_cast<size_t>(analysisFrameSize / 2), 0.0f); + cmndfBuffer.assign(static_cast<size_t>(analysisFrameSize / 2), 1.0f); + autocorrelationBuffer.assign(static_cast<size_t>(analysisFrameSize), 0.0f); + prefixEnergyBuffer.assign(static_cast<size_t>(analysisFrameSize + 1), 0.0f); + windowedFrameBuffer.assign(static_cast<size_t>(analysisFrameSize), 0.0f); + fftBuffer.assign(static_cast<size_t>(analysisFftSize), {}); +} - // Step 1+2: Difference function + cumulative mean normalization - yinBuffer[0] = 1.0f; - float runningSum = 0.0f; +namespace +{ +static void applyHannWindow(const float* inputFrame, + int frameSize, + const std::vector<float>& window, + std::vector<float>& outputFrame) +{ + outputFrame.resize(static_cast<size_t>(frameSize)); + for (int i = 0; i < frameSize; ++i) + outputFrame[static_cast<size_t>(i)] = inputFrame[i] * window[static_cast<size_t>(i)]; +} + +static void computeDirectDifference(const std::vector<float>& frame, + int frameSize, + std::vector<float>& differenceBuffer) +{ + const int halfSize = frameSize / 2; + differenceBuffer.assign(static_cast<size_t>(halfSize), 0.0f); for (int tau = 1; tau < halfSize; ++tau) { + const int overlap = frameSize - tau; float sum = 0.0f; - for (int j = 0; j < halfSize; ++j) + for (int j = 0; j < overlap; ++j) { - float delta = frame[j] - frame[j + tau]; + const float delta = frame[static_cast<size_t>(j)] - frame[static_cast<size_t>(j + tau)]; sum += delta * delta; } - runningSum += sum; - yinBuffer[static_cast<size_t>(tau)] = (runningSum > 0.0f) - ? sum * static_cast<float>(tau) / runningSum - : 0.0f; + differenceBuffer[static_cast<size_t>(tau)] = std::max(0.0f, sum); } +} - // Step 3: Find first dip below threshold - int bestTau = -1; - float bestVal = sensitivity; +static void computeDifferenceFft(const std::vector<float>& frame, + int frameSize, + juce::dsp::FFT& fft, + std::vector<juce::dsp::Complex<float>>& fftBuffer, + std::vector<float>& autocorrelationBuffer, + std::vector<float>& prefixEnergyBuffer, + std::vector<float>& differenceBuffer) +{ + const int fftSize = static_cast<int>(fftBuffer.size()); + const int halfSize = frameSize / 2; - for (int tau = tauMin; tau <= tauMax; ++tau) + std::fill(fftBuffer.begin(), fftBuffer.end(), juce::dsp::Complex<float>(0.0f, 0.0f)); + for (int i = 0; i < frameSize; ++i) + fftBuffer[static_cast<size_t>(i)] = juce::dsp::Complex<float>(frame[static_cast<size_t>(i)], 0.0f); + + fft.perform(fftBuffer.data(), fftBuffer.data(), false); + for (int i = 0; i < fftSize; ++i) + fftBuffer[static_cast<size_t>(i)] *= std::conj(fftBuffer[static_cast<size_t>(i)]); + fft.perform(fftBuffer.data(), fftBuffer.data(), true); + + autocorrelationBuffer.assign(static_cast<size_t>(frameSize), 0.0f); + for (int i = 0; i < frameSize; ++i) + autocorrelationBuffer[static_cast<size_t>(i)] = fftBuffer[static_cast<size_t>(i)].real() / static_cast<float>(fftSize); + + prefixEnergyBuffer.assign(static_cast<size_t>(frameSize + 1), 0.0f); + for (int i = 0; i < frameSize; ++i) + prefixEnergyBuffer[static_cast<size_t>(i + 1)] = prefixEnergyBuffer[static_cast<size_t>(i)] + + frame[static_cast<size_t>(i)] * frame[static_cast<size_t>(i)]; + + differenceBuffer.assign(static_cast<size_t>(halfSize), 0.0f); + for (int tau = 1; tau < halfSize; ++tau) + { + const int overlap = frameSize - tau; + const float energyLeft = prefixEnergyBuffer[static_cast<size_t>(overlap)]; + const float energyRight = prefixEnergyBuffer[static_cast<size_t>(frameSize)] - prefixEnergyBuffer[static_cast<size_t>(tau)]; + const float diff = energyLeft + energyRight - 2.0f * autocorrelationBuffer[static_cast<size_t>(tau)]; + differenceBuffer[static_cast<size_t>(tau)] = std::max(0.0f, diff); + } +} + +static void computeCmndf(const std::vector<float>& differenceBuffer, + std::vector<float>& cmndfBuffer) +{ + cmndfBuffer.assign(differenceBuffer.size(), 1.0f); + if (differenceBuffer.size() <= 1) + return; + + float runningSum = 0.0f; + for (size_t tau = 1; tau < differenceBuffer.size(); ++tau) + { + runningSum += differenceBuffer[tau]; + cmndfBuffer[tau] = runningSum > 0.0f + ? differenceBuffer[tau] * static_cast<float>(tau) / runningSum + : 1.0f; + } +} + +static FrameObservation extractFrameObservation(const std::vector<float>& cmndfBuffer, + int tauMin, + int tauMax, + double sampleRate, + float rmsDB, + float sensitivity) +{ + FrameObservation observation; + observation.rmsDB = rmsDB; + + if (tauMax <= tauMin || tauMin <= 0 || tauMax >= static_cast<int>(cmndfBuffer.size())) + return observation; + + int globalBestTau = tauMin; + float globalBestCmndf = cmndfBuffer[static_cast<size_t>(tauMin)]; + for (int tau = tauMin + 1; tau <= tauMax; ++tau) + { + const float value = cmndfBuffer[static_cast<size_t>(tau)]; + if (value < globalBestCmndf) + { + globalBestCmndf = value; + globalBestTau = tau; + } + } + + observation.bestCmndf = globalBestCmndf; + + constexpr std::array<float, 5> thresholds { 0.06f, 0.08f, 0.10f, 0.14f, 0.20f }; + std::vector<int> candidateTaus; + std::vector<int> voteCounts(static_cast<size_t>(cmndfBuffer.size()), 0); + + // pYIN-style threshold voting: each threshold votes for the first dip it would have chosen. + for (const float threshold : thresholds) { - if (yinBuffer[static_cast<size_t>(tau)] < bestVal) + for (int tau = tauMin; tau <= tauMax; ++tau) { - while (tau + 1 <= tauMax && yinBuffer[static_cast<size_t>(tau + 1)] < yinBuffer[static_cast<size_t>(tau)]) + if (cmndfBuffer[static_cast<size_t>(tau)] >= threshold) + continue; + + while (tau + 1 <= tauMax + && cmndfBuffer[static_cast<size_t>(tau + 1)] < cmndfBuffer[static_cast<size_t>(tau)]) + { ++tau; - bestTau = tau; - bestVal = yinBuffer[static_cast<size_t>(tau)]; + } + + ++voteCounts[static_cast<size_t>(tau)]; break; } } - if (bestTau < 0) + // Keep additional local minima as fallbacks, but threshold-voted minima get priority. + for (int tau = std::max(tauMin, 2); tau < std::min(tauMax, static_cast<int>(cmndfBuffer.size()) - 1); ++tau) + { + const float center = cmndfBuffer[static_cast<size_t>(tau)]; + const float left = cmndfBuffer[static_cast<size_t>(tau - 1)]; + const float right = cmndfBuffer[static_cast<size_t>(tau + 1)]; + const bool localMinimum = center <= left && center <= right; + if (! localMinimum || center > 0.28f) + continue; + candidateTaus.push_back(tau); + } + + for (int tau = tauMin; tau <= tauMax; ++tau) + { + if (voteCounts[static_cast<size_t>(tau)] > 0) + candidateTaus.push_back(tau); + } + + if (candidateTaus.empty()) + candidateTaus.push_back(globalBestTau); + + std::sort(candidateTaus.begin(), candidateTaus.end(), [&] (int a, int b) + { + const int voteA = voteCounts[static_cast<size_t>(a)]; + const int voteB = voteCounts[static_cast<size_t>(b)]; + if (voteA != voteB) + return voteA > voteB; + return cmndfBuffer[static_cast<size_t>(a)] < cmndfBuffer[static_cast<size_t>(b)]; + }); + candidateTaus.erase(std::unique(candidateTaus.begin(), candidateTaus.end()), candidateTaus.end()); + + std::vector<float> rawScores; + rawScores.reserve(candidateTaus.size()); + + for (const int tau : candidateTaus) + { + const float refinedTau = parabolicInterpolation(cmndfBuffer, tau); + const float frequency = refinedTau > 0.0f ? static_cast<float>(sampleRate / refinedTau) : 0.0f; + if (frequency <= 0.0f) + continue; + + const int thresholdHits = voteCounts[static_cast<size_t>(tau)]; + const float periodicityScore = std::exp(-10.0f * cmndfBuffer[static_cast<size_t>(tau)]); + const float thresholdScore = 0.15f + 0.85f * (static_cast<float>(thresholdHits) / static_cast<float>(thresholds.size())); + const float shorterPeriodBias = std::sqrt(static_cast<float>(tauMin) / static_cast<float>(std::max(tau, tauMin))); + + PitchCandidate candidate; + candidate.frequency = frequency; + candidate.midi = hzToMidi(frequency); + candidate.confidence = clamp01((1.0f - cmndfBuffer[static_cast<size_t>(tau)]) * std::max(0.35f, thresholdScore)); + candidate.tau = tau; + candidate.cmndf = cmndfBuffer[static_cast<size_t>(tau)]; + observation.candidates.push_back(candidate); + rawScores.push_back(periodicityScore * thresholdScore * shorterPeriodBias); + } + + if (observation.candidates.empty()) + return observation; + + const float energyProbability = clamp01((rmsDB + 60.0f) / 24.0f); + float periodicityProbability = clamp01((0.30f - globalBestCmndf) / 0.22f); + if (globalBestCmndf <= sensitivity) + periodicityProbability = std::max(periodicityProbability, 0.75f); + + observation.voicedProbability = clamp01(0.45f * energyProbability + 0.55f * periodicityProbability); + observation.unvoicedProbability = clamp01(1.0f - observation.voicedProbability); + + const float scoreSum = std::accumulate(rawScores.begin(), rawScores.end(), 0.0f); + if (scoreSum <= 1.0e-6f) + { + const float uniformProbability = observation.voicedProbability / static_cast<float>(observation.candidates.size()); + for (auto& candidate : observation.candidates) + candidate.probability = uniformProbability; + } + else + { + for (size_t i = 0; i < observation.candidates.size(); ++i) + observation.candidates[i].probability = observation.voicedProbability * (rawScores[i] / scoreSum); + } + + constexpr size_t maxCandidates = 5; + if (observation.candidates.size() > maxCandidates) + observation.candidates.resize(maxCandidates); + + return observation; +} + +static float transitionPenalty(bool previousVoiced, + bool currentVoiced, + float previousMidi, + float currentMidi) +{ + if (! previousVoiced && ! currentVoiced) + return 0.05f; + + if (previousVoiced != currentVoiced) + return 0.55f; + + const float semitoneDelta = std::abs(currentMidi - previousMidi); + float penalty = 0.08f * semitoneDelta; + + if (semitoneDelta > 2.5f) + penalty += 0.18f * (semitoneDelta - 2.5f); + if (semitoneDelta > 7.0f) + penalty += 0.70f + 0.22f * (semitoneDelta - 7.0f); + if (std::abs(semitoneDelta - 12.0f) < 0.85f) + penalty += 2.0f; + + return penalty; +} + +static std::vector<DecodedFrame> decodePitchTrack(const std::vector<FrameObservation>& observations) +{ + if (observations.empty()) + return {}; + + std::vector<std::vector<DecoderCell>> cells(observations.size()); + std::vector<std::vector<bool>> stateVoiced(observations.size()); + std::vector<std::vector<float>> stateMidi(observations.size()); + std::vector<std::vector<float>> stateFrequency(observations.size()); + std::vector<std::vector<float>> stateConfidence(observations.size()); + + for (size_t frameIndex = 0; frameIndex < observations.size(); ++frameIndex) + { + const auto& observation = observations[frameIndex]; + const size_t stateCount = observation.candidates.size() + 1; + cells[frameIndex].assign(stateCount, {}); + stateVoiced[frameIndex].assign(stateCount, false); + stateMidi[frameIndex].assign(stateCount, 0.0f); + stateFrequency[frameIndex].assign(stateCount, 0.0f); + stateConfidence[frameIndex].assign(stateCount, 0.0f); + + cells[frameIndex][0].score = safeLogProbability(observation.unvoicedProbability); + stateVoiced[frameIndex][0] = false; + + for (size_t candidateIndex = 0; candidateIndex < observation.candidates.size(); ++candidateIndex) + { + const auto& candidate = observation.candidates[candidateIndex]; + const size_t stateIndex = candidateIndex + 1; + cells[frameIndex][stateIndex].score = safeLogProbability(candidate.probability); + stateVoiced[frameIndex][stateIndex] = true; + stateMidi[frameIndex][stateIndex] = candidate.midi; + stateFrequency[frameIndex][stateIndex] = candidate.frequency; + stateConfidence[frameIndex][stateIndex] = clamp01(candidate.confidence * observation.voicedProbability); + } + } + + for (size_t frameIndex = 1; frameIndex < observations.size(); ++frameIndex) { - bestVal = yinBuffer[static_cast<size_t>(tauMin)]; - bestTau = tauMin; - for (int tau = tauMin + 1; tau <= tauMax; ++tau) + for (size_t stateIndex = 0; stateIndex < cells[frameIndex].size(); ++stateIndex) { - if (yinBuffer[static_cast<size_t>(tau)] < bestVal) + float bestScore = -std::numeric_limits<float>::infinity(); + int bestPreviousState = -1; + + for (size_t previousState = 0; previousState < cells[frameIndex - 1].size(); ++previousState) { - bestVal = yinBuffer[static_cast<size_t>(tau)]; - bestTau = tau; + const float penalty = transitionPenalty(stateVoiced[frameIndex - 1][previousState], + stateVoiced[frameIndex][stateIndex], + stateMidi[frameIndex - 1][previousState], + stateMidi[frameIndex][stateIndex]); + const float score = cells[frameIndex - 1][previousState].score + + cells[frameIndex][stateIndex].score + - penalty; + + if (score > bestScore) + { + bestScore = score; + bestPreviousState = static_cast<int>(previousState); + } } + + cells[frameIndex][stateIndex].score = bestScore; + cells[frameIndex][stateIndex].previousState = bestPreviousState; } } - // Step 4: Parabolic interpolation - float refinedTau = static_cast<float>(bestTau); - if (bestTau > 0 && bestTau < halfSize - 1) + int bestFinalState = 0; + float bestFinalScore = cells.back()[0].score; + for (size_t stateIndex = 1; stateIndex < cells.back().size(); ++stateIndex) + { + if (cells.back()[stateIndex].score > bestFinalScore) + { + bestFinalScore = cells.back()[stateIndex].score; + bestFinalState = static_cast<int>(stateIndex); + } + } + + std::vector<int> bestStates(observations.size(), 0); + bestStates.back() = bestFinalState; + for (size_t frameIndex = observations.size() - 1; frameIndex > 0; --frameIndex) + { + const int previousState = cells[frameIndex][static_cast<size_t>(bestStates[frameIndex])].previousState; + bestStates[frameIndex - 1] = previousState >= 0 ? previousState : 0; + } + + std::vector<DecodedFrame> decoded(observations.size()); + for (size_t frameIndex = 0; frameIndex < observations.size(); ++frameIndex) + { + const int stateIndex = bestStates[frameIndex]; + DecodedFrame frame; + frame.voiced = stateVoiced[frameIndex][static_cast<size_t>(stateIndex)]; + frame.midi = stateMidi[frameIndex][static_cast<size_t>(stateIndex)]; + frame.frequency = stateFrequency[frameIndex][static_cast<size_t>(stateIndex)]; + frame.confidence = frame.voiced + ? stateConfidence[frameIndex][static_cast<size_t>(stateIndex)] + : observations[frameIndex].unvoicedProbability; + decoded[frameIndex] = frame; + } + + return decoded; +} +} + +float PitchAnalyzer::analyzeFrame(const float* frameData, int frameSize, double sampleRate, + float& outConfidence) +{ + const int tauMin = std::max(2, static_cast<int>(sampleRate / maxFreq)); + const int tauMax = std::min(frameSize / 2 - 1, static_cast<int>(sampleRate / minFreq)); + if (tauMax <= tauMin || frameSize != analysisFrameSize) { - float s0 = yinBuffer[static_cast<size_t>(bestTau - 1)]; - float s1 = yinBuffer[static_cast<size_t>(bestTau)]; - float s2 = yinBuffer[static_cast<size_t>(bestTau + 1)]; - float denom = 2.0f * (2.0f * s1 - s2 - s0); - if (std::abs(denom) > 1e-10f) - refinedTau += (s2 - s0) / denom; + outConfidence = 0.0f; + return 0.0f; } - outConfidence = juce::jlimit(0.0f, 1.0f, 1.0f - bestVal); - float freq = (refinedTau > 0.0f) ? static_cast<float>(sr) / refinedTau : 0.0f; + float sumSquares = 0.0f; + for (int i = 0; i < frameSize; ++i) + sumSquares += frameData[i] * frameData[i]; + const float rms = std::sqrt(sumSquares / static_cast<float>(frameSize)); + const float rmsDB = rms > 0.0f ? 20.0f * std::log10(rms) : -100.0f; + + applyHannWindow(frameData, frameSize, analysisWindow, windowedFrameBuffer); + const bool useFftYin = shouldUseFftYin(); + if (useFftYin) + computeDifferenceFft(windowedFrameBuffer, frameSize, *fft, fftBuffer, autocorrelationBuffer, prefixEnergyBuffer, differenceBuffer); + else + computeDirectDifference(windowedFrameBuffer, frameSize, differenceBuffer); - if (freq < minFreq || freq > maxFreq) + if (useFftYin && shouldCompareDirectYin()) + { + computeDirectDifference(windowedFrameBuffer, frameSize, directDifferenceBuffer); + float maxDifference = 0.0f; + for (size_t i = 1; i < differenceBuffer.size(); ++i) + maxDifference = std::max(maxDifference, std::abs(differenceBuffer[i] - directDifferenceBuffer[i])); + juce::Logger::writeToLog("[PitchAnalyzer] FFT/direct YIN difference max=" + juce::String(maxDifference, 6)); + } + + computeCmndf(differenceBuffer, cmndfBuffer); + yinBuffer = cmndfBuffer; + + const auto observation = extractFrameObservation(cmndfBuffer, tauMin, tauMax, sampleRate, rmsDB, sensitivity); + if (observation.candidates.empty()) { outConfidence = 0.0f; return 0.0f; } - return freq; + const auto bestIt = std::max_element(observation.candidates.begin(), observation.candidates.end(), + [] (const PitchCandidate& a, const PitchCandidate& b) + { + return a.probability < b.probability; + }); + outConfidence = clamp01(bestIt->confidence * observation.voicedProbability); + return bestIt->frequency; } -PitchAnalyzer::AnalysisResult PitchAnalyzer::analyzeClip( - const float *audioData, int numSamples, double sampleRate, const juce::String &clipId) +PitchAnalyzer::AnalysisResult PitchAnalyzer::analyzeClip(const float* audioData, int numSamples, + double sampleRate, const juce::String& clipId, + std::function<bool()> shouldCancel) { AnalysisResult result; result.clipId = clipId; result.sampleRate = sampleRate; - // Offline: adaptive hop size based on clip length to keep analysis time reasonable - constexpr int frameSize = 2048; - // Short clips (<30s): high res hop=256 (~5.8ms at 44.1kHz) - // Medium clips (30-120s): hop=512 (~11.6ms) - // Long clips (>120s): hop=1024 (~23ms) — still good enough for graphical editing const double durationSec = static_cast<double>(numSamples) / sampleRate; const int hopSize = (durationSec > 120.0) ? 1024 : (durationSec > 30.0) ? 512 : 256; result.hopSize = hopSize; + const int frameSize = analysisFrameSize; const int numFrames = (numSamples - frameSize) / hopSize + 1; if (numFrames <= 0) return result; + const int tauMin = std::max(2, static_cast<int>(sampleRate / maxFreq)); + const int tauMax = std::min(frameSize / 2 - 1, static_cast<int>(sampleRate / minFreq)); + + std::vector<FrameObservation> observations; + observations.reserve(static_cast<size_t>(numFrames)); result.frames.reserve(static_cast<size_t>(numFrames)); - for (int i = 0; i < numFrames; ++i) + for (int frameIndex = 0; frameIndex < numFrames; ++frameIndex) { - int offset = i * hopSize; - const float *frameData = audioData + offset; + if (shouldCancel && shouldCancel()) + return result; + + const int offset = frameIndex * hopSize; + const float* frame = audioData + offset; - // RMS - float sumSq = 0.0f; - for (int j = 0; j < frameSize; ++j) - sumSq += frameData[j] * frameData[j]; - float rms = std::sqrt(sumSq / static_cast<float>(frameSize)); - float rmsDB = rms > 0.0f ? 20.0f * std::log10(rms) : -100.0f; + float sumSquares = 0.0f; + for (int i = 0; i < frameSize; ++i) + sumSquares += frame[i] * frame[i]; + const float rms = std::sqrt(sumSquares / static_cast<float>(frameSize)); + const float rmsDB = rms > 0.0f ? 20.0f * std::log10(rms) : -100.0f; - float conf = 0.0f; - float freq = 0.0f; + FrameObservation observation; + observation.rmsDB = rmsDB; - // Skip analysis for very quiet frames if (rmsDB > -60.0f) - freq = analyzeFrame(frameData, frameSize, sampleRate, conf); + { + applyHannWindow(frame, frameSize, analysisWindow, windowedFrameBuffer); + const bool useFftYin = shouldUseFftYin(); + if (useFftYin) + computeDifferenceFft(windowedFrameBuffer, frameSize, *fft, fftBuffer, autocorrelationBuffer, prefixEnergyBuffer, differenceBuffer); + else + computeDirectDifference(windowedFrameBuffer, frameSize, differenceBuffer); - float midi = (freq > 0.0f && conf > 0.3f) ? hzToMidi(freq) : 0.0f; + if (useFftYin && shouldCompareDirectYin()) + { + computeDirectDifference(windowedFrameBuffer, frameSize, directDifferenceBuffer); + float maxDifference = 0.0f; + for (size_t i = 1; i < differenceBuffer.size(); ++i) + maxDifference = std::max(maxDifference, std::abs(differenceBuffer[i] - directDifferenceBuffer[i])); + juce::Logger::writeToLog("[PitchAnalyzer] FFT/direct YIN difference max=" + juce::String(maxDifference, 6) + + " frame=" + juce::String(frameIndex)); + } - // Voiced/unvoiced classification: - // - Silence: rmsDB < -55 dB - // - Unvoiced (sibilant/breath): has energy but low pitch confidence - // - Voiced: clear pitch detected with good confidence - bool isVoiced = (midi > 0.0f && conf >= 0.3f && rmsDB > -55.0f); + computeCmndf(differenceBuffer, cmndfBuffer); + observation = extractFrameObservation(cmndfBuffer, tauMin, tauMax, sampleRate, rmsDB, sensitivity); + } - PitchFrame pf; - pf.time = static_cast<float>(offset) / static_cast<float>(sampleRate); - pf.frequency = freq; - pf.midiNote = midi; - pf.confidence = conf; - pf.rmsDB = rmsDB; - pf.voiced = isVoiced; - result.frames.push_back(pf); + observations.push_back(observation); } - // Segment into notes - result.notes = segmentNotes(result.frames, hopSize, sampleRate); + if (shouldCancel && shouldCancel()) + return result; + + const auto decodedFrames = decodePitchTrack(observations); + for (size_t frameIndex = 0; frameIndex < decodedFrames.size(); ++frameIndex) + { + const auto& decoded = decodedFrames[frameIndex]; + PitchFrame frame; + frame.time = static_cast<float>(static_cast<int>(frameIndex) * hopSize) / static_cast<float>(sampleRate); + frame.frequency = decoded.voiced ? decoded.frequency : 0.0f; + frame.midiNote = decoded.voiced ? decoded.midi : 0.0f; + frame.confidence = clamp01(decoded.confidence); + frame.rmsDB = observations[frameIndex].rmsDB; + frame.voiced = decoded.voiced; + result.frames.push_back(frame); + } + result.notes = segmentNotes(result.frames, hopSize, sampleRate, &result.boundaryCandidates); return result; } -std::vector<PitchAnalyzer::PitchNote> PitchAnalyzer::segmentNotes( - const std::vector<PitchFrame> &frames, int /*hopSize*/, double /*sampleRate*/) +std::vector<PitchAnalyzer::PitchNote> PitchAnalyzer::segmentNotes(const std::vector<PitchFrame>& frames, + int /*hopSize*/, + double /*sampleRate*/, + std::vector<PitchBoundaryCandidate>* boundaryCandidates) { std::vector<PitchNote> notes; + if (boundaryCandidates != nullptr) + boundaryCandidates->clear(); if (frames.empty()) return notes; - const float minNoteDuration = 0.05f; // 50 ms minimum note - const float pitchJumpThreshold = 1.5f; // semitones — tolerates vibrato (±1 sem), - // splits on genuine melodic note changes - const float confidenceThreshold = 0.40f; // relaxed from 0.5 — avoids ending a note on a - // momentarily low-confidence frame mid-vibrato - const float silenceThreshold = -50.0f; // dB - const int stableFrames = 3; // pitch deviation must be sustained this many - // consecutive frames before triggering a split - // (prevents vibrato peaks from splitting notes) - const float energyDipSplitDB = 10.0f; // dB drop from running avg triggers a note split - const int energyDipMinFrames = 2; // energy must dip for at least this many frames + const float minNoteDuration = 0.05f; + const float pitchJumpCandidateThreshold = 1.5f; + const float confidenceThreshold = 0.25f; + const float silenceThreshold = -50.0f; + const float energyDipSplitDB = 10.0f; + const float energyDipSplitSec = 0.030f; + const float dropoutBridgeSec = 0.080f; + const float hardUnvoicedGapSec = 0.090f; int noteStartIdx = -1; float noteSum = 0.0f; int noteCount = 0; int noteId = 0; - int deviationRun = 0; // consecutive frames exceeding pitchJumpThreshold - float runningEnergySum = 0.0f; // running energy sum for current note - int energyDipRun = 0; // consecutive frames with significant energy dip + float runningEnergySum = 0.0f; + int energyDipRun = 0; + int inactiveRun = 0; + int lastVoicedIdx = -1; - auto finalizeNote = [&](int endIdx) - { - if (noteStartIdx < 0 || noteCount == 0) - return; + const float frameStepSec = frames.size() >= 2 + ? std::max (0.001f, frames[1].time - frames[0].time) + : 0.01f; - float avgMidi = noteSum / static_cast<float>(noteCount); - float startTime = frames[static_cast<size_t>(noteStartIdx)].time; - float endTime = frames[static_cast<size_t>(endIdx)].time; - float duration = endTime - startTime; + auto makeNote = [&](int startIdx, int endIdx) -> PitchNote + { + float sum = 0.0f; + int count = 0; + for (int i = startIdx; i <= endIdx && i < static_cast<int>(frames.size()); ++i) + { + const float midi = frames[static_cast<size_t>(i)].midiNote; + if (midi > 0.0f) + { + sum += midi; + ++count; + } + } - if (duration < minNoteDuration) - return; + const float avgMidi = count > 0 ? sum / static_cast<float>(count) : 0.0f; + const float startTime = frames[static_cast<size_t>(startIdx)].time; + const float endTime = frames[static_cast<size_t>(endIdx)].time; PitchNote note; note.id = "note_" + juce::String(noteId++); note.startTime = startTime; note.endTime = endTime; + note.effectiveStartTime = startTime; + note.effectiveEndTime = endTime; note.detectedPitch = avgMidi; - note.correctedPitch = avgMidi; // No auto-tune by default — user applies correction explicitly + note.correctedPitch = avgMidi; note.driftCorrectionAmount = 0.0f; note.vibratoDepth = 1.0f; note.vibratoRate = 0.0f; @@ -221,131 +666,589 @@ std::vector<PitchAnalyzer::PitchNote> PitchAnalyzer::segmentNotes( note.transitionOut = 0.0f; note.formantShift = 0.0f; note.gain = 0.0f; - - // Store per-frame drift (deviation from note center) - for (int i = noteStartIdx; i <= endIdx && i < static_cast<int>(frames.size()); ++i) + note.voiced = true; + note.wordGroupId = {}; + note.entryBoundaryKind = "unknown"; + note.exitBoundaryKind = "unknown"; + note.entryBoundaryReason = {}; + note.exitBoundaryReason = {}; + note.entryBoundaryScore = 0.0f; + note.exitBoundaryScore = 0.0f; + + for (int i = startIdx; i <= endIdx && i < static_cast<int>(frames.size()); ++i) { - float midi = frames[static_cast<size_t>(i)].midiNote; + const float midi = frames[static_cast<size_t>(i)].midiNote; note.pitchDrift.push_back(midi > 0.0f ? midi - avgMidi : 0.0f); } + return note; + }; + + auto isHardBoundaryFrame = [&](int idx) -> bool + { + if (idx < 0 || idx >= static_cast<int>(frames.size())) + return true; + + const auto& f = frames[static_cast<size_t>(idx)]; + return ! f.voiced + || f.midiNote <= 0.0f + || f.confidence < confidenceThreshold * 0.8f + || f.rmsDB <= silenceThreshold; + }; + + auto hasHardBoundaryNear = [&](int centerIdx) -> bool + { + const int radius = std::max(1, static_cast<int>(std::round(0.025f / frameStepSec))); + for (int i = centerIdx - radius; i <= centerIdx + radius; ++i) + { + if (isHardBoundaryFrame(i)) + return true; + } + return false; + }; + + const int energyDipMinFrames = std::max(2, static_cast<int>(std::ceil(energyDipSplitSec / frameStepSec))); + const int hardUnvoicedGapFrames = std::max(1, static_cast<int>(std::ceil(hardUnvoicedGapSec / frameStepSec))); + + auto resetOpenNote = [&]() + { + noteStartIdx = -1; + noteSum = 0.0f; + noteCount = 0; + runningEnergySum = 0.0f; + energyDipRun = 0; + inactiveRun = 0; + lastVoicedIdx = -1; + }; + + auto finalizeNote = [&](int endIdx) + { + if (noteStartIdx < 0 || noteCount == 0) + return; + + const float startTime = frames[static_cast<size_t>(noteStartIdx)].time; + const float endTime = frames[static_cast<size_t>(endIdx)].time; + const float duration = endTime - startTime; + + if (duration < minNoteDuration) + return; + + auto note = makeNote(noteStartIdx, endIdx); + const bool hardEntry = hasHardBoundaryNear(noteStartIdx - 1); + const bool hardExit = hasHardBoundaryNear(endIdx + 1); + note.entryBoundaryKind = hardEntry ? "hard_word_like" : "unknown"; + note.entryBoundaryReason = hardEntry ? "voiced_region_start" : "continuous_pitch_segment_start"; + note.entryBoundaryScore = hardEntry ? 1.0f : 0.0f; + note.exitBoundaryKind = hardExit ? "hard_word_like" : "unknown"; + note.exitBoundaryReason = hardExit ? "voiced_region_end" : "continuous_pitch_segment_end"; + note.exitBoundaryScore = hardExit ? 1.0f : 0.0f; notes.push_back(std::move(note)); }; for (int i = 0; i < static_cast<int>(frames.size()); ++i) { - const auto &f = frames[static_cast<size_t>(i)]; - bool isVoiced = f.midiNote > 0.0f && f.confidence >= confidenceThreshold && f.rmsDB > silenceThreshold; + const auto& frame = frames[static_cast<size_t>(i)]; + const bool isVoiced = frame.voiced + && frame.midiNote > 0.0f + && frame.confidence >= confidenceThreshold + && frame.rmsDB > silenceThreshold; - if (!isVoiced) + if (! isVoiced) { - // End current note at silence/unvoiced if (noteStartIdx >= 0) { - finalizeNote(i - 1); - noteStartIdx = -1; - noteSum = 0.0f; - noteCount = 0; - deviationRun = 0; - runningEnergySum = 0.0f; - energyDipRun = 0; + ++inactiveRun; + if (inactiveRun >= hardUnvoicedGapFrames) + { + finalizeNote(std::max(noteStartIdx, lastVoicedIdx)); + resetOpenNote(); + } } continue; } if (noteStartIdx < 0) { - // Start new note noteStartIdx = i; - noteSum = f.midiNote; + noteSum = frame.midiNote; noteCount = 1; - deviationRun = 0; - runningEnergySum = f.rmsDB; + runningEnergySum = frame.rmsDB; + energyDipRun = 0; + inactiveRun = 0; + lastVoicedIdx = i; + continue; + } + + if (inactiveRun > 0) + { + const float inactiveSec = static_cast<float>(inactiveRun) * frameStepSec; + if (inactiveSec > dropoutBridgeSec) + { + finalizeNote(std::max(noteStartIdx, lastVoicedIdx)); + noteStartIdx = i; + noteSum = frame.midiNote; + noteCount = 1; + runningEnergySum = frame.rmsDB; + energyDipRun = 0; + inactiveRun = 0; + lastVoicedIdx = i; + continue; + } + + inactiveRun = 0; energyDipRun = 0; } + + const float avgEnergy = runningEnergySum / static_cast<float>(noteCount); + const float energyDrop = avgEnergy - frame.rmsDB; + + if (energyDrop > energyDipSplitDB) + { + ++energyDipRun; + if (energyDipRun >= energyDipMinFrames) + { + finalizeNote(i - energyDipRun); + resetOpenNote(); + continue; + } + } else { - // --- Energy-dip-based splitting (syllable/word boundaries) --- - // If energy drops significantly below the note's running average, - // this likely indicates a consonant or breath between syllables. - float avgEnergy = runningEnergySum / static_cast<float>(noteCount); - float energyDrop = avgEnergy - f.rmsDB; + energyDipRun = 0; + } + + noteSum += frame.midiNote; + runningEnergySum += frame.rmsDB; + ++noteCount; + lastVoicedIdx = i; + } + + if (noteStartIdx >= 0) + finalizeNote(std::max(noteStartIdx, lastVoicedIdx)); + + std::vector<float> smoothedMidi(frames.size(), 0.0f); + for (int i = 0; i < static_cast<int>(frames.size()); ++i) + { + std::vector<float> local; + local.reserve(5); + for (int k = std::max(0, i - 2); k <= std::min(static_cast<int>(frames.size()) - 1, i + 2); ++k) + { + const auto& f = frames[static_cast<size_t>(k)]; + if (f.voiced && f.midiNote > 0.0f && f.confidence >= confidenceThreshold && f.rmsDB > silenceThreshold) + local.push_back(f.midiNote); + } + + if (! local.empty()) + { + const auto middle = local.begin() + static_cast<std::ptrdiff_t>(local.size() / 2); + std::nth_element(local.begin(), middle, local.end()); + smoothedMidi[static_cast<size_t>(i)] = *middle; + } + } + + struct CornerCandidate + { + int frameIndex = -1; + juce::String kind = "unknown"; + juce::String reason; + float score = 0.0f; + bool destructiveSplitAllowed = false; + }; - if (energyDrop > energyDipSplitDB) + auto noteFrameIndexForTime = [&](float time, bool preferEnd) -> int + { + int best = 0; + float bestDelta = std::numeric_limits<float>::max(); + for (int i = 0; i < static_cast<int>(frames.size()); ++i) + { + const float delta = std::abs(frames[static_cast<size_t>(i)].time - time); + if (delta < bestDelta) { - ++energyDipRun; - if (energyDipRun >= energyDipMinFrames) - { - // Sustained energy dip → syllable boundary, split here - finalizeNote(i - energyDipRun); - noteStartIdx = -1; - noteSum = 0.0f; - noteCount = 0; - deviationRun = 0; - runningEnergySum = 0.0f; - energyDipRun = 0; - // Don't start new note on the dip frame — let the loop - // naturally pick up when energy rises again - continue; - } + bestDelta = delta; + best = i; + } + } + if (preferEnd) + return std::min(best, static_cast<int>(frames.size()) - 1); + return std::max(0, best); + }; + + auto averageRange = [&](const std::vector<float>& values, int start, int end, float fallback) -> float + { + double sum = 0.0; + int count = 0; + for (int i = std::max(0, start); i <= std::min(static_cast<int>(values.size()) - 1, end); ++i) + { + const float value = values[static_cast<size_t>(i)]; + if (value > 0.0f) + { + sum += value; + ++count; + } + } + return count > 0 ? static_cast<float>(sum / static_cast<double>(count)) : fallback; + }; + + auto averageEnergy = [&](int start, int end) -> float + { + double sum = 0.0; + int count = 0; + for (int i = std::max(0, start); i <= std::min(static_cast<int>(frames.size()) - 1, end); ++i) + { + sum += frames[static_cast<size_t>(i)].rmsDB; + ++count; + } + return count > 0 ? static_cast<float>(sum / static_cast<double>(count)) : -100.0f; + }; + + auto averageConfidence = [&](int start, int end) -> float + { + double sum = 0.0; + int count = 0; + for (int i = std::max(0, start); i <= std::min(static_cast<int>(frames.size()) - 1, end); ++i) + { + sum += frames[static_cast<size_t>(i)].confidence; + ++count; + } + return count > 0 ? static_cast<float>(sum / static_cast<double>(count)) : 0.0f; + }; + + auto hasNearbyUnvoiced = [&](int center, int radiusFrames) -> bool + { + for (int i = std::max(0, center - radiusFrames); i <= std::min(static_cast<int>(frames.size()) - 1, center + radiusFrames); ++i) + { + const auto& f = frames[static_cast<size_t>(i)]; + if (! f.voiced || f.confidence < confidenceThreshold || f.rmsDB <= silenceThreshold || f.midiNote <= 0.0f) + return true; + } + return false; + }; + + auto isVibratoLikeCorner = [&](int center, int noteStartFrame, int noteEndFrame) -> bool + { + const int radius = std::max(4, static_cast<int>(std::round(0.125f / frameStepSec))); + const int start = std::max(noteStartFrame, center - radius); + const int end = std::min(noteEndFrame, center + radius); + if (end - start < 8) + return false; + + int reversals = 0; + int lastSign = 0; + float minMidi = std::numeric_limits<float>::max(); + float maxMidi = 0.0f; + float minEnergy = std::numeric_limits<float>::max(); + float maxEnergy = -100.0f; + + for (int i = start + 1; i <= end; ++i) + { + const float prev = smoothedMidi[static_cast<size_t>(i - 1)]; + const float here = smoothedMidi[static_cast<size_t>(i)]; + if (prev <= 0.0f || here <= 0.0f) + continue; + + minMidi = std::min(minMidi, here); + maxMidi = std::max(maxMidi, here); + minEnergy = std::min(minEnergy, frames[static_cast<size_t>(i)].rmsDB); + maxEnergy = std::max(maxEnergy, frames[static_cast<size_t>(i)].rmsDB); + + const float slope = (here - prev) / frameStepSec; + const int sign = slope > 2.0f ? 1 : (slope < -2.0f ? -1 : 0); + if (sign == 0) + continue; + if (lastSign != 0 && sign != lastSign) + ++reversals; + lastSign = sign; + } + + const float pitchRange = maxMidi > minMidi ? maxMidi - minMidi : 0.0f; + const float energyRange = maxEnergy > minEnergy ? maxEnergy - minEnergy : 0.0f; + return reversals >= 3 && pitchRange <= 1.2f && energyRange <= 4.0f; + }; + + auto findCornerCandidatesForNote = [&](const PitchNote& note) -> std::vector<CornerCandidate> + { + std::vector<CornerCandidate> candidates; + const int startFrame = noteFrameIndexForTime(note.startTime, false); + const int endFrame = noteFrameIndexForTime(note.endTime, true); + const int minEdgeFrames = std::max(2, static_cast<int>(std::round(0.035f / frameStepSec))); + const int minSplitSpacingFrames = std::max(4, static_cast<int>(std::round(0.080f / frameStepSec))); + const int slopeFrames = std::max(3, static_cast<int>(std::round(0.040f / frameStepSec))); + const int cueFrames = std::max(2, static_cast<int>(std::round(0.020f / frameStepSec))); + int lastAccepted = -minSplitSpacingFrames * 2; + + for (int i = startFrame + minEdgeFrames; i <= endFrame - minEdgeFrames; ++i) + { + if (i - lastAccepted < minSplitSpacingFrames) + continue; + + const int leftIndex = std::max(startFrame, i - slopeFrames); + const int rightIndex = std::min(endFrame, i + slopeFrames); + const float leftMidi = smoothedMidi[static_cast<size_t>(leftIndex)]; + const float centerMidi = smoothedMidi[static_cast<size_t>(i)]; + const float rightMidi = smoothedMidi[static_cast<size_t>(rightIndex)]; + if (leftMidi <= 0.0f || centerMidi <= 0.0f || rightMidi <= 0.0f) + continue; + + const float leftSlope = (centerMidi - leftMidi) / std::max(frameStepSec, frames[static_cast<size_t>(i)].time - frames[static_cast<size_t>(leftIndex)].time); + const float rightSlope = (rightMidi - centerMidi) / std::max(frameStepSec, frames[static_cast<size_t>(rightIndex)].time - frames[static_cast<size_t>(i)].time); + const bool directionFlip = (leftSlope > 5.0f && rightSlope < -5.0f) + || (leftSlope < -5.0f && rightSlope > 5.0f); + if (! directionFlip) + continue; + + const float slopeDelta = std::abs(leftSlope - rightSlope); + const float leftAvg = averageRange(smoothedMidi, leftIndex, i - 1, leftMidi); + const float rightAvg = averageRange(smoothedMidi, i + 1, rightIndex, rightMidi); + const float cornerProminence = std::min(std::abs(centerMidi - leftAvg), std::abs(centerMidi - rightAvg)); + const float sidePitchChange = std::abs(rightAvg - leftAvg); + if (slopeDelta < 18.0f || cornerProminence < 0.35f) + continue; + + const float leftEnergy = averageEnergy(leftIndex, i - cueFrames); + const float rightEnergy = averageEnergy(i + cueFrames, rightIndex); + const float centerEnergy = averageEnergy(i - cueFrames, i + cueFrames); + const float energyDip = std::max(leftEnergy, rightEnergy) - centerEnergy; + const float leftConfidence = averageConfidence(leftIndex, i - cueFrames); + const float rightConfidence = averageConfidence(i + cueFrames, rightIndex); + const float centerConfidence = averageConfidence(i - cueFrames, i + cueFrames); + const float confidenceDip = std::max(leftConfidence, rightConfidence) - centerConfidence; + const bool nearbyUnvoiced = hasNearbyUnvoiced(i, cueFrames); + const bool strongPitchCue = cornerProminence >= 0.75f || sidePitchChange >= 0.55f; + const bool hasCompanionCue = energyDip >= 3.0f || confidenceDip >= 0.12f || nearbyUnvoiced || strongPitchCue; + + if (! hasCompanionCue) + continue; + if (isVibratoLikeCorner(i, startFrame, endFrame)) + { + CornerCandidate candidate; + candidate.frameIndex = i; + candidate.kind = "internal_vibrato"; + candidate.reason = "pitch_corner_vibrato_suppressed"; + candidate.score = juce::jlimit(0.0f, 1.0f, 0.18f + std::min(0.42f, slopeDelta / 140.0f)); + candidate.destructiveSplitAllowed = false; + candidates.push_back(candidate); + lastAccepted = i; + continue; + } + + CornerCandidate candidate; + candidate.frameIndex = i; + candidate.score = juce::jlimit(0.0f, 1.0f, + 0.34f * std::min(1.0f, slopeDelta / 42.0f) + + 0.28f * std::min(1.0f, cornerProminence / 1.2f) + + 0.18f * std::min(1.0f, energyDip / 8.0f) + + 0.12f * std::min(1.0f, confidenceDip / 0.35f) + + (nearbyUnvoiced ? 0.08f : 0.0f)); + + if (energyDip >= 3.0f || confidenceDip >= 0.18f || nearbyUnvoiced) + { + candidate.kind = "hard_word_like"; + candidate.reason = "pitch_corner_with_energy_or_confidence_cue"; + candidate.destructiveSplitAllowed = candidate.score >= 0.50f + && (energyDip >= 6.0f || confidenceDip >= 0.18f || nearbyUnvoiced); } else { - energyDipRun = 0; + candidate.kind = "soft_legato"; + candidate.reason = "pitch_corner_continuous_voiced"; + candidate.destructiveSplitAllowed = false; + } + + candidates.push_back(candidate); + lastAccepted = i; + } + + return candidates; + }; + + auto findPitchDeviationCandidatesForNote = [&](const PitchNote& note) -> std::vector<CornerCandidate> + { + std::vector<CornerCandidate> candidates; + const int startFrame = noteFrameIndexForTime(note.startTime, false); + const int endFrame = noteFrameIndexForTime(note.endTime, true); + const int minEdgeFrames = std::max(2, static_cast<int>(std::round(0.035f / frameStepSec))); + const int minRunFrames = std::max(4, static_cast<int>(std::round(0.050f / frameStepSec))); + const int minSpacingFrames = std::max(6, static_cast<int>(std::round(0.120f / frameStepSec))); + const int noteSpan = endFrame - startFrame; + if (noteSpan < minEdgeFrames * 2 + minRunFrames) + return candidates; + + float dynamicAverage = 0.0f; + int runStart = -1; + int runLength = 0; + float runMaxDeviation = 0.0f; + float runArea = 0.0f; + int lastAccepted = -minSpacingFrames * 2; + + auto finishRun = [&](int runEnd) + { + if (runStart < 0 || runLength < minRunFrames || runStart - lastAccepted < minSpacingFrames) + return; + if (runStart <= startFrame + minEdgeFrames || runEnd >= endFrame - minEdgeFrames) + return; + + const int center = runStart + std::max(0, (runEnd - runStart) / 2); + const bool vibratoLike = isVibratoLikeCorner(center, startFrame, endFrame); + CornerCandidate candidate; + candidate.frameIndex = center; + candidate.kind = vibratoLike ? "internal_vibrato" : "internal_bend"; + candidate.reason = vibratoLike ? "pitch_hysteresis_vibrato_suppressed" : "pitch_hysteresis_deviation_internal_bend"; + candidate.score = juce::jlimit(0.0f, 1.0f, + 0.35f * std::min(1.0f, runMaxDeviation / 3.0f) + + 0.35f * std::min(1.0f, static_cast<float>(runLength) * frameStepSec / 0.180f) + + 0.30f * std::min(1.0f, runArea / 10.0f)); + candidate.destructiveSplitAllowed = false; + candidates.push_back(candidate); + lastAccepted = center; + }; + + for (int i = startFrame; i <= endFrame; ++i) + { + const float midi = smoothedMidi[static_cast<size_t>(i)]; + const auto& frame = frames[static_cast<size_t>(i)]; + if (midi <= 0.0f || ! frame.voiced || frame.confidence < confidenceThreshold || frame.rmsDB <= silenceThreshold) + { + finishRun(i - 1); + runStart = -1; + runLength = 0; + runMaxDeviation = 0.0f; + runArea = 0.0f; + continue; } - // --- Pitch-jump-based splitting --- - float avgSoFar = noteSum / static_cast<float>(noteCount); - float deviation = std::abs(f.midiNote - avgSoFar); + if (dynamicAverage <= 0.0f) + { + dynamicAverage = midi; + continue; + } - if (deviation > pitchJumpThreshold) + const float deviation = std::abs(midi - dynamicAverage); + if (deviation > pitchJumpCandidateThreshold) { - ++deviationRun; - if (deviationRun >= stableFrames) + if (runStart < 0) { - // Sustained pitch jump → genuine note change - finalizeNote(i - stableFrames); - noteStartIdx = i - stableFrames + 1; - // Recompute sum for the new note's initial frames - noteSum = 0.0f; - noteCount = 0; - runningEnergySum = 0.0f; - for (int k = noteStartIdx; k <= i; ++k) - { - noteSum += frames[static_cast<size_t>(k)].midiNote; - runningEnergySum += frames[static_cast<size_t>(k)].rmsDB; - ++noteCount; - } - deviationRun = 0; - energyDipRun = 0; + runStart = i; + runLength = 0; + runMaxDeviation = 0.0f; + runArea = 0.0f; } - // else: don't split yet — might be a vibrato transient + ++runLength; + runMaxDeviation = std::max(runMaxDeviation, deviation); + runArea += std::max(0.0f, deviation - pitchJumpCandidateThreshold); } else { - deviationRun = 0; - noteSum += f.midiNote; - runningEnergySum += f.rmsDB; - ++noteCount; + finishRun(i - 1); + runStart = -1; + runLength = 0; + runMaxDeviation = 0.0f; + runArea = 0.0f; + dynamicAverage = 0.985f * dynamicAverage + 0.015f * midi; } } + + finishRun(endFrame); + return candidates; + }; + + const bool applyCornerSplits = juce::SystemStats::getEnvironmentVariable( + "OPENSTUDIO_ANALYZER_APPLY_CORNER_SPLITS", {}).trim() == "1"; + + std::vector<PitchNote> cornerSplitNotes; + cornerSplitNotes.reserve(notes.size()); + for (const auto& note : notes) + { + auto candidates = findPitchDeviationCandidatesForNote(note); + auto cornerCandidates = findCornerCandidatesForNote(note); + candidates.insert(candidates.end(), cornerCandidates.begin(), cornerCandidates.end()); + std::sort(candidates.begin(), candidates.end(), [](const CornerCandidate& a, const CornerCandidate& b) + { + return a.frameIndex < b.frameIndex; + }); + + if (boundaryCandidates != nullptr) + { + for (const auto& candidate : candidates) + { + PitchBoundaryCandidate publicCandidate; + publicCandidate.id = "boundary_" + juce::String(static_cast<int>(boundaryCandidates->size())); + publicCandidate.sourceNoteId = note.id; + publicCandidate.time = frames[static_cast<size_t>(candidate.frameIndex)].time; + publicCandidate.kind = candidate.kind; + publicCandidate.reason = candidate.reason; + publicCandidate.score = candidate.score; + publicCandidate.destructiveSplitAllowed = candidate.destructiveSplitAllowed + && (candidate.kind == "hard_word_like" || applyCornerSplits); + boundaryCandidates->push_back(std::move(publicCandidate)); + } + } + + const bool hasDefaultAcousticSplit = std::any_of(candidates.begin(), candidates.end(), [](const CornerCandidate& candidate) + { + return candidate.kind == "hard_word_like" && candidate.destructiveSplitAllowed; + }); + + if (! applyCornerSplits && ! hasDefaultAcousticSplit) + { + cornerSplitNotes.push_back(note); + continue; + } + + if (candidates.empty()) + { + cornerSplitNotes.push_back(note); + continue; + } + + int segmentStart = noteFrameIndexForTime(note.startTime, false); + const int noteEnd = noteFrameIndexForTime(note.endTime, true); + juce::String entryKind = note.entryBoundaryKind; + juce::String entryReason = note.entryBoundaryReason; + float entryScore = note.entryBoundaryScore; + + for (const auto& candidate : candidates) + { + const bool shouldSplit = candidate.destructiveSplitAllowed + && (candidate.kind == "hard_word_like" || applyCornerSplits); + if (! shouldSplit) + continue; + + const int leftEnd = candidate.frameIndex - 1; + const float leftDuration = frames[static_cast<size_t>(leftEnd)].time - frames[static_cast<size_t>(segmentStart)].time; + const float rightDuration = frames[static_cast<size_t>(noteEnd)].time - frames[static_cast<size_t>(candidate.frameIndex)].time; + if (leftDuration < minNoteDuration || rightDuration < minNoteDuration) + continue; + + auto left = makeNote(segmentStart, leftEnd); + left.entryBoundaryKind = entryKind; + left.entryBoundaryReason = entryReason; + left.entryBoundaryScore = entryScore; + left.exitBoundaryKind = candidate.kind; + left.exitBoundaryReason = candidate.reason; + left.exitBoundaryScore = candidate.score; + cornerSplitNotes.push_back(std::move(left)); + + segmentStart = candidate.frameIndex; + entryKind = candidate.kind; + entryReason = candidate.reason; + entryScore = candidate.score; + } + + auto tail = makeNote(segmentStart, noteEnd); + tail.entryBoundaryKind = entryKind; + tail.entryBoundaryReason = entryReason; + tail.entryBoundaryScore = entryScore; + tail.exitBoundaryKind = note.exitBoundaryKind; + tail.exitBoundaryReason = note.exitBoundaryReason; + tail.exitBoundaryScore = note.exitBoundaryScore; + cornerSplitNotes.push_back(std::move(tail)); } - // Finalize last note - if (noteStartIdx >= 0) - finalizeNote(static_cast<int>(frames.size()) - 1); - - // ------------------------------------------------------------------------- - // Merge pass: collapse adjacent notes ONLY across very brief detection - // dropouts (1-2 frames of YIN uncertainty), NOT across real syllable gaps. - // Conditions: - // • gap < 40 ms (only detection glitches, not consonants/breath) - // • pitch centers within 1.0 semitone (clearly same note) - // • no significant energy dip in the gap frames - // ------------------------------------------------------------------------- - const float mergeGapSec = 0.04f; // 40 ms — only merge across detection glitches - const float mergePitchSem = 1.0f; // semitones - const float mergeEnergyDipDB = 8.0f; // don't merge if gap has energy dip > this + notes = std::move(cornerSplitNotes); + + const float mergeGapSec = 0.04f; + const float mergePitchSem = 1.0f; + const float mergeEnergyDipDB = 8.0f; bool merged = true; while (merged) @@ -353,55 +1256,99 @@ std::vector<PitchAnalyzer::PitchNote> PitchAnalyzer::segmentNotes( merged = false; for (int i = 0; i + 1 < static_cast<int>(notes.size()); ++i) { - auto &a = notes[static_cast<size_t>(i)]; - auto &b = notes[static_cast<size_t>(i + 1)]; - float gap = b.startTime - a.endTime; - float pitchDiff = std::abs(a.detectedPitch - b.detectedPitch); - - if (gap >= 0.0f && gap < mergeGapSec && pitchDiff < mergePitchSem) + auto& a = notes[static_cast<size_t>(i)]; + auto& b = notes[static_cast<size_t>(i + 1)]; + const float gap = b.startTime - a.endTime; + const float pitchDiff = std::abs(a.detectedPitch - b.detectedPitch); + const bool boundaryProtected = a.exitBoundaryKind == "hard_word_like" + && b.entryBoundaryKind == "hard_word_like" + && a.exitBoundaryScore >= 0.25f + && b.entryBoundaryScore >= 0.25f; + + if (gap < 0.0f || gap >= mergeGapSec || pitchDiff >= mergePitchSem || boundaryProtected) + continue; + + bool hasEnergyDip = false; + for (size_t frameIndex = 0; frameIndex < frames.size(); ++frameIndex) { - // Check for energy dip in gap frames — if there's a real dip, don't merge - bool hasEnergyDip = false; - for (size_t fi = 0; fi < frames.size(); ++fi) + if (frames[frameIndex].time < a.endTime || frames[frameIndex].time > b.startTime) + continue; + + float leftEnergy = -60.0f; + float rightEnergy = -60.0f; + if (frameIndex > 0) + leftEnergy = frames[frameIndex - 1].rmsDB; + if (frameIndex < frames.size()) + rightEnergy = frames[frameIndex].rmsDB; + const float referenceEnergy = std::max(leftEnergy, rightEnergy); + + if ((referenceEnergy - frames[frameIndex].rmsDB) > mergeEnergyDipDB) { - if (frames[fi].time >= a.endTime && frames[fi].time <= b.startTime) - { - // Get energy of the boundary frames of both notes - float aEndEnergy = -60.0f; - float bStartEnergy = -60.0f; - if (fi > 0) aEndEnergy = frames[fi - 1].rmsDB; - if (fi < frames.size()) bStartEnergy = frames[fi].rmsDB; - float refEnergy = std::max(aEndEnergy, bStartEnergy); - - if ((refEnergy - frames[fi].rmsDB) > mergeEnergyDipDB) - { - hasEnergyDip = true; - break; - } - } + hasEnergyDip = true; + break; } + } - if (hasEnergyDip) - continue; + if (hasEnergyDip) + continue; + + const float aDuration = a.endTime - a.startTime; + const float bDuration = b.endTime - b.startTime; + const float totalDuration = aDuration + bDuration; + const float newPitch = totalDuration > 0.0f + ? (a.detectedPitch * aDuration + b.detectedPitch * bDuration) / totalDuration + : a.detectedPitch; + + a.endTime = b.endTime; + a.effectiveEndTime = b.effectiveEndTime; + a.detectedPitch = newPitch; + a.correctedPitch = newPitch; + a.exitBoundaryKind = b.exitBoundaryKind; + a.exitBoundaryReason = b.exitBoundaryReason; + a.exitBoundaryScore = b.exitBoundaryScore; + a.pitchDrift.insert(a.pitchDrift.end(), b.pitchDrift.begin(), b.pitchDrift.end()); + + notes.erase(notes.begin() + i + 1); + merged = true; + break; + } + } - // Weighted average pitch (by duration as proxy for frame count) - float aDur = a.endTime - a.startTime; - float bDur = b.endTime - b.startTime; - float totalDur = aDur + bDur; - float newPitch = (totalDur > 0.0f) - ? (a.detectedPitch * aDur + b.detectedPitch * bDur) / totalDur - : a.detectedPitch; + int groupId = 0; + for (int i = 0; i < static_cast<int>(notes.size()); ++i) + { + auto& note = notes[static_cast<size_t>(i)]; + note.id = "note_" + juce::String(i); + + bool startsNewGroup = i == 0; + if (i > 0) + { + const auto& previous = notes[static_cast<size_t>(i - 1)]; + const float gap = note.startTime - previous.endTime; + const float pitchDiff = std::abs(note.detectedPitch - previous.detectedPitch); + const bool hardBoundary = previous.exitBoundaryKind == "hard_word_like" + && note.entryBoundaryKind == "hard_word_like" + && std::max(previous.exitBoundaryScore, note.entryBoundaryScore) >= 0.50f; + startsNewGroup = gap > 0.04f || hardBoundary || pitchDiff > 1.0f; + } - a.endTime = b.endTime; - a.detectedPitch = newPitch; - a.correctedPitch = newPitch; + if (startsNewGroup && i > 0) + ++groupId; - // Append drift frames from b - a.pitchDrift.insert(a.pitchDrift.end(), b.pitchDrift.begin(), b.pitchDrift.end()); + note.wordGroupId = "word_" + juce::String(groupId); + } - notes.erase(notes.begin() + i + 1); - merged = true; - break; // restart scan after any merge + if (boundaryCandidates != nullptr) + { + for (auto& candidate : *boundaryCandidates) + { + for (const auto& note : notes) + { + if (candidate.time >= note.startTime && candidate.time <= note.endTime) + { + candidate.sourceNoteId = note.id; + break; + } } } } @@ -409,18 +1356,13 @@ std::vector<PitchAnalyzer::PitchNote> PitchAnalyzer::segmentNotes( return notes; } -// ============================================================================ -// JSON serialization -// ============================================================================ - -juce::var PitchAnalyzer::resultToJSON(const AnalysisResult &result) +juce::var PitchAnalyzer::resultToJSON(const AnalysisResult& result) { juce::DynamicObject::Ptr root = new juce::DynamicObject(); root->setProperty("clipId", result.clipId); root->setProperty("sampleRate", result.sampleRate); root->setProperty("hopSize", result.hopSize); - // Frames — send as flat arrays for efficiency juce::Array<juce::var> frameTimes, frameMidi, frameConf, frameRms, frameVoiced; frameTimes.ensureStorageAllocated(static_cast<int>(result.frames.size())); frameMidi.ensureStorageAllocated(static_cast<int>(result.frames.size())); @@ -428,101 +1370,150 @@ juce::var PitchAnalyzer::resultToJSON(const AnalysisResult &result) frameRms.ensureStorageAllocated(static_cast<int>(result.frames.size())); frameVoiced.ensureStorageAllocated(static_cast<int>(result.frames.size())); - for (const auto &f : result.frames) + for (const auto& frame : result.frames) { - frameTimes.add(static_cast<double>(f.time)); - frameMidi.add(static_cast<double>(f.midiNote)); - frameConf.add(static_cast<double>(f.confidence)); - frameRms.add(static_cast<double>(f.rmsDB)); - frameVoiced.add(f.voiced); + frameTimes.add(static_cast<double>(frame.time)); + frameMidi.add(static_cast<double>(frame.midiNote)); + frameConf.add(static_cast<double>(frame.confidence)); + frameRms.add(static_cast<double>(frame.rmsDB)); + frameVoiced.add(frame.voiced); } - juce::DynamicObject::Ptr framesObj = new juce::DynamicObject(); - framesObj->setProperty("times", frameTimes); - framesObj->setProperty("midi", frameMidi); - framesObj->setProperty("confidence", frameConf); - framesObj->setProperty("rms", frameRms); - framesObj->setProperty("voiced", frameVoiced); - root->setProperty("frames", juce::var(framesObj.get())); + juce::DynamicObject::Ptr framesObject = new juce::DynamicObject(); + framesObject->setProperty("times", frameTimes); + framesObject->setProperty("midi", frameMidi); + framesObject->setProperty("confidence", frameConf); + framesObject->setProperty("rms", frameRms); + framesObject->setProperty("voiced", frameVoiced); + root->setProperty("frames", juce::var(framesObject.get())); - // Notes juce::Array<juce::var> notesList; - for (const auto &n : result.notes) - { - juce::DynamicObject::Ptr noteObj = new juce::DynamicObject(); - noteObj->setProperty("id", n.id); - noteObj->setProperty("startTime", static_cast<double>(n.startTime)); - noteObj->setProperty("endTime", static_cast<double>(n.endTime)); - noteObj->setProperty("detectedPitch", static_cast<double>(n.detectedPitch)); - noteObj->setProperty("correctedPitch", static_cast<double>(n.correctedPitch)); - noteObj->setProperty("driftCorrectionAmount", static_cast<double>(n.driftCorrectionAmount)); - noteObj->setProperty("vibratoDepth", static_cast<double>(n.vibratoDepth)); - noteObj->setProperty("vibratoRate", static_cast<double>(n.vibratoRate)); - noteObj->setProperty("transitionIn", static_cast<double>(n.transitionIn)); - noteObj->setProperty("transitionOut", static_cast<double>(n.transitionOut)); - noteObj->setProperty("formantShift", static_cast<double>(n.formantShift)); - noteObj->setProperty("gain", static_cast<double>(n.gain)); - noteObj->setProperty("voiced", n.voiced); - - // Downsample drift to max ~50 points per note for efficient rendering + for (const auto& note : result.notes) + { + juce::DynamicObject::Ptr noteObject = new juce::DynamicObject(); + noteObject->setProperty("id", note.id); + noteObject->setProperty("startTime", static_cast<double>(note.startTime)); + noteObject->setProperty("endTime", static_cast<double>(note.endTime)); + noteObject->setProperty("effectiveStartTime", static_cast<double>(note.effectiveStartTime)); + noteObject->setProperty("effectiveEndTime", static_cast<double>(note.effectiveEndTime)); + noteObject->setProperty("detectedPitch", static_cast<double>(note.detectedPitch)); + noteObject->setProperty("correctedPitch", static_cast<double>(note.correctedPitch)); + noteObject->setProperty("driftCorrectionAmount", static_cast<double>(note.driftCorrectionAmount)); + noteObject->setProperty("vibratoDepth", static_cast<double>(note.vibratoDepth)); + noteObject->setProperty("vibratoRate", static_cast<double>(note.vibratoRate)); + noteObject->setProperty("transitionIn", static_cast<double>(note.transitionIn)); + noteObject->setProperty("transitionOut", static_cast<double>(note.transitionOut)); + noteObject->setProperty("formantShift", static_cast<double>(note.formantShift)); + noteObject->setProperty("gain", static_cast<double>(note.gain)); + noteObject->setProperty("voiced", note.voiced); + noteObject->setProperty("wordGroupId", note.wordGroupId.isEmpty() ? note.id : note.wordGroupId); + noteObject->setProperty("entryBoundaryKind", note.entryBoundaryKind.isEmpty() ? "unknown" : note.entryBoundaryKind); + noteObject->setProperty("exitBoundaryKind", note.exitBoundaryKind.isEmpty() ? "unknown" : note.exitBoundaryKind); + noteObject->setProperty("entryBoundaryReason", note.entryBoundaryReason); + noteObject->setProperty("exitBoundaryReason", note.exitBoundaryReason); + noteObject->setProperty("entryBoundaryScore", static_cast<double>(note.entryBoundaryScore)); + noteObject->setProperty("exitBoundaryScore", static_cast<double>(note.exitBoundaryScore)); + juce::Array<juce::var> drift; const int maxDriftPoints = 50; - const int driftSize = static_cast<int>(n.pitchDrift.size()); + const int driftSize = static_cast<int>(note.pitchDrift.size()); if (driftSize <= maxDriftPoints) { - for (float d : n.pitchDrift) - drift.add(static_cast<double>(d)); + for (const float value : note.pitchDrift) + drift.add(static_cast<double>(value)); } else { const float step = static_cast<float>(driftSize) / static_cast<float>(maxDriftPoints); - for (int di = 0; di < maxDriftPoints; ++di) + for (int driftIndex = 0; driftIndex < maxDriftPoints; ++driftIndex) { - int idx = static_cast<int>(static_cast<float>(di) * step); - drift.add(static_cast<double>(n.pitchDrift[static_cast<size_t>(idx)])); + const int sourceIndex = static_cast<int>(static_cast<float>(driftIndex) * step); + drift.add(static_cast<double>(note.pitchDrift[static_cast<size_t>(sourceIndex)])); } } - noteObj->setProperty("pitchDrift", drift); + noteObject->setProperty("pitchDrift", drift); - notesList.add(juce::var(noteObj.get())); + notesList.add(juce::var(noteObject.get())); } root->setProperty("notes", notesList); + juce::Array<juce::var> boundaryList; + for (const auto& candidate : result.boundaryCandidates) + { + juce::DynamicObject::Ptr candidateObject = new juce::DynamicObject(); + candidateObject->setProperty("id", candidate.id); + candidateObject->setProperty("sourceNoteId", candidate.sourceNoteId); + candidateObject->setProperty("time", static_cast<double>(candidate.time)); + candidateObject->setProperty("kind", candidate.kind.isEmpty() ? "unknown" : candidate.kind); + candidateObject->setProperty("reason", candidate.reason); + candidateObject->setProperty("score", static_cast<double>(candidate.score)); + candidateObject->setProperty("destructiveSplitAllowed", candidate.destructiveSplitAllowed); + boundaryList.add(juce::var(candidateObject.get())); + } + root->setProperty("boundaryCandidates", boundaryList); + return juce::var(root.get()); } -std::vector<PitchAnalyzer::PitchNote> PitchAnalyzer::notesFromJSON(const juce::var &json) +std::vector<PitchAnalyzer::PitchNote> PitchAnalyzer::notesFromJSON(const juce::var& json) { std::vector<PitchNote> notes; - if (auto *arr = json.getArray()) + if (auto* array = json.getArray()) { - for (const auto &item : *arr) + for (const auto& item : *array) { - if (auto *obj = item.getDynamicObject()) + if (auto* object = item.getDynamicObject()) { - PitchNote n; - n.id = obj->getProperty("id").toString(); - n.startTime = static_cast<float>(static_cast<double>(obj->getProperty("startTime"))); - n.endTime = static_cast<float>(static_cast<double>(obj->getProperty("endTime"))); - n.detectedPitch = static_cast<float>(static_cast<double>(obj->getProperty("detectedPitch"))); - n.correctedPitch = static_cast<float>(static_cast<double>(obj->getProperty("correctedPitch"))); - n.driftCorrectionAmount = static_cast<float>(static_cast<double>(obj->getProperty("driftCorrectionAmount"))); - n.vibratoDepth = static_cast<float>(static_cast<double>(obj->getProperty("vibratoDepth"))); - n.vibratoRate = static_cast<float>(static_cast<double>(obj->getProperty("vibratoRate"))); - n.transitionIn = static_cast<float>(static_cast<double>(obj->getProperty("transitionIn"))); - n.transitionOut = static_cast<float>(static_cast<double>(obj->getProperty("transitionOut"))); - n.formantShift = static_cast<float>(static_cast<double>(obj->getProperty("formantShift"))); - n.gain = static_cast<float>(static_cast<double>(obj->getProperty("gain"))); - n.voiced = obj->hasProperty("voiced") ? static_cast<bool>(obj->getProperty("voiced")) : true; - - if (auto *driftArr = obj->getProperty("pitchDrift").getArray()) + PitchNote note; + note.id = object->getProperty("id").toString(); + note.startTime = static_cast<float>(static_cast<double>(object->getProperty("startTime"))); + note.endTime = static_cast<float>(static_cast<double>(object->getProperty("endTime"))); + note.effectiveStartTime = object->hasProperty("effectiveStartTime") + ? static_cast<float>(static_cast<double>(object->getProperty("effectiveStartTime"))) + : note.startTime; + note.effectiveEndTime = object->hasProperty("effectiveEndTime") + ? static_cast<float>(static_cast<double>(object->getProperty("effectiveEndTime"))) + : note.endTime; + note.detectedPitch = static_cast<float>(static_cast<double>(object->getProperty("detectedPitch"))); + note.correctedPitch = static_cast<float>(static_cast<double>(object->getProperty("correctedPitch"))); + note.driftCorrectionAmount = static_cast<float>(static_cast<double>(object->getProperty("driftCorrectionAmount"))); + note.vibratoDepth = static_cast<float>(static_cast<double>(object->getProperty("vibratoDepth"))); + note.vibratoRate = static_cast<float>(static_cast<double>(object->getProperty("vibratoRate"))); + note.transitionIn = static_cast<float>(static_cast<double>(object->getProperty("transitionIn"))); + note.transitionOut = static_cast<float>(static_cast<double>(object->getProperty("transitionOut"))); + note.formantShift = static_cast<float>(static_cast<double>(object->getProperty("formantShift"))); + note.gain = static_cast<float>(static_cast<double>(object->getProperty("gain"))); + note.voiced = object->hasProperty("voiced") ? static_cast<bool>(object->getProperty("voiced")) : true; + note.wordGroupId = object->hasProperty("wordGroupId") + ? object->getProperty("wordGroupId").toString() + : note.id; + note.entryBoundaryKind = object->hasProperty("entryBoundaryKind") + ? object->getProperty("entryBoundaryKind").toString() + : "unknown"; + note.exitBoundaryKind = object->hasProperty("exitBoundaryKind") + ? object->getProperty("exitBoundaryKind").toString() + : "unknown"; + note.entryBoundaryReason = object->hasProperty("entryBoundaryReason") + ? object->getProperty("entryBoundaryReason").toString() + : juce::String(); + note.exitBoundaryReason = object->hasProperty("exitBoundaryReason") + ? object->getProperty("exitBoundaryReason").toString() + : juce::String(); + note.entryBoundaryScore = object->hasProperty("entryBoundaryScore") + ? static_cast<float>(static_cast<double>(object->getProperty("entryBoundaryScore"))) + : 0.0f; + note.exitBoundaryScore = object->hasProperty("exitBoundaryScore") + ? static_cast<float>(static_cast<double>(object->getProperty("exitBoundaryScore"))) + : 0.0f; + + if (auto* driftArray = object->getProperty("pitchDrift").getArray()) { - for (const auto &d : *driftArr) - n.pitchDrift.push_back(static_cast<float>(static_cast<double>(d))); + for (const auto& value : *driftArray) + note.pitchDrift.push_back(static_cast<float>(static_cast<double>(value))); } - notes.push_back(std::move(n)); + notes.push_back(std::move(note)); } } } @@ -530,40 +1521,44 @@ std::vector<PitchAnalyzer::PitchNote> PitchAnalyzer::notesFromJSON(const juce::v return notes; } -std::vector<PitchAnalyzer::PitchFrame> PitchAnalyzer::framesFromJSON(const juce::var &json) +std::vector<PitchAnalyzer::PitchFrame> PitchAnalyzer::framesFromJSON(const juce::var& json) { std::vector<PitchFrame> frames; - if (auto *obj = json.getDynamicObject()) - { - auto *times = obj->getProperty("times").getArray(); - auto *midi = obj->getProperty("midi").getArray(); - auto *conf = obj->getProperty("confidence").getArray(); - auto *rms = obj->getProperty("rms").getArray(); - auto *voiced = obj->getProperty("voiced").getArray(); - - if (times == nullptr || midi == nullptr) - return frames; - - const int count = times->size(); - frames.reserve(static_cast<size_t>(count)); - - for (int i = 0; i < count; ++i) - { - PitchFrame f; - f.time = static_cast<float>(static_cast<double>((*times)[i])); - f.midiNote = static_cast<float>(static_cast<double>((*midi)[i])); - f.frequency = (f.midiNote > 0.0f) ? 440.0f * std::pow(2.0f, (f.midiNote - 69.0f) / 12.0f) : 0.0f; - f.confidence = (conf != nullptr && i < conf->size()) - ? static_cast<float>(static_cast<double>((*conf)[i])) - : 0.5f; - f.rmsDB = (rms != nullptr && i < rms->size()) - ? static_cast<float>(static_cast<double>((*rms)[i])) - : -60.0f; - f.voiced = (voiced != nullptr && i < voiced->size()) - ? static_cast<bool>((*voiced)[i]) - : true; - frames.push_back(f); + if (auto* object = json.getDynamicObject()) + { + const auto timesVar = object->getProperty("times"); + const auto midiVar = object->getProperty("midi"); + const auto confidenceVar = object->getProperty("confidence"); + const auto rmsVar = object->getProperty("rms"); + const auto voicedVar = object->getProperty("voiced"); + + auto* timesArray = timesVar.getArray(); + auto* midiArray = midiVar.getArray(); + auto* confidenceArray = confidenceVar.getArray(); + auto* rmsArray = rmsVar.getArray(); + auto* voicedArray = voicedVar.getArray(); + + if (timesArray && midiArray && confidenceArray && rmsArray) + { + const int frameCount = std::min({ timesArray->size(), midiArray->size(), confidenceArray->size(), rmsArray->size() }); + frames.reserve(static_cast<size_t>(frameCount)); + + for (int index = 0; index < frameCount; ++index) + { + PitchFrame frame; + frame.time = static_cast<float>(static_cast<double>((*timesArray)[index])); + frame.midiNote = static_cast<float>(static_cast<double>((*midiArray)[index])); + frame.frequency = frame.midiNote > 0.0f + ? 440.0f * std::pow(2.0f, (frame.midiNote - 69.0f) / 12.0f) + : 0.0f; + frame.confidence = static_cast<float>(static_cast<double>((*confidenceArray)[index])); + frame.rmsDB = static_cast<float>(static_cast<double>((*rmsArray)[index])); + frame.voiced = voicedArray && index < voicedArray->size() + ? static_cast<bool>((*voicedArray)[index]) + : (frame.midiNote > 0.0f && frame.confidence >= 0.25f); + frames.push_back(frame); + } } } diff --git a/Source/PitchAnalyzer.h b/Source/PitchAnalyzer.h index 914c275..fd984bc 100644 --- a/Source/PitchAnalyzer.h +++ b/Source/PitchAnalyzer.h @@ -3,6 +3,8 @@ #include <JuceHeader.h> #include <vector> #include <string> +#include <memory> +#include <functional> /** * PitchAnalyzer — Offline pitch contour analysis for graphical editing. @@ -36,6 +38,8 @@ class PitchAnalyzer juce::String id; // unique ID float startTime; // seconds float endTime; // seconds + float effectiveStartTime; // seconds, includes rendered pitch shoulder + float effectiveEndTime; // seconds, includes rendered pitch shoulder float detectedPitch; // average MIDI note (fractional) float correctedPitch; // target (initially = detected, user edits) float driftCorrectionAmount; // 0 = original, 1 = straight @@ -46,9 +50,27 @@ class PitchAnalyzer float formantShift; // semitones float gain; // dB adjustment bool voiced = true; // true = pitched vocal, false = unvoiced segment + juce::String wordGroupId; // stable editable word/phrase group + juce::String entryBoundaryKind; // unknown | hard_word_like | soft_legato | internal_bend | internal_vibrato + juce::String exitBoundaryKind; // unknown | hard_word_like | soft_legato | internal_bend | internal_vibrato + juce::String entryBoundaryReason; + juce::String exitBoundaryReason; + float entryBoundaryScore = 0.0f; + float exitBoundaryScore = 0.0f; std::vector<float> pitchDrift; // per-frame deviation from note center }; + struct PitchBoundaryCandidate + { + juce::String id; + juce::String sourceNoteId; + float time = 0.0f; + juce::String kind = "unknown"; // hard_word_like | soft_legato | internal_bend | internal_vibrato + juce::String reason; + float score = 0.0f; + bool destructiveSplitAllowed = false; + }; + struct AnalysisResult { juce::String clipId; @@ -56,6 +78,7 @@ class PitchAnalyzer int hopSize; std::vector<PitchFrame> frames; std::vector<PitchNote> notes; + std::vector<PitchBoundaryCandidate> boundaryCandidates; }; /** @@ -68,7 +91,8 @@ class PitchAnalyzer * @return Full analysis result with frames and notes */ AnalysisResult analyzeClip(const float* audioData, int numSamples, - double sampleRate, const juce::String& clipId); + double sampleRate, const juce::String& clipId, + std::function<bool()> shouldCancel = {}); // Analysis parameters void setMinFrequency(float hz) { minFreq = hz; } @@ -89,16 +113,30 @@ class PitchAnalyzer float maxFreq = 1000.0f; float sensitivity = 0.15f; + static constexpr int analysisFrameSize = 2048; + static constexpr int analysisFftOrder = 12; // 4096-point FFT for 2048-sample frames + static constexpr int analysisFftSize = 1 << analysisFftOrder; + // YIN on a single frame float analyzeFrame(const float* frame, int frameSize, double sampleRate, float& outConfidence); // Note segmentation from frame data std::vector<PitchNote> segmentNotes(const std::vector<PitchFrame>& frames, - int hopSize, double sampleRate); + int hopSize, double sampleRate, + std::vector<PitchBoundaryCandidate>* boundaryCandidates = nullptr); // YIN buffer (reused across frames) std::vector<float> yinBuffer; + std::vector<float> analysisWindow; + std::vector<float> differenceBuffer; + std::vector<float> directDifferenceBuffer; + std::vector<float> cmndfBuffer; + std::vector<float> autocorrelationBuffer; + std::vector<float> prefixEnergyBuffer; + std::vector<float> windowedFrameBuffer; + std::vector<juce::dsp::Complex<float>> fftBuffer; + std::unique_ptr<juce::dsp::FFT> fft; JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(PitchAnalyzer) }; diff --git a/Source/PitchDetector.cpp b/Source/PitchDetector.cpp index 71131ab..fce7afd 100644 --- a/Source/PitchDetector.cpp +++ b/Source/PitchDetector.cpp @@ -84,7 +84,7 @@ void PitchDetector::processSamples(const float* samples, int numSamples) * * 1. Compute difference function d(tau) * 2. Compute cumulative mean normalized difference d'(tau) - * 3. Find first minimum below threshold (pYIN approach) + * 3. Find first minimum below threshold (classic YIN first-dip heuristic) * 4. Parabolic interpolation for sub-sample accuracy */ float PitchDetector::runYIN(const float* frame, int size) @@ -133,7 +133,7 @@ float PitchDetector::runYIN(const float* frame, int size) bestTau = tau; bestVal = yinBuffer[static_cast<size_t>(tau)]; - break; // Take first minimum below threshold (pYIN heuristic) + break; // Take first minimum below threshold (YIN first-dip heuristic) } } diff --git a/Source/PitchDetector.h b/Source/PitchDetector.h index 7504483..2d4fba2 100644 --- a/Source/PitchDetector.h +++ b/Source/PitchDetector.h @@ -5,10 +5,12 @@ #include <atomic> /** - * PitchDetector — pYIN-based fundamental frequency estimation. + * PitchDetector — YIN-style real-time fundamental frequency estimation. * - * Operates on mono audio frames. Uses the probabilistic YIN (pYIN) algorithm - * for better accuracy with octave errors and noisy signals. + * Operates on mono audio frames. This is a lightweight YIN-style detector + * used for live tracking and UI feedback. It does not yet implement the + * offline pYIN-style candidate generation and temporal decoding used by the + * pitch-editor analysis path. * * Thread-safe: analyzeFrame() is called from audio thread, * results read from UI thread via atomics. @@ -59,7 +61,7 @@ class PitchDetector float maxFreq = 1000.0f; float sensitivityThreshold = 0.15f; // YIN threshold - // pYIN internals + // YIN internals std::vector<float> yinBuffer; float runYIN(const float* frame, int size); float parabolicInterpolation(int tauEstimate) const; diff --git a/Source/PitchResynthesizer.cpp b/Source/PitchResynthesizer.cpp index a737bda..406232a 100644 --- a/Source/PitchResynthesizer.cpp +++ b/Source/PitchResynthesizer.cpp @@ -1,21 +1,27 @@ #include "PitchResynthesizer.h" -#include "SignalsmithShifter.h" +#include "LpcEnvelopeTransfer.h" +#include "OwnPitchEngine.h" #include <juce_core/juce_core.h> +#include <juce_dsp/juce_dsp.h> #include <cmath> #include <algorithm> +#include <complex> #include <cstring> #include <limits> #include <numeric> +static bool shouldEnablePitchEditorFormantDebugLogs() +{ #if JUCE_DEBUG -static constexpr bool kPitchEditorFormantDebugLogs = true; + return true; #else -static constexpr bool kPitchEditorFormantDebugLogs = false; + return juce::SystemStats::getEnvironmentVariable ("OPENSTUDIO_PITCH_DEBUG", {}).trim() == "1"; #endif +} static void logPitchEditorFormant(const juce::String& message) { - if (kPitchEditorFormantDebugLogs) + if (shouldEnablePitchEditorFormantDebugLogs()) juce::Logger::writeToLog ("[pitchEditor.formant] " + message); } @@ -48,609 +54,5060 @@ static float smoothstep01 (float x) return x * x * (3.0f - 2.0f * x); } +static float minimumJerk01 (float x) +{ + x = juce::jlimit (0.0f, 1.0f, x); + return x * x * x * (10.0f + x * (-15.0f + 6.0f * x)); +} + +static float equalPowerFadeIn (float x) +{ + x = juce::jlimit (0.0f, 1.0f, x); + return std::sin (0.5f * juce::MathConstants<float>::pi * x); +} + +static float equalPowerFadeOut (float x) +{ + x = juce::jlimit (0.0f, 1.0f, x); + return std::cos (0.5f * juce::MathConstants<float>::pi * x); +} + static float mapRequestedFormantRatio (float requestedRatio) { requestedRatio = juce::jlimit (0.25f, 4.0f, requestedRatio); if (requestedRatio >= 1.0f) - return std::pow (requestedRatio, 1.10f); - return std::pow (requestedRatio, 1.08f); + return std::pow (requestedRatio, 1.00f); + return std::pow (requestedRatio, 1.18f); } -// --------------------------------------------------------------------------- -// Formant post-correction — second pass after Signalsmith Stretch. -// -// Uses log-domain moving average for spectral envelope estimation. -// No IFFT/FFT round-trip = no normalization ambiguity, guaranteed correct. -// Smoothing width adapts to detected pitch (3× fundamental spacing in bins). -// -// This is a RESIDUAL corrector: Signalsmith's built-in compensatePitch=true -// does first-pass correction; this catches what the library missed. -// --------------------------------------------------------------------------- -static void applyFormantPostCorrection ( - std::vector<std::vector<float>>& output, - const float* const* originalInput, - int numChannels, - int numSamples, - double sampleRate, - const std::vector<float>& ratios, - const std::vector<float>& detectedPitchHz) +static float sanitizeFiniteFloat (float value, float fallback = 0.0f) { - const float ratioThreshold = 0.06f; - bool hasShift = false; - for (size_t i = 0; i < ratios.size() && ! hasShift; i += 256) - if (std::abs (ratios[i] - 1.0f) > ratioThreshold) hasShift = true; - if (! hasShift) return; + return std::isfinite (value) ? value : fallback; +} - const int fftOrder = 12; - const int fftSize = 1 << fftOrder; - const int hopLen = fftSize / 4; - const int halfBins = fftSize / 2 + 1; - const float pi = juce::MathConstants<float>::pi; - const float binHz = static_cast<float> (sampleRate) / static_cast<float> (fftSize); +static float getEnvFloat (const char* name, float fallback) +{ + const auto value = juce::SystemStats::getEnvironmentVariable (name, {}).trim(); + if (value.isEmpty()) + return fallback; + return sanitizeFiniteFloat (value.getFloatValue(), fallback); +} - juce::dsp::FFT fft (fftOrder); +static int getEnvInt (const char* name, int fallback) +{ + const auto value = juce::SystemStats::getEnvironmentVariable (name, {}).trim(); + if (value.isEmpty()) + return fallback; + return value.getIntValue(); +} - std::vector<float> hannWin (static_cast<size_t> (fftSize)); - for (int i = 0; i < fftSize; ++i) - hannWin[static_cast<size_t> (i)] = 0.5f * (1.0f - std::cos ( - 2.0f * pi * static_cast<float> (i) / static_cast<float> (fftSize - 1))); +static juce::File getPitchLayerDumpDirectory() +{ + auto path = juce::SystemStats::getEnvironmentVariable ("OPENSTUDIO_VSF_LAYER_DUMP_DIR", {}).trim(); + if (path.isEmpty()) + path = juce::SystemStats::getEnvironmentVariable ("OPENSTUDIO_PITCH_LAYER_DUMP_DIR", {}).trim(); + return path.isEmpty() ? juce::File() : juce::File (path); +} - std::vector<float> origFFTBuf (static_cast<size_t> (fftSize * 2)); - std::vector<float> outFFTBuf (static_cast<size_t> (fftSize * 2)); - std::vector<float> origLogMag (static_cast<size_t> (halfBins)); - std::vector<float> outLogMag (static_cast<size_t> (halfBins)); - std::vector<float> origEnv (static_cast<size_t> (halfBins)); - std::vector<float> outEnv (static_cast<size_t> (halfBins)); +static bool writePitchLayerDumpWav ( + const juce::File& directory, + const juce::String& name, + const std::vector<std::vector<float>>& channels, + double sampleRate) +{ + if (directory == juce::File() || channels.empty() || sampleRate <= 0.0) + return false; - // Log-domain moving average spectral envelope estimation. - // Smooths log magnitudes with a window of ~3× the fundamental frequency - // spacing in bins. This averages over ~3 harmonics, yielding a smooth - // formant envelope. Log domain = geometric mean, which follows formant - // peaks without being pulled by individual harmonics. - auto computeEnvelope = [&] (const float* fftBufData, int smoothHalfW, - const std::vector<float>& logMag, - std::vector<float>& env) - { - juce::ignoreUnused (fftBufData); - for (int i = 0; i < halfBins; ++i) - { - int lo = std::max (0, i - smoothHalfW); - int hi = std::min (halfBins - 1, i + smoothHalfW); - float sum = 0.0f; - for (int j = lo; j <= hi; ++j) - sum += logMag[static_cast<size_t> (j)]; - env[static_cast<size_t> (i)] = std::exp (sum / static_cast<float> (hi - lo + 1)); - } - }; + const int numChannels = static_cast<int> (channels.size()); + const int numSamples = static_cast<int> (channels.front().size()); + if (numChannels <= 0 || numSamples <= 0) + return false; + + directory.createDirectory(); + const auto file = directory.getChildFile (name + ".wav"); + file.deleteFile(); + juce::AudioBuffer<float> buffer (numChannels, numSamples); + buffer.clear(); for (int ch = 0; ch < numChannels; ++ch) { - auto& out = output[static_cast<size_t> (ch)]; - const float* orig = originalInput[ch]; + if (channels[static_cast<size_t> (ch)].empty()) + continue; + const int copyCount = std::min (numSamples, static_cast<int> (channels[static_cast<size_t> (ch)].size())); + buffer.copyFrom (ch, 0, channels[static_cast<size_t> (ch)].data(), copyCount); + } - std::vector<float> corrected (static_cast<size_t> (numSamples), 0.0f); - std::vector<float> winSum (static_cast<size_t> (numSamples), 0.0f); + juce::WavAudioFormat format; + std::unique_ptr<juce::FileOutputStream> stream (file.createOutputStream()); + if (stream == nullptr) + return false; - for (int pos = 0; pos < numSamples; pos += hopLen) - { - float avgRatio = 0.0f; - int cnt = 0; - for (int i = pos; i < std::min (pos + fftSize, numSamples); ++i) - { - if (static_cast<size_t> (i) < ratios.size()) - { - avgRatio += ratios[static_cast<size_t> (i)]; - ++cnt; - } - } - avgRatio = (cnt > 0) ? avgRatio / static_cast<float> (cnt) : 1.0f; + std::unique_ptr<juce::AudioFormatWriter> writer ( + format.createWriterFor (stream.get(), sampleRate, static_cast<unsigned int> (numChannels), 24, {}, 0)); + if (writer == nullptr) + return false; - if (std::abs (avgRatio - 1.0f) <= ratioThreshold) - { - for (int i = 0; i < fftSize; ++i) - { - int idx = pos + i; - if (idx >= 0 && idx < numSamples) - { - float w = hannWin[static_cast<size_t> (i)]; - corrected[static_cast<size_t> (idx)] += out[static_cast<size_t> (idx)] * w; - winSum[static_cast<size_t> (idx)] += w * w; - } - } - continue; - } + stream.release(); + return writer->writeFromAudioSampleBuffer (buffer, 0, numSamples); +} - // Window and FFT both signals - std::fill (origFFTBuf.begin(), origFFTBuf.end(), 0.0f); - std::fill (outFFTBuf.begin(), outFFTBuf.end(), 0.0f); - for (int i = 0; i < fftSize; ++i) - { - int idx = pos + i; - if (idx >= 0 && idx < numSamples) - { - float w = hannWin[static_cast<size_t> (i)]; - origFFTBuf[static_cast<size_t> (i)] = orig[idx] * w; - outFFTBuf[static_cast<size_t> (i)] = out[static_cast<size_t> (idx)] * w; - } - } - fft.performRealOnlyForwardTransform (origFFTBuf.data(), true); - fft.performRealOnlyForwardTransform (outFFTBuf.data(), true); +static float getEffectiveNoteStartTime (const PitchAnalyzer::PitchNote& note) +{ + return std::min (note.startTime, note.effectiveStartTime); +} - // Compute log magnitudes - for (int i = 0; i < halfBins; ++i) - { - float ore = origFFTBuf[static_cast<size_t> (i * 2)]; - float oim = origFFTBuf[static_cast<size_t> (i * 2 + 1)]; - origLogMag[static_cast<size_t> (i)] = std::log (std::sqrt (ore * ore + oim * oim) + 1e-10f); +static float getEffectiveNoteEndTime (const PitchAnalyzer::PitchNote& note) +{ + return std::max (note.endTime, note.effectiveEndTime); +} - float sre = outFFTBuf[static_cast<size_t> (i * 2)]; - float sim = outFFTBuf[static_cast<size_t> (i * 2 + 1)]; - outLogMag[static_cast<size_t> (i)] = std::log (std::sqrt (sre * sre + sim * sim) + 1e-10f); - } +static bool hasPitchStyleEdit (const PitchAnalyzer::PitchNote& note) +{ + if (std::abs (note.correctedPitch - note.detectedPitch) > 0.01f) + return true; + if (note.driftCorrectionAmount > 0.01f) + return true; + if (std::abs (note.vibratoDepth - 1.0f) > 0.01f) + return true; + for (const auto drift : note.pitchDrift) + if (std::abs (drift) > 0.01f) + return true; + return false; +} - // Adaptive smoothing width: ~3× fundamental spacing in bins. - // For 200Hz voice at 44.1kHz/4096: 200/10.77 ≈ 18.6 bins, 3× = 56 bins half-width ~28 - float avgPitch = 0.0f; - int pitchCnt = 0; - if (! detectedPitchHz.empty()) - { - for (int i = pos; i < std::min (pos + fftSize, numSamples); ++i) - if (static_cast<size_t> (i) < detectedPitchHz.size() - && detectedPitchHz[static_cast<size_t> (i)] > 0.0f) - { - avgPitch += detectedPitchHz[static_cast<size_t> (i)]; - ++pitchCnt; - } - if (pitchCnt > 0) avgPitch /= static_cast<float> (pitchCnt); - } - // Smoothing width: 5× fundamental spacing in bins. - // Wide enough to average over ~5 harmonics → captures only the - // broad formant shape, never follows individual harmonic peaks. - // This is critical: if the envelope tracks harmonics, the correction - // cuts/boosts harmonics individually → artificial sound. - int smoothHalfW; - if (avgPitch > 50.0f) - smoothHalfW = std::max (15, static_cast<int> (2.5f * avgPitch / binHz)); - else - smoothHalfW = 50; // fallback: ~540Hz at 44.1kHz/4096 - smoothHalfW = std::min (smoothHalfW, halfBins / 3); +static std::vector<std::vector<float>> copyInputChannels (const float* const* input, + int numChannels, + int numSamples) +{ + std::vector<std::vector<float>> result (static_cast<size_t> (std::max (0, numChannels))); + for (int ch = 0; ch < numChannels; ++ch) + { + if (input == nullptr || input[ch] == nullptr || numSamples <= 0) + result[static_cast<size_t> (ch)].assign (static_cast<size_t> (std::max (0, numSamples)), 0.0f); + else + result[static_cast<size_t> (ch)].assign (input[ch], input[ch] + numSamples); + } + return result; +} + +struct PitchEditDirectionSummary +{ + juce::String name = "none"; + bool hasUpward = false; + bool hasDownward = false; + bool hasMixed = false; + float averageShiftSemitones = 0.0f; +}; - computeEnvelope (origFFTBuf.data(), smoothHalfW, origLogMag, origEnv); - computeEnvelope (outFFTBuf.data(), smoothHalfW, outLogMag, outEnv); +static PitchEditDirectionSummary getPitchEditDirectionSummary (const std::vector<PitchAnalyzer::PitchNote>& notes) +{ + PitchEditDirectionSummary summary; + float sum = 0.0f; + int count = 0; + for (const auto& note : notes) + { + if (! hasPitchStyleEdit (note)) + continue; - // Correction strength — gentle residual on top of library's first pass. - // Kept moderate to avoid destroying harmonics. - float shiftMag = std::abs (avgRatio - 1.0f); - float maxStrength = (avgRatio < 1.0f) ? 0.75f : 0.55f; - float strength = juce::jlimit (0.0f, maxStrength, - (shiftMag - ratioThreshold) * 2.5f); + const float shift = note.correctedPitch - note.detectedPitch; + if (shift > 0.01f) + summary.hasUpward = true; + else if (shift < -0.01f) + summary.hasDownward = true; + else + continue; - // Measure energy before correction - float energyBefore = 0.0f; - for (int i = 0; i < halfBins; ++i) - { - float re = outFFTBuf[static_cast<size_t> (i * 2)]; - float im = outFFTBuf[static_cast<size_t> (i * 2 + 1)]; - energyBefore += re * re + im * im; - } + sum += shift; + ++count; + } - // Apply spectral envelope correction - for (int i = 0; i < halfBins; ++i) - { - float gain = std::pow (origEnv[static_cast<size_t> (i)] - / (outEnv[static_cast<size_t> (i)] + 1e-10f), strength); - gain = juce::jlimit (0.5f, 2.0f, gain); // ±6dB max — reshape, never remove harmonics - outFFTBuf[static_cast<size_t> (i * 2)] *= gain; - outFFTBuf[static_cast<size_t> (i * 2 + 1)] *= gain; - } + if (count > 0) + summary.averageShiftSemitones = sum / static_cast<float> (count); + summary.hasMixed = summary.hasUpward && summary.hasDownward; + if (summary.hasMixed) + summary.name = "mixed"; + else if (summary.hasDownward) + summary.name = "downward"; + else if (summary.hasUpward) + summary.name = "upward"; + return summary; +} - // Preserve total spectral energy - float energyAfter = 0.0f; - for (int i = 0; i < halfBins; ++i) - { - float re = outFFTBuf[static_cast<size_t> (i * 2)]; - float im = outFFTBuf[static_cast<size_t> (i * 2 + 1)]; - energyAfter += re * re + im * im; - } - if (energyAfter > 1e-10f) - { - float scale = std::sqrt (energyBefore / energyAfter); - for (int i = 0; i < halfBins; ++i) - { - outFFTBuf[static_cast<size_t> (i * 2)] *= scale; - outFFTBuf[static_cast<size_t> (i * 2 + 1)] *= scale; - } - } +static constexpr bool kEnablePitchOnlyStageB = true; - fft.performRealOnlyInverseTransform (outFFTBuf.data()); +enum class PitchOnlyRendererBranch +{ + SimpleCe33, + CurrentAdvanced, + HybridReset, + HybridStructural, + AdaptiveSelector, + EngineV2Program, + IslandNative, + IslandNativePsola, + PsolaCore, + ModelCore, + VocalSourceFilterHq, + OwnEnginePitchOnly, + OwnEngineFormantOnly, + OwnEnginePitchPlusFormant, + EngineV3FullClip, + EngineV3FullClipLpc, + EngineV3FullClipLpcTransient +}; - for (int i = 0; i < fftSize; ++i) - { - int idx = pos + i; - if (idx >= 0 && idx < numSamples) - { - float w = hannWin[static_cast<size_t> (i)]; - corrected[static_cast<size_t> (idx)] += outFFTBuf[static_cast<size_t> (i)] * w; - winSum[static_cast<size_t> (idx)] += w * w; - } - } - } +enum class PitchOnlyRecoveryPath +{ + LegacyNatural, + NeutralFormantMinimal, + CurrentExperimental +}; - for (int s = 0; s < numSamples; ++s) - { - if (winSum[static_cast<size_t> (s)] > 0.001f) - out[static_cast<size_t> (s)] = corrected[static_cast<size_t> (s)] - / winSum[static_cast<size_t> (s)]; - } - } +static PitchOnlyRecoveryPath getPitchOnlyRecoveryPath() +{ + return PitchOnlyRecoveryPath::CurrentExperimental; } -static void applyExplicitFormantWarp ( - std::vector<std::vector<float>>& output, - const float* const* originalInput, - int numChannels, - int numSamples, - double sampleRate, - const std::vector<float>& formantRatios, - const std::vector<float>& detectedPitchHz, - float warpIntensity, - bool formantOnly, - PitchResynthesizer::RenderQuality renderQuality, - std::function<bool()> shouldCancel) +static const char* getPitchOnlyRecoveryPathName (PitchOnlyRecoveryPath path) { - const bool previewFast = renderQuality == PitchResynthesizer::RenderQuality::PreviewFast; - const float ratioThreshold = 0.03f; - bool hasShift = false; - float requestedMinRatio = 1.0f; - float requestedMaxRatio = 1.0f; + juce::ignoreUnused (path); + return "native_vsf_only"; +} - for (size_t i = 0; i < formantRatios.size(); i += 256) +static PitchOnlyRendererBranch getPitchOnlyRendererBranch() +{ + const auto branch = juce::SystemStats::getEnvironmentVariable ("OPENSTUDIO_PITCH_RENDERER_BRANCH", {}) + .trim() + .toLowerCase(); + + if (branch == "pitch_only_vocal_source_filter_hq" + || branch == "vocal_source_filter_hq" + || branch == "vocal_source_filter" + || branch == "source_filter_hq") { - const float ratio = formantRatios[i]; - requestedMinRatio = std::min (requestedMinRatio, ratio); - requestedMaxRatio = std::max (requestedMaxRatio, ratio); - if (std::abs (ratio - 1.0f) > ratioThreshold) - hasShift = true; + return PitchOnlyRendererBranch::VocalSourceFilterHq; } - if (! hasShift) - return; + if (branch.isNotEmpty()) + logPitchEditorFormant ("ignoring retired pitch renderer branch override: " + branch); - const int fftOrder = formantOnly - ? (previewFast ? 10 : 11) - : (previewFast ? 11 : 12); - const int fftSize = 1 << fftOrder; - const int hopLen = previewFast ? (fftSize / 2) : (fftSize / 4); - const int halfBins = fftSize / 2 + 1; - const float pi = juce::MathConstants<float>::pi; - const float binHz = static_cast<float> (sampleRate) / static_cast<float> (fftSize); + return PitchOnlyRendererBranch::VocalSourceFilterHq; +} - juce::dsp::FFT fft (fftOrder); +static const char* getPitchOnlyRendererBranchName (PitchOnlyRendererBranch branch) +{ + juce::ignoreUnused (branch); + return "pitch_only_vocal_source_filter_hq"; +} - std::vector<float> hannWin (static_cast<size_t> (fftSize)); - for (int i = 0; i < fftSize; ++i) - hannWin[static_cast<size_t> (i)] = 0.5f * (1.0f - std::cos ( - 2.0f * pi * static_cast<float> (i) / static_cast<float> (fftSize - 1))); +static bool isEngineV3Branch (PitchOnlyRendererBranch branch) +{ + return branch == PitchOnlyRendererBranch::EngineV3FullClip + || branch == PitchOnlyRendererBranch::EngineV3FullClipLpc + || branch == PitchOnlyRendererBranch::EngineV3FullClipLpcTransient; +} - std::vector<float> origFFTBuf (static_cast<size_t> (fftSize * 2)); - std::vector<float> outFFTBuf (static_cast<size_t> (fftSize * 2)); - std::vector<float> preWarpFFTBuf (static_cast<size_t> (fftSize * 2)); - std::vector<float> origLogMag (static_cast<size_t> (halfBins)); - std::vector<float> outLogMag (static_cast<size_t> (halfBins)); - std::vector<float> origEnv (static_cast<size_t> (halfBins)); - std::vector<float> outEnv (static_cast<size_t> (halfBins)); - std::vector<float> targetEnv (static_cast<size_t> (halfBins)); +static bool shouldUseEngineV3LpcTransfer (PitchOnlyRendererBranch branch) +{ + juce::ignoreUnused (branch); + return false; +} - auto computeEnvelope = [&] (const std::vector<float>& logMag, - int smoothHalfW, - std::vector<float>& env) - { - for (int i = 0; i < halfBins; ++i) - { - const int lo = std::max (0, i - smoothHalfW); - const int hi = std::min (halfBins - 1, i + smoothHalfW); - float sum = 0.0f; - for (int j = lo; j <= hi; ++j) - sum += logMag[static_cast<size_t> (j)]; - env[static_cast<size_t> (i)] = std::exp (sum / static_cast<float> (hi - lo + 1)); - } - }; +static bool shouldUseEngineV3TransientBypass (PitchOnlyRendererBranch branch) +{ + return branch == PitchOnlyRendererBranch::EngineV3FullClipLpcTransient; +} - int shiftedBlocks = 0; - float appliedMinRatio = std::numeric_limits<float>::max(); - float appliedMaxRatio = 0.0f; - float avgVoicedBlendApplied = 0.0f; - float minSmoothedVoicedBlend = std::numeric_limits<float>::max(); - float maxSmoothedVoicedBlend = 0.0f; - float minEffectiveStrength = std::numeric_limits<float>::max(); - float maxEffectiveStrength = 0.0f; - float totalEnvelopeDeltaBefore = 0.0f; - float totalEnvelopeDeltaAfter = 0.0f; - int totalEnvelopeDeltaBins = 0; +juce::String PitchResynthesizer::getRequestedPitchRendererBranchName() +{ + return juce::String (getPitchOnlyRendererBranchName (getPitchOnlyRendererBranch())); +} - for (int ch = 0; ch < numChannels; ++ch) +static float getPitchOnlyStageBWetScale (PitchOnlyRendererBranch rendererBranch, bool downwardShift) +{ + const auto specificName = downwardShift + ? "OPENSTUDIO_PITCH_STAGEB_SCALE_DOWN" + : "OPENSTUDIO_PITCH_STAGEB_SCALE_UP"; + const auto specificValue = juce::SystemStats::getEnvironmentVariable (specificName, {}).trim(); + if (specificValue.isNotEmpty()) + return juce::jlimit (0.0f, 4.0f, specificValue.getFloatValue()); + + const auto globalValue = juce::SystemStats::getEnvironmentVariable ("OPENSTUDIO_PITCH_STAGEB_SCALE", {}).trim(); + if (globalValue.isNotEmpty()) + return juce::jlimit (0.0f, 4.0f, globalValue.getFloatValue()); + + if (rendererBranch == PitchOnlyRendererBranch::SimpleCe33) + return 0.0f; + + if (rendererBranch == PitchOnlyRendererBranch::HybridReset + || rendererBranch == PitchOnlyRendererBranch::HybridStructural) + return downwardShift ? 0.45f : 0.20f; + + return downwardShift ? 1.5f : 1.25f; +} + +static OwnPitchEngine::Quality toOwnEngineQuality (PitchResynthesizer::RenderQuality renderQuality) +{ + return renderQuality == PitchResynthesizer::RenderQuality::PreviewFast + ? OwnPitchEngine::Quality::PreviewFast + : OwnPitchEngine::Quality::FinalHQ; +} + +static float getAverageEditedNoteDurationSec (const std::vector<PitchAnalyzer::PitchNote>& notes) +{ + float durationSum = 0.0f; + int durationCount = 0; + for (const auto& note : notes) { - auto& out = output[static_cast<size_t> (ch)]; - const float* orig = originalInput[ch]; + if (! hasPitchStyleEdit (note)) + continue; + durationSum += std::max (0.0f, note.endTime - note.startTime); + ++durationCount; + } - std::vector<float> corrected (static_cast<size_t> (numSamples), 0.0f); - std::vector<float> winSum (static_cast<size_t> (numSamples), 0.0f); - float prevVoicedBlend = 0.0f; - float prevStrength = 0.0f; - float prevOriginalBlend = 0.0f; + return durationCount > 0 ? durationSum / static_cast<float> (durationCount) : 0.0f; +} - for (int pos = 0; pos < numSamples; pos += hopLen) +struct HybridBridgeDiagnostics +{ + bool bridgeUsed = false; + bool bridgeFallbackUsed = false; + int bridgeStartSample = 0; + int bridgeLengthSamples = 0; + int bridgeAlignmentLagSamples = 0; + float bridgeCorrelationScore = 0.0f; + float bridgeGainDeltaDb = 0.0f; +}; + +struct HybridBodyReplacementDiagnostics +{ + bool bodyReplacementUsed = false; + bool bodyReplacementFallbackUsed = false; + int entryLockStartSample = 0; + int entryLockLengthSamples = 0; + int exitLockStartSample = 0; + int renderedBodyStartSample = 0; + int renderedBodyEndSample = 0; +}; + +struct HybridStructuralBlendResult +{ + std::vector<std::vector<float>> output; + HybridBridgeDiagnostics diagnostics; + HybridBodyReplacementDiagnostics bodyReplacementDiagnostics; + struct IslandNativeDiagnostics + { + bool islandNativeUsed = false; + bool islandNativeFallbackUsed = false; + int islandRenderStartSample = 0; + int islandRenderEndSample = 0; + float transientMaskPeak = 0.0f; + float voicedCoreMaskPeak = 0.0f; + } islandNativeDiagnostics; + struct HpssDiagnostics + { + bool hpssUsed = false; + bool hpssFallbackUsed = false; + int islandRenderStartSample = 0; + int islandRenderEndSample = 0; + float transientMaskPeak = 0.0f; + float harmonicMaskPeak = 0.0f; + float aperiodicMaskPeak = 0.0f; + bool spectralEnvelopeCorrectionUsed = false; + } hpssDiagnostics; +}; + +struct EngineV2ScaffoldDiagnostics +{ + bool engineV2Used = false; + bool engineV2FallbackUsed = false; + int transitionCount = 0; + int firstTransitionStartSample = 0; + int lastTransitionEndSample = 0; + float harmonicSupportPeak = 0.0f; + float residualSupportPeak = 0.0f; + float envelopeSupportPeak = 0.0f; +}; + +static bool isUpwardEditedNote (const PitchAnalyzer::PitchNote& note) +{ + return hasPitchStyleEdit (note) && (note.correctedPitch - note.detectedPitch) > 0.01f; +} + +static EngineV2ScaffoldDiagnostics buildEngineV2ScaffoldDiagnostics ( + const OwnPitchEngine::SharedAnalysis& analysis, + const std::vector<PitchAnalyzer::PitchNote>& notes, + double sampleRate) +{ + juce::ignoreUnused (notes); + EngineV2ScaffoldDiagnostics diagnostics; + + if (analysis.islands.empty() || sampleRate <= 0.0) + { + diagnostics.engineV2FallbackUsed = true; + return diagnostics; + } + + for (const auto& island : analysis.islands) + { + bool islandHasEligibleNote = false; + for (const auto& note : island.notes) { - if (shouldCancel && shouldCancel()) - return; - float avgFormantRatio = 0.0f; - int ratioCount = 0; - float frameEnergy = 0.0f; - int frameSamples = 0; - for (int i = pos; i < std::min (pos + fftSize, numSamples); ++i) + if (! isUpwardEditedNote (note)) + continue; + islandHasEligibleNote = true; + break; + } + + if (! islandHasEligibleNote) + continue; + + diagnostics.engineV2Used = true; + diagnostics.transitionCount += 1; + + const int transitionLeadSamples = std::max (1, static_cast<int> (std::round (0.030 * sampleRate))); + const int transitionTailSamples = std::max (1, static_cast<int> (std::round (0.060 * sampleRate))); + const int absoluteBodyStartSample = island.contextStartSample + island.bodyStartSample; + const int absoluteBodyEndSample = island.contextStartSample + island.bodyEndSample; + const int transitionStart = std::max (island.renderStartSample, absoluteBodyStartSample - transitionLeadSamples); + const int transitionEnd = std::min (island.renderEndSample, absoluteBodyEndSample + transitionTailSamples); + + if (diagnostics.transitionCount == 1) + { + diagnostics.firstTransitionStartSample = transitionStart; + diagnostics.lastTransitionEndSample = transitionEnd; + } + else + { + diagnostics.firstTransitionStartSample = std::min (diagnostics.firstTransitionStartSample, transitionStart); + diagnostics.lastTransitionEndSample = std::max (diagnostics.lastTransitionEndSample, transitionEnd); + } + + const int localStart = std::max (0, transitionStart - island.contextStartSample); + const int localEnd = std::min (static_cast<int> (island.voicedMask.size()), transitionEnd - island.contextStartSample); + + for (int i = localStart; i < localEnd; ++i) + { + const float voicedSupport = i < static_cast<int> (island.voicedMask.size()) + ? island.voicedMask[static_cast<size_t> (i)] + : 0.0f; + const float envelope = i < static_cast<int> (island.amplitudeEnvelope.size()) + ? island.amplitudeEnvelope[static_cast<size_t> (i)] + : 0.0f; + diagnostics.harmonicSupportPeak = std::max (diagnostics.harmonicSupportPeak, voicedSupport); + diagnostics.envelopeSupportPeak = std::max (diagnostics.envelopeSupportPeak, envelope); + } + + diagnostics.residualSupportPeak = std::max ( + diagnostics.residualSupportPeak, + std::max (island.residualModel.suggestedMix, + std::max (island.residualModel.voicedMix, island.residualModel.highBandMix))); + + if (! island.spectralEnvelopeModel.averageMagnitude.empty()) + { + const auto peakIt = std::max_element (island.spectralEnvelopeModel.averageMagnitude.begin(), + island.spectralEnvelopeModel.averageMagnitude.end()); + if (peakIt != island.spectralEnvelopeModel.averageMagnitude.end()) + diagnostics.envelopeSupportPeak = std::max (diagnostics.envelopeSupportPeak, *peakIt); + } + } + + if (! diagnostics.engineV2Used) + diagnostics.engineV2FallbackUsed = true; + + return diagnostics; +} + +struct EngineV2ProgramRenderResult +{ + std::vector<std::vector<float>> output; + EngineV2ScaffoldDiagnostics diagnostics; + bool spectralEnvelopeCorrectionUsed = false; + bool transientBypassUsed = false; + bool residualCarryUsed = false; + int cepstralCutoffUsed = 0; + int fftSizeUsed = 0; + int hopSizeUsed = 0; +}; + +static const PitchAnalyzer::PitchNote* findEngineV2LeadUpwardNote ( + const OwnPitchEngine::NoteIslandAnalysis& island) +{ + const PitchAnalyzer::PitchNote* best = nullptr; + float bestShift = 0.0f; + for (const auto& note : island.notes) + { + const float shift = note.correctedPitch - note.detectedPitch; + if (shift <= 0.01f) + continue; + if (best == nullptr || shift > bestShift) + { + best = ¬e; + bestShift = shift; + } + } + return best; +} + +static std::vector<float> buildEngineV2SpectralFlatnessMask ( + const std::vector<float>& monoSignal, + int startSample, + int endSample, + double sampleRate, + float flatnessCenter = 0.38f, + float flatnessWidth = 0.20f, + float minRms = 0.003f, + float rmsWidth = 0.020f) +{ + const int signalSamples = static_cast<int> (monoSignal.size()); + std::vector<float> mask (static_cast<size_t> (signalSamples), 0.0f); + std::vector<float> weight (static_cast<size_t> (signalSamples), 0.0f); + if (signalSamples <= 0 || endSample <= startSample || sampleRate <= 0.0) + return mask; + + const int fftOrder = 9; + const int fftSize = 1 << fftOrder; + const int halfBins = fftSize / 2 + 1; + const int hopSize = fftSize / 4; + juce::dsp::FFT fft (fftOrder); + std::vector<float> hann (static_cast<size_t> (fftSize), 0.0f); + for (int i = 0; i < fftSize; ++i) + hann[static_cast<size_t> (i)] = 0.5f * (1.0f - std::cos ( + juce::MathConstants<float>::twoPi * static_cast<float> (i) / static_cast<float> (fftSize - 1))); + + std::vector<juce::dsp::Complex<float>> fftIn (static_cast<size_t> (fftSize)); + std::vector<juce::dsp::Complex<float>> fftOut (static_cast<size_t> (fftSize)); + const int paddedStart = std::max (0, startSample - fftSize / 2); + const int paddedEnd = std::min (signalSamples, endSample + fftSize / 2); + + for (int pos = paddedStart; pos < paddedEnd; pos += hopSize) + { + double logSum = 0.0; + double magSum = 0.0; + double rmsSum = 0.0; + int rmsCount = 0; + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + const float sample = (idx >= 0 && idx < signalSamples) + ? monoSignal[static_cast<size_t> (idx)] * hann[static_cast<size_t> (i)] + : 0.0f; + fftIn[static_cast<size_t> (i)] = { sample, 0.0f }; + rmsSum += static_cast<double> (sample) * sample; + ++rmsCount; + } + + fft.perform (fftIn.data(), fftOut.data(), false); + for (int bin = 1; bin < halfBins; ++bin) + { + const float mag = std::max (1.0e-8f, std::abs (fftOut[static_cast<size_t> (bin)])); + logSum += std::log (mag); + magSum += mag; + } + + const float rms = rmsCount > 0 ? std::sqrt (static_cast<float> (rmsSum / static_cast<double> (rmsCount))) : 0.0f; + const float geometricMean = std::exp (static_cast<float> (logSum / static_cast<double> (std::max (1, halfBins - 1)))); + const float arithmeticMean = static_cast<float> (magSum / static_cast<double> (std::max (1, halfBins - 1))); + const float flatness = arithmeticMean > 1.0e-8f ? geometricMean / arithmeticMean : 0.0f; + const float energyGate = smoothstep01 ((rms - minRms) / std::max (1.0e-5f, rmsWidth)); + const float bypass = smoothstep01 ((flatness - flatnessCenter) / std::max (0.02f, flatnessWidth)) * energyGate; + const int frameStart = std::max (startSample, pos); + const int frameEnd = std::min (endSample, pos + hopSize); + for (int s = frameStart; s < frameEnd; ++s) + { + mask[static_cast<size_t> (s)] += bypass; + weight[static_cast<size_t> (s)] += 1.0f; + } + } + + for (size_t i = 0; i < mask.size(); ++i) + { + if (weight[i] > 0.0f) + mask[i] = juce::jlimit (0.0f, 1.0f, mask[i] / weight[i]); + } + return mask; +} + +static bool applyEngineV2CepstralEnvelopeRestore ( + std::vector<float>& processed, + const std::vector<float>& original, + const std::vector<float>& voicedSupport, + const std::vector<float>& detectedPitchHz, + int coreStartSample, + int coreEndSample, + double sampleRate, + int* usedLifterCutoffOut = nullptr, + int* usedFftSizeOut = nullptr, + int* usedHopSizeOut = nullptr, + float lifterScale = 0.36f, + float correctionStrengthBase = 0.54f, + float correctionStrengthSlope = 0.18f, + float maxCorrectionDb = 0.0f, + int fixedLifterCutoff = 0); + +struct AdaptiveBoundaryCorrectionResult +{ + std::vector<std::vector<float>> output; + bool used = false; + bool spectralEnvelopeCorrectionUsed = false; + bool transientBypassUsed = false; + bool residualCarryUsed = false; + int cepstralCutoffUsed = 0; + int fftSizeUsed = 0; + int hopSizeUsed = 0; + float transientMaskPeak = 0.0f; + float voicedSupportPeak = 0.0f; +}; + +static AdaptiveBoundaryCorrectionResult applyAdaptiveBoundaryCorrections ( + const float* const* originalInput, + const std::vector<std::vector<float>>& adaptiveOutput, + const OwnPitchEngine::SharedAnalysis& analysis, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& pitchRatios, + int numChannels, + int numSamples, + double sampleRate) +{ + juce::ignoreUnused (notes); + + AdaptiveBoundaryCorrectionResult result; + result.output = adaptiveOutput; + + if (analysis.islands.empty() || numChannels <= 0 || numSamples <= 0 || sampleRate <= 0.0) + return result; + + const float flatnessCenter = getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_FLATNESS_CENTER", 0.31f); + const float flatnessWidth = getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_FLATNESS_WIDTH", 0.15f); + const float minRms = getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_MIN_RMS", 0.0035f); + const float rmsWidth = getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_RMS_WIDTH", 0.018f); + const float entryCrossfadeMs = juce::jlimit (5.0f, 15.0f, getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_ENTRY_CROSSFADE_MS", 8.0f)); + const float exitCrossfadeMs = juce::jlimit (5.0f, 18.0f, getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_EXIT_CROSSFADE_MS", 15.0f)); + const float entryBiasMs = juce::jlimit (-8.0f, 8.0f, getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_ENTRY_BIAS_MS", -5.0f)); + const float entryCoreWet = juce::jlimit (0.04f, 0.28f, getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_ENTRY_CORE_WET", 0.065f)); + const float exitCoreWet = juce::jlimit (0.02f, 0.20f, getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_EXIT_CORE_WET", 0.035f)); + const float entryResidualWet = juce::jlimit (0.0f, 0.08f, getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_ENTRY_RESIDUAL_WET", 0.0f)); + const float exitResidualWet = juce::jlimit (0.0f, 0.06f, getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_EXIT_RESIDUAL_WET", 0.0f)); + const float entryLeadMs = juce::jlimit (6.0f, 28.0f, getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_ENTRY_PRE_MS", 8.0f)); + const float entryTailMs = juce::jlimit (18.0f, 80.0f, getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_ENTRY_POST_MS", 30.0f)); + const float exitLeadMs = juce::jlimit (14.0f, 70.0f, getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_EXIT_PRE_MS", 26.0f)); + const float exitTailMs = juce::jlimit (4.0f, 20.0f, getEnvFloat ("OPENSTUDIO_PITCH_ADAPTIVE_EXIT_POST_MS", 8.0f)); + const float cepstralLifterScale = juce::jlimit (0.24f, 0.60f, getEnvFloat ("OPENSTUDIO_PITCH_CEPSTRAL_LIFTER_SCALE", 0.36f)); + const float cepstralStrengthBase = juce::jlimit (0.30f, 0.80f, getEnvFloat ("OPENSTUDIO_PITCH_CEPSTRAL_STRENGTH_BASE", 0.54f)); + const float cepstralStrengthSlope = juce::jlimit (0.05f, 0.35f, getEnvFloat ("OPENSTUDIO_PITCH_CEPSTRAL_STRENGTH_SLOPE", 0.18f)); + const int entryLeadSamples = std::max (1, static_cast<int> (std::round (entryLeadMs * 0.001 * sampleRate))); + const int entryTailSamples = std::max (1, static_cast<int> (std::round (entryTailMs * 0.001 * sampleRate))); + const int exitLeadSamples = std::max (1, static_cast<int> (std::round (exitLeadMs * 0.001 * sampleRate))); + const int exitTailSamples = std::max (1, static_cast<int> (std::round (exitTailMs * 0.001 * sampleRate))); + + auto applyWindow = [&] ( + const OwnPitchEngine::NoteIslandAnalysis& island, + int windowStart, + int windowEnd, + int focusCenterSample, + bool isEntryWindow) + { + const int transitionStart = juce::jlimit (0, numSamples, windowStart); + const int transitionEnd = juce::jlimit (transitionStart, numSamples, windowEnd); + const int transitionSamples = transitionEnd - transitionStart; + if (transitionSamples <= 64) + return; + const float windowCrossfadeMs = isEntryWindow ? entryCrossfadeMs : exitCrossfadeMs; + const float windowCoreWet = isEntryWindow ? entryCoreWet : exitCoreWet; + const float windowResidualWet = isEntryWindow ? entryResidualWet : exitResidualWet; + const int outerFadeSamples = std::max (1, static_cast<int> (std::round (windowCrossfadeMs * 0.001 * sampleRate))); + + const int localIslandStart = juce::jlimit (0, static_cast<int> (island.monoSignal.size()), transitionStart - island.contextStartSample); + const int localIslandEnd = juce::jlimit (localIslandStart, static_cast<int> (island.monoSignal.size()), transitionEnd - island.contextStartSample); + auto flatnessMask = buildEngineV2SpectralFlatnessMask ( + island.monoSignal, + localIslandStart, + localIslandEnd, + sampleRate, + flatnessCenter, + flatnessWidth, + minRms, + rmsWidth); + + const float maxEnvelope = (localIslandEnd > localIslandStart && ! island.amplitudeEnvelope.empty()) + ? *std::max_element (island.amplitudeEnvelope.begin() + localIslandStart, + island.amplitudeEnvelope.begin() + localIslandEnd) + : 0.0f; + const float envThreshold = std::max (0.007f, maxEnvelope * 0.16f); + + std::vector<std::vector<float>> localOriginal (static_cast<size_t> (numChannels), std::vector<float> (static_cast<size_t> (transitionSamples), 0.0f)); + std::vector<const float*> localInputPtrs (static_cast<size_t> (numChannels), nullptr); + for (int ch = 0; ch < numChannels; ++ch) + { + for (int i = 0; i < transitionSamples; ++i) + localOriginal[static_cast<size_t> (ch)][static_cast<size_t> (i)] = originalInput[ch][transitionStart + i]; + localInputPtrs[static_cast<size_t> (ch)] = localOriginal[static_cast<size_t> (ch)].data(); + } + + std::vector<float> localRatios (static_cast<size_t> (transitionSamples), 1.0f); + std::vector<float> localDetectedPitchHz (static_cast<size_t> (transitionSamples), island.core.meanF0Hz); + std::vector<float> localVoicedSupport (static_cast<size_t> (transitionSamples), 0.0f); + std::vector<float> localTransientMask (static_cast<size_t> (transitionSamples), 0.0f); + std::vector<float> localResidualWeight (static_cast<size_t> (transitionSamples), 0.0f); + std::vector<float> localCoreFocus (static_cast<size_t> (transitionSamples), 0.0f); + + const float centerSampleF = static_cast<float> (focusCenterSample); + for (int i = 0; i < transitionSamples; ++i) + { + const int absoluteSample = transitionStart + i; + if (absoluteSample >= 0 && absoluteSample < static_cast<int> (pitchRatios.size())) + localRatios[static_cast<size_t> (i)] = pitchRatios[static_cast<size_t> (absoluteSample)]; + + const int islandIndex = localIslandStart + i; + const float voiced = islandIndex >= 0 && islandIndex < static_cast<int> (island.voicedMask.size()) + ? island.voicedMask[static_cast<size_t> (islandIndex)] + : 0.0f; + const float consonant = islandIndex >= 0 && islandIndex < static_cast<int> (island.consonantMask.size()) + ? island.consonantMask[static_cast<size_t> (islandIndex)] + : 0.0f; + const float envelope = islandIndex >= 0 && islandIndex < static_cast<int> (island.amplitudeEnvelope.size()) + ? island.amplitudeEnvelope[static_cast<size_t> (islandIndex)] + : 0.0f; + const float f0 = islandIndex >= 0 && islandIndex < static_cast<int> (island.f0TrackHz.size()) + ? island.f0TrackHz[static_cast<size_t> (islandIndex)] + : island.core.meanF0Hz; + localDetectedPitchHz[static_cast<size_t> (i)] = f0 > 0.0f ? f0 : island.core.meanF0Hz; + + const float biasedOffsetSec = (static_cast<float> (absoluteSample) - centerSampleF) / static_cast<float> (sampleRate) + + 0.001f * entryBiasMs * (isEntryWindow ? 1.0f : -0.5f); + const float focusIn = smoothstep01 ((biasedOffsetSec + 0.012f) / 0.015f); + const float focusOut = 1.0f - smoothstep01 ((biasedOffsetSec - 0.052f) / 0.026f); + const float coreFocus = juce::jlimit (0.0f, 1.0f, focusIn * focusOut); + const float boundaryBodyFocus = isEntryWindow + ? smoothstep01 ((biasedOffsetSec + 0.004f) / 0.019f) + : (1.0f - smoothstep01 ((biasedOffsetSec - 0.008f) / 0.026f)); + const float transientMask = juce::jlimit (0.0f, 1.0f, + std::max ( + std::max (islandIndex < static_cast<int> (flatnessMask.size()) ? flatnessMask[static_cast<size_t> (islandIndex)] : 0.0f, + consonant * 0.88f), + isEntryWindow + ? (1.0f - smoothstep01 ((biasedOffsetSec + 0.010f) / 0.020f)) + : (1.0f - smoothstep01 (((-biasedOffsetSec) + 0.008f) / 0.020f)))); + const float envelopeGate = smoothstep01 ((envelope - envThreshold) / std::max (envThreshold * 1.8f, 1.0e-4f)); + const float voicedSupport = juce::jlimit (0.0f, 1.0f, + voiced * envelopeGate * coreFocus * boundaryBodyFocus * (1.0f - 0.96f * transientMask)); + const float residualWeight = juce::jlimit (0.0f, windowResidualWet, + voicedSupport * voicedSupport * boundaryBodyFocus * island.residualModel.highBandMix * 0.35f); + + localTransientMask[static_cast<size_t> (i)] = transientMask; + localVoicedSupport[static_cast<size_t> (i)] = voicedSupport; + localResidualWeight[static_cast<size_t> (i)] = residualWeight; + localCoreFocus[static_cast<size_t> (i)] = coreFocus; + result.transientMaskPeak = std::max (result.transientMaskPeak, transientMask); + result.voicedSupportPeak = std::max (result.voicedSupportPeak, voicedSupport); + result.transientBypassUsed = result.transientBypassUsed || transientMask > 0.20f; + } + + auto voicedCore = copyInputChannels (localInputPtrs.data(), numChannels, transitionSamples); + + int cepstralCutoffUsed = 0; + int fftSizeUsed = 0; + int hopSizeUsed = 0; + const int localCoreLeadSamples = isEntryWindow ? entryLeadSamples : exitLeadSamples; + const int localCoreTailSamples = isEntryWindow ? entryTailSamples : exitTailSamples; + const int localCoreStart = juce::jlimit (0, transitionSamples, focusCenterSample - transitionStart - localCoreLeadSamples); + const int localCoreEnd = juce::jlimit (localCoreStart, transitionSamples, focusCenterSample - transitionStart + localCoreTailSamples); + for (int ch = 0; ch < numChannels; ++ch) + { + result.spectralEnvelopeCorrectionUsed = applyEngineV2CepstralEnvelopeRestore ( + voicedCore[static_cast<size_t> (ch)], + localOriginal[static_cast<size_t> (ch)], + localVoicedSupport, + localDetectedPitchHz, + localCoreStart, + localCoreEnd, + sampleRate, + &cepstralCutoffUsed, + &fftSizeUsed, + &hopSizeUsed, + cepstralLifterScale, + cepstralStrengthBase, + cepstralStrengthSlope) || result.spectralEnvelopeCorrectionUsed; + } + result.cepstralCutoffUsed = std::max (result.cepstralCutoffUsed, cepstralCutoffUsed); + result.fftSizeUsed = std::max (result.fftSizeUsed, fftSizeUsed); + result.hopSizeUsed = std::max (result.hopSizeUsed, hopSizeUsed); + + if (! island.residualModel.voicedHighBandResidual.empty()) + { + for (int ch = 0; ch < numChannels; ++ch) { - if (static_cast<size_t> (i) < formantRatios.size()) + for (int i = 0; i < transitionSamples; ++i) { - avgFormantRatio += formantRatios[static_cast<size_t> (i)]; - ++ratioCount; + const int islandIndex = localIslandStart + i; + if (islandIndex < 0 || islandIndex >= static_cast<int> (island.residualModel.voicedHighBandResidual.size())) + continue; + voicedCore[static_cast<size_t> (ch)][static_cast<size_t> (i)] += + island.residualModel.voicedHighBandResidual[static_cast<size_t> (islandIndex)] + * localResidualWeight[static_cast<size_t> (i)]; } - const float sample = orig[i]; - frameEnergy += sample * sample; - ++frameSamples; } - avgFormantRatio = ratioCount > 0 ? avgFormantRatio / static_cast<float> (ratioCount) : 1.0f; - const float frameRms = frameSamples > 0 ? std::sqrt (frameEnergy / static_cast<float> (frameSamples)) : 0.0f; + result.residualCarryUsed = true; + } - if (std::abs (avgFormantRatio - 1.0f) <= ratioThreshold) + for (int ch = 0; ch < numChannels; ++ch) + { + for (int i = 0; i < transitionSamples; ++i) { - for (int i = 0; i < fftSize; ++i) + const int absoluteSample = transitionStart + i; + const float baseSample = result.output[static_cast<size_t> (ch)][static_cast<size_t> (absoluteSample)]; + const float originalSample = localOriginal[static_cast<size_t> (ch)][static_cast<size_t> (i)]; + const float renderedSample = voicedCore[static_cast<size_t> (ch)][static_cast<size_t> (i)]; + const float transientWeight = localTransientMask[static_cast<size_t> (i)]; + const float voicedSupport = localVoicedSupport[static_cast<size_t> (i)]; + const float correctionWeight = windowCoreWet * voicedSupport; + const float transientKeep = juce::jlimit (0.0f, 1.0f, + std::max (transientWeight, 1.0f - smoothstep01 ((localCoreFocus[static_cast<size_t> (i)] - 0.18f) / 0.52f))); + const float residualDelta = (renderedSample - baseSample) * localResidualWeight[static_cast<size_t> (i)]; + const float entryTimingBlend = isEntryWindow + ? smoothstep01 ((static_cast<float> (i) / static_cast<float> (std::max (1, transitionSamples - 1)) - 0.08f) / 0.18f) + : 1.0f; + const float outerBlendIn = i < outerFadeSamples ? smoothstep01 (static_cast<float> (i) / static_cast<float> (outerFadeSamples)) : 1.0f; + const int fromEnd = transitionSamples - 1 - i; + const float outerBlendOut = fromEnd < outerFadeSamples + ? smoothstep01 (static_cast<float> (fromEnd) / static_cast<float> (outerFadeSamples)) + : 1.0f; + const float outerBlend = std::min (outerBlendIn, outerBlendOut) * entryTimingBlend; + const float delta = (originalSample - baseSample) * transientKeep + + (renderedSample - baseSample) * correctionWeight + + residualDelta; + result.output[static_cast<size_t> (ch)][static_cast<size_t> (absoluteSample)] = + baseSample + delta * outerBlend; + } + } + + result.used = true; + }; + + for (const auto& island : analysis.islands) + { + for (const auto& note : island.notes) + { + if (! hasPitchStyleEdit (note)) + continue; + + const int absoluteEffectiveStart = island.contextStartSample + + static_cast<int> (std::floor (getEffectiveNoteStartTime (note) * sampleRate)); + const int absoluteNoteStart = island.contextStartSample + + static_cast<int> (std::floor (note.startTime * sampleRate)); + const int absoluteNoteEnd = island.contextStartSample + + static_cast<int> (std::ceil (note.endTime * sampleRate)); + const int absoluteEffectiveEnd = island.contextStartSample + + static_cast<int> (std::ceil (getEffectiveNoteEndTime (note) * sampleRate)); + + applyWindow ( + island, + std::max (island.renderStartSample, absoluteEffectiveStart - entryLeadSamples), + std::min (island.renderEndSample, absoluteNoteStart + entryTailSamples), + absoluteNoteStart, + true); + + applyWindow ( + island, + std::max (island.renderStartSample, absoluteNoteEnd - exitLeadSamples), + std::min (island.renderEndSample, absoluteEffectiveEnd + exitTailSamples), + absoluteNoteEnd, + false); + } + } + + if (result.used) + { + logPitchEditorFormant ("adaptive boundary correction" + + juce::String (" flatnessCenter=") + juce::String (flatnessCenter, 3) + + " flatnessWidth=" + juce::String (flatnessWidth, 3) + + " minRms=" + juce::String (minRms, 4) + + " entryCrossfadeMs=" + juce::String (entryCrossfadeMs, 2) + + " exitCrossfadeMs=" + juce::String (exitCrossfadeMs, 2) + + " entryBiasMs=" + juce::String (entryBiasMs, 2) + + " coreWet=" + juce::String (entryCoreWet, 3) + "/" + juce::String (exitCoreWet, 3) + + " residualWet=" + juce::String (entryResidualWet, 3) + "/" + juce::String (exitResidualWet, 3) + + " entryPrePostMs=" + juce::String (entryLeadMs, 2) + "/" + juce::String (entryTailMs, 2) + + " exitPrePostMs=" + juce::String (exitLeadMs, 2) + "/" + juce::String (exitTailMs, 2) + + " lifterScale=" + juce::String (cepstralLifterScale, 3) + + " cepstralStrength=" + juce::String (cepstralStrengthBase, 3) + "+" + juce::String (cepstralStrengthSlope, 3) + "*voiced" + + " transientPeak=" + juce::String (result.transientMaskPeak, 3) + + " voicedPeak=" + juce::String (result.voicedSupportPeak, 3) + + " lifter=" + juce::String (result.cepstralCutoffUsed) + + " fft/hop=" + juce::String (result.fftSizeUsed) + "/" + juce::String (result.hopSizeUsed)); + } + + return result; +} + +static std::vector<float> computeCepstralEnvelope ( + const std::vector<float>& magnitude, + juce::dsp::FFT& fft, + int fftSize, + int lifterCutoff) +{ + const int halfBins = fftSize / 2 + 1; + std::vector<juce::dsp::Complex<float>> logSpectrum (static_cast<size_t> (fftSize), { 0.0f, 0.0f }); + std::vector<juce::dsp::Complex<float>> cepstrum (static_cast<size_t> (fftSize), { 0.0f, 0.0f }); + std::vector<juce::dsp::Complex<float>> lifted (static_cast<size_t> (fftSize), { 0.0f, 0.0f }); + std::vector<juce::dsp::Complex<float>> smoothedSpectrum (static_cast<size_t> (fftSize), { 0.0f, 0.0f }); + + for (int bin = 0; bin < halfBins; ++bin) + logSpectrum[static_cast<size_t> (bin)] = { std::log (std::max (1.0e-7f, magnitude[static_cast<size_t> (bin)])), 0.0f }; + for (int bin = 1; bin < halfBins - 1; ++bin) + logSpectrum[static_cast<size_t> (fftSize - bin)] = logSpectrum[static_cast<size_t> (bin)]; + + fft.perform (logSpectrum.data(), cepstrum.data(), true); + const float inverseScale = 1.0f / static_cast<float> (fftSize); + for (int i = 0; i < fftSize; ++i) + { + const bool keep = i <= lifterCutoff || i >= fftSize - lifterCutoff; + lifted[static_cast<size_t> (i)] = keep ? cepstrum[static_cast<size_t> (i)] * inverseScale + : juce::dsp::Complex<float> { 0.0f, 0.0f }; + } + + fft.perform (lifted.data(), smoothedSpectrum.data(), false); + std::vector<float> envelope (static_cast<size_t> (halfBins), 1.0f); + for (int bin = 0; bin < halfBins; ++bin) + envelope[static_cast<size_t> (bin)] = std::exp (smoothedSpectrum[static_cast<size_t> (bin)].real()); + return envelope; +} + +static float estimateMedianPositiveF0Hz ( + const std::vector<float>& detectedPitchHz, + const std::vector<float>& voicedSupport, + int startSample, + int endSample) +{ + std::vector<float> values; + values.reserve (static_cast<size_t> (std::max (0, endSample - startSample))); + const int boundedEnd = std::min (static_cast<int> (detectedPitchHz.size()), endSample); + for (int i = std::max (0, startSample); i < boundedEnd; ++i) + { + const float f0 = detectedPitchHz[static_cast<size_t> (i)]; + const float support = static_cast<size_t> (i) < voicedSupport.size() ? voicedSupport[static_cast<size_t> (i)] : 0.0f; + if (f0 > 40.0f && support > 0.08f) + values.push_back (f0); + } + + if (values.empty()) + return 0.0f; + + const auto middle = values.begin() + static_cast<std::ptrdiff_t> (values.size() / 2); + std::nth_element (values.begin(), middle, values.end()); + return *middle; +} + +static bool applyEngineV2CepstralEnvelopeRestore ( + std::vector<float>& processed, + const std::vector<float>& original, + const std::vector<float>& voicedSupport, + const std::vector<float>& detectedPitchHz, + int coreStartSample, + int coreEndSample, + double sampleRate, + int* usedLifterCutoffOut, + int* usedFftSizeOut, + int* usedHopSizeOut, + float lifterScale, + float correctionStrengthBase, + float correctionStrengthSlope, + float maxCorrectionDb, + int fixedLifterCutoff) +{ + if (processed.empty() || original.size() != processed.size() || coreEndSample <= coreStartSample || sampleRate <= 0.0) + return false; + + const int defaultFftOrder = processed.size() < 2600 ? 10 : 11; + const int fftOrder = juce::jlimit (10, 12, getEnvInt ("OPENSTUDIO_PITCH_CEPSTRAL_FFT_ORDER", defaultFftOrder)); + const int fftSize = 1 << fftOrder; + const int halfBins = fftSize / 2 + 1; + const int hopDivisor = juce::jlimit (4, 8, getEnvInt ("OPENSTUDIO_PITCH_CEPSTRAL_HOP_DIVISOR", 8)); + const int hopSize = std::max (1, fftSize / hopDivisor); + const float medianF0Hz = estimateMedianPositiveF0Hz (detectedPitchHz, voicedSupport, coreStartSample, coreEndSample); + const float periodSamples = medianF0Hz > 40.0f + ? static_cast<float> (sampleRate / medianF0Hz) + : static_cast<float> (sampleRate / 180.0); + const int computedLifterCutoff = static_cast<int> (std::round (periodSamples * lifterScale)); + const int lifterCutoff = juce::jlimit (8, fftSize / 6, fixedLifterCutoff > 0 ? fixedLifterCutoff : computedLifterCutoff); + const float maxCorrectionGain = maxCorrectionDb > 0.0f + ? std::pow (10.0f, juce::jlimit (0.5f, 6.0f, maxCorrectionDb) / 20.0f) + : 0.0f; + const float minCorrectionGain = maxCorrectionGain > 0.0f ? 1.0f / maxCorrectionGain : 0.0f; + juce::dsp::FFT fft (fftOrder); + std::vector<float> hann (static_cast<size_t> (fftSize), 0.0f); + for (int i = 0; i < fftSize; ++i) + hann[static_cast<size_t> (i)] = 0.5f * (1.0f - std::cos ( + juce::MathConstants<float>::twoPi * static_cast<float> (i) / static_cast<float> (fftSize - 1))); + + std::vector<juce::dsp::Complex<float>> origIn (static_cast<size_t> (fftSize)); + std::vector<juce::dsp::Complex<float>> procIn (static_cast<size_t> (fftSize)); + std::vector<juce::dsp::Complex<float>> origFft (static_cast<size_t> (fftSize)); + std::vector<juce::dsp::Complex<float>> procFft (static_cast<size_t> (fftSize)); + std::vector<juce::dsp::Complex<float>> ifftOut (static_cast<size_t> (fftSize)); + std::vector<float> overlapAdd (processed.size(), 0.0f); + std::vector<float> windowSum (processed.size(), 0.0f); + bool used = false; + if (usedLifterCutoffOut != nullptr) + *usedLifterCutoffOut = lifterCutoff; + if (usedFftSizeOut != nullptr) + *usedFftSizeOut = fftSize; + if (usedHopSizeOut != nullptr) + *usedHopSizeOut = hopSize; + + for (int pos = std::max (0, coreStartSample - fftSize / 2); + pos < std::min (static_cast<int> (processed.size()), coreEndSample + fftSize / 2); + pos += hopSize) + { + float voicedAverage = 0.0f; + int voicedCount = 0; + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + const float origSample = (idx >= 0 && idx < static_cast<int> (original.size())) ? original[static_cast<size_t> (idx)] : 0.0f; + const float procSample = (idx >= 0 && idx < static_cast<int> (processed.size())) ? processed[static_cast<size_t> (idx)] : 0.0f; + origIn[static_cast<size_t> (i)] = { origSample * hann[static_cast<size_t> (i)], 0.0f }; + procIn[static_cast<size_t> (i)] = { procSample * hann[static_cast<size_t> (i)], 0.0f }; + if (idx >= coreStartSample && idx < coreEndSample && static_cast<size_t> (idx) < voicedSupport.size()) + { + voicedAverage += voicedSupport[static_cast<size_t> (idx)]; + ++voicedCount; + } + } + voicedAverage = voicedCount > 0 ? voicedAverage / static_cast<float> (voicedCount) : 0.0f; + + if (voicedAverage < 0.15f) + { + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + if (idx >= 0 && idx < static_cast<int> (processed.size())) { - const int idx = pos + i; - if (idx >= 0 && idx < numSamples) - { - const float w = hannWin[static_cast<size_t> (i)]; - corrected[static_cast<size_t> (idx)] += out[static_cast<size_t> (idx)] * w; - winSum[static_cast<size_t> (idx)] += w * w; - } + const float w = hann[static_cast<size_t> (i)]; + overlapAdd[static_cast<size_t> (idx)] += processed[static_cast<size_t> (idx)] * w; + windowSum[static_cast<size_t> (idx)] += w * w; } - continue; } + continue; + } + + fft.perform (origIn.data(), origFft.data(), false); + fft.perform (procIn.data(), procFft.data(), false); + std::vector<float> origMagnitude (static_cast<size_t> (halfBins), 1.0f); + std::vector<float> procMagnitude (static_cast<size_t> (halfBins), 1.0f); + float energyBefore = 0.0f; + float energyAfter = 0.0f; + + for (int bin = 0; bin < halfBins; ++bin) + { + origMagnitude[static_cast<size_t> (bin)] = std::max (1.0e-7f, std::abs (origFft[static_cast<size_t> (bin)])); + procMagnitude[static_cast<size_t> (bin)] = std::max (1.0e-7f, std::abs (procFft[static_cast<size_t> (bin)])); + energyBefore += procMagnitude[static_cast<size_t> (bin)] * procMagnitude[static_cast<size_t> (bin)]; + } + + const auto origEnvelope = computeCepstralEnvelope (origMagnitude, fft, fftSize, lifterCutoff); + const auto procEnvelope = computeCepstralEnvelope (procMagnitude, fft, fftSize, lifterCutoff); + const float correctionStrength = juce::jlimit (0.24f, 0.92f, correctionStrengthBase + correctionStrengthSlope * voicedAverage); + + for (int bin = 0; bin < halfBins; ++bin) + { + const float envRatio = std::pow ( + (origEnvelope[static_cast<size_t> (bin)] + 1.0e-6f) / (procEnvelope[static_cast<size_t> (bin)] + 1.0e-6f), + correctionStrength); + const float freqNorm = static_cast<float> (bin) / static_cast<float> (std::max (1, halfBins - 1)); + const float cappedGain = maxCorrectionGain > 0.0f + ? juce::jlimit (minCorrectionGain, maxCorrectionGain, envRatio) + : (freqNorm > 0.68f + ? juce::jlimit (0.78f, 1.18f, envRatio) + : juce::jlimit (0.70f, 1.32f, envRatio)); + procFft[static_cast<size_t> (bin)] *= cappedGain; + energyAfter += std::norm (procFft[static_cast<size_t> (bin)]); + if (bin > 0 && bin < halfBins - 1) + procFft[static_cast<size_t> (fftSize - bin)] = std::conj (procFft[static_cast<size_t> (bin)]); + } + + if (energyAfter > 1.0e-8f) + { + const float energyScale = std::pow (energyBefore / energyAfter, 0.12f); + for (int bin = 0; bin < halfBins; ++bin) + { + procFft[static_cast<size_t> (bin)] *= energyScale; + if (bin > 0 && bin < halfBins - 1) + procFft[static_cast<size_t> (fftSize - bin)] = std::conj (procFft[static_cast<size_t> (bin)]); + } + } + + fft.perform (procFft.data(), ifftOut.data(), true); + const float inverseScale = 1.0f / static_cast<float> (fftSize); + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + if (idx >= 0 && idx < static_cast<int> (processed.size())) + { + const float w = hann[static_cast<size_t> (i)]; + overlapAdd[static_cast<size_t> (idx)] += ifftOut[static_cast<size_t> (i)].real() * inverseScale * w; + windowSum[static_cast<size_t> (idx)] += w * w; + } + } + used = true; + } + + if (used) + { + for (int i = coreStartSample; i < coreEndSample && i < static_cast<int> (processed.size()); ++i) + { + if (windowSum[static_cast<size_t> (i)] > 1.0e-4f && voicedSupport[static_cast<size_t> (i)] > 0.08f) + processed[static_cast<size_t> (i)] = overlapAdd[static_cast<size_t> (i)] / windowSum[static_cast<size_t> (i)]; + } + } + + return used; +} + +static bool applyVoicedCoreEnvelopeTransferForPitchOnly ( + std::vector<std::vector<float>>& output, + const float* const* input, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchFrame>& frames, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& detectedPitchHz, + float upwardEnvelopeMix, + float downwardEnvelopeMix, + float maxCorrectionDb, + int fixedLifterCutoff, + int* usedLifterCutoffOut, + int* usedFftSizeOut, + int* usedHopSizeOut, + float* usedEnvelopeMixOut) +{ + if (output.empty() || input == nullptr || numChannels <= 0 || numSamples <= 0 || sampleRate <= 0.0) + return false; + + std::vector<float> voicedSupport (static_cast<size_t> (numSamples), 0.0f); + int hopSamples = 256; + if (frames.size() >= 2) + hopSamples = std::max (1, static_cast<int> (std::round ((frames[1].time - frames[0].time) * sampleRate))); + + for (const auto& frame : frames) + { + if (! frame.voiced || frame.midiNote <= 0.0f || frame.frequency <= 0.0f) + continue; + + const float confidenceSupport = smoothstep01 ((frame.confidence - 0.18f) / 0.42f); + const float energySupport = smoothstep01 ((frame.rmsDB + 54.0f) / 18.0f); + const float support = juce::jlimit (0.0f, 1.0f, confidenceSupport * energySupport); + const int start = juce::jlimit (0, numSamples, static_cast<int> (std::floor (frame.time * sampleRate))); + const int end = juce::jlimit (start, numSamples, start + hopSamples); + for (int i = start; i < end; ++i) + voicedSupport[static_cast<size_t> (i)] = std::max (voicedSupport[static_cast<size_t> (i)], support); + } + + bool used = false; + std::vector<std::vector<float>> original (static_cast<size_t> (numChannels)); + for (int ch = 0; ch < numChannels; ++ch) + original[static_cast<size_t> (ch)].assign (input[ch], input[ch] + numSamples); + + for (const auto& note : notes) + { + if (! hasPitchStyleEdit (note)) + continue; + + const float notePitchDelta = note.correctedPitch - note.detectedPitch; + const bool upward = notePitchDelta > 0.01f; + const bool downward = notePitchDelta < -0.01f; + if (! upward && ! downward) + continue; + + const int noteStart = juce::jlimit (0, numSamples, static_cast<int> (std::round (note.startTime * sampleRate))); + const int noteEnd = juce::jlimit (noteStart, numSamples, static_cast<int> (std::round (note.endTime * sampleRate))); + const int duration = noteEnd - noteStart; + if (duration <= static_cast<int> (std::round (0.055 * sampleRate))) + continue; + + const int guard = juce::jlimit ( + static_cast<int> (std::round (0.010 * sampleRate)), + static_cast<int> (std::round (0.030 * sampleRate)), + duration / 5); + const int coreStart = juce::jlimit (noteStart, noteEnd, noteStart + guard); + const int coreEnd = juce::jlimit (coreStart, noteEnd, noteEnd - guard); + if (coreEnd - coreStart <= static_cast<int> (std::round (0.035 * sampleRate))) + continue; + + for (int ch = 0; ch < numChannels; ++ch) + { + std::vector<float> beforeCore; + beforeCore.reserve (static_cast<size_t> (coreEnd - coreStart)); + for (int i = coreStart; i < coreEnd; ++i) + beforeCore.push_back (output[static_cast<size_t> (ch)][static_cast<size_t> (i)]); + + int lifter = 0; + int fftSize = 0; + int hopSize = 0; + const bool channelUsed = applyEngineV2CepstralEnvelopeRestore ( + output[static_cast<size_t> (ch)], + original[static_cast<size_t> (ch)], + voicedSupport, + detectedPitchHz, + coreStart, + coreEnd, + sampleRate, + &lifter, + &fftSize, + &hopSize, + juce::jlimit (0.28f, 0.55f, getEnvFloat ("OPENSTUDIO_PITCH_CORE_ENVELOPE_LIFTER_SCALE", 0.36f)), + juce::jlimit (0.28f, 0.74f, getEnvFloat ("OPENSTUDIO_PITCH_CORE_ENVELOPE_STRENGTH_BASE", 0.48f)), + juce::jlimit (0.03f, 0.26f, getEnvFloat ("OPENSTUDIO_PITCH_CORE_ENVELOPE_STRENGTH_SLOPE", 0.12f)), + maxCorrectionDb, + fixedLifterCutoff); + if (channelUsed) + { + const char* mixEnvName = upward + ? "OPENSTUDIO_PITCH_CORE_UP_ENVELOPE_MIX" + : "OPENSTUDIO_PITCH_CORE_DOWN_ENVELOPE_MIX"; + const float defaultMix = upward ? upwardEnvelopeMix : downwardEnvelopeMix; + const float mix = juce::jlimit ( + 0.0f, 0.55f, getEnvFloat (mixEnvName, getEnvFloat ("OPENSTUDIO_PITCH_CORE_ENVELOPE_MIX", defaultMix))); + for (int i = coreStart; i < coreEnd; ++i) + { + const size_t local = static_cast<size_t> (i - coreStart); + const float support = static_cast<size_t> (i) < voicedSupport.size() + ? smoothstep01 (voicedSupport[static_cast<size_t> (i)]) + : 0.0f; + auto& sample = output[static_cast<size_t> (ch)][static_cast<size_t> (i)]; + sample = beforeCore[local] + (sample - beforeCore[local]) * mix * support; + } + if (usedEnvelopeMixOut != nullptr) + *usedEnvelopeMixOut = std::max (*usedEnvelopeMixOut, mix); + } + used = channelUsed || used; + if (usedLifterCutoffOut != nullptr) + *usedLifterCutoffOut = std::max (*usedLifterCutoffOut, lifter); + if (usedFftSizeOut != nullptr) + *usedFftSizeOut = std::max (*usedFftSizeOut, fftSize); + if (usedHopSizeOut != nullptr) + *usedHopSizeOut = std::max (*usedHopSizeOut, hopSize); + } + } + + return used; +} + +static EngineV2ProgramRenderResult renderEngineV2Program ( + const float* const* originalInput, + const std::vector<std::vector<float>>& adaptiveOutput, + const OwnPitchEngine::SharedAnalysis& analysis, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& pitchRatios, + int numChannels, + int numSamples, + double sampleRate) +{ + juce::ignoreUnused (notes); + EngineV2ProgramRenderResult result; + result.output = adaptiveOutput; + result.diagnostics = buildEngineV2ScaffoldDiagnostics (analysis, notes, sampleRate); + + if (analysis.islands.empty()) + { + result.diagnostics.engineV2FallbackUsed = true; + return result; + } + + for (const auto& island : analysis.islands) + { + const auto* leadNote = findEngineV2LeadUpwardNote (island); + if (leadNote == nullptr) + continue; + + result.diagnostics.engineV2Used = true; + const float entryLeadMs = juce::jlimit (10.0f, 40.0f, getEnvFloat ("OPENSTUDIO_ENGINEV2_ENTRY_LEAD_MS", 18.0f)); + const float entryTailMs = juce::jlimit (30.0f, 110.0f, getEnvFloat ("OPENSTUDIO_ENGINEV2_ENTRY_TAIL_MS", 68.0f)); + const float transitionOuterFadeMs = juce::jlimit (6.0f, 16.0f, getEnvFloat ("OPENSTUDIO_ENGINEV2_OUTER_FADE_MS", 8.0f)); + const float transientFlatnessCenter = getEnvFloat ("OPENSTUDIO_ENGINEV2_FLATNESS_CENTER", 0.33f); + const float transientFlatnessWidth = getEnvFloat ("OPENSTUDIO_ENGINEV2_FLATNESS_WIDTH", 0.15f); + const float transientMinRms = getEnvFloat ("OPENSTUDIO_ENGINEV2_MIN_RMS", 0.0032f); + const float transientRmsWidth = getEnvFloat ("OPENSTUDIO_ENGINEV2_RMS_WIDTH", 0.018f); + const float entryBiasMs = juce::jlimit (-8.0f, 4.0f, getEnvFloat ("OPENSTUDIO_ENGINEV2_ENTRY_BIAS_MS", -3.5f)); + const float coreWet = juce::jlimit (0.05f, 0.22f, getEnvFloat ("OPENSTUDIO_ENGINEV2_CORE_WET", 0.12f)); + const float residualWet = juce::jlimit (0.0f, 0.04f, getEnvFloat ("OPENSTUDIO_ENGINEV2_RESIDUAL_WET", 0.0f)); + const float cepstralLifterScale = juce::jlimit (0.24f, 0.60f, getEnvFloat ("OPENSTUDIO_ENGINEV2_CEPSTRAL_LIFTER_SCALE", 0.34f)); + const float cepstralStrengthBase = juce::jlimit (0.30f, 0.80f, getEnvFloat ("OPENSTUDIO_ENGINEV2_CEPSTRAL_STRENGTH_BASE", 0.52f)); + const float cepstralStrengthSlope = juce::jlimit (0.05f, 0.35f, getEnvFloat ("OPENSTUDIO_ENGINEV2_CEPSTRAL_STRENGTH_SLOPE", 0.16f)); + const int absoluteNoteStart = island.contextStartSample + + static_cast<int> (std::floor (leadNote->startTime * sampleRate)); + const int absoluteEffectiveStart = island.contextStartSample + + static_cast<int> (std::floor (getEffectiveNoteStartTime (*leadNote) * sampleRate)); + const int transitionLeadSamples = std::max (1, static_cast<int> (std::round (entryLeadMs * 0.001 * sampleRate))); + const int transitionTailSamples = std::max (1, static_cast<int> (std::round (entryTailMs * 0.001 * sampleRate))); + const int transitionStart = juce::jlimit (0, numSamples, + std::max (island.renderStartSample, absoluteEffectiveStart - transitionLeadSamples)); + const int transitionEnd = juce::jlimit (transitionStart, numSamples, + std::min (island.renderEndSample, absoluteNoteStart + transitionTailSamples)); + const int transitionSamples = transitionEnd - transitionStart; + if (transitionSamples <= 64) + continue; + + const int localIslandStart = juce::jlimit (0, static_cast<int> (island.monoSignal.size()), transitionStart - island.contextStartSample); + const int localIslandEnd = juce::jlimit (localIslandStart, static_cast<int> (island.monoSignal.size()), transitionEnd - island.contextStartSample); + const int localCoreStart = juce::jlimit (0, transitionSamples, absoluteNoteStart - transitionStart); + const int localCoreEnd = juce::jlimit (localCoreStart, transitionSamples, + absoluteNoteStart + std::min (transitionTailSamples, + std::max (1, static_cast<int> (std::round (0.085 * sampleRate)))) - transitionStart); + + std::vector<std::vector<float>> localOriginal (static_cast<size_t> (numChannels), std::vector<float> (static_cast<size_t> (transitionSamples), 0.0f)); + std::vector<const float*> localInputPtrs (static_cast<size_t> (numChannels), nullptr); + for (int ch = 0; ch < numChannels; ++ch) + { + for (int i = 0; i < transitionSamples; ++i) + localOriginal[static_cast<size_t> (ch)][static_cast<size_t> (i)] = originalInput[ch][transitionStart + i]; + localInputPtrs[static_cast<size_t> (ch)] = localOriginal[static_cast<size_t> (ch)].data(); + } + + std::vector<float> localRatios (static_cast<size_t> (transitionSamples), 1.0f); + std::vector<float> localDetectedPitchHz (static_cast<size_t> (transitionSamples), island.core.meanF0Hz); + std::vector<float> localVoicedSupport (static_cast<size_t> (transitionSamples), 0.0f); + std::vector<float> localTransientMask (static_cast<size_t> (transitionSamples), 0.0f); + std::vector<float> localResidualWeight (static_cast<size_t> (transitionSamples), 0.0f); + std::vector<float> localEntryFocus (static_cast<size_t> (transitionSamples), 0.0f); + auto flatnessMask = buildEngineV2SpectralFlatnessMask ( + island.monoSignal, + localIslandStart, + localIslandEnd, + sampleRate, + transientFlatnessCenter, + transientFlatnessWidth, + transientMinRms, + transientRmsWidth); + const float maxEnvelope = island.amplitudeEnvelope.empty() + ? 0.0f + : *std::max_element (island.amplitudeEnvelope.begin() + localIslandStart, + island.amplitudeEnvelope.begin() + localIslandEnd); + const float envThreshold = std::max (0.008f, maxEnvelope * 0.18f); + + for (int i = 0; i < transitionSamples; ++i) + { + const int absoluteSample = transitionStart + i; + if (absoluteSample >= 0 && absoluteSample < static_cast<int> (pitchRatios.size())) + localRatios[static_cast<size_t> (i)] = pitchRatios[static_cast<size_t> (absoluteSample)]; + + const int islandIndex = localIslandStart + i; + const float voiced = islandIndex >= 0 && islandIndex < static_cast<int> (island.voicedMask.size()) + ? island.voicedMask[static_cast<size_t> (islandIndex)] + : 0.0f; + const float consonant = islandIndex >= 0 && islandIndex < static_cast<int> (island.consonantMask.size()) + ? island.consonantMask[static_cast<size_t> (islandIndex)] + : 0.0f; + const float envelope = islandIndex >= 0 && islandIndex < static_cast<int> (island.amplitudeEnvelope.size()) + ? island.amplitudeEnvelope[static_cast<size_t> (islandIndex)] + : 0.0f; + const float f0 = islandIndex >= 0 && islandIndex < static_cast<int> (island.f0TrackHz.size()) + ? island.f0TrackHz[static_cast<size_t> (islandIndex)] + : island.core.meanF0Hz; + localDetectedPitchHz[static_cast<size_t> (i)] = f0 > 0.0f ? f0 : island.core.meanF0Hz; + + const float relativeSec = static_cast<float> (absoluteSample - absoluteNoteStart) / static_cast<float> (sampleRate) + + 0.001f * entryBiasMs; + const float focusIn = smoothstep01 ((relativeSec + 0.004f) / 0.014f); + const float focusHoldOut = 1.0f - smoothstep01 ((relativeSec - 0.044f) / 0.020f); + const float entryFocus = juce::jlimit (0.0f, 1.0f, focusIn * focusHoldOut); + const float transientMask = juce::jlimit (0.0f, 1.0f, + std::max (std::max (islandIndex < static_cast<int> (flatnessMask.size()) ? flatnessMask[static_cast<size_t> (islandIndex)] : 0.0f, + consonant * 0.82f), + 1.0f - smoothstep01 ((relativeSec - 0.016f) / 0.018f))); + const float envelopeGate = smoothstep01 ((envelope - envThreshold) / std::max (envThreshold * 2.0f, 1.0e-4f)); + const float voicedSupport = juce::jlimit (0.0f, 1.0f, + voiced * envelopeGate * entryFocus * (1.0f - 0.97f * transientMask)); + const float residualWeight = juce::jlimit (0.0f, residualWet, + voicedSupport * voicedSupport * entryFocus * island.residualModel.highBandMix * 0.22f); + localTransientMask[static_cast<size_t> (i)] = transientMask; + localVoicedSupport[static_cast<size_t> (i)] = voicedSupport; + localResidualWeight[static_cast<size_t> (i)] = residualWeight; + localEntryFocus[static_cast<size_t> (i)] = entryFocus; + result.diagnostics.harmonicSupportPeak = std::max (result.diagnostics.harmonicSupportPeak, voicedSupport); + result.diagnostics.residualSupportPeak = std::max (result.diagnostics.residualSupportPeak, residualWeight); + result.diagnostics.envelopeSupportPeak = std::max (result.diagnostics.envelopeSupportPeak, envelopeGate); + result.transientBypassUsed = result.transientBypassUsed || transientMask > 0.20f; + } + + auto voicedCore = copyInputChannels (localInputPtrs.data(), numChannels, transitionSamples); + + int cepstralCutoffUsed = 0; + int fftSizeUsed = 0; + int hopSizeUsed = 0; + for (int ch = 0; ch < numChannels; ++ch) + { + result.spectralEnvelopeCorrectionUsed = applyEngineV2CepstralEnvelopeRestore ( + voicedCore[static_cast<size_t> (ch)], + localOriginal[static_cast<size_t> (ch)], + localVoicedSupport, + localDetectedPitchHz, + localCoreStart, + localCoreEnd, + sampleRate, + &cepstralCutoffUsed, + &fftSizeUsed, + &hopSizeUsed, + cepstralLifterScale, + cepstralStrengthBase, + cepstralStrengthSlope) || result.spectralEnvelopeCorrectionUsed; + } + result.cepstralCutoffUsed = std::max (result.cepstralCutoffUsed, cepstralCutoffUsed); + result.fftSizeUsed = std::max (result.fftSizeUsed, fftSizeUsed); + result.hopSizeUsed = std::max (result.hopSizeUsed, hopSizeUsed); + + if (! island.residualModel.voicedHighBandResidual.empty()) + { + for (int ch = 0; ch < numChannels; ++ch) + { + for (int i = 0; i < transitionSamples; ++i) + { + const int islandIndex = localIslandStart + i; + if (islandIndex < 0 || islandIndex >= static_cast<int> (island.residualModel.voicedHighBandResidual.size())) + continue; + voicedCore[static_cast<size_t> (ch)][static_cast<size_t> (i)] += + island.residualModel.voicedHighBandResidual[static_cast<size_t> (islandIndex)] + * localResidualWeight[static_cast<size_t> (i)]; + } + } + result.residualCarryUsed = true; + } + + const int outerFadeSamples = std::max (1, static_cast<int> (std::round (transitionOuterFadeMs * 0.001 * sampleRate))); + for (int ch = 0; ch < numChannels; ++ch) + { + for (int i = 0; i < transitionSamples; ++i) + { + const int absoluteSample = transitionStart + i; + const float baseSample = result.output[static_cast<size_t> (ch)][static_cast<size_t> (absoluteSample)]; + const float originalSample = localOriginal[static_cast<size_t> (ch)][static_cast<size_t> (i)]; + const float renderedSample = voicedCore[static_cast<size_t> (ch)][static_cast<size_t> (i)]; + const float transientWeight = localTransientMask[static_cast<size_t> (i)]; + const float entryFocus = localEntryFocus[static_cast<size_t> (i)]; + const float stableVoiced = localVoicedSupport[static_cast<size_t> (i)] * smoothstep01 (localVoicedSupport[static_cast<size_t> (i)]); + const float coreWeight = coreWet * stableVoiced * entryFocus; + const float transientKeep = juce::jlimit (0.0f, 1.0f, + std::max (transientWeight, 1.0f - smoothstep01 ((entryFocus - 0.10f) / 0.60f))); + const float residualDelta = (renderedSample - baseSample) * localResidualWeight[static_cast<size_t> (i)] * 0.06f; + const float outerBlendIn = i < outerFadeSamples ? smoothstep01 (static_cast<float> (i) / static_cast<float> (outerFadeSamples)) : 1.0f; + const int fromEnd = transitionSamples - 1 - i; + const float outerBlendOut = fromEnd < outerFadeSamples + ? smoothstep01 (static_cast<float> (fromEnd) / static_cast<float> (outerFadeSamples)) + : 1.0f; + const float outerBlend = std::min (outerBlendIn, outerBlendOut); + const float delta = (originalSample - baseSample) * transientKeep + + (renderedSample - baseSample) * coreWeight + + residualDelta; + result.output[static_cast<size_t> (ch)][static_cast<size_t> (absoluteSample)] = + baseSample + delta * outerBlend; + } + } + } + + if (! result.diagnostics.engineV2Used) + result.diagnostics.engineV2FallbackUsed = true; + + return result; +} + +static std::vector<float> buildHybridBridgeMono ( + const std::vector<std::vector<float>>& audio, + int numChannels, + int numSamples) +{ + std::vector<float> mono (static_cast<size_t> (numSamples), 0.0f); + if (numChannels <= 0 || numSamples <= 0) + return mono; + + const float scale = 1.0f / static_cast<float> (numChannels); + for (int ch = 0; ch < numChannels; ++ch) + { + if (static_cast<size_t> (ch) >= audio.size()) + continue; + const auto& channel = audio[static_cast<size_t> (ch)]; + const int available = std::min (numSamples, static_cast<int> (channel.size())); + for (int s = 0; s < available; ++s) + mono[static_cast<size_t> (s)] += channel[static_cast<size_t> (s)] * scale; + } + + return mono; +} +static float computeHybridBridgeRms ( + const std::vector<float>& signal, + int startSample, + int lengthSamples) +{ + if (signal.empty() || lengthSamples <= 0) + return 0.0f; + + const int start = juce::jlimit (0, static_cast<int> (signal.size()), startSample); + const int end = juce::jlimit (start, static_cast<int> (signal.size()), start + lengthSamples); + if (end <= start) + return 0.0f; + + double sum = 0.0; + for (int s = start; s < end; ++s) + { + const float v = signal[static_cast<size_t> (s)]; + sum += static_cast<double> (v) * v; + } + + return std::sqrt (static_cast<float> (sum / static_cast<double> (std::max (1, end - start)))); +} + +struct HybridBridgeAlignmentResult +{ + bool found = false; + int lagSamples = 0; + float score = 0.0f; +}; + +static HybridBridgeAlignmentResult findHybridBridgeAlignment ( + const std::vector<float>& legacyMono, + const std::vector<float>& ownMono, + int bridgeStartSample, + int bridgeLengthSamples, + int searchRadiusSamples) +{ + HybridBridgeAlignmentResult result; + if (legacyMono.empty() || ownMono.empty() || bridgeLengthSamples < 8) + return result; + + const int compareLength = std::max (8, bridgeLengthSamples); + const int legacyStart = juce::jlimit (0, static_cast<int> (legacyMono.size()) - compareLength, bridgeStartSample); + + for (int lag = -searchRadiusSamples; lag <= searchRadiusSamples; ++lag) + { + const int ownStart = legacyStart + lag; + if (ownStart < 0 || ownStart + compareLength >= static_cast<int> (ownMono.size())) + continue; + + double dot = 0.0; + double energyLegacy = 0.0; + double energyOwn = 0.0; + int slopeAgreeCount = 0; + + for (int i = 0; i < compareLength; ++i) + { + const float a = legacyMono[static_cast<size_t> (legacyStart + i)]; + const float b = ownMono[static_cast<size_t> (ownStart + i)]; + dot += static_cast<double> (a) * b; + energyLegacy += static_cast<double> (a) * a; + energyOwn += static_cast<double> (b) * b; + + if (i > 0) + { + const float da = a - legacyMono[static_cast<size_t> (legacyStart + i - 1)]; + const float db = b - ownMono[static_cast<size_t> (ownStart + i - 1)]; + if ((da >= 0.0f && db >= 0.0f) || (da <= 0.0f && db <= 0.0f)) + ++slopeAgreeCount; + } + } + + if (energyLegacy <= 1.0e-8 || energyOwn <= 1.0e-8) + continue; + + const float normCorr = juce::jlimit (-1.0f, 1.0f, + static_cast<float> (dot / std::sqrt (energyLegacy * energyOwn))); + const float corrScore = 0.5f * (normCorr + 1.0f); + const float slopeScore = compareLength > 1 + ? static_cast<float> (slopeAgreeCount) / static_cast<float> (compareLength - 1) + : 0.0f; + const float rmsLegacy = std::sqrt (static_cast<float> (energyLegacy / compareLength)); + const float rmsOwn = std::sqrt (static_cast<float> (energyOwn / compareLength)); + const float rmsScore = (std::min (rmsLegacy, rmsOwn) + 1.0e-4f) / (std::max (rmsLegacy, rmsOwn) + 1.0e-4f); + const float totalScore = 0.50f * corrScore + 0.30f * slopeScore + 0.20f * rmsScore; + + if (! result.found || totalScore > result.score) + { + result.found = true; + result.lagSamples = lag; + result.score = totalScore; + } + } + + const int maxSafeLagSamples = std::max (4, searchRadiusSamples); + if (! result.found || result.score < 0.68f || std::abs (result.lagSamples) > maxSafeLagSamples) + return {}; + + return result; +} + +static bool hasStableEntryWindow ( + const OwnPitchEngine::NoteIslandAnalysis& island, + int startSample, + int sustainSamples, + float voicedThreshold, + float envThreshold) +{ + if (island.voicedMask.empty() || island.amplitudeEnvelope.empty() || island.f0TrackHz.empty()) + return false; + + const int endSample = startSample + sustainSamples; + if (startSample < 0 + || endSample > static_cast<int> (island.voicedMask.size()) + || endSample > static_cast<int> (island.amplitudeEnvelope.size()) + || endSample > static_cast<int> (island.f0TrackHz.size())) + { + return false; + } + + float firstF0 = 0.0f; + for (int s = startSample; s < endSample; ++s) + { + const float voiced = island.voicedMask[static_cast<size_t> (s)]; + const float env = island.amplitudeEnvelope[static_cast<size_t> (s)]; + const float f0 = island.f0TrackHz[static_cast<size_t> (s)]; + if (voiced < voicedThreshold || env < envThreshold || f0 <= 40.0f) + return false; + + if (firstF0 <= 0.0f) + firstF0 = f0; + else if (std::abs (1200.0f * std::log2 (std::max (f0, 1.0f) / std::max (firstF0, 1.0f))) > 45.0f) + return false; + } + + return true; +} + +static bool hasStableVoicedF0Window ( + const OwnPitchEngine::NoteIslandAnalysis& island, + int startSample, + int sustainSamples, + float voicedThreshold, + float envThreshold, + float maxF0DriftCents) +{ + if (! hasStableEntryWindow (island, startSample, sustainSamples, voicedThreshold, envThreshold)) + return false; + + if (island.f0TrackHz.empty()) + return false; + + float minF0 = std::numeric_limits<float>::max(); + float maxF0 = 0.0f; + for (int s = startSample; s < startSample + sustainSamples; ++s) + { + if (s < 0 || s >= static_cast<int> (island.f0TrackHz.size())) + return false; + const float f0 = island.f0TrackHz[static_cast<size_t> (s)]; + if (f0 < 40.0f) + return false; + minF0 = std::min (minF0, f0); + maxF0 = std::max (maxF0, f0); + } + + if (minF0 <= 0.0f || maxF0 <= 0.0f) + return false; + + const float driftCents = std::abs (1200.0f * std::log2 (maxF0 / minF0)); + return driftCents <= maxF0DriftCents; +} + +static const OwnPitchEngine::NoteIslandAnalysis* findHybridReplacementIsland ( + const OwnPitchEngine::SharedAnalysis& analysis, + int noteBodyStartAbsolute, + int noteBodyEndAbsolute) +{ + const OwnPitchEngine::NoteIslandAnalysis* best = nullptr; + int bestOverlap = 0; + + for (const auto& island : analysis.islands) + { + const int islandBodyStart = island.contextStartSample + island.bodyStartSample; + const int islandBodyEnd = island.contextStartSample + island.bodyEndSample; + const int overlap = std::max (0, std::min (noteBodyEndAbsolute, islandBodyEnd) - std::max (noteBodyStartAbsolute, islandBodyStart)); + if (overlap > bestOverlap) + { + best = &island; + bestOverlap = overlap; + } + } + + return best; +} + +static HybridStructuralBlendResult blendHybridStructuralOutputs ( + const float* const* originalInput, + const std::vector<std::vector<float>>& legacyOutput, + const std::vector<std::vector<float>>& ownOutput, + const OwnPitchEngine::SharedAnalysis& ownAnalysis, + PitchOnlyRendererBranch rendererBranch, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes) +{ + HybridStructuralBlendResult result; + auto& blended = result.output; + blended = legacyOutput; + if (legacyOutput.size() != ownOutput.size() || numChannels <= 0 || numSamples <= 0) + return result; + + std::vector<float> ownWeight (static_cast<size_t> (numSamples), 0.0f); + std::vector<float> replacementWeight (static_cast<size_t> (numSamples), 0.0f); + std::vector<float> islandNativeWeight (static_cast<size_t> (numSamples), 0.0f); + std::vector<std::vector<float>> replacementSignal (static_cast<size_t> (numChannels), + std::vector<float> (static_cast<size_t> (numSamples), 0.0f)); + std::vector<std::vector<float>> islandNativeSignal (static_cast<size_t> (numChannels), + std::vector<float> (static_cast<size_t> (numSamples), 0.0f)); + const auto legacyMono = buildHybridBridgeMono (legacyOutput, numChannels, numSamples); + const auto ownMono = buildHybridBridgeMono (ownOutput, numChannels, numSamples); + for (const auto& note : notes) + { + if (! hasPitchStyleEdit (note)) + continue; + + const float pitchRatio = std::pow (2.0f, (note.correctedPitch - note.detectedPitch) / 12.0f); + const bool downwardShift = pitchRatio < 0.999f; + const float bodyDurationSec = std::max (0.0f, note.endTime - note.startTime); + const bool longBody = bodyDurationSec >= 0.90f; + if (downwardShift || longBody) + continue; + + const int renderStart = juce::jlimit (0, numSamples, static_cast<int> (std::floor (getEffectiveNoteStartTime (note) * sampleRate))); + const int renderEnd = juce::jlimit (renderStart, numSamples, static_cast<int> (std::ceil (getEffectiveNoteEndTime (note) * sampleRate))); + const int bodyStart = juce::jlimit (renderStart, renderEnd, static_cast<int> (std::floor (note.startTime * sampleRate))); + const int bodyEnd = juce::jlimit (bodyStart, renderEnd, static_cast<int> (std::ceil (note.endTime * sampleRate))); + const int coreEntryProtect = std::max (1, static_cast<int> (std::round (0.050 * sampleRate))); + const int coreExitProtect = std::max (1, static_cast<int> (std::round (0.060 * sampleRate))); + const int coreStart = juce::jlimit (bodyStart, bodyEnd, bodyStart + coreEntryProtect); + const int coreEnd = juce::jlimit (coreStart, bodyEnd, bodyEnd - coreExitProtect); + bool usedBodyReplacement = false; + bool bodyReplacementFallbackUsed = false; + const bool enableBodyReplacement = false; + const auto* replacementIsland = findHybridReplacementIsland (ownAnalysis, bodyStart, bodyEnd); + const bool enableIslandNative = rendererBranch == PitchOnlyRendererBranch::IslandNative + || rendererBranch == PitchOnlyRendererBranch::IslandNativePsola; + const bool useIslandNativePsolaCore = rendererBranch == PitchOnlyRendererBranch::IslandNativePsola; + if (enableIslandNative) + { + if (replacementIsland == nullptr) + { + if (! result.islandNativeDiagnostics.islandNativeUsed) + result.islandNativeDiagnostics.islandNativeFallbackUsed = true; + } + else + { + const int islandRenderStart = juce::jlimit (renderStart, renderEnd, replacementIsland->contextStartSample); + const int islandRenderEnd = juce::jlimit (islandRenderStart, renderEnd, replacementIsland->contextEndSample); + const int localBodyStart = juce::jlimit (0, static_cast<int> (replacementIsland->monoSignal.size()), bodyStart - replacementIsland->contextStartSample); + const int localBodyEnd = juce::jlimit (localBodyStart, static_cast<int> (replacementIsland->monoSignal.size()), bodyEnd - replacementIsland->contextStartSample); + const int onsetHoldSamples = std::max (1, static_cast<int> (std::round (0.025 * sampleRate))); + const int exitHoldSamples = std::max (1, static_cast<int> (std::round (0.020 * sampleRate))); + const int outerEntryFadeSamples = std::max (1, static_cast<int> (std::round (0.012 * sampleRate))); + const int outerExitFadeSamples = std::max (1, static_cast<int> (std::round (0.010 * sampleRate))); + const int baseSustainSamples = std::max (1, static_cast<int> (std::round (0.010 * sampleRate))); + float maxBodyEnv = 0.0f; + if (! replacementIsland->amplitudeEnvelope.empty() && localBodyEnd > localBodyStart) + { + maxBodyEnv = *std::max_element (replacementIsland->amplitudeEnvelope.begin() + localBodyStart, + replacementIsland->amplitudeEnvelope.begin() + localBodyEnd); + } + const float envThreshold = std::max (0.010f, maxBodyEnv * 0.18f); + const float baseVoicedThreshold = 0.60f; + const int minimumCoreSamples = std::max (8, static_cast<int> (std::round (0.025 * sampleRate))); + const auto findStableCoreSpan = [&] (float voicedThreshold, int sustainSamples, int& stableStartLocal, int& stableEndLocal) + { + stableStartLocal = -1; + stableEndLocal = -1; + const int coreSearchStart = juce::jlimit (localBodyStart, localBodyEnd, localBodyStart + onsetHoldSamples); + const int coreSearchEnd = juce::jlimit (coreSearchStart, localBodyEnd, localBodyEnd - exitHoldSamples); + if (coreSearchEnd - coreSearchStart < sustainSamples) + return false; + + for (int sample = coreSearchStart; sample + sustainSamples <= coreSearchEnd; ++sample) + { + if (hasStableVoicedF0Window (*replacementIsland, sample, sustainSamples, voicedThreshold, envThreshold, 35.0f)) + { + stableStartLocal = sample; + break; + } + } + + for (int sample = coreSearchEnd; sample - sustainSamples >= coreSearchStart; --sample) + { + if (hasStableVoicedF0Window (*replacementIsland, sample - sustainSamples, sustainSamples, voicedThreshold, envThreshold, 35.0f)) + { + stableEndLocal = sample; + break; + } + } + + return stableStartLocal >= 0 + && stableEndLocal > stableStartLocal + && (stableEndLocal - stableStartLocal) >= minimumCoreSamples; + }; + + int stableCoreStartLocal = -1; + int stableCoreEndLocal = -1; + bool hasStableCore = findStableCoreSpan (baseVoicedThreshold, baseSustainSamples, stableCoreStartLocal, stableCoreEndLocal); + if (! hasStableCore && useIslandNativePsolaCore) + hasStableCore = findStableCoreSpan (0.55f, + std::max (1, static_cast<int> (std::round (0.008 * sampleRate))), + stableCoreStartLocal, + stableCoreEndLocal); + + if (hasStableCore) + { + const int stableCoreStart = replacementIsland->contextStartSample + stableCoreStartLocal; + const int stableCoreEnd = replacementIsland->contextStartSample + stableCoreEndLocal; + std::vector<std::vector<float>> islandCoreSignal; + bool hasIslandCoreSignal = ! useIslandNativePsolaCore; + + if (useIslandNativePsolaCore) + { + std::vector<int> sourceEpochs; + for (const int epoch : replacementIsland->epochs) + { + if (epoch >= stableCoreStartLocal && epoch <= stableCoreEndLocal) + sourceEpochs.push_back (replacementIsland->contextStartSample + epoch); + } + + if (sourceEpochs.size() >= 5) + { + double sourcePeriodSum = 0.0; + for (size_t i = 1; i < sourceEpochs.size(); ++i) + sourcePeriodSum += static_cast<double> (sourceEpochs[i] - sourceEpochs[i - 1]); + + const double averageSourcePeriod = sourcePeriodSum / static_cast<double> (std::max<size_t> (1, sourceEpochs.size() - 1)); + const double targetPeriod = std::max (8.0, averageSourcePeriod / std::max (0.25f, pitchRatio)); + std::vector<int> synthesisEpochs; + for (double pos = static_cast<double> (stableCoreStart); pos < static_cast<double> (stableCoreEnd); pos += targetPeriod) + synthesisEpochs.push_back (static_cast<int> (std::round (pos))); + + if (synthesisEpochs.size() >= 5) + { + islandCoreSignal.assign (static_cast<size_t> (numChannels), + std::vector<float> (static_cast<size_t> (numSamples), 0.0f)); + std::vector<float> coreNorm (static_cast<size_t> (numSamples), 0.0f); + + for (size_t i = 0; i < synthesisEpochs.size(); ++i) + { + const int sourceIndex = synthesisEpochs.size() <= 1 + ? 0 + : juce::jlimit (0, + static_cast<int> (sourceEpochs.size()) - 1, + static_cast<int> (std::round ( + static_cast<double> (i) * static_cast<double> (sourceEpochs.size() - 1) + / static_cast<double> (synthesisEpochs.size() - 1)))); + const int sourceEpoch = sourceEpochs[static_cast<size_t> (sourceIndex)]; + const int localSourceEpoch = juce::jlimit (0, + static_cast<int> (replacementIsland->f0TrackHz.size()) - 1, + sourceEpoch - replacementIsland->contextStartSample); + const float sourceF0Hz = replacementIsland->f0TrackHz.empty() + ? std::max (55.0f, midiToHz (note.detectedPitch)) + : std::max (55.0f, replacementIsland->f0TrackHz[static_cast<size_t> (localSourceEpoch)]); + const int sourcePeriod = juce::jlimit (20, 960, + static_cast<int> (std::round (sampleRate / sourceF0Hz))); + const int halfWindow = juce::jlimit (20, 960, + static_cast<int> (std::round (1.1 * static_cast<double> (sourcePeriod)))); + const int destEpoch = synthesisEpochs[static_cast<size_t> (i)]; + + for (int n = -halfWindow; n <= halfWindow; ++n) + { + const int srcIndex = sourceEpoch + n; + const int dstIndex = destEpoch + n; + if (srcIndex < 0 || srcIndex >= numSamples || dstIndex < stableCoreStart || dstIndex >= stableCoreEnd) + continue; + + const float phase = static_cast<float> (n + halfWindow) / static_cast<float> (2 * halfWindow + 1); + const float window = 0.5f - 0.5f * std::cos (2.0f * juce::MathConstants<float>::pi * phase); + coreNorm[static_cast<size_t> (dstIndex)] += window; + for (int ch = 0; ch < numChannels; ++ch) + islandCoreSignal[static_cast<size_t> (ch)][static_cast<size_t> (dstIndex)] += originalInput[ch][srcIndex] * window; + } + } + + hasIslandCoreSignal = false; + for (int s = stableCoreStart; s < stableCoreEnd; ++s) + { + const float norm = coreNorm[static_cast<size_t> (s)]; + if (norm > 1.0e-4f) + { + hasIslandCoreSignal = true; + for (int ch = 0; ch < numChannels; ++ch) + islandCoreSignal[static_cast<size_t> (ch)][static_cast<size_t> (s)] /= norm; + } + } + + if (hasIslandCoreSignal) + { + const int rmsWindowEnd = std::min (stableCoreEnd, stableCoreStart + static_cast<int> (std::round (0.020 * sampleRate))); + float replacementRms = 0.0f; + float originalRms = 0.0f; + int rmsCount = 0; + for (int s = stableCoreStart; s < rmsWindowEnd; ++s) + { + const float replacement = islandCoreSignal[0][static_cast<size_t> (s)]; + const float original = originalInput[0][s]; + replacementRms += replacement * replacement; + originalRms += original * original; + ++rmsCount; + } + + if (rmsCount > 0) + { + replacementRms = std::sqrt (replacementRms / static_cast<float> (rmsCount)); + originalRms = std::sqrt (originalRms / static_cast<float> (rmsCount)); + const float gainDb = (replacementRms > 1.0e-5f && originalRms > 1.0e-5f) + ? juce::jlimit (-1.0f, 1.0f, 20.0f * std::log10 (originalRms / replacementRms)) + : 0.0f; + const float gain = std::pow (10.0f, gainDb / 20.0f); + for (int ch = 0; ch < numChannels; ++ch) + for (int s = stableCoreStart; s < stableCoreEnd; ++s) + islandCoreSignal[static_cast<size_t> (ch)][static_cast<size_t> (s)] *= gain; + } + } + } + } + } + if (hasIslandCoreSignal) + { + float transientMaskPeak = 0.0f; + float voicedCoreMaskPeak = 0.0f; + const int stabilityProbeSpan = std::max (1, static_cast<int> (std::round (0.008 * sampleRate))); + + for (int s = islandRenderStart; s < islandRenderEnd; ++s) + { + const int localIndex = juce::jlimit (0, static_cast<int> (replacementIsland->monoSignal.size()) - 1, + s - replacementIsland->contextStartSample); + const float voiced = replacementIsland->voicedMask.empty() + ? 0.0f + : sanitizeFiniteFloat (replacementIsland->voicedMask[static_cast<size_t> (localIndex)]); + const float consonant = replacementIsland->consonantMask.empty() + ? 0.0f + : sanitizeFiniteFloat (replacementIsland->consonantMask[static_cast<size_t> (localIndex)]); + const float env = replacementIsland->amplitudeEnvelope.empty() + ? 0.0f + : sanitizeFiniteFloat (replacementIsland->amplitudeEnvelope[static_cast<size_t> (localIndex)]); + const float onsetTransient = s < stableCoreStart + ? 1.0f - juce::jlimit (0.0f, 1.0f, + static_cast<float> (s - bodyStart) / static_cast<float> (std::max (1, stableCoreStart - bodyStart))) + : 0.0f; + const float exitTransient = s >= stableCoreEnd + ? juce::jlimit (0.0f, 1.0f, + static_cast<float> (s - stableCoreEnd) / static_cast<float> (std::max (1, bodyEnd - stableCoreEnd))) + : 0.0f; + const int localProbeStart = juce::jlimit (localBodyStart, localBodyEnd, + localIndex - stabilityProbeSpan / 2); + const int localProbeSpan = std::max (1, std::min (stabilityProbeSpan, localBodyEnd - localProbeStart)); + const bool stableLocalF0 = hasStableVoicedF0Window (*replacementIsland, + localProbeStart, + localProbeSpan, + baseVoicedThreshold, + envThreshold, + 35.0f); + const float f0UnstableMask = stableLocalF0 ? 0.0f : 0.75f; + const float exitTransientMask = std::max (exitTransient, + s >= bodyEnd - exitHoldSamples ? 1.0f - juce::jlimit (0.0f, 1.0f, + static_cast<float> ((bodyEnd - s)) / static_cast<float> (std::max (1, exitHoldSamples))) : 0.0f); + const float transientMask = juce::jlimit (0.0f, 1.0f, + std::max ({ consonant, 1.0f - voiced, onsetTransient, f0UnstableMask, exitTransientMask })); + float voicedCoreMask = 0.0f; + if (s >= stableCoreStart && s < stableCoreEnd && env >= envThreshold) + voicedCoreMask = juce::jlimit (0.0f, 1.0f, (voiced - baseVoicedThreshold) / std::max (0.05f, 1.0f - baseVoicedThreshold)); + + const float coreWeight = juce::jlimit (0.0f, 1.0f, voicedCoreMask * (1.0f - 0.85f * transientMask)); + const float originalWeight = transientMask; + const float weightSum = std::max (0.001f, originalWeight + coreWeight); + transientMaskPeak = std::max (transientMaskPeak, transientMask); + voicedCoreMaskPeak = std::max (voicedCoreMaskPeak, coreWeight); + const float outerWeight = s < islandRenderStart + outerEntryFadeSamples + ? equalPowerFadeIn (static_cast<float> (s - islandRenderStart) / static_cast<float> (std::max (1, outerEntryFadeSamples))) + : (s >= islandRenderEnd - outerExitFadeSamples + ? equalPowerFadeOut (static_cast<float> (s - (islandRenderEnd - outerExitFadeSamples)) / static_cast<float> (std::max (1, outerExitFadeSamples))) + : 1.0f); + islandNativeWeight[static_cast<size_t> (s)] = std::max (islandNativeWeight[static_cast<size_t> (s)], outerWeight); + + for (int ch = 0; ch < numChannels; ++ch) + { + const float original = originalInput[ch][s]; + const float rendered = useIslandNativePsolaCore + ? islandCoreSignal[static_cast<size_t> (ch)][static_cast<size_t> (s)] + : ownOutput[static_cast<size_t> (ch)][static_cast<size_t> (s)]; + islandNativeSignal[static_cast<size_t> (ch)][static_cast<size_t> (s)] = + (original * originalWeight + rendered * coreWeight) / weightSum; + } + } + + if (! result.islandNativeDiagnostics.islandNativeUsed) + { + result.islandNativeDiagnostics.islandNativeUsed = true; + result.islandNativeDiagnostics.islandRenderStartSample = islandRenderStart; + result.islandNativeDiagnostics.islandRenderEndSample = islandRenderEnd; + result.islandNativeDiagnostics.transientMaskPeak = transientMaskPeak; + result.islandNativeDiagnostics.voicedCoreMaskPeak = voicedCoreMaskPeak; + } + + continue; + } + if (! result.islandNativeDiagnostics.islandNativeUsed) + result.islandNativeDiagnostics.islandNativeFallbackUsed = true; + } + + if (! result.islandNativeDiagnostics.islandNativeUsed) + result.islandNativeDiagnostics.islandNativeFallbackUsed = true; + } + } + if (enableBodyReplacement && replacementIsland != nullptr) + { + const int localBodyStart = juce::jlimit (0, static_cast<int> (replacementIsland->monoSignal.size()), bodyStart - replacementIsland->contextStartSample); + const int localBodyEnd = juce::jlimit (localBodyStart, static_cast<int> (replacementIsland->monoSignal.size()), bodyEnd - replacementIsland->contextStartSample); + const int entrySearchOffset = std::max (1, static_cast<int> (std::round (0.008 * sampleRate))); + const int entrySustainSamples = std::max (1, static_cast<int> (std::round (0.010 * sampleRate))); + const int exitSustainSamples = std::max (1, static_cast<int> (std::round (0.010 * sampleRate))); + const int entrySearchStart = juce::jlimit (localBodyStart, localBodyEnd, localBodyStart + entrySearchOffset); + const int entrySearchEnd = juce::jlimit (entrySearchStart, localBodyEnd, localBodyStart + static_cast<int> (std::round (0.090 * sampleRate))); + const int exitSearchStart = juce::jlimit (localBodyStart, localBodyEnd, localBodyEnd - static_cast<int> (std::round (0.090 * sampleRate))); + const int dryExitSamples = std::max (1, static_cast<int> (std::round (0.024 * sampleRate))); + const int exitSearchEnd = juce::jlimit (exitSearchStart, localBodyEnd, localBodyEnd - dryExitSamples); + float maxBodyEnv = 0.0f; + if (! replacementIsland->amplitudeEnvelope.empty() && localBodyEnd > localBodyStart) + { + maxBodyEnv = *std::max_element (replacementIsland->amplitudeEnvelope.begin() + localBodyStart, + replacementIsland->amplitudeEnvelope.begin() + localBodyEnd); + } + + const float envThreshold = std::max (0.010f, maxBodyEnv * 0.18f); + const float voicedThreshold = 0.60f; + int stableEntryLocal = -1; + for (int sample = entrySearchStart; sample + entrySustainSamples <= entrySearchEnd; ++sample) + { + if (hasStableVoicedF0Window (*replacementIsland, sample, entrySustainSamples, voicedThreshold, envThreshold, 35.0f)) + { + stableEntryLocal = sample; + break; + } + } + + int stableExitLocal = -1; + for (int sample = exitSearchEnd; sample - exitSustainSamples >= exitSearchStart; --sample) + { + if (hasStableVoicedF0Window (*replacementIsland, sample - exitSustainSamples, exitSustainSamples, voicedThreshold, envThreshold, 35.0f)) + { + stableExitLocal = sample; + break; + } + } + + if (stableEntryLocal >= 0 && stableExitLocal >= 0) + { + const int entryCrossfadeSamples = std::max (1, static_cast<int> (std::round (0.012 * sampleRate))); + const int exitCrossfadeSamples = std::max (1, static_cast<int> (std::round (0.008 * sampleRate))); + const int absoluteStableEntry = replacementIsland->contextStartSample + stableEntryLocal; + const int absoluteStableExit = replacementIsland->contextStartSample + stableExitLocal; + const int entryLockStart = juce::jlimit (bodyStart, bodyEnd, absoluteStableEntry - entryCrossfadeSamples); + const int renderedBodyStart = juce::jlimit (entryLockStart + 1, bodyEnd, absoluteStableEntry + entryCrossfadeSamples); + const int exitLockStart = juce::jlimit (renderedBodyStart + 1, bodyEnd, absoluteStableExit - exitCrossfadeSamples); + const int renderedBodyEnd = juce::jlimit (renderedBodyStart + 1, bodyEnd, exitLockStart); + const int earlyContinuationSamples = std::max (1, static_cast<int> (std::round (0.045 * sampleRate))); + const int continuationBodyEnd = juce::jlimit (renderedBodyStart + 1, renderedBodyEnd, renderedBodyStart + earlyContinuationSamples); + const int continuationFadeEnd = juce::jlimit (continuationBodyEnd + 1, bodyEnd, + std::min (bodyEnd, continuationBodyEnd + static_cast<int> (std::round (0.012 * sampleRate)))); + + if (continuationBodyEnd - renderedBodyStart >= std::max (4, static_cast<int> (std::round (0.018 * sampleRate)))) + { + const float entryF0Hz = replacementIsland->f0TrackHz.empty() + ? std::max (55.0f, midiToHz (note.detectedPitch)) + : std::max (55.0f, replacementIsland->f0TrackHz[static_cast<size_t> (stableEntryLocal)]); + const int entryPeriodSamples = juce::jlimit (20, 960, + static_cast<int> (std::round (sampleRate / entryF0Hz))); + const int anchorWindowStartLocal = std::max (localBodyStart, stableEntryLocal - 5 * entryPeriodSamples); + const int anchorWindowEndLocal = std::min (localBodyEnd, stableEntryLocal + 2 * entryPeriodSamples); + + std::vector<int> sourceEpochs; + sourceEpochs.reserve (8); + for (const int epoch : replacementIsland->epochs) + { + if (epoch >= anchorWindowStartLocal && epoch <= anchorWindowEndLocal) + sourceEpochs.push_back (replacementIsland->contextStartSample + epoch); + } + + if (sourceEpochs.size() > 4) + sourceEpochs.erase (sourceEpochs.begin(), sourceEpochs.end() - 4); + + if (sourceEpochs.size() >= 3) + { + double sourcePeriodSum = 0.0; + for (size_t i = 1; i < sourceEpochs.size(); ++i) + sourcePeriodSum += static_cast<double> (sourceEpochs[i] - sourceEpochs[i - 1]); + const double averageSourcePeriod = sourcePeriodSum / static_cast<double> (std::max<size_t> (1, sourceEpochs.size() - 1)); + const double targetPeriod = std::max (8.0, averageSourcePeriod / std::max (0.25f, pitchRatio)); + + std::vector<int> synthesisEpochs; + for (double pos = static_cast<double> (renderedBodyStart); pos < static_cast<double> (continuationBodyEnd); pos += targetPeriod) + synthesisEpochs.push_back (static_cast<int> (std::round (pos))); + + if (synthesisEpochs.size() >= 5) + { + std::vector<std::vector<float>> localReplacement (static_cast<size_t> (numChannels), + std::vector<float> (static_cast<size_t> (numSamples), 0.0f)); + std::vector<float> localNorm (static_cast<size_t> (numSamples), 0.0f); + + for (size_t i = 0; i < synthesisEpochs.size(); ++i) + { + const int sourceIndex = static_cast<int> (i % sourceEpochs.size()); + const int sourceEpoch = sourceEpochs[static_cast<size_t> (sourceIndex)]; + const int localSourceEpoch = juce::jlimit (0, static_cast<int> (replacementIsland->f0TrackHz.size()) - 1, + sourceEpoch - replacementIsland->contextStartSample); + const float sourceF0Hz = replacementIsland->f0TrackHz.empty() + ? std::max (55.0f, midiToHz (note.detectedPitch)) + : std::max (55.0f, replacementIsland->f0TrackHz[static_cast<size_t> (localSourceEpoch)]); + const int sourcePeriod = juce::jlimit (20, 960, + static_cast<int> (std::round (sampleRate / sourceF0Hz))); + const int halfWindow = juce::jlimit (20, 960, + static_cast<int> (std::round (1.1 * static_cast<double> (sourcePeriod)))); + const int destEpoch = synthesisEpochs[static_cast<size_t> (i)]; + + for (int n = -halfWindow; n <= halfWindow; ++n) + { + const int srcIndex = sourceEpoch + n; + const int dstIndex = destEpoch + n; + if (srcIndex < 0 || srcIndex >= numSamples || dstIndex < entryLockStart || dstIndex >= continuationFadeEnd) + continue; + + const float phase = static_cast<float> (n + halfWindow) / static_cast<float> (2 * halfWindow + 1); + const float window = 0.5f - 0.5f * std::cos (2.0f * juce::MathConstants<float>::pi * phase); + localNorm[static_cast<size_t> (dstIndex)] += window; + for (int ch = 0; ch < numChannels; ++ch) + localReplacement[static_cast<size_t> (ch)][static_cast<size_t> (dstIndex)] += originalInput[ch][srcIndex] * window; + } + } + + bool hasReplacementSignal = false; + for (int s = entryLockStart; s < continuationFadeEnd; ++s) + { + const float norm = localNorm[static_cast<size_t> (s)]; + if (norm > 1.0e-4f) + { + hasReplacementSignal = true; + for (int ch = 0; ch < numChannels; ++ch) + localReplacement[static_cast<size_t> (ch)][static_cast<size_t> (s)] /= norm; + } + } + + if (hasReplacementSignal) + { + const int rmsWindowEnd = std::min (continuationBodyEnd, renderedBodyStart + static_cast<int> (std::round (0.020 * sampleRate))); + float replacementRms = 0.0f; + float legacyRms = 0.0f; + int rmsCount = 0; + for (int s = renderedBodyStart; s < rmsWindowEnd; ++s) + { + const float replacement = localReplacement[0][static_cast<size_t> (s)]; + const float legacy = legacyOutput[0][static_cast<size_t> (s)]; + replacementRms += replacement * replacement; + legacyRms += legacy * legacy; + ++rmsCount; + } + + if (rmsCount > 0) + { + replacementRms = std::sqrt (replacementRms / static_cast<float> (rmsCount)); + legacyRms = std::sqrt (legacyRms / static_cast<float> (rmsCount)); + const float gainDb = (replacementRms > 1.0e-5f && legacyRms > 1.0e-5f) + ? juce::jlimit (-1.0f, 1.0f, 20.0f * std::log10 (legacyRms / replacementRms)) + : 0.0f; + const float gain = std::pow (10.0f, gainDb / 20.0f); + for (int ch = 0; ch < numChannels; ++ch) + for (int s = entryLockStart; s < continuationFadeEnd; ++s) + localReplacement[static_cast<size_t> (ch)][static_cast<size_t> (s)] *= gain; + } + + usedBodyReplacement = true; + for (int s = bodyStart; s < bodyEnd; ++s) + { + float weight = 0.0f; + if (s < entryLockStart || s >= continuationFadeEnd) + { + weight = 0.0f; + } + else if (s >= renderedBodyStart && s < continuationBodyEnd) + { + weight = 1.0f; + } + else if (s >= entryLockStart && s < renderedBodyStart) + { + const float t = static_cast<float> (s - entryLockStart) + / static_cast<float> (std::max (1, renderedBodyStart - entryLockStart)); + weight = equalPowerFadeIn (t); + } + else if (s >= continuationBodyEnd && s < continuationFadeEnd) + { + const float t = static_cast<float> (s - continuationBodyEnd) + / static_cast<float> (std::max (1, continuationFadeEnd - continuationBodyEnd)); + weight = equalPowerFadeOut (t); + } + + replacementWeight[static_cast<size_t> (s)] = std::max (replacementWeight[static_cast<size_t> (s)], weight); + for (int ch = 0; ch < numChannels; ++ch) + replacementSignal[static_cast<size_t> (ch)][static_cast<size_t> (s)] = localReplacement[static_cast<size_t> (ch)][static_cast<size_t> (s)]; + } + + if (! result.bodyReplacementDiagnostics.bodyReplacementUsed) + { + result.bodyReplacementDiagnostics.bodyReplacementUsed = true; + result.bodyReplacementDiagnostics.entryLockStartSample = entryLockStart; + result.bodyReplacementDiagnostics.entryLockLengthSamples = renderedBodyStart - entryLockStart; + result.bodyReplacementDiagnostics.exitLockStartSample = continuationBodyEnd; + result.bodyReplacementDiagnostics.renderedBodyStartSample = renderedBodyStart; + result.bodyReplacementDiagnostics.renderedBodyEndSample = continuationBodyEnd; + } + } + else + { + bodyReplacementFallbackUsed = true; + } + } + else + { + bodyReplacementFallbackUsed = true; + } + } + else + { + bodyReplacementFallbackUsed = true; + } + + } + else + { + bodyReplacementFallbackUsed = true; + } + } + else + { + bodyReplacementFallbackUsed = true; + } + } + else + { + bodyReplacementFallbackUsed = true; + } + + if (bodyReplacementFallbackUsed && ! result.bodyReplacementDiagnostics.bodyReplacementUsed) + result.bodyReplacementDiagnostics.bodyReplacementFallbackUsed = true; + + if (usedBodyReplacement) + continue; + + const float localTargetHz = std::max (midiToHz (note.correctedPitch), 55.0f); + const int localPeriodSamples = std::max (1, static_cast<int> (std::round (sampleRate / localTargetHz))); + const int bridgeLengthSamples = std::max ( + static_cast<int> (std::round (0.008 * sampleRate)), + std::min (static_cast<int> (std::round (0.018 * sampleRate)), + static_cast<int> (std::round (1.5 * static_cast<double> (localPeriodSamples))))); + const int searchRadiusSamples = std::max ( + 1, + std::min (static_cast<int> (std::round (0.0015 * sampleRate)), + static_cast<int> (std::round (0.35 * static_cast<double> (localPeriodSamples))))); + const int bridgeStart = coreStart; + const int bridgeEnd = juce::jlimit (bridgeStart, coreEnd, bridgeStart + bridgeLengthSamples); + const bool enableExperimentalOnsetBridge = false; + const bool canUseBridge = enableExperimentalOnsetBridge && (bridgeEnd - bridgeStart >= 8); + const float coreOwnWeightCap = 0.82f; + const int fallbackAttackSettle = std::max (1, static_cast<int> (std::round (0.020 * sampleRate))); + const int fallbackFullCoreStart = juce::jlimit (coreStart, coreEnd, coreStart + fallbackAttackSettle); + const float fallbackAttackOwnWeightCap = 0.62f; + HybridBridgeAlignmentResult alignment; + float bridgeGainDb = 0.0f; + bool bridgeUsed = false; + bool bridgeFallbackUsed = false; + + if (canUseBridge) + { + alignment = findHybridBridgeAlignment ( + legacyMono, + ownMono, + bridgeStart, + bridgeEnd - bridgeStart, + searchRadiusSamples); + + if (alignment.found) + { + const int rmsWindowSamples = std::max ( + static_cast<int> (std::round (0.004 * sampleRate)), + std::min (static_cast<int> (std::round (0.006 * sampleRate)), bridgeEnd - bridgeStart)); + const float legacyRms = computeHybridBridgeRms (legacyMono, bridgeStart, rmsWindowSamples); + const float ownRms = computeHybridBridgeRms (ownMono, bridgeStart + alignment.lagSamples, rmsWindowSamples); + if (legacyRms > 1.0e-5f && ownRms > 1.0e-5f) + bridgeGainDb = juce::jlimit (-1.5f, 1.5f, 20.0f * std::log10 (legacyRms / ownRms)); + bridgeUsed = true; + } + else + { + bridgeFallbackUsed = true; + } + } + + if ((bridgeUsed || bridgeFallbackUsed) && ! result.diagnostics.bridgeUsed && ! result.diagnostics.bridgeFallbackUsed) + { + result.diagnostics.bridgeUsed = bridgeUsed; + result.diagnostics.bridgeFallbackUsed = bridgeFallbackUsed; + result.diagnostics.bridgeStartSample = bridgeStart; + result.diagnostics.bridgeLengthSamples = bridgeEnd - bridgeStart; + result.diagnostics.bridgeAlignmentLagSamples = alignment.lagSamples; + result.diagnostics.bridgeCorrelationScore = alignment.score; + result.diagnostics.bridgeGainDeltaDb = bridgeGainDb; + } + + for (int s = renderStart; s < renderEnd; ++s) + { + float weight = 0.0f; + if (bridgeUsed && s >= bodyStart && s < bridgeStart) + { + const float t = static_cast<float> (s - bodyStart) / static_cast<float> (std::max (1, bridgeStart - bodyStart)); + weight = fallbackAttackOwnWeightCap * smoothstep01 (t); + } + else if (bridgeUsed && s >= bridgeStart && s < bridgeEnd) + { + weight = 0.0f; + } + else if (s >= (bridgeUsed ? bridgeEnd : fallbackFullCoreStart) && s < coreEnd) + { + weight = coreOwnWeightCap; + } + else if (! bridgeUsed && s >= coreStart && s < fallbackFullCoreStart) + { + const float t = static_cast<float> (s - coreStart) / static_cast<float> (std::max (1, fallbackFullCoreStart - coreStart)); + weight = fallbackAttackOwnWeightCap + (coreOwnWeightCap - fallbackAttackOwnWeightCap) * smoothstep01 (t); + } + else if (! bridgeUsed && s >= bodyStart && s < coreStart) + { + const float t = static_cast<float> (s - bodyStart) / static_cast<float> (std::max (1, coreStart - bodyStart)); + weight = fallbackAttackOwnWeightCap * smoothstep01 (t); + } + else if (s >= coreEnd && s < bodyEnd) + { + const float t = static_cast<float> (bodyEnd - s) / static_cast<float> (std::max (1, bodyEnd - coreEnd)); + weight = coreOwnWeightCap * smoothstep01 (t); + } + ownWeight[static_cast<size_t> (s)] = std::max (ownWeight[static_cast<size_t> (s)], weight); + } + } + + for (int ch = 0; ch < numChannels; ++ch) + { + if (ownOutput[static_cast<size_t> (ch)].size() != blended[static_cast<size_t> (ch)].size()) + continue; + + for (int s = 0; s < numSamples; ++s) + { + const float legacy = legacyOutput[static_cast<size_t> (ch)][static_cast<size_t> (s)]; + float sample = legacy; + const float islandBlend = islandNativeWeight[static_cast<size_t> (s)]; + if (islandBlend > 0.0f) + { + const float islandSample = islandNativeSignal[static_cast<size_t> (ch)][static_cast<size_t> (s)]; + sample = legacy * (1.0f - islandBlend) + islandSample * islandBlend; + } + const float ownBlend = ownWeight[static_cast<size_t> (s)]; + if (islandBlend <= 0.0f && ownBlend > 0.0f) + { + const float own = ownOutput[static_cast<size_t> (ch)][static_cast<size_t> (s)]; + sample = legacy * (1.0f - ownBlend) + own * ownBlend; + } + const float replacementBlend = replacementWeight[static_cast<size_t> (s)]; + if (replacementBlend > 0.0f) + { + const float replacement = replacementSignal[static_cast<size_t> (ch)][static_cast<size_t> (s)]; + sample = sample * (1.0f - replacementBlend) + replacement * replacementBlend; + } + blended[static_cast<size_t> (ch)][static_cast<size_t> (s)] = sample; + } + } + + if (result.diagnostics.bridgeUsed) + { + const int bridgeStart = result.diagnostics.bridgeStartSample; + const int bridgeEnd = juce::jlimit (bridgeStart, numSamples, bridgeStart + result.diagnostics.bridgeLengthSamples); + const float ownGain = std::pow (10.0f, result.diagnostics.bridgeGainDeltaDb / 20.0f); + const float coreOwnWeightCap = 0.82f; + + for (int ch = 0; ch < numChannels; ++ch) + { + if (static_cast<size_t> (ch) >= legacyOutput.size() || static_cast<size_t> (ch) >= ownOutput.size()) + continue; + + const auto& legacyChannel = legacyOutput[static_cast<size_t> (ch)]; + const auto& ownChannel = ownOutput[static_cast<size_t> (ch)]; + auto& outChannel = blended[static_cast<size_t> (ch)]; + for (int s = bridgeStart; s < bridgeEnd; ++s) + { + const int ownIndex = juce::jlimit ( + 0, + static_cast<int> (ownChannel.size()) - 1, + s + result.diagnostics.bridgeAlignmentLagSamples); + const float t = static_cast<float> (s - bridgeStart) / static_cast<float> (std::max (1, bridgeEnd - bridgeStart)); + const float legacy = legacyChannel[static_cast<size_t> (s)]; + const float ownCurrent = ownChannel[static_cast<size_t> (s)]; + const float ownAligned = ownChannel[static_cast<size_t> (ownIndex)] * ownGain; + const float ownBridge = + ownCurrent * equalPowerFadeOut (t) + ownAligned * equalPowerFadeIn (t); + const float bridgeWeight = 0.62f + (coreOwnWeightCap - 0.62f) * smoothstep01 (t); + outChannel[static_cast<size_t> (s)] = + legacy * (1.0f - bridgeWeight) + ownBridge * bridgeWeight; + } + } + } + + return result; +} + +static std::vector<std::vector<float>> composeAdaptiveSelectorOutput ( + const std::vector<std::vector<float>>& hybridOutput, + const std::vector<std::vector<float>>& simpleOutput, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes) +{ + auto output = hybridOutput; + if (output.size() != simpleOutput.size() || numChannels <= 0 || numSamples <= 0 || sampleRate <= 0.0) + return output; + + const int entryFadeSamples = std::max (1, static_cast<int> (std::round (0.012 * sampleRate))); + const int exitFadeSamples = std::max (1, static_cast<int> (std::round (0.010 * sampleRate))); + std::vector<float> simpleWeight (static_cast<size_t> (numSamples), 0.0f); + + for (const auto& note : notes) + { + if (! hasPitchStyleEdit (note)) + continue; + + const float pitchRatio = std::pow (2.0f, (note.correctedPitch - note.detectedPitch) / 12.0f); + const bool upwardShift = pitchRatio > 1.001f; + const float bodyDurationSec = std::max (0.0f, note.endTime - note.startTime); + const bool longBody = bodyDurationSec >= 0.90f; + if (! upwardShift || ! longBody) + continue; + + const int noteStart = juce::jlimit (0, numSamples, static_cast<int> (std::floor (getEffectiveNoteStartTime (note) * sampleRate))); + const int noteEnd = juce::jlimit (noteStart, numSamples, static_cast<int> (std::ceil (getEffectiveNoteEndTime (note) * sampleRate))); + if (noteEnd <= noteStart) + continue; + + for (int s = noteStart; s < noteEnd; ++s) + { + float weight = 1.0f; + if (s < noteStart + entryFadeSamples) + { + const float t = static_cast<float> (s - noteStart) / static_cast<float> (std::max (1, entryFadeSamples)); + weight = equalPowerFadeIn (t); + } + if (s >= noteEnd - exitFadeSamples) + { + const float t = static_cast<float> (s - (noteEnd - exitFadeSamples)) / static_cast<float> (std::max (1, exitFadeSamples)); + weight = std::min (weight, equalPowerFadeOut (t)); + } + simpleWeight[static_cast<size_t> (s)] = std::max (simpleWeight[static_cast<size_t> (s)], weight); + } + } + + for (int ch = 0; ch < numChannels; ++ch) + { + for (int s = 0; s < numSamples; ++s) + { + const float w = simpleWeight[static_cast<size_t> (s)]; + if (w <= 1.0e-4f) + continue; + output[static_cast<size_t> (ch)][static_cast<size_t> (s)] = + hybridOutput[static_cast<size_t> (ch)][static_cast<size_t> (s)] * (1.0f - w) + + simpleOutput[static_cast<size_t> (ch)][static_cast<size_t> (s)] * w; + } + } + + return output; +} + +static void applyAdaptiveDownwardOwnBlend ( + std::vector<std::vector<float>>& output, + const std::vector<std::vector<float>>& ownOutput, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes, + float maxOwnWeight) +{ + if (output.size() != ownOutput.size() || numChannels <= 0 || numSamples <= 0 || sampleRate <= 0.0) + return; + + const int entryProtectSamples = std::max (1, static_cast<int> (std::round (0.040 * sampleRate))); + const int exitProtectSamples = std::max (1, static_cast<int> (std::round (0.050 * sampleRate))); + + for (const auto& note : notes) + { + if (! hasPitchStyleEdit (note)) + continue; + + const float pitchRatio = std::pow (2.0f, (note.correctedPitch - note.detectedPitch) / 12.0f); + const bool downwardShift = pitchRatio < 0.999f; + const float bodyDurationSec = std::max (0.0f, note.endTime - note.startTime); + const bool shortBody = bodyDurationSec > 0.0f && bodyDurationSec < 0.80f; + if (! downwardShift || ! shortBody) + continue; + + const int bodyStart = juce::jlimit (0, numSamples, static_cast<int> (std::floor (note.startTime * sampleRate))); + const int bodyEnd = juce::jlimit (bodyStart, numSamples, static_cast<int> (std::ceil (note.endTime * sampleRate))); + const int coreStart = juce::jlimit (bodyStart, bodyEnd, bodyStart + entryProtectSamples); + const int coreEnd = juce::jlimit (coreStart, bodyEnd, bodyEnd - exitProtectSamples); + if (coreEnd <= coreStart) + continue; + + for (int s = bodyStart; s < bodyEnd; ++s) + { + float ownWeight = 0.0f; + if (s >= coreStart && s < coreEnd) + { + ownWeight = maxOwnWeight; + } + else if (s >= bodyStart && s < coreStart) + { + const float t = static_cast<float> (s - bodyStart) / static_cast<float> (std::max (1, coreStart - bodyStart)); + ownWeight = maxOwnWeight * equalPowerFadeIn (t); + } + else if (s >= coreEnd && s < bodyEnd) + { + const float t = static_cast<float> (s - coreEnd) / static_cast<float> (std::max (1, bodyEnd - coreEnd)); + ownWeight = maxOwnWeight * equalPowerFadeOut (t); + } + + if (ownWeight <= 1.0e-4f) + continue; + + for (int ch = 0; ch < numChannels; ++ch) + { + output[static_cast<size_t> (ch)][static_cast<size_t> (s)] = + output[static_cast<size_t> (ch)][static_cast<size_t> (s)] * (1.0f - ownWeight) + + ownOutput[static_cast<size_t> (ch)][static_cast<size_t> (s)] * ownWeight; + } + } + } +} + +struct PitchRenderIsland +{ + int renderStartSample = 0; + int renderEndSample = 0; + int contextStartSample = 0; + int contextEndSample = 0; + std::vector<PitchAnalyzer::PitchNote> notes; +}; + +struct PitchOnlyCoreRegion +{ + int bodyStartSample = 0; + int bodyEndSample = 0; + int coreStartSample = -1; + int coreEndSample = -1; + float pitchRatio = 1.0f; + bool upwardShift = true; +}; + +struct PitchOnlyStageBStats +{ + bool ran = false; + bool failedClosed = false; + int regionCount = 0; + int longestCoreSamples = 0; + float wetCap = 0.0f; + int pitchMarkCount = 0; + juce::String branchName; +}; + +static PitchOnlyStageBStats applyPitchOnlyCoreSerialCorrection ( + std::vector<std::vector<float>>& output, + const float* const* originalInput, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& detectedPitchHz, + PitchResynthesizer::RenderQuality renderQuality, + PitchOnlyRendererBranch rendererBranch, + std::function<bool()> shouldCancel); + +static std::vector<float> buildPitchOnlyCoreMask ( + const float* const* originalInput, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& detectedPitchHz); + +static int determineHopSize (const std::vector<PitchAnalyzer::PitchFrame>& frames, double sampleRate) +{ + if (frames.size() >= 2) + { + const float dt = frames[1].time - frames[0].time; + return std::max (1, static_cast<int> (dt * sampleRate)); + } + return 256; +} + +static std::vector<PitchAnalyzer::PitchFrame> sliceFramesForWindow ( + const std::vector<PitchAnalyzer::PitchFrame>& frames, + double windowStartSec, + double windowEndSec) +{ + std::vector<PitchAnalyzer::PitchFrame> windowFrames; + windowFrames.reserve (frames.size()); + for (const auto& f : frames) + { + const double ft = static_cast<double> (f.time); + if (ft >= windowStartSec - 0.5 && ft <= windowEndSec + 0.5) + { + auto wf = f; + wf.time = static_cast<float> (ft - windowStartSec); + windowFrames.push_back (wf); + } + } + return windowFrames; +} + +static std::vector<float> buildDetectedPitchCurveHz ( + int numSamples, + double sampleRate, + int hopSize, + const std::vector<PitchAnalyzer::PitchFrame>& frames) +{ + std::vector<float> detectedPitchHz (static_cast<size_t> (numSamples), 0.0f); + for (size_t fi = 0; fi < frames.size(); ++fi) + { + const auto& f = frames[fi]; + if (f.frequency <= 0.0f) + continue; + + const int sampleStart = static_cast<int> (f.time * sampleRate); + const int sampleEnd = std::min (numSamples, sampleStart + hopSize); + for (int s = std::max (0, sampleStart); s < sampleEnd; ++s) + detectedPitchHz[static_cast<size_t> (s)] = f.frequency; + } + return detectedPitchHz; +} + +static std::vector<float> stabilizePitchOnlyFormantBaseHz ( + const std::vector<float>& detectedPitchHz, + double sampleRate) +{ + std::vector<float> stabilized = detectedPitchHz; + if (stabilized.empty()) + return stabilized; + + std::vector<int> voicedIndices; + voicedIndices.reserve (stabilized.size() / 64 + 1); + for (int i = 0; i < static_cast<int> (stabilized.size()); ++i) + { + if (stabilized[static_cast<size_t> (i)] > 0.0f) + voicedIndices.push_back (i); + } + + if (voicedIndices.empty()) + return stabilized; + + const int firstVoiced = voicedIndices.front(); + const int lastVoiced = voicedIndices.back(); + for (int i = 0; i < firstVoiced; ++i) + stabilized[static_cast<size_t> (i)] = stabilized[static_cast<size_t> (firstVoiced)]; + for (int i = lastVoiced + 1; i < static_cast<int> (stabilized.size()); ++i) + stabilized[static_cast<size_t> (i)] = stabilized[static_cast<size_t> (lastVoiced)]; + + for (size_t vi = 1; vi < voicedIndices.size(); ++vi) + { + const int left = voicedIndices[vi - 1]; + const int right = voicedIndices[vi]; + if (right <= left + 1) + continue; + + const float leftHz = stabilized[static_cast<size_t> (left)]; + const float rightHz = stabilized[static_cast<size_t> (right)]; + const int gapLen = right - left - 1; + for (int g = 1; g <= gapLen; ++g) + { + const float t = static_cast<float> (g) / static_cast<float> (gapLen + 1); + stabilized[static_cast<size_t> (left + g)] = leftHz + (rightHz - leftHz) * t; + } + } + + const int smoothRadius = std::max (1, static_cast<int> (std::round (0.006 * sampleRate))); + std::vector<float> smoothed = stabilized; + for (int i = 0; i < static_cast<int> (stabilized.size()); ++i) + { + int count = 0; + double weightedSum = 0.0; + double weightTotal = 0.0; + for (int j = std::max (0, i - smoothRadius); j <= std::min (static_cast<int> (stabilized.size()) - 1, i + smoothRadius); ++j) + { + const float hz = stabilized[static_cast<size_t> (j)]; + if (hz <= 0.0f) + continue; + const double dist = static_cast<double> (j - i); + const double weight = std::exp (-0.5 * (dist * dist) / std::max (1.0, static_cast<double> (smoothRadius * smoothRadius) * 0.35)); + weightedSum += static_cast<double> (hz) * weight; + weightTotal += weight; + ++count; + } + if (count > 0 && weightTotal > 0.0) + smoothed[static_cast<size_t> (i)] = static_cast<float> (weightedSum / weightTotal); + } + + return smoothed; +} + +static std::vector<PitchRenderIsland> buildPitchRenderIslands ( + int numSamples, + double sampleRate, + PitchResynthesizer::RenderQuality renderQuality, + const std::vector<PitchAnalyzer::PitchNote>& notes) +{ + struct CandidateNote + { + float renderStartSec = 0.0f; + float renderEndSec = 0.0f; + PitchAnalyzer::PitchNote note; + }; + + std::vector<CandidateNote> candidates; + candidates.reserve (notes.size()); + for (const auto& note : notes) + { + if (! hasPitchStyleEdit (note)) + continue; + + const float renderStartSec = getEffectiveNoteStartTime (note); + const float renderEndSec = getEffectiveNoteEndTime (note); + if (renderEndSec <= renderStartSec) + continue; + + candidates.push_back ({ renderStartSec, renderEndSec, note }); + } + + std::sort (candidates.begin(), candidates.end(), [] (const auto& a, const auto& b) + { + return a.renderStartSec < b.renderStartSec; + }); + + const float contextPadSec = renderQuality == PitchResynthesizer::RenderQuality::PreviewFast ? 0.14f : 0.20f; + const float mergeGapSec = 0.02f; + + std::vector<PitchRenderIsland> islands; + for (const auto& candidate : candidates) + { + const int renderStartSample = juce::jlimit (0, numSamples - 1, + static_cast<int> (std::floor (candidate.renderStartSec * sampleRate))); + const int renderEndSample = juce::jlimit (0, numSamples, + static_cast<int> (std::ceil (candidate.renderEndSec * sampleRate))); + if (renderEndSample <= renderStartSample) + continue; + + if (! islands.empty()) + { + auto& island = islands.back(); + const double islandRenderEndSec = static_cast<double> (island.renderEndSample) / sampleRate; + if (candidate.renderStartSec <= islandRenderEndSec + mergeGapSec) + { + island.renderStartSample = std::min (island.renderStartSample, renderStartSample); + island.renderEndSample = std::max (island.renderEndSample, renderEndSample); + island.notes.push_back (candidate.note); + continue; + } + } + + PitchRenderIsland island; + island.renderStartSample = renderStartSample; + island.renderEndSample = renderEndSample; + island.notes.push_back (candidate.note); + islands.push_back (std::move (island)); + } + + for (auto& island : islands) + { + island.contextStartSample = juce::jlimit (0, numSamples, + island.renderStartSample - static_cast<int> (std::round (contextPadSec * sampleRate))); + island.contextEndSample = juce::jlimit (0, numSamples, + island.renderEndSample + static_cast<int> (std::round (contextPadSec * sampleRate))); + } + + return islands; +} + +static void splicePitchIsland ( + std::vector<std::vector<float>>& output, + const std::vector<std::vector<float>>& correctedIsland, + const float* const* originalInput, + int numChannels, + int renderStartSample, + int renderEndSample, + int contextStartSample, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& islandNotes, + PitchOnlyRendererBranch rendererBranch) +{ + const int renderSamples = std::max (0, renderEndSample - renderStartSample); + if (renderSamples <= 0) + return; + + const bool hybridReset = rendererBranch == PitchOnlyRendererBranch::HybridReset + || rendererBranch == PitchOnlyRendererBranch::HybridStructural; + const int localRenderStart = renderStartSample - contextStartSample; + const int entryXfadeLen = std::min (hybridReset ? 576 : 384, renderSamples / 5); + const int exitXfadeLen = std::min (384, renderSamples / 6); + std::vector<float> noteWetMask (static_cast<size_t> (renderSamples), 0.0f); + + for (const auto& note : islandNotes) + { + const bool upwardShift = note.correctedPitch >= note.detectedPitch; + const int effectiveStart = juce::jlimit (renderStartSample, renderEndSample, + static_cast<int> (std::floor (getEffectiveNoteStartTime (note) * sampleRate))); + const int bodyStart = juce::jlimit (renderStartSample, renderEndSample, + static_cast<int> (std::floor (note.startTime * sampleRate))); + const int bodyEnd = juce::jlimit (renderStartSample, renderEndSample, + static_cast<int> (std::ceil (note.endTime * sampleRate))); + const int effectiveEnd = juce::jlimit (renderStartSample, renderEndSample, + static_cast<int> (std::ceil (getEffectiveNoteEndTime (note) * sampleRate))); + + if (effectiveEnd <= effectiveStart) + continue; + + const int bodyLen = std::max (1, bodyEnd - bodyStart); + const double entryShoulderSec = hybridReset + ? (upwardShift ? 0.024 : 0.030) + : 0.038; + const double exitShoulderSec = hybridReset + ? (upwardShift ? 0.034 : 0.042) + : 0.050; + const int maxAudibleEntryShoulder = std::max (1, std::min ( + static_cast<int> (std::round (entryShoulderSec * sampleRate)), bodyLen / 3)); + const int maxAudibleExitShoulder = std::max (1, std::min ( + static_cast<int> (std::round (exitShoulderSec * sampleRate)), bodyLen / 3)); + const int shoulderStart = std::max (effectiveStart, bodyStart - maxAudibleEntryShoulder); + const int shoulderEnd = std::min (effectiveEnd, bodyEnd + maxAudibleExitShoulder); + const int riseLen = std::max (1, bodyStart - shoulderStart); + const int fallLen = std::max (1, shoulderEnd - bodyEnd); + const double entryProtectSec = hybridReset + ? (upwardShift ? 0.040 : 0.034) + : 0.028; + const double exitProtectSec = hybridReset + ? (upwardShift ? 0.046 : 0.040) + : 0.034; + const int entryProtect = std::max (1, std::min ( + static_cast<int> (std::round (entryProtectSec * sampleRate)), bodyLen / 4)); + const int exitProtect = std::max (1, std::min ( + static_cast<int> (std::round (exitProtectSec * sampleRate)), bodyLen / 4)); + const float entryProtectFloor = hybridReset + ? (upwardShift ? 0.54f : 0.68f) + : 0.78f; + const float exitProtectFloor = hybridReset + ? (upwardShift ? 0.60f : 0.74f) + : 0.80f; + + for (int destIndex = shoulderStart; destIndex < shoulderEnd; ++destIndex) + { + float noteBlend = 1.0f; + if (destIndex < bodyStart) + { + const float t = static_cast<float> (destIndex - shoulderStart) / static_cast<float> (riseLen); + noteBlend = smoothstep01 (t); + } + else if (destIndex < bodyStart + entryProtect) + { + const float t = static_cast<float> (destIndex - bodyStart) / static_cast<float> (entryProtect); + noteBlend = juce::jmap (smoothstep01 (t), entryProtectFloor, 1.0f); + } + + if (destIndex >= bodyEnd) + { + const float t = static_cast<float> (shoulderEnd - 1 - destIndex) / static_cast<float> (fallLen); + noteBlend = std::min (noteBlend, smoothstep01 (t)); + } + else if (destIndex >= bodyEnd - exitProtect) + { + const float t = static_cast<float> (bodyEnd - 1 - destIndex) / static_cast<float> (exitProtect); + noteBlend = std::min (noteBlend, juce::jmap (smoothstep01 (t), exitProtectFloor, 1.0f)); + } + + const int localIndex = destIndex - renderStartSample; + if (localIndex >= 0 && localIndex < renderSamples) + noteWetMask[static_cast<size_t> (localIndex)] = std::max (noteWetMask[static_cast<size_t> (localIndex)], noteBlend); + } + } + + for (int ch = 0; ch < numChannels; ++ch) + { + auto& out = output[static_cast<size_t> (ch)]; + const auto& corrected = correctedIsland[static_cast<size_t> (ch)]; + const float* orig = originalInput[ch]; + + for (int i = 0; i < renderSamples; ++i) + { + const int destIndex = renderStartSample + i; + const int sourceIndex = localRenderStart + i; + if (destIndex < 0 || destIndex >= static_cast<int> (out.size())) + continue; + if (sourceIndex < 0 || sourceIndex >= static_cast<int> (corrected.size())) + continue; + + float blend = noteWetMask[static_cast<size_t> (i)]; + if (entryXfadeLen > 0 && i < entryXfadeLen) + { + const float edgeBlend = 0.5f * (1.0f - std::cos (juce::MathConstants<float>::pi + * static_cast<float> (i) / static_cast<float> (entryXfadeLen))); + blend = std::min (blend, edgeBlend); + } + const int distFromEnd = renderSamples - 1 - i; + if (exitXfadeLen > 0 && distFromEnd < exitXfadeLen) + { + const float edgeBlend = 0.5f * (1.0f - std::cos (juce::MathConstants<float>::pi + * static_cast<float> (distFromEnd) / static_cast<float> (exitXfadeLen))); + blend = std::min (blend, edgeBlend); + } + + const float dry = orig[destIndex]; + const float wet = corrected[static_cast<size_t> (sourceIndex)]; + out[static_cast<size_t> (destIndex)] = dry * (1.0f - blend) + wet * blend; + } + } +} + +static std::vector<std::vector<float>> renderPitchOnlyIslands ( + const float* const* input, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchFrame>& frames, + const std::vector<PitchAnalyzer::PitchNote>& notes, + PitchOnlyRendererBranch rendererBranch, + PitchResynthesizer::RenderQuality renderQuality, + std::function<bool()> shouldCancel) +{ + std::vector<std::vector<float>> output (static_cast<size_t> (numChannels)); + for (int ch = 0; ch < numChannels; ++ch) + output[static_cast<size_t> (ch)].assign (input[ch], input[ch] + numSamples); + + const auto islands = buildPitchRenderIslands (numSamples, sampleRate, renderQuality, notes); + if (islands.empty()) + return output; + + int processedIslands = 0; + int processedNotes = 0; + int stageBRanIslands = 0; + int stageBFailedClosedIslands = 0; + PitchResynthesizer islandResynth; + + for (const auto& island : islands) + { + if (shouldCancel && shouldCancel()) + return {}; + + const int contextSamples = island.contextEndSample - island.contextStartSample; + if (contextSamples <= 0) + continue; + + const double contextStartSec = static_cast<double> (island.contextStartSample) / sampleRate; + const double contextEndSec = static_cast<double> (island.contextEndSample) / sampleRate; + auto islandFrames = sliceFramesForWindow (frames, contextStartSec, contextEndSec); + if (islandFrames.empty()) + continue; + + auto islandNotes = island.notes; + for (auto& note : islandNotes) + { + note.startTime -= static_cast<float> (contextStartSec); + note.endTime -= static_cast<float> (contextStartSec); + note.effectiveStartTime -= static_cast<float> (contextStartSec); + note.effectiveEndTime -= static_cast<float> (contextStartSec); + } + + const int islandHopSize = determineHopSize (islandFrames, sampleRate); + auto islandRatios = islandResynth.buildCorrectionCurve ( + contextSamples, sampleRate, islandFrames, islandNotes, islandHopSize); + bool islandHasPitchShift = false; + for (size_t i = 0; i < islandRatios.size() && ! islandHasPitchShift; i += 128) + islandHasPitchShift = std::abs (islandRatios[i] - 1.0f) > 0.001f; + if (! islandHasPitchShift) + continue; + + auto islandDetectedPitchHz = buildDetectedPitchCurveHz ( + contextSamples, sampleRate, islandHopSize, islandFrames); + islandDetectedPitchHz = stabilizePitchOnlyFormantBaseHz (islandDetectedPitchHz, sampleRate); + + std::vector<const float*> islandInput (static_cast<size_t> (numChannels)); + for (int ch = 0; ch < numChannels; ++ch) + islandInput[static_cast<size_t> (ch)] = input[ch] + island.contextStartSample; + + auto correctedIsland = copyInputChannels (islandInput.data(), numChannels, contextSamples); + if (rendererBranch == PitchOnlyRendererBranch::HybridReset + || rendererBranch == PitchOnlyRendererBranch::HybridStructural) + { + logPitchEditorFormant ("using hybrid reset guided pitch-only carrier + minimal note-core stageB"); + } + if (shouldCancel && shouldCancel()) + return {}; + if (correctedIsland.empty()) + continue; + + if (kEnablePitchOnlyStageB) + { + const auto stageBStats = applyPitchOnlyCoreSerialCorrection (correctedIsland, + islandInput.data(), + numChannels, + contextSamples, + sampleRate, + islandNotes, + islandDetectedPitchHz, + renderQuality, + rendererBranch, + shouldCancel); + if (stageBStats.ran) + ++stageBRanIslands; + if (stageBStats.failedClosed) + ++stageBFailedClosedIslands; + logPitchEditorFormant ("pitch-only island stageB branch=" + stageBStats.branchName + + " ran=" + juce::String (stageBStats.ran ? "true" : "false") + + " failedClosed=" + juce::String (stageBStats.failedClosed ? "true" : "false") + + " regions=" + juce::String (stageBStats.regionCount) + + " pitchMarks=" + juce::String (stageBStats.pitchMarkCount) + + " longestCoreMs=" + juce::String (sampleRate > 0.0 + ? 1000.0 * static_cast<double> (stageBStats.longestCoreSamples) / sampleRate : 0.0, 1) + + " wetCap=" + juce::String (stageBStats.wetCap, 3)); + if (shouldCancel && shouldCancel()) + return {}; + } + + splicePitchIsland (output, correctedIsland, input, numChannels, + island.renderStartSample, island.renderEndSample, + island.contextStartSample, sampleRate, island.notes, rendererBranch); + ++processedIslands; + processedNotes += static_cast<int> (island.notes.size()); + } + + logPitchEditorFormant ("rendered pitch-only note islands=" + juce::String (processedIslands) + + " notes=" + juce::String (processedNotes) + + " branch=" + juce::String (getPitchOnlyRendererBranchName (rendererBranch)) + + " stageB=" + juce::String (kEnablePitchOnlyStageB ? "enabled" : "disabled") + + " stageBRanIslands=" + juce::String (stageBRanIslands) + + " stageBFailedClosedIslands=" + juce::String (stageBFailedClosedIslands) + + " renderQuality=" + juce::String (renderQuality == PitchResynthesizer::RenderQuality::PreviewFast + ? "preview_fast" : "final_hq")); + + return output; +} + +static void applyExplicitFormantWarp ( + std::vector<std::vector<float>>& output, + const float* const* originalInput, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<float>& formantRatios, + const std::vector<float>& detectedPitchHz, + float warpIntensity, + bool formantOnly, + PitchResynthesizer::RenderQuality renderQuality, + std::function<bool()> shouldCancel) +{ + const bool previewFast = renderQuality == PitchResynthesizer::RenderQuality::PreviewFast; + const float ratioThreshold = 0.03f; + bool hasShift = false; + float requestedMinRatio = 1.0f; + float requestedMaxRatio = 1.0f; + + for (size_t i = 0; i < formantRatios.size(); i += 256) + { + const float ratio = formantRatios[i]; + requestedMinRatio = std::min (requestedMinRatio, ratio); + requestedMaxRatio = std::max (requestedMaxRatio, ratio); + if (std::abs (ratio - 1.0f) > ratioThreshold) + hasShift = true; + } + + if (! hasShift) + return; + + const int fftOrder = formantOnly + ? (previewFast ? 10 : 11) + : (previewFast ? 11 : 12); + const int fftSize = 1 << fftOrder; + const int hopLen = previewFast ? (fftSize / 2) : (fftSize / 4); + const int halfBins = fftSize / 2 + 1; + const float pi = juce::MathConstants<float>::pi; + const float binHz = static_cast<float> (sampleRate) / static_cast<float> (fftSize); + + juce::dsp::FFT fft (fftOrder); + + std::vector<float> hannWin (static_cast<size_t> (fftSize)); + for (int i = 0; i < fftSize; ++i) + hannWin[static_cast<size_t> (i)] = 0.5f * (1.0f - std::cos ( + 2.0f * pi * static_cast<float> (i) / static_cast<float> (fftSize - 1))); + + std::vector<float> origFFTBuf (static_cast<size_t> (fftSize * 2)); + std::vector<float> outFFTBuf (static_cast<size_t> (fftSize * 2)); + std::vector<float> preWarpFFTBuf (static_cast<size_t> (fftSize * 2)); + std::vector<float> origLogMag (static_cast<size_t> (halfBins)); + std::vector<float> outLogMag (static_cast<size_t> (halfBins)); + std::vector<float> origEnv (static_cast<size_t> (halfBins)); + std::vector<float> outEnv (static_cast<size_t> (halfBins)); + std::vector<float> targetEnv (static_cast<size_t> (halfBins)); + + auto computeEnvelope = [&] (const std::vector<float>& logMag, + int smoothHalfW, + std::vector<float>& env) + { + for (int i = 0; i < halfBins; ++i) + { + const int lo = std::max (0, i - smoothHalfW); + const int hi = std::min (halfBins - 1, i + smoothHalfW); + float sum = 0.0f; + for (int j = lo; j <= hi; ++j) + sum += logMag[static_cast<size_t> (j)]; + env[static_cast<size_t> (i)] = std::exp (sum / static_cast<float> (hi - lo + 1)); + } + }; + + int shiftedBlocks = 0; + float appliedMinRatio = std::numeric_limits<float>::max(); + float appliedMaxRatio = 0.0f; + float avgVoicedBlendApplied = 0.0f; + float minSmoothedVoicedBlend = std::numeric_limits<float>::max(); + float maxSmoothedVoicedBlend = 0.0f; + float minEffectiveStrength = std::numeric_limits<float>::max(); + float maxEffectiveStrength = 0.0f; + float totalEnvelopeDeltaBefore = 0.0f; + float totalEnvelopeDeltaAfter = 0.0f; + int totalEnvelopeDeltaBins = 0; + + for (int ch = 0; ch < numChannels; ++ch) + { + auto& out = output[static_cast<size_t> (ch)]; + const float* orig = originalInput[ch]; + + std::vector<float> corrected (static_cast<size_t> (numSamples), 0.0f); + std::vector<float> winSum (static_cast<size_t> (numSamples), 0.0f); + float prevVoicedBlend = 0.0f; + float prevStrength = 0.0f; + float prevOriginalBlend = 0.0f; + + for (int pos = 0; pos < numSamples; pos += hopLen) + { + if (shouldCancel && shouldCancel()) + return; + float avgFormantRatio = 0.0f; + int ratioCount = 0; + float frameEnergy = 0.0f; + int frameSamples = 0; + for (int i = pos; i < std::min (pos + fftSize, numSamples); ++i) + { + if (static_cast<size_t> (i) < formantRatios.size()) + { + avgFormantRatio += formantRatios[static_cast<size_t> (i)]; + ++ratioCount; + } + const float sample = orig[i]; + frameEnergy += sample * sample; + ++frameSamples; + } + avgFormantRatio = ratioCount > 0 ? avgFormantRatio / static_cast<float> (ratioCount) : 1.0f; + const float frameRms = frameSamples > 0 ? std::sqrt (frameEnergy / static_cast<float> (frameSamples)) : 0.0f; + + if (std::abs (avgFormantRatio - 1.0f) <= ratioThreshold) + { + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + if (idx >= 0 && idx < numSamples) + { + const float w = hannWin[static_cast<size_t> (i)]; + corrected[static_cast<size_t> (idx)] += out[static_cast<size_t> (idx)] * w; + winSum[static_cast<size_t> (idx)] += w * w; + } + } + continue; + } + + ++shiftedBlocks; + + std::fill (origFFTBuf.begin(), origFFTBuf.end(), 0.0f); + std::fill (outFFTBuf.begin(), outFFTBuf.end(), 0.0f); + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + if (idx >= 0 && idx < numSamples) + { + const float w = hannWin[static_cast<size_t> (i)]; + origFFTBuf[static_cast<size_t> (i)] = orig[idx] * w; + outFFTBuf[static_cast<size_t> (i)] = out[static_cast<size_t> (idx)] * w; + } + } + + fft.performRealOnlyForwardTransform (origFFTBuf.data(), true); + fft.performRealOnlyForwardTransform (outFFTBuf.data(), true); + std::copy (outFFTBuf.begin(), outFFTBuf.end(), preWarpFFTBuf.begin()); + + for (int i = 0; i < halfBins; ++i) + { + const float ore = origFFTBuf[static_cast<size_t> (i * 2)]; + const float oim = origFFTBuf[static_cast<size_t> (i * 2 + 1)]; + origLogMag[static_cast<size_t> (i)] = std::log (std::sqrt (ore * ore + oim * oim) + 1e-10f); + + const float sre = outFFTBuf[static_cast<size_t> (i * 2)]; + const float sim = outFFTBuf[static_cast<size_t> (i * 2 + 1)]; + outLogMag[static_cast<size_t> (i)] = std::log (std::sqrt (sre * sre + sim * sim) + 1e-10f); + } + + float avgPitch = 0.0f; + int pitchCount = 0; + int voicedSamples = 0; + if (! detectedPitchHz.empty()) + { + for (int i = pos; i < std::min (pos + fftSize, numSamples); ++i) + { + if (static_cast<size_t> (i) < detectedPitchHz.size() + && detectedPitchHz[static_cast<size_t> (i)] > 0.0f) + { + avgPitch += detectedPitchHz[static_cast<size_t> (i)]; + ++pitchCount; + ++voicedSamples; + } + } + if (pitchCount > 0) + avgPitch /= static_cast<float> (pitchCount); + } + + const float rawVoicedBlend = smoothstep01 (static_cast<float> (voicedSamples) + / static_cast<float> (std::max (1, std::min (fftSize, numSamples - pos))) + * (formantOnly ? 1.48f : 1.36f)); + const float voicedBlend = (pos == 0) + ? rawVoicedBlend + : ((formantOnly ? 0.82f : 0.76f) * prevVoicedBlend + + (formantOnly ? 0.18f : 0.24f) * rawVoicedBlend); + prevVoicedBlend = voicedBlend; + avgVoicedBlendApplied += voicedBlend; + minSmoothedVoicedBlend = std::min (minSmoothedVoicedBlend, voicedBlend); + maxSmoothedVoicedBlend = std::max (maxSmoothedVoicedBlend, voicedBlend); + + int smoothHalfW; + if (avgPitch > 50.0f) + smoothHalfW = std::max (previewFast ? (formantOnly ? 10 : 12) : (formantOnly ? 14 : 16), + static_cast<int> ((formantOnly ? (previewFast ? 1.35f : 1.75f) : (previewFast ? 1.85f : 2.45f)) * avgPitch / binHz)); + else + smoothHalfW = previewFast ? (formantOnly ? 24 : 30) : (formantOnly ? 36 : 50); + smoothHalfW = std::min (smoothHalfW, halfBins / 3); + + computeEnvelope (origLogMag, smoothHalfW, origEnv); + computeEnvelope (outLogMag, smoothHalfW, outEnv); + + const bool lowerFormant = avgFormantRatio < 1.0f; + const bool upperFormant = avgFormantRatio > 1.0f; + const float effectiveShiftSemitones = std::abs (12.0f * std::log2 (std::max (avgFormantRatio, 1.0e-4f))); + const float bassFocus = formantOnly && avgPitch > 0.0f + ? smoothstep01 (((lowerFormant ? 210.0f : 185.0f) - avgPitch) / (lowerFormant ? 105.0f : 115.0f)) + : 0.0f; + const float lowRegisterBoost = formantOnly && avgPitch > 0.0f + ? (1.0f + (lowerFormant ? (previewFast ? 0.32f : 0.44f) : (previewFast ? 0.20f : 0.30f)) * bassFocus) + : 1.0f; + const float voicedStrengthBias = formantOnly + ? (0.48f + 0.52f * voicedBlend) + : (0.28f + 0.72f * voicedBlend); + const float rawStrength = juce::jlimit (0.0f, + formantOnly + ? (lowerFormant + ? (previewFast ? 1.52f : 1.74f) + : (previewFast ? 1.26f : 1.46f)) + : (lowerFormant + ? (previewFast ? 1.22f : 1.48f) + : (previewFast ? 1.08f : 1.34f)), + (0.76f + effectiveShiftSemitones * 0.20f) + * voicedStrengthBias * warpIntensity * lowRegisterBoost); + const float strength = (pos == 0) + ? rawStrength + : ((formantOnly ? 0.78f : 0.72f) * prevStrength + + (formantOnly ? 0.22f : 0.28f) * rawStrength); + prevStrength = strength; + minEffectiveStrength = std::min (minEffectiveStrength, strength); + maxEffectiveStrength = std::max (maxEffectiveStrength, strength); + appliedMinRatio = std::min (appliedMinRatio, avgFormantRatio); + appliedMaxRatio = std::max (appliedMaxRatio, avgFormantRatio); + + float energyBefore = 0.0f; + for (int i = 0; i < halfBins; ++i) + { + const float re = outFFTBuf[static_cast<size_t> (i * 2)]; + const float im = outFFTBuf[static_cast<size_t> (i * 2 + 1)]; + energyBefore += re * re + im * im; + } + + for (int i = 0; i < halfBins; ++i) + { + const float freqHz = static_cast<float> (i) * binHz; + const float lowAnchor = std::max (formantOnly ? 65.0f : 90.0f, avgPitch > 0.0f ? avgPitch * (formantOnly ? 0.78f : 0.95f) : (formantOnly ? 90.0f : 120.0f)); + const float lowFull = std::max (formantOnly ? 220.0f : 320.0f, avgPitch > 0.0f ? avgPitch * (formantOnly ? 1.65f : 2.15f) : (formantOnly ? 300.0f : 430.0f)); + const float lowBandBase = formantOnly + ? (0.58f + 0.12f * bassFocus) + : 0.30f; + const float lowBandLift = formantOnly + ? (0.42f - 0.10f * bassFocus) + : 0.70f; + const float lowBandWeight = lowBandBase + lowBandLift + * smoothstep01 ((freqHz - lowAnchor) / std::max (45.0f, lowFull - lowAnchor)); + const float detailProtect = smoothstep01 ((freqHz - 3400.0f) / 2200.0f); + const float airProtection = 1.0f - (formantOnly + ? (lowerFormant ? 0.26f : 0.10f) + : (lowerFormant ? 0.34f : 0.20f)) + * smoothstep01 ((freqHz - (lowerFormant ? 4600.0f : 5600.0f)) / (lowerFormant ? 2400.0f : 3000.0f)); + const float perBinWeight = juce::jlimit (0.0f, 1.0f, voicedStrengthBias * lowBandWeight * airProtection); + + targetEnv[static_cast<size_t> (i)] = interpolateEnvelopeBin ( + origEnv, static_cast<float> (i) / avgFormantRatio); + + const float targetBlend = juce::jlimit (0.0f, 1.10f, + perBinWeight * (formantOnly ? (previewFast ? 1.06f : 1.10f) : 1.02f)); + const float blendedTargetEnv = outEnv[static_cast<size_t> (i)] + + (targetEnv[static_cast<size_t> (i)] - outEnv[static_cast<size_t> (i)]) + * targetBlend; + const float sourceEnv = outEnv[static_cast<size_t> (i)] + 1.0e-10f; + const float targetValue = blendedTargetEnv + 1.0e-10f; + totalEnvelopeDeltaBefore += std::abs (std::log (targetValue / sourceEnv)); + + float gain = std::pow (targetValue / sourceEnv, + strength); + float minGain = juce::jmap (voicedBlend, 0.0f, 1.0f, + detailProtect > 0.25f ? 0.84f : 0.72f, + detailProtect > 0.25f ? (lowerFormant ? 0.64f : 0.72f) : (lowerFormant ? (previewFast ? 0.50f : 0.40f) : (previewFast ? 0.60f : 0.52f))); + float maxGain = juce::jmap (voicedBlend, 0.0f, 1.0f, + detailProtect > 0.25f ? (upperFormant ? 1.28f : 1.16f) : (upperFormant ? 1.64f : 1.48f), + detailProtect > 0.25f ? (upperFormant ? 1.62f : 1.48f) : (upperFormant ? (previewFast ? 2.18f : 2.58f) : (previewFast ? 1.92f : 2.26f))); + if (formantOnly && bassFocus > 0.0f && freqHz < std::max (420.0f, avgPitch > 0.0f ? avgPitch * 3.1f : 420.0f)) + maxGain *= (1.0f + 0.16f * bassFocus); + gain = juce::jlimit (minGain, maxGain, gain); + totalEnvelopeDeltaAfter += std::abs (std::log (targetValue / (sourceEnv * gain))); + ++totalEnvelopeDeltaBins; + outFFTBuf[static_cast<size_t> (i * 2)] *= gain; + outFFTBuf[static_cast<size_t> (i * 2 + 1)] *= gain; + } + + float energyAfter = 0.0f; + for (int i = 0; i < halfBins; ++i) + { + const float re = outFFTBuf[static_cast<size_t> (i * 2)]; + const float im = outFFTBuf[static_cast<size_t> (i * 2 + 1)]; + energyAfter += re * re + im * im; + } + if (energyAfter > 1.0e-10f) + { + const float energyRatio = energyBefore / energyAfter; + const float scale = std::pow (energyRatio, formantOnly ? 0.24f : 0.24f); + for (int i = 0; i < halfBins; ++i) + { + outFFTBuf[static_cast<size_t> (i * 2)] *= scale; + outFFTBuf[static_cast<size_t> (i * 2 + 1)] *= scale; + } + } + + for (int i = 0; i < halfBins; ++i) + { + const float freqHz = static_cast<float> (i) * binHz; + const float detailProtect = smoothstep01 ((freqHz - 3200.0f) / 2000.0f); + const float detailKeep = juce::jlimit (0.0f, 0.72f, + detailProtect * (0.20f + + 0.46f * (1.0f - voicedBlend) + + 0.18f * smoothstep01 ((0.028f - frameRms) / 0.018f)) + * (lowerFormant ? 0.82f : 1.04f)); + if (detailKeep <= 0.001f) + continue; + outFFTBuf[static_cast<size_t> (i * 2)] = + outFFTBuf[static_cast<size_t> (i * 2)] * (1.0f - detailKeep) + + preWarpFFTBuf[static_cast<size_t> (i * 2)] * detailKeep; + outFFTBuf[static_cast<size_t> (i * 2 + 1)] = + outFFTBuf[static_cast<size_t> (i * 2 + 1)] * (1.0f - detailKeep) + + preWarpFFTBuf[static_cast<size_t> (i * 2 + 1)] * detailKeep; + } + + fft.performRealOnlyInverseTransform (outFFTBuf.data()); + + const float rawOriginalBlend = formantOnly + ? juce::jlimit (0.10f, 0.50f, + 0.10f + + (1.0f - voicedBlend) * (lowerFormant ? 0.28f : 0.20f) + + smoothstep01 ((0.032f - frameRms) / 0.020f) * (lowerFormant ? 0.16f : 0.10f)) + : 0.0f; + const float originalBlend = (pos == 0) + ? rawOriginalBlend + : (0.82f * prevOriginalBlend + 0.18f * rawOriginalBlend); + prevOriginalBlend = originalBlend; + + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + if (idx >= 0 && idx < numSamples) + { + const float w = hannWin[static_cast<size_t> (i)]; + const float processedSample = outFFTBuf[static_cast<size_t> (i)] * (1.0f - originalBlend) + + orig[idx] * originalBlend; + corrected[static_cast<size_t> (idx)] += processedSample * w; + winSum[static_cast<size_t> (idx)] += w * w; + } + } + } + + for (int s = 0; s < numSamples; ++s) + { + if (winSum[static_cast<size_t> (s)] > 0.001f) + out[static_cast<size_t> (s)] = corrected[static_cast<size_t> (s)] + / winSum[static_cast<size_t> (s)]; + } + } + + logPitchEditorFormant ("explicit formant warp blocks=" + juce::String (shiftedBlocks) + + " requestedRatio=[" + juce::String (requestedMinRatio, 3) + "," + juce::String (requestedMaxRatio, 3) + "]" + + " appliedRatio=[" + juce::String (appliedMinRatio == std::numeric_limits<float>::max() ? 1.0f : appliedMinRatio, 3) + + "," + juce::String (appliedMaxRatio, 3) + "]" + + " avgVoicedBlend=" + juce::String (shiftedBlocks > 0 ? avgVoicedBlendApplied / static_cast<float> (shiftedBlocks) : 0.0f, 3) + + " voicedBlendRange=[" + juce::String (minSmoothedVoicedBlend == std::numeric_limits<float>::max() ? 0.0f : minSmoothedVoicedBlend, 3) + + "," + juce::String (maxSmoothedVoicedBlend, 3) + "]" + + " strengthRange=[" + juce::String (minEffectiveStrength == std::numeric_limits<float>::max() ? 0.0f : minEffectiveStrength, 3) + + "," + juce::String (maxEffectiveStrength, 3) + "]" + + " warpIntensity=" + juce::String (warpIntensity, 3) + + " renderQuality=" + juce::String (previewFast ? "preview_fast" : "final_hq") + + " mode=" + juce::String (formantOnly ? "formant-only" : "mixed") + + " envDeltaBefore=" + juce::String (totalEnvelopeDeltaBins > 0 ? totalEnvelopeDeltaBefore / static_cast<float> (totalEnvelopeDeltaBins) : 0.0f, 4) + + " envDeltaAfter=" + juce::String (totalEnvelopeDeltaBins > 0 ? totalEnvelopeDeltaAfter / static_cast<float> (totalEnvelopeDeltaBins) : 0.0f, 4)); +} + +static void applyPitchEnvelopeAnchorMatch ( + std::vector<std::vector<float>>& output, + const float* const* originalInput, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& detectedPitchHz, + PitchOnlyRendererBranch rendererBranch, + PitchResynthesizer::RenderQuality renderQuality, + std::function<bool()> shouldCancel) +{ + if (detectedPitchHz.empty()) + return; + + const bool previewFast = renderQuality == PitchResynthesizer::RenderQuality::PreviewFast; + const bool hybridReset = rendererBranch == PitchOnlyRendererBranch::HybridReset + || rendererBranch == PitchOnlyRendererBranch::HybridStructural; + const int fftOrder = previewFast ? 10 : 12; + const int fftSize = 1 << fftOrder; + const int hopLen = fftSize / 4; + const int halfBins = fftSize / 2 + 1; + const float pi = juce::MathConstants<float>::pi; + const float binHz = static_cast<float> (sampleRate) / static_cast<float> (fftSize); + const float blockDurSec = static_cast<float> (fftSize) / static_cast<float> (sampleRate); + + juce::dsp::FFT fft (fftOrder); + std::vector<float> hannWin (static_cast<size_t> (fftSize)); + for (int i = 0; i < fftSize; ++i) + hannWin[static_cast<size_t> (i)] = 0.5f * (1.0f - std::cos ( + 2.0f * pi * static_cast<float> (i) / static_cast<float> (fftSize - 1))); + + auto gaussian = [] (float x, float centre, float width) + { + const float safeWidth = std::max (1.0f, width); + const float d = (x - centre) / safeWidth; + return std::exp (-0.5f * d * d); + }; + + auto computeEnvelope = [&] (const std::vector<float>& logMag, + int smoothHalfW, + std::vector<float>& env) + { + for (int i = 0; i < halfBins; ++i) + { + const int lo = std::max (0, i - smoothHalfW); + const int hi = std::min (halfBins - 1, i + smoothHalfW); + float sum = 0.0f; + for (int j = lo; j <= hi; ++j) + sum += logMag[static_cast<size_t> (j)]; + env[static_cast<size_t> (i)] = std::exp (sum / static_cast<float> (hi - lo + 1)); + } + }; + + int processedBlocks = 0; + float avgAppliedStrength = 0.0f; + float avgVoicedBlend = 0.0f; + int invalidStrengthBlocks = 0; + bool loggedInvalidStrength = false; + + for (int ch = 0; ch < numChannels; ++ch) + { + auto& out = output[static_cast<size_t> (ch)]; + const float* orig = originalInput[ch]; + std::vector<float> corrected (static_cast<size_t> (numSamples), 0.0f); + std::vector<float> winSum (static_cast<size_t> (numSamples), 0.0f); + std::vector<float> origFFTBuf (static_cast<size_t> (fftSize * 2), 0.0f); + std::vector<float> outFFTBuf (static_cast<size_t> (fftSize * 2), 0.0f); + std::vector<float> outPreMatchFFTBuf (static_cast<size_t> (fftSize * 2), 0.0f); + std::vector<float> origLogMag (static_cast<size_t> (halfBins), 0.0f); + std::vector<float> outLogMag (static_cast<size_t> (halfBins), 0.0f); + std::vector<float> origEnv (static_cast<size_t> (halfBins), 1.0f); + std::vector<float> outEnv (static_cast<size_t> (halfBins), 1.0f); + + for (int pos = 0; pos < numSamples; pos += hopLen) + { + if (shouldCancel && shouldCancel()) + return; + + const float blockStartSec = static_cast<float> (pos) / static_cast<float> (sampleRate); + const float blockEndSec = static_cast<float> (std::min (numSamples, pos + fftSize)) / static_cast<float> (sampleRate); + const float blockCenterSec = 0.5f * (blockStartSec + blockEndSec); + const int blockSampleCount = std::max (1, std::min (fftSize, numSamples - pos)); + + float localPitchWeight = 0.0f; + float avgPitchShiftSt = 0.0f; + const PitchAnalyzer::PitchNote* dominantNote = nullptr; + float dominantOverlap = 0.0f; + for (const auto& note : notes) + { + if (! hasPitchStyleEdit (note)) + continue; + + const float noteStart = getEffectiveNoteStartTime (note); + const float noteEnd = getEffectiveNoteEndTime (note); + const float overlap = std::max (0.0f, std::min (blockEndSec, noteEnd) - std::max (blockStartSec, noteStart)); + if (overlap <= 0.0f) + continue; + + const float overlapWeight = overlap / std::max (0.001f, blockDurSec); + localPitchWeight += overlapWeight; + avgPitchShiftSt += (note.correctedPitch - note.detectedPitch) * overlapWeight; + if (overlapWeight > dominantOverlap) + { + dominantOverlap = overlapWeight; + dominantNote = ¬e; + } + } + + if (localPitchWeight <= 0.001f) + { + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + if (idx >= 0 && idx < numSamples) + { + const float w = hannWin[static_cast<size_t> (i)]; + corrected[static_cast<size_t> (idx)] += out[static_cast<size_t> (idx)] * w; + winSum[static_cast<size_t> (idx)] += w * w; + } + } + continue; + } + + ++processedBlocks; + avgPitchShiftSt /= std::max (0.001f, localPitchWeight); + avgPitchShiftSt = sanitizeFiniteFloat (avgPitchShiftSt, 0.0f); + const float pitchStrength = sanitizeFiniteFloat (std::min (1.0f, std::abs (avgPitchShiftSt) / 4.0f), 0.0f); + + float avgPitchHz = 0.0f; + int pitchCount = 0; + for (int i = pos; i < std::min (pos + fftSize, numSamples); ++i) + { + if (static_cast<size_t> (i) < detectedPitchHz.size() && detectedPitchHz[static_cast<size_t> (i)] > 0.0f) + { + avgPitchHz += detectedPitchHz[static_cast<size_t> (i)]; + ++pitchCount; + } + } + if (pitchCount > 0) + avgPitchHz /= static_cast<float> (pitchCount); + avgPitchHz = sanitizeFiniteFloat (avgPitchHz, 0.0f); + const float voicedBlend = sanitizeFiniteFloat ( + smoothstep01 (static_cast<float> (pitchCount) / static_cast<float> (blockSampleCount) * 1.35f), 0.0f); + avgVoicedBlend += voicedBlend; + + if (voicedBlend < 0.36f || avgPitchHz < 70.0f) + { + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + if (idx >= 0 && idx < numSamples) + { + const float w = hannWin[static_cast<size_t> (i)]; + corrected[static_cast<size_t> (idx)] += out[static_cast<size_t> (idx)] * w; + winSum[static_cast<size_t> (idx)] += w * w; + } + } + continue; + } + + const float noteProgress = dominantNote != nullptr + ? juce::jlimit (0.0f, 1.0f, + (blockCenterSec - dominantNote->startTime) + / std::max (0.025f, dominantNote->endTime - dominantNote->startTime)) + : 0.5f; + const float dominantNoteDurationSec = dominantNote != nullptr + ? std::max (0.025f, dominantNote->endTime - dominantNote->startTime) + : 0.0f; + const float safeNoteProgress = sanitizeFiniteFloat (noteProgress, 0.5f); + const float noteMidFocus = sanitizeFiniteFloat ( + std::pow (std::sin (pi * juce::jlimit (0.0f, 1.0f, safeNoteProgress)), 1.12f), 0.0f); + const float noteShoulderEase = sanitizeFiniteFloat (0.72f + 0.20f * noteMidFocus, 0.72f); + const bool upwardShift = avgPitchShiftSt >= 0.0f; + const float shortUpwardNoteBoost = upwardShift + ? sanitizeFiniteFloat (1.0f + 0.12f * smoothstep01 ((0.90f - dominantNoteDurationSec) / 0.35f), 1.0f) + : 1.0f; + const float hqBoost = upwardShift + ? (previewFast ? 0.86f : 0.94f) + : (previewFast ? 0.96f : 1.14f); + + std::fill (origFFTBuf.begin(), origFFTBuf.end(), 0.0f); + std::fill (outFFTBuf.begin(), outFFTBuf.end(), 0.0f); + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + if (idx >= 0 && idx < numSamples) + { + const float w = hannWin[static_cast<size_t> (i)]; + origFFTBuf[static_cast<size_t> (i)] = orig[idx] * w; + outFFTBuf[static_cast<size_t> (i)] = out[static_cast<size_t> (idx)] * w; + } + } + + fft.performRealOnlyForwardTransform (origFFTBuf.data(), true); + fft.performRealOnlyForwardTransform (outFFTBuf.data(), true); + std::copy (outFFTBuf.begin(), outFFTBuf.end(), outPreMatchFFTBuf.begin()); + + for (int i = 0; i < halfBins; ++i) + { + const float ore = origFFTBuf[static_cast<size_t> (i * 2)]; + const float oim = origFFTBuf[static_cast<size_t> (i * 2 + 1)]; + origLogMag[static_cast<size_t> (i)] = std::log (std::sqrt (ore * ore + oim * oim) + 1.0e-10f); + + const float sre = outFFTBuf[static_cast<size_t> (i * 2)]; + const float sim = outFFTBuf[static_cast<size_t> (i * 2 + 1)]; + outLogMag[static_cast<size_t> (i)] = std::log (std::sqrt (sre * sre + sim * sim) + 1.0e-10f); + } + + int smoothHalfW; + if (avgPitchHz > 50.0f) + smoothHalfW = std::max (previewFast ? 10 : 14, + static_cast<int> ((previewFast ? 1.8f : 2.4f) * avgPitchHz / std::max (1.0f, binHz))); + else + smoothHalfW = previewFast ? 24 : 36; + smoothHalfW = std::min (smoothHalfW, halfBins / 3); + + computeEnvelope (origLogMag, smoothHalfW, origEnv); + computeEnvelope (outLogMag, smoothHalfW, outEnv); + + float energyBefore = 0.0f; + float energyAfter = 0.0f; + for (int i = 0; i < halfBins; ++i) + { + const float re = outFFTBuf[static_cast<size_t> (i * 2)]; + const float im = outFFTBuf[static_cast<size_t> (i * 2 + 1)]; + energyBefore += re * re + im * im; + } + + const float blockBlend = juce::jlimit (0.0f, 1.0f, localPitchWeight); + const float downshiftGuardBoost = upwardShift ? 1.0f : 1.22f; + const float rawStrength = ((0.34f + 0.20f * pitchStrength) + * (0.32f + 0.78f * voicedBlend) + * noteShoulderEase + * hqBoost + * shortUpwardNoteBoost + * downshiftGuardBoost) * (hybridReset ? 0.82f : 1.0f); + if (! std::isfinite (rawStrength)) + { + ++invalidStrengthBlocks; + if (! loggedInvalidStrength) + { + loggedInvalidStrength = true; + logPitchEditorFormant ("pitch envelope anchor invalid rawStrength=" + + juce::String (rawStrength) + + " avgPitchShiftSt=" + juce::String (avgPitchShiftSt) + + " pitchStrength=" + juce::String (pitchStrength) + + " voicedBlend=" + juce::String (voicedBlend) + + " noteProgress=" + juce::String (safeNoteProgress) + + " noteShoulderEase=" + juce::String (noteShoulderEase) + + " localPitchWeight=" + juce::String (localPitchWeight) + + " dominantStart=" + juce::String (dominantNote != nullptr ? dominantNote->startTime : -1.0f) + + " dominantEnd=" + juce::String (dominantNote != nullptr ? dominantNote->endTime : -1.0f)); + } + } + const float strength = juce::jlimit (0.0f, + upwardShift ? (previewFast ? 0.32f : 0.42f) + : (previewFast ? 0.40f : 0.62f), + sanitizeFiniteFloat (rawStrength, 0.0f)); + avgAppliedStrength += strength; + + for (int i = 1; i < halfBins; ++i) + { + const float freqHz = static_cast<float> (i) * binHz; + float harmonicMask = 0.0f; + if (avgPitchHz > 55.0f && freqHz > avgPitchHz) + { + const float harmonicIndex = freqHz / avgPitchHz; + const float nearestHarmonic = std::max (1.0f, std::round (harmonicIndex)); + const float dist = std::abs (harmonicIndex - nearestHarmonic); + const float sigma = upwardShift && hybridReset + ? (previewFast ? 0.14f : 0.10f) + : (previewFast ? 0.18f : 0.13f); + harmonicMask = std::exp (-0.5f * (dist / sigma) * (dist / sigma)); + harmonicMask *= smoothstep01 ((freqHz - 160.0f) / 220.0f); + } + harmonicMask = sanitizeFiniteFloat (harmonicMask, 0.0f); + + const float midLift = upwardShift + ? ((hybridReset ? 1.08f : 1.02f) * gaussian (freqHz, 1080.0f, 480.0f) + + (hybridReset ? 0.74f : 0.66f) * gaussian (freqHz, 1850.0f, 620.0f) + + (hybridReset ? 0.10f : 0.06f) * gaussian (freqHz, 2850.0f, 720.0f)) + : (1.20f * gaussian (freqHz, 950.0f, 520.0f) + + 1.05f * gaussian (freqHz, 1700.0f, 680.0f) + + 0.46f * gaussian (freqHz, 2850.0f, 780.0f)); + const float lowTrim = upwardShift + ? ((hybridReset ? 0.22f : 0.28f) * gaussian (freqHz, 360.0f, 180.0f) + + (hybridReset ? 0.16f : 0.20f) * gaussian (freqHz, 760.0f, 260.0f)) + : (0.12f * gaussian (freqHz, 260.0f, 150.0f) + + 0.08f * gaussian (freqHz, 620.0f, 230.0f)); + const float airProtect = upwardShift + ? (hybridReset + ? (1.0f - 0.48f * smoothstep01 ((freqHz - 3400.0f) / 1400.0f)) + : (1.0f - 0.60f * smoothstep01 ((freqHz - 3200.0f) / 1200.0f))) + : (1.0f - 0.76f * smoothstep01 ((freqHz - 3500.0f) / 1200.0f)); + const float downshiftBandGuard = upwardShift + ? 1.0f + : (smoothstep01 ((freqHz - 180.0f) / 160.0f) + * (1.0f - 0.70f * smoothstep01 ((freqHz - 3500.0f) / 1200.0f))); + const float envelopeError = sanitizeFiniteFloat (std::log ((origEnv[static_cast<size_t> (i)] + 1.0e-10f) + / (outEnv[static_cast<size_t> (i)] + 1.0e-10f)), 0.0f); + const float envelopeWeight = sanitizeFiniteFloat ((0.38f + 0.62f * harmonicMask) + * (0.35f + 0.65f * voicedBlend) + * (0.82f + midLift - lowTrim) + * airProtect + * downshiftBandGuard, 0.0f); + const float gain = sanitizeFiniteFloat (juce::jlimit ( + upwardShift + ? (previewFast ? 0.88f : 0.86f) + : (previewFast ? 0.86f : 0.82f), + upwardShift + ? (previewFast ? 1.12f : 1.14f) + : (previewFast ? 1.14f : 1.18f), + std::pow (10.0f, (envelopeError * strength * envelopeWeight * blockBlend) / 2.302585093f)), 1.0f); + + outFFTBuf[static_cast<size_t> (i * 2)] *= gain; + outFFTBuf[static_cast<size_t> (i * 2 + 1)] *= gain; + } + + for (int i = 0; i < halfBins; ++i) + { + const float re = outFFTBuf[static_cast<size_t> (i * 2)]; + const float im = outFFTBuf[static_cast<size_t> (i * 2 + 1)]; + energyAfter += re * re + im * im; + } + if (energyAfter > 1.0e-10f) + { + const float scale = sanitizeFiniteFloat (std::pow (energyBefore / energyAfter, 0.08f), 1.0f); + for (int i = 0; i < halfBins; ++i) + { + outFFTBuf[static_cast<size_t> (i * 2)] *= scale; + outFFTBuf[static_cast<size_t> (i * 2 + 1)] *= scale; + } + } + + // Keep a little of the pre-match fine detail so the anchor pass + // preserves the source-filter "move excitation, keep colour" feel + // without over-smoothing the note. + for (int i = 0; i < halfBins; ++i) + { + const float freqHz = static_cast<float> (i) * binHz; + const float detailKeepBase = (0.05f + 0.07f * smoothstep01 ((freqHz - 3000.0f) / 1800.0f)) + * (1.0f - 0.70f * voicedBlend); + const float detailKeep = juce::jlimit (0.0f, hybridReset ? (upwardShift ? 0.14f : 0.06f) : 0.12f, + detailKeepBase); + if (detailKeep <= 0.001f) + continue; + outFFTBuf[static_cast<size_t> (i * 2)] = + outFFTBuf[static_cast<size_t> (i * 2)] * (1.0f - detailKeep) + + outPreMatchFFTBuf[static_cast<size_t> (i * 2)] * detailKeep; + outFFTBuf[static_cast<size_t> (i * 2 + 1)] = + outFFTBuf[static_cast<size_t> (i * 2 + 1)] * (1.0f - detailKeep) + + outPreMatchFFTBuf[static_cast<size_t> (i * 2 + 1)] * detailKeep; + } + + fft.performRealOnlyInverseTransform (outFFTBuf.data()); + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + if (idx >= 0 && idx < numSamples) + { + const float w = hannWin[static_cast<size_t> (i)]; + corrected[static_cast<size_t> (idx)] += outFFTBuf[static_cast<size_t> (i)] * w; + winSum[static_cast<size_t> (idx)] += w * w; + } + } + } + + for (int s = 0; s < numSamples; ++s) + { + if (winSum[static_cast<size_t> (s)] > 0.001f) + out[static_cast<size_t> (s)] = corrected[static_cast<size_t> (s)] / winSum[static_cast<size_t> (s)]; + } + } + + logPitchEditorFormant ("pitch envelope anchor blocks=" + juce::String (processedBlocks) + + " avgStrength=" + juce::String (processedBlocks > 0 ? avgAppliedStrength / static_cast<float> (processedBlocks) : 0.0f, 3) + + " avgVoicedBlend=" + juce::String (processedBlocks > 0 ? avgVoicedBlend / static_cast<float> (processedBlocks) : 0.0f, 3) + + " invalidStrengthBlocks=" + juce::String (invalidStrengthBlocks) + + " renderQuality=" + juce::String (previewFast ? "preview_fast" : "final_hq")); +} + +static void applyReferenceResidualMatch ( + std::vector<std::vector<float>>& output, + const float* const* originalInput, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& formantRatios, + const std::vector<float>& detectedPitchHz, + bool hasPitchShift, + bool hasExplicitFormant, + PitchResynthesizer::RenderQuality renderQuality, + std::function<bool()> shouldCancel, + float intensityScale = 1.0f) +{ + if (! hasPitchShift && ! hasExplicitFormant) + return; + + const bool previewFast = renderQuality == PitchResynthesizer::RenderQuality::PreviewFast; + const int fftOrder = previewFast ? 10 : 12; + const int fftSize = 1 << fftOrder; + const int hopLen = fftSize / 4; + const int halfBins = fftSize / 2 + 1; + const float pi = juce::MathConstants<float>::pi; + const float binHz = static_cast<float> (sampleRate) / static_cast<float> (fftSize); + const float blockDurSec = static_cast<float> (fftSize) / static_cast<float> (sampleRate); + + juce::dsp::FFT fft (fftOrder); + std::vector<float> hannWin (static_cast<size_t> (fftSize)); + for (int i = 0; i < fftSize; ++i) + hannWin[static_cast<size_t> (i)] = 0.5f * (1.0f - std::cos ( + 2.0f * pi * static_cast<float> (i) / static_cast<float> (fftSize - 1))); + + auto gaussian = [] (float x, float centre, float width) + { + const float safeWidth = std::max (1.0f, width); + const float d = (x - centre) / safeWidth; + return std::exp (-0.5f * d * d); + }; + + int processedBlocks = 0; + int harmonicDrivenBlocks = 0; + + for (int ch = 0; ch < numChannels; ++ch) + { + auto& out = output[static_cast<size_t> (ch)]; + const float* orig = originalInput[ch]; + std::vector<float> corrected (static_cast<size_t> (numSamples), 0.0f); + std::vector<float> winSum (static_cast<size_t> (numSamples), 0.0f); + std::vector<float> origFFTBuf (static_cast<size_t> (fftSize * 2), 0.0f); + std::vector<float> outFFTBuf (static_cast<size_t> (fftSize * 2), 0.0f); + + for (int pos = 0; pos < numSamples; pos += hopLen) + { + if (shouldCancel && shouldCancel()) + return; + + const float blockStartSec = static_cast<float> (pos) / static_cast<float> (sampleRate); + const float blockEndSec = static_cast<float> (std::min (numSamples, pos + fftSize)) / static_cast<float> (sampleRate); + const float blockCenterSec = 0.5f * (blockStartSec + blockEndSec); + const int blockSampleCount = std::max (1, std::min (fftSize, numSamples - pos)); + + float dominantOverlap = 0.0f; + const PitchAnalyzer::PitchNote* dominantNote = nullptr; + float avgPitchShiftSt = 0.0f; + float pitchWeight = 0.0f; + float localEditWeight = 0.0f; + + for (const auto& note : notes) + { + const float noteStart = getEffectiveNoteStartTime (note); + const float noteEnd = getEffectiveNoteEndTime (note); + const float overlap = std::max (0.0f, std::min (blockEndSec, noteEnd) - std::max (blockStartSec, noteStart)); + if (overlap <= 0.0f) + continue; + + const float overlapWeight = overlap / std::max (0.001f, blockDurSec); + const bool pitchEdited = hasPitchStyleEdit (note); + const bool formantEdited = std::abs (note.formantShift) > 0.01f || hasExplicitFormant; + if (! pitchEdited && ! formantEdited) + continue; + + localEditWeight += overlapWeight; + if (pitchEdited) + { + pitchWeight += overlapWeight; + avgPitchShiftSt += (note.correctedPitch - note.detectedPitch) * overlapWeight; + } + if (overlapWeight > dominantOverlap) + { + dominantOverlap = overlapWeight; + dominantNote = ¬e; + } + } + + if (localEditWeight <= 0.001f) + { + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + if (idx >= 0 && idx < numSamples) + { + const float w = hannWin[static_cast<size_t> (i)]; + corrected[static_cast<size_t> (idx)] += out[static_cast<size_t> (idx)] * w; + winSum[static_cast<size_t> (idx)] += w * w; + } + } + continue; + } + + ++processedBlocks; + avgPitchShiftSt = pitchWeight > 0.0f ? avgPitchShiftSt / pitchWeight : 0.0f; + + float avgPitchHz = 0.0f; + int pitchCount = 0; + for (int i = pos; i < std::min (pos + fftSize, numSamples); ++i) + { + if (static_cast<size_t> (i) < detectedPitchHz.size() && detectedPitchHz[static_cast<size_t> (i)] > 0.0f) + { + avgPitchHz += detectedPitchHz[static_cast<size_t> (i)]; + ++pitchCount; + } + } + if (pitchCount > 0) + avgPitchHz /= static_cast<float> (pitchCount); + const float voicedFraction = static_cast<float> (pitchCount) / static_cast<float> (blockSampleCount); + + float avgFormantRatio = 1.0f; + if (! formantRatios.empty()) + { + float sumRatio = 0.0f; + int ratioCount = 0; + for (int i = pos; i < std::min (pos + fftSize, numSamples); ++i) + { + if (static_cast<size_t> (i) < formantRatios.size()) + { + sumRatio += formantRatios[static_cast<size_t> (i)]; + ++ratioCount; + } + } + if (ratioCount > 0) + avgFormantRatio = sumRatio / static_cast<float> (ratioCount); + } + + // Global formant references behave like a voiced timbre field, not only + // a note-local transform. If no edited note overlaps this block but the + // block is voiced and a formant shift is active, still apply the sign- + // specific residual match so off-note voiced audio follows the sample. + if (hasExplicitFormant && voicedFraction > 0.12f) + localEditWeight = std::max (localEditWeight, voicedFraction * 0.78f); + + const bool lowerFormant = avgFormantRatio < 0.995f; + const bool upperFormant = avgFormantRatio > 1.005f; + const float formantStrength = std::min (1.4f, std::abs (12.0f * std::log2 (std::max (avgFormantRatio, 1.0e-4f))) / 1.8f); + const float pitchStrength = std::min (1.0f, std::abs (avgPitchShiftSt) / 4.0f); + const float noteProgress = dominantNote != nullptr + ? juce::jlimit (0.0f, 1.0f, (blockCenterSec - dominantNote->startTime) + / std::max (0.025f, dominantNote->endTime - dominantNote->startTime)) + : 0.5f; + const float noteMidFocus = std::pow (std::sin (pi * juce::jlimit (0.0f, 1.0f, noteProgress)), 0.9f); + const float noteEarlyFocus = std::pow (1.0f - juce::jlimit (0.0f, 1.0f, noteProgress), 0.8f); + const float noteLateFocus = std::pow (juce::jlimit (0.0f, 1.0f, noteProgress), 0.8f); + const float blockBlend = juce::jlimit (0.0f, 1.0f, localEditWeight); + const float hqBoost = previewFast ? 0.88f : 1.12f; + + std::fill (origFFTBuf.begin(), origFFTBuf.end(), 0.0f); + std::fill (outFFTBuf.begin(), outFFTBuf.end(), 0.0f); + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + if (idx >= 0 && idx < numSamples) + { + const float w = hannWin[static_cast<size_t> (i)]; + origFFTBuf[static_cast<size_t> (i)] = orig[idx] * w; + outFFTBuf[static_cast<size_t> (i)] = out[static_cast<size_t> (idx)] * w; + } + } + + fft.performRealOnlyForwardTransform (origFFTBuf.data(), true); + fft.performRealOnlyForwardTransform (outFFTBuf.data(), true); + + float energyBefore = 0.0f; + float energyAfter = 0.0f; + for (int i = 0; i < halfBins; ++i) + { + const float re = outFFTBuf[static_cast<size_t> (i * 2)]; + const float im = outFFTBuf[static_cast<size_t> (i * 2 + 1)]; + energyBefore += re * re + im * im; + } + + bool harmonicDriven = false; + const float harmonicSigma = previewFast ? 0.18f : 0.14f; + for (int i = 1; i < halfBins; ++i) + { + const float freqHz = static_cast<float> (i) * binHz; + float harmonicMask = 0.0f; + if (avgPitchHz > 55.0f && freqHz > avgPitchHz) + { + const float harmonicIndex = freqHz / avgPitchHz; + const float nearestHarmonic = std::max (1.0f, std::round (harmonicIndex)); + const float dist = std::abs (harmonicIndex - nearestHarmonic); + harmonicMask = std::exp (-0.5f * (dist / harmonicSigma) * (dist / harmonicSigma)); + harmonicMask *= smoothstep01 ((freqHz - 180.0f) / 220.0f); + } + + float dbDelta = 0.0f; + if (pitchStrength > 0.001f) + { + const float harmonicTilt = 0.24f + + 0.38f * gaussian (freqHz, 1800.0f, 900.0f) + + 0.70f * gaussian (freqHz, 3600.0f, 1400.0f) + + 0.42f * gaussian (freqHz, 6200.0f, 1500.0f); + const float lowMidTrim = 0.18f * gaussian (freqHz, 520.0f, 260.0f); + dbDelta += pitchStrength * noteMidFocus * hqBoost + * harmonicMask * (harmonicTilt - lowMidTrim); + } + + if (upperFormant) + { + const float formantBand = 0.64f * gaussian (freqHz, 1500.0f, 560.0f) + + 0.72f * gaussian (freqHz, 2050.0f, 620.0f) + + 1.14f * gaussian (freqHz, 5950.0f, 1080.0f) + - 0.92f * gaussian (freqHz, 7900.0f, 760.0f) + - 0.18f * gaussian (freqHz, 340.0f, 180.0f); + const float broadband = -0.12f * smoothstep01 ((freqHz - 7600.0f) / 2600.0f); + const float timeLaw = 0.70f + 0.30f * noteMidFocus + 0.62f * noteLateFocus; + dbDelta += formantStrength * hqBoost * timeLaw * (formantBand + broadband) * (0.28f + 0.72f * harmonicMask); + } + else if (lowerFormant) + { + const float formantBand = 0.86f * gaussian (freqHz, 1180.0f, 820.0f) + + 0.56f * gaussian (freqHz, 1850.0f, 520.0f) + + 1.10f * gaussian (freqHz, 5450.0f, 1320.0f) + + 0.66f * gaussian (freqHz, 6750.0f, 1180.0f) + - 0.98f * gaussian (freqHz, 10800.0f, 980.0f); + const float broadband = -0.16f * smoothstep01 ((freqHz - 6800.0f) / 2000.0f) + - 0.08f * smoothstep01 ((freqHz - 3200.0f) / 1800.0f); + const float timeLaw = 0.74f + 0.68f * noteEarlyFocus + 0.20f * noteMidFocus; + dbDelta += formantStrength * hqBoost * timeLaw * (formantBand + broadband) * (0.34f + 0.66f * harmonicMask); + } + + dbDelta *= intensityScale; + if (std::abs (dbDelta) > 0.001f) + { + harmonicDriven = harmonicDriven || harmonicMask > 0.15f; + const float gain = juce::jlimit (0.55f, 1.85f, std::pow (10.0f, (dbDelta * blockBlend) / 20.0f)); + outFFTBuf[static_cast<size_t> (i * 2)] *= gain; + outFFTBuf[static_cast<size_t> (i * 2 + 1)] *= gain; + } + } + + for (int i = 0; i < halfBins; ++i) + { + const float re = outFFTBuf[static_cast<size_t> (i * 2)]; + const float im = outFFTBuf[static_cast<size_t> (i * 2 + 1)]; + energyAfter += re * re + im * im; + } + if (energyAfter > 1.0e-10f) + { + const float scale = std::pow (energyBefore / energyAfter, hasExplicitFormant ? 0.22f : 0.18f); + for (int i = 0; i < halfBins; ++i) + { + outFFTBuf[static_cast<size_t> (i * 2)] *= scale; + outFFTBuf[static_cast<size_t> (i * 2 + 1)] *= scale; + } + } + + if (harmonicDriven) + ++harmonicDrivenBlocks; + + fft.performRealOnlyInverseTransform (outFFTBuf.data()); + for (int i = 0; i < fftSize; ++i) + { + const int idx = pos + i; + if (idx >= 0 && idx < numSamples) + { + const float w = hannWin[static_cast<size_t> (i)]; + corrected[static_cast<size_t> (idx)] += outFFTBuf[static_cast<size_t> (i)] * w; + winSum[static_cast<size_t> (idx)] += w * w; + } + } + } + + for (int s = 0; s < numSamples; ++s) + { + if (winSum[static_cast<size_t> (s)] > 0.001f) + out[static_cast<size_t> (s)] = corrected[static_cast<size_t> (s)] / winSum[static_cast<size_t> (s)]; + } + } + + logPitchEditorFormant ("reference residual match blocks=" + juce::String (processedBlocks) + + " harmonicDrivenBlocks=" + juce::String (harmonicDrivenBlocks) + + " hasPitch=" + juce::String (hasPitchShift ? "true" : "false") + + " hasFormant=" + juce::String (hasExplicitFormant ? "true" : "false") + + " intensityScale=" + juce::String (intensityScale, 3) + + " renderQuality=" + juce::String (previewFast ? "preview_fast" : "final_hq")); +} + +static std::vector<PitchOnlyCoreRegion> derivePitchOnlyCoreRegions ( + const float* const* originalInput, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& detectedPitchHz) +{ + std::vector<PitchOnlyCoreRegion> regions; + if (detectedPitchHz.empty()) + return regions; + + const auto hasStableVoicedSupport = [&] (int sampleIndex, int radiusSamples) + { + int supported = 0; + int checked = 0; + for (int s = std::max (0, sampleIndex - radiusSamples); + s <= std::min (numSamples - 1, sampleIndex + radiusSamples); ++s) + { + ++checked; + if (static_cast<size_t> (s) < detectedPitchHz.size() + && detectedPitchHz[static_cast<size_t> (s)] > 0.0f) + { + ++supported; + } + } + + return checked > 0 && supported * 10 >= checked * 7; + }; + + for (const auto& note : notes) + { + if (! hasPitchStyleEdit (note)) + continue; + + PitchOnlyCoreRegion region; + const int startSample = juce::jlimit (0, numSamples, static_cast<int> (std::floor (note.startTime * sampleRate))); + const int endSample = juce::jlimit (0, numSamples, static_cast<int> (std::ceil (note.endTime * sampleRate))); + if (endSample <= startSample) + continue; + region.bodyStartSample = startSample; + region.bodyEndSample = endSample; + const float detectedHz = midiToHz (note.detectedPitch); + const float correctedHz = midiToHz (note.correctedPitch); + region.pitchRatio = detectedHz > 0.0f + ? juce::jlimit (0.25f, 4.0f, correctedHz / detectedHz) + : 1.0f; + region.upwardShift = region.pitchRatio >= 1.0f; + + const int bodyLen = std::max (1, endSample - startSample); + const int edgeProtectIn = std::max (1, std::min (static_cast<int> (std::round (0.032 * sampleRate)), bodyLen / 3)); + const int edgeProtectOut = std::max (1, std::min (static_cast<int> (std::round (0.038 * sampleRate)), bodyLen / 3)); + const int coreSearchStart = startSample + edgeProtectIn; + const int coreSearchEnd = endSample - edgeProtectOut; + if (coreSearchEnd <= coreSearchStart) + continue; + + const int voicedRadius = std::max (1, static_cast<int> (std::round (0.008 * sampleRate))); + int coreStart = -1; + int coreEnd = -1; + for (int s = coreSearchStart; s < coreSearchEnd; ++s) + { + if (static_cast<size_t> (s) < detectedPitchHz.size() + && detectedPitchHz[static_cast<size_t> (s)] > 0.0f + && hasStableVoicedSupport (s, voicedRadius)) + { + coreStart = s; + break; + } + } + for (int s = coreSearchEnd - 1; s >= coreSearchStart; --s) + { + if (static_cast<size_t> (s) < detectedPitchHz.size() + && detectedPitchHz[static_cast<size_t> (s)] > 0.0f + && hasStableVoicedSupport (s, voicedRadius)) + { + coreEnd = s + 1; + break; + } + } + + if (coreStart < 0 || coreEnd <= coreStart) + continue; + + const int coreLen = coreEnd - coreStart; + const int minimumCoreLen = std::max (1, static_cast<int> (std::round (0.055 * sampleRate))); + if (coreLen < minimumCoreLen) + continue; + + double energySum = 0.0; + int energyCount = 0; + for (int s = coreStart; s < coreEnd; ++s) + { + float sampleAbs = 0.0f; + for (int ch = 0; ch < numChannels; ++ch) + sampleAbs = std::max (sampleAbs, std::abs (originalInput[ch][s])); + energySum += sampleAbs * sampleAbs; + ++energyCount; + } + const float noteRms = energyCount > 0 ? std::sqrt (static_cast<float> (energySum / static_cast<double> (energyCount))) : 0.0f; + if (noteRms <= 0.0025f) + continue; + + region.coreStartSample = coreStart; + region.coreEndSample = coreEnd; + regions.push_back (region); + } + + return regions; +} + +static std::vector<float> buildPitchOnlyCoreMask ( + const float* const* originalInput, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& detectedPitchHz) +{ + std::vector<float> coreMask (static_cast<size_t> (numSamples), 0.0f); + if (detectedPitchHz.empty()) + return coreMask; + + const auto regions = derivePitchOnlyCoreRegions (originalInput, numChannels, numSamples, sampleRate, notes, detectedPitchHz); + if (regions.empty()) + return coreMask; + + const auto hasStableVoicedSupport = [&] (int sampleIndex, int radiusSamples) + { + int supported = 0; + int checked = 0; + for (int s = std::max (0, sampleIndex - radiusSamples); + s <= std::min (numSamples - 1, sampleIndex + radiusSamples); ++s) + { + ++checked; + if (static_cast<size_t> (s) < detectedPitchHz.size() + && detectedPitchHz[static_cast<size_t> (s)] > 0.0f) + { + ++supported; + } + } + return checked > 0 && supported * 10 >= checked * 7; + }; + + for (const auto& region : regions) + { + const int coreStart = region.coreStartSample; + const int coreEnd = region.coreEndSample; + const int coreLen = coreEnd - coreStart; + const int voicedRadius = std::max (1, static_cast<int> (std::round (0.008 * sampleRate))); + + double energySum = 0.0; + int energyCount = 0; + for (int s = coreStart; s < coreEnd; ++s) + { + float sampleAbs = 0.0f; + for (int ch = 0; ch < numChannels; ++ch) + sampleAbs = std::max (sampleAbs, std::abs (originalInput[ch][s])); + energySum += sampleAbs * sampleAbs; + ++energyCount; + } + const float noteRms = energyCount > 0 ? std::sqrt (static_cast<float> (energySum / static_cast<double> (energyCount))) : 0.0f; + if (noteRms <= 1.0e-4f) + continue; + + for (int s = coreStart; s < coreEnd; ++s) + { + if (static_cast<size_t> (s) >= detectedPitchHz.size() + || detectedPitchHz[static_cast<size_t> (s)] <= 0.0f + || ! hasStableVoicedSupport (s, voicedRadius)) + { + continue; + } + + float sampleAbs = 0.0f; + for (int ch = 0; ch < numChannels; ++ch) + sampleAbs = std::max (sampleAbs, std::abs (originalInput[ch][s])); + + const float coreProgress = juce::jlimit (0.0f, 1.0f, + static_cast<float> (s - coreStart) / static_cast<float> (std::max (1, coreLen - 1))); + const float coreFocus = std::pow (std::sin (juce::MathConstants<float>::pi * coreProgress), 1.16f); + const float energyGate = smoothstep01 ((sampleAbs - noteRms * 0.30f) / std::max (noteRms * 0.44f, 1.0e-4f)); + const float regionMask = juce::jlimit (0.0f, 0.72f, coreFocus * energyGate); + if (regionMask <= 0.0001f) + continue; + + coreMask[static_cast<size_t> (s)] = std::max (coreMask[static_cast<size_t> (s)], regionMask); + } + } + + return coreMask; +} + +static bool applyPitchOnlyCoreRmsTrim ( + std::vector<std::vector<float>>& output, + const float* const* originalInput, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& detectedPitchHz, + float maxTrimDb, + bool lowMidOnly, + float* appliedTrimDbOut) +{ + if (output.empty() || originalInput == nullptr || numChannels <= 0 || numSamples <= 0 || sampleRate <= 0.0) + return false; + + const auto regions = derivePitchOnlyCoreRegions (originalInput, numChannels, numSamples, sampleRate, notes, detectedPitchHz); + if (regions.empty()) + return false; + + const auto coreMask = buildPitchOnlyCoreMask (originalInput, numChannels, numSamples, sampleRate, notes, detectedPitchHz); + if (coreMask.empty()) + return false; + + std::vector<float> trimMask (static_cast<size_t> (numSamples), 0.0f); + const int requestedFadeSamples = std::max (1, static_cast<int> (std::round (0.025 * sampleRate))); + for (const auto& region : regions) + { + const int coreStart = juce::jlimit (0, numSamples, region.coreStartSample); + const int coreEnd = juce::jlimit (coreStart, numSamples, region.coreEndSample); + const int coreLen = coreEnd - coreStart; + if (coreLen <= 1) + continue; + + const int fadeSamples = std::max (1, std::min (requestedFadeSamples, coreLen / 2)); + for (int s = coreStart; s < coreEnd; ++s) + { + const float fromStart = static_cast<float> (s - coreStart) / static_cast<float> (fadeSamples); + const float fromEnd = static_cast<float> (coreEnd - 1 - s) / static_cast<float> (fadeSamples); + const float edge = smoothstep01 (std::min (fromStart, fromEnd)); + const float core = static_cast<size_t> (s) < coreMask.size() + ? smoothstep01 (coreMask[static_cast<size_t> (s)] / 0.72f) + : 0.0f; + trimMask[static_cast<size_t> (s)] = std::max (trimMask[static_cast<size_t> (s)], edge * core); + } + } + + double sourceEnergy = 0.0; + double outputEnergy = 0.0; + double weightSum = 0.0; + for (int ch = 0; ch < numChannels; ++ch) + { + if (static_cast<size_t> (ch) >= output.size()) + continue; + + for (int s = 0; s < numSamples; ++s) + { + const float weight = trimMask[static_cast<size_t> (s)]; + if (weight <= 0.0001f) + continue; + + const float sourceSample = originalInput[ch][s]; + const float renderedSample = static_cast<size_t> (s) < output[static_cast<size_t> (ch)].size() + ? output[static_cast<size_t> (ch)][static_cast<size_t> (s)] + : 0.0f; + sourceEnergy += static_cast<double> (sourceSample) * static_cast<double> (sourceSample) * weight; + outputEnergy += static_cast<double> (renderedSample) * static_cast<double> (renderedSample) * weight; + weightSum += weight; + } + } + + if (weightSum <= 1.0 || sourceEnergy <= 1.0e-10 || outputEnergy <= 1.0e-10) + return false; + + const double sourceRms = std::sqrt (sourceEnergy / weightSum); + const double outputRms = std::sqrt (outputEnergy / weightSum); + const float requestedTrimDb = static_cast<float> (20.0 * std::log10 (sourceRms / outputRms)); + const float trimDb = juce::jlimit (-std::abs (maxTrimDb), std::abs (maxTrimDb), requestedTrimDb); + if (std::abs (trimDb) < 0.02f) + { + if (appliedTrimDbOut != nullptr) + *appliedTrimDbOut = 0.0f; + return false; + } + + const float gain = std::pow (10.0f, trimDb / 20.0f); + const float lowpassCutoffHz = juce::jlimit (900.0f, 3200.0f, getEnvFloat ("OPENSTUDIO_PITCH_CORE_RMS_LOWMID_CUTOFF_HZ", 2200.0f)); + const float highCompensation = lowMidOnly + ? juce::jlimit (0.0f, 1.4f, getEnvFloat ("OPENSTUDIO_PITCH_CORE_RMS_HIGH_COMP", 1.4f)) + : 0.0f; + const float lowpassCoeff = static_cast<float> (std::exp ( + -juce::MathConstants<double>::twoPi * static_cast<double> (lowpassCutoffHz) / sampleRate)); + for (int ch = 0; ch < numChannels; ++ch) + { + if (static_cast<size_t> (ch) >= output.size()) + continue; + + std::vector<float> lowMid; + if (lowMidOnly) + { + lowMid = output[static_cast<size_t> (ch)]; + float state = 0.0f; + for (auto& sample : lowMid) + { + state = (1.0f - lowpassCoeff) * sample + lowpassCoeff * state; + sample = state; + } + state = 0.0f; + for (auto it = lowMid.rbegin(); it != lowMid.rend(); ++it) + { + state = (1.0f - lowpassCoeff) * *it + lowpassCoeff * state; + *it = state; + } + } + + for (int s = 0; s < numSamples && static_cast<size_t> (s) < output[static_cast<size_t> (ch)].size(); ++s) + { + const float weight = trimMask[static_cast<size_t> (s)]; + if (weight <= 0.0001f) + continue; + + if (lowMidOnly) + { + const float current = output[static_cast<size_t> (ch)][static_cast<size_t> (s)]; + const float lowMidSample = lowMid[static_cast<size_t> (s)]; + const float highSample = current - lowMidSample; + output[static_cast<size_t> (ch)][static_cast<size_t> (s)] += (gain - 1.0f) + * (lowMidSample - highCompensation * highSample) + * weight; + } + else + { + const float localGain = 1.0f + (gain - 1.0f) * weight; + output[static_cast<size_t> (ch)][static_cast<size_t> (s)] *= localGain; + } + } + } + + if (appliedTrimDbOut != nullptr) + *appliedTrimDbOut = trimDb; + return true; +} + +static bool applyPitchOnlyCoreAperiodicTexture ( + std::vector<std::vector<float>>& output, + const float* const* originalInput, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& detectedPitchHz, + bool downwardOnly, + float* mixUsedOut) +{ + if (output.empty() || originalInput == nullptr || numChannels <= 0 || numSamples <= 0 || sampleRate <= 0.0) + return false; + + const float defaultMix = downwardOnly ? 0.20f : 0.30f; + const float mix = juce::jlimit (0.0f, 0.30f, getEnvFloat ( + downwardOnly ? "OPENSTUDIO_PITCH_CORE_TEXTURE_MIX_DOWN" : "OPENSTUDIO_PITCH_CORE_TEXTURE_MIX_UP", + defaultMix)); + if (mix <= 0.001f) + return false; + + const float cutoffHz = juce::jlimit (2600.0f, 9000.0f, getEnvFloat ( + downwardOnly ? "OPENSTUDIO_PITCH_CORE_TEXTURE_CUTOFF_HZ_DOWN" : "OPENSTUDIO_PITCH_CORE_TEXTURE_CUTOFF_HZ_UP", + downwardOnly ? 5200.0f : 4200.0f)); + + auto coreMask = buildPitchOnlyCoreMask (originalInput, numChannels, numSamples, sampleRate, notes, detectedPitchHz); + if (coreMask.empty()) + return false; + + float maskPeak = 0.0f; + for (auto& mask : coreMask) + { + mask = smoothstep01 (mask / 0.72f); + maskPeak = std::max (maskPeak, mask); + } + if (maskPeak <= 0.001f) + return false; + + const float coeff = static_cast<float> (std::exp ( + -juce::MathConstants<double>::twoPi * static_cast<double> (cutoffHz) / sampleRate)); + + auto makeZeroPhaseLowpass = [coeff] (const std::vector<float>& source) + { + std::vector<float> low = source; + float state = 0.0f; + for (auto& sample : low) + { + state = (1.0f - coeff) * sample + coeff * state; + sample = state; + } + + state = 0.0f; + for (auto it = low.rbegin(); it != low.rend(); ++it) + { + state = (1.0f - coeff) * *it + coeff * state; + *it = state; + } + return low; + }; + + bool changed = false; + for (int ch = 0; ch < numChannels; ++ch) + { + if (static_cast<size_t> (ch) >= output.size() + || output[static_cast<size_t> (ch)].size() < static_cast<size_t> (numSamples)) + continue; + + std::vector<float> source (static_cast<size_t> (numSamples), 0.0f); + for (int s = 0; s < numSamples; ++s) + source[static_cast<size_t> (s)] = originalInput[ch][s]; + + const auto sourceLow = makeZeroPhaseLowpass (source); + const auto outputLow = makeZeroPhaseLowpass (output[static_cast<size_t> (ch)]); + auto& dst = output[static_cast<size_t> (ch)]; + for (int s = 0; s < numSamples; ++s) + { + const float mask = coreMask[static_cast<size_t> (s)]; + if (mask <= 0.001f) + continue; + + const float sourceHigh = source[static_cast<size_t> (s)] - sourceLow[static_cast<size_t> (s)]; + const float renderedHigh = dst[static_cast<size_t> (s)] - outputLow[static_cast<size_t> (s)]; + dst[static_cast<size_t> (s)] += (sourceHigh - renderedHigh) * (mix * mask); + changed = true; + } + } + + if (changed && mixUsedOut != nullptr) + *mixUsedOut = mix; + return changed; +} + +static bool applyPitchOnlyCoreDirectionalLevel ( + std::vector<std::vector<float>>& output, + const float* const* originalInput, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& detectedPitchHz, + bool downwardOnly, + float* appliedDbOut) +{ + if (output.empty() || originalInput == nullptr || numChannels <= 0 || numSamples <= 0 || sampleRate <= 0.0) + return false; + + const float defaultDb = downwardOnly ? -0.90f : 1.05f; + const float gainDb = juce::jlimit (-1.5f, 1.5f, getEnvFloat ( + downwardOnly ? "OPENSTUDIO_PITCH_CORE_LEVEL_DB_DOWN" : "OPENSTUDIO_PITCH_CORE_LEVEL_DB_UP", + defaultDb)); + if (std::abs (gainDb) <= 0.01f) + return false; + + auto coreMask = buildPitchOnlyCoreMask (originalInput, numChannels, numSamples, sampleRate, notes, detectedPitchHz); + if (coreMask.empty()) + return false; + + float maskPeak = 0.0f; + for (auto& mask : coreMask) + { + mask = smoothstep01 (mask / 0.72f); + maskPeak = std::max (maskPeak, mask); + } + if (maskPeak <= 0.001f) + return false; + + const float gain = std::pow (10.0f, gainDb / 20.0f); + bool changed = false; + for (int ch = 0; ch < numChannels; ++ch) + { + if (static_cast<size_t> (ch) >= output.size()) + continue; + + auto& dst = output[static_cast<size_t> (ch)]; + for (int s = 0; s < numSamples && static_cast<size_t> (s) < dst.size(); ++s) + { + const float mask = coreMask[static_cast<size_t> (s)]; + if (mask <= 0.001f) + continue; + + dst[static_cast<size_t> (s)] *= 1.0f + (gain - 1.0f) * mask; + changed = true; + } + } + + if (changed && appliedDbOut != nullptr) + *appliedDbOut = gainDb; + return changed; +} + +static std::vector<float> buildPitchOnlyMonoReference ( + const float* const* originalInput, + int numChannels, + int numSamples) +{ + std::vector<float> mono (static_cast<size_t> (numSamples), 0.0f); + if (numChannels <= 0 || numSamples <= 0) + return mono; + + const float channelScale = 1.0f / static_cast<float> (numChannels); + for (int ch = 0; ch < numChannels; ++ch) + { + const auto* src = originalInput[ch]; + for (int s = 0; s < numSamples; ++s) + mono[static_cast<size_t> (s)] += src[s] * channelScale; + } + + return mono; +} + +static int estimateLocalPitchPeriodSamples ( + const std::vector<float>& detectedPitchHz, + int sampleIndex, + double sampleRate, + int numSamples) +{ + if (detectedPitchHz.empty() || numSamples <= 0 || sampleRate <= 0.0) + return static_cast<int> (std::round (sampleRate / 220.0)); + + const int safeIndex = juce::jlimit (0, numSamples - 1, sampleIndex); + float pitchHz = detectedPitchHz[static_cast<size_t> (safeIndex)]; + if (pitchHz <= 0.0f) + { + const int searchRadius = std::max (1, static_cast<int> (std::round (0.012 * sampleRate))); + for (int delta = 1; delta <= searchRadius; ++delta) + { + const int left = safeIndex - delta; + if (left >= 0 && detectedPitchHz[static_cast<size_t> (left)] > 0.0f) + { + pitchHz = detectedPitchHz[static_cast<size_t> (left)]; + break; + } + + const int right = safeIndex + delta; + if (right < numSamples && detectedPitchHz[static_cast<size_t> (right)] > 0.0f) + { + pitchHz = detectedPitchHz[static_cast<size_t> (right)]; + break; + } + } + } + + if (pitchHz <= 0.0f) + pitchHz = 220.0f; + + return juce::jlimit (24, 2048, static_cast<int> (std::round (sampleRate / pitchHz))); +} + +static int findStrongestAbsolutePeak ( + const std::vector<float>& mono, + int expectedSample, + int radiusSamples, + int searchStart, + int searchEnd) +{ + if (mono.empty()) + return -1; + + const int lo = std::max (searchStart, expectedSample - radiusSamples); + const int hi = std::min (searchEnd, expectedSample + radiusSamples); + if (hi <= lo) + return -1; + + float bestValue = -1.0f; + int bestIndex = -1; + for (int s = lo; s <= hi; ++s) + { + const float value = std::abs (mono[static_cast<size_t> (s)]); + if (value > bestValue) + { + bestValue = value; + bestIndex = s; + } + } + + return bestIndex; +} + +static std::vector<int> buildPitchOnlyAnalysisMarks ( + const std::vector<float>& mono, + const std::vector<float>& detectedPitchHz, + const PitchOnlyCoreRegion& region, + double sampleRate, + int numSamples) +{ + std::vector<int> marks; + if (region.coreEndSample <= region.coreStartSample || mono.empty()) + return marks; + + const int seedPeriod = estimateLocalPitchPeriodSamples (detectedPitchHz, + (region.coreStartSample + region.coreEndSample) / 2, + sampleRate, + numSamples); + const int seedExpected = region.coreStartSample + seedPeriod / 2; + const int seed = findStrongestAbsolutePeak (mono, + seedExpected, + std::max (8, seedPeriod / 2), + region.coreStartSample, + region.coreEndSample - 1); + if (seed < 0) + return marks; + + marks.push_back (seed); + + int cursor = seed; + while (true) + { + const int localPeriod = estimateLocalPitchPeriodSamples (detectedPitchHz, cursor, sampleRate, numSamples); + const int expected = cursor + localPeriod; + if (expected >= region.coreEndSample) + break; + + const int next = findStrongestAbsolutePeak (mono, + expected, + std::max (8, static_cast<int> (std::round (localPeriod * 0.40f))), + std::max (region.coreStartSample, cursor + std::max (4, localPeriod / 3)), + region.coreEndSample - 1); + if (next <= cursor) + break; + + marks.push_back (next); + cursor = next; + if (static_cast<int> (marks.size()) > 4096) + break; + } + + cursor = seed; + while (true) + { + const int localPeriod = estimateLocalPitchPeriodSamples (detectedPitchHz, cursor, sampleRate, numSamples); + const int expected = cursor - localPeriod; + if (expected <= region.coreStartSample) + break; + + const int prev = findStrongestAbsolutePeak (mono, + expected, + std::max (8, static_cast<int> (std::round (localPeriod * 0.40f))), + region.coreStartSample, + std::min (region.coreEndSample - 1, cursor - std::max (4, localPeriod / 3))); + if (prev < region.coreStartSample || prev >= cursor) + break; + + marks.insert (marks.begin(), prev); + cursor = prev; + if (static_cast<int> (marks.size()) > 4096) + break; + } + + return marks; +} + +static int findNearestAnalysisMarkIndex (const std::vector<int>& marks, float targetSample) +{ + if (marks.empty()) + return -1; + + const auto it = std::lower_bound (marks.begin(), marks.end(), static_cast<int> (std::round (targetSample))); + if (it == marks.begin()) + return 0; + if (it == marks.end()) + return static_cast<int> (marks.size()) - 1; + + const int hi = static_cast<int> (std::distance (marks.begin(), it)); + const int lo = hi - 1; + return std::abs (marks[static_cast<size_t> (hi)] - targetSample) + < std::abs (marks[static_cast<size_t> (lo)] - targetSample) + ? hi + : lo; +} + +static bool applyPitchOnlyCorePsolaCorrection ( + std::vector<std::vector<float>>& output, + const float* const* originalInput, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<PitchOnlyCoreRegion>& regions, + const std::vector<float>& detectedPitchHz, + const std::vector<float>& coreMask, + PitchResynthesizer::RenderQuality renderQuality, + bool addEnvelopeTrim, + PitchOnlyStageBStats& stats, + std::function<bool()> shouldCancel) +{ + const float wetCap = addEnvelopeTrim + ? (renderQuality == PitchResynthesizer::RenderQuality::PreviewFast ? 0.96f : 1.00f) + : (renderQuality == PitchResynthesizer::RenderQuality::PreviewFast ? 0.90f : 0.96f); + stats.wetCap = wetCap; + + auto mono = buildPitchOnlyMonoReference (originalInput, numChannels, numSamples); + if (mono.empty()) + return false; + + std::vector<std::vector<float>> psolaOutput (static_cast<size_t> (numChannels), + std::vector<float> (static_cast<size_t> (numSamples), 0.0f)); + std::vector<float> psolaWeight (static_cast<size_t> (numSamples), 0.0f); + + int totalPitchMarks = 0; + int appliedRegions = 0; + + for (const auto& region : regions) + { + if (shouldCancel && shouldCancel()) + return false; + + if (region.coreStartSample < 0 + || region.coreEndSample <= region.coreStartSample + || std::abs (region.pitchRatio - 1.0f) < 1.0e-3f) + { + continue; + } + + const auto marks = buildPitchOnlyAnalysisMarks (mono, detectedPitchHz, region, sampleRate, numSamples); + if (static_cast<int> (marks.size()) < 4) + continue; + + totalPitchMarks += static_cast<int> (marks.size()); + ++appliedRegions; - ++shiftedBlocks; + const float targetPeriod = std::max (12.0f, + static_cast<float> (estimateLocalPitchPeriodSamples (detectedPitchHz, + (region.coreStartSample + region.coreEndSample) / 2, + sampleRate, + numSamples))) + / std::max (0.25f, region.pitchRatio); - std::fill (origFFTBuf.begin(), origFFTBuf.end(), 0.0f); - std::fill (outFFTBuf.begin(), outFFTBuf.end(), 0.0f); - for (int i = 0; i < fftSize; ++i) - { - const int idx = pos + i; - if (idx >= 0 && idx < numSamples) - { - const float w = hannWin[static_cast<size_t> (i)]; - origFFTBuf[static_cast<size_t> (i)] = orig[idx] * w; - outFFTBuf[static_cast<size_t> (i)] = out[static_cast<size_t> (idx)] * w; - } - } + std::vector<float> synthesisMarks; + synthesisMarks.reserve (marks.size() * 2); + float synthesisCursor = static_cast<float> (marks.front()); + while (synthesisCursor < static_cast<float> (region.coreEndSample)) + { + if (synthesisCursor >= static_cast<float> (region.coreStartSample)) + synthesisMarks.push_back (synthesisCursor); + synthesisCursor += targetPeriod; + if (synthesisMarks.size() > 8192) + break; + } - fft.performRealOnlyForwardTransform (origFFTBuf.data(), true); - fft.performRealOnlyForwardTransform (outFFTBuf.data(), true); - std::copy (outFFTBuf.begin(), outFFTBuf.end(), preWarpFFTBuf.begin()); + if (synthesisMarks.size() < 3) + continue; - for (int i = 0; i < halfBins; ++i) - { - const float ore = origFFTBuf[static_cast<size_t> (i * 2)]; - const float oim = origFFTBuf[static_cast<size_t> (i * 2 + 1)]; - origLogMag[static_cast<size_t> (i)] = std::log (std::sqrt (ore * ore + oim * oim) + 1e-10f); + for (const float synthMark : synthesisMarks) + { + const int markIndex = findNearestAnalysisMarkIndex (marks, synthMark); + if (markIndex < 0) + continue; - const float sre = outFFTBuf[static_cast<size_t> (i * 2)]; - const float sim = outFFTBuf[static_cast<size_t> (i * 2 + 1)]; - outLogMag[static_cast<size_t> (i)] = std::log (std::sqrt (sre * sre + sim * sim) + 1e-10f); - } + const int analysisMark = marks[static_cast<size_t> (markIndex)]; + const int localPeriod = estimateLocalPitchPeriodSamples (detectedPitchHz, analysisMark, sampleRate, numSamples); + const int halfWindow = juce::jlimit (24, 1024, static_cast<int> (std::round (localPeriod * 1.35f))); + const int destCenter = static_cast<int> (std::round (synthMark)); - float avgPitch = 0.0f; - int pitchCount = 0; - int voicedSamples = 0; - if (! detectedPitchHz.empty()) + for (int n = -halfWindow; n <= halfWindow; ++n) { - for (int i = pos; i < std::min (pos + fftSize, numSamples); ++i) + const int srcIndex = analysisMark + n; + const int dstIndex = destCenter + n; + if (srcIndex < region.coreStartSample || srcIndex >= region.coreEndSample + || dstIndex < region.coreStartSample || dstIndex >= region.coreEndSample) { - if (static_cast<size_t> (i) < detectedPitchHz.size() - && detectedPitchHz[static_cast<size_t> (i)] > 0.0f) - { - avgPitch += detectedPitchHz[static_cast<size_t> (i)]; - ++pitchCount; - ++voicedSamples; - } + continue; } - if (pitchCount > 0) - avgPitch /= static_cast<float> (pitchCount); + + const float phase = static_cast<float> (n + halfWindow) / static_cast<float> (2 * halfWindow + 1); + const float window = 0.5f - 0.5f * std::cos (2.0f * juce::MathConstants<float>::pi * phase); + psolaWeight[static_cast<size_t> (dstIndex)] += window; + for (int ch = 0; ch < numChannels; ++ch) + psolaOutput[static_cast<size_t> (ch)][static_cast<size_t> (dstIndex)] += originalInput[ch][srcIndex] * window; } + } + } - const float rawVoicedBlend = smoothstep01 (static_cast<float> (voicedSamples) - / static_cast<float> (std::max (1, std::min (fftSize, numSamples - pos))) - * (formantOnly ? 1.48f : 1.36f)); - const float voicedBlend = (pos == 0) - ? rawVoicedBlend - : ((formantOnly ? 0.82f : 0.76f) * prevVoicedBlend - + (formantOnly ? 0.18f : 0.24f) * rawVoicedBlend); - prevVoicedBlend = voicedBlend; - avgVoicedBlendApplied += voicedBlend; - minSmoothedVoicedBlend = std::min (minSmoothedVoicedBlend, voicedBlend); - maxSmoothedVoicedBlend = std::max (maxSmoothedVoicedBlend, voicedBlend); + if (appliedRegions == 0) + return false; - int smoothHalfW; - if (avgPitch > 50.0f) - smoothHalfW = std::max (previewFast ? (formantOnly ? 10 : 12) : (formantOnly ? 14 : 16), - static_cast<int> ((formantOnly ? (previewFast ? 1.35f : 1.75f) : (previewFast ? 1.85f : 2.45f)) * avgPitch / binHz)); - else - smoothHalfW = previewFast ? (formantOnly ? 24 : 30) : (formantOnly ? 36 : 50); - smoothHalfW = std::min (smoothHalfW, halfBins / 3); + for (int ch = 0; ch < numChannels; ++ch) + { + auto& out = output[static_cast<size_t> (ch)]; + auto psola = psolaOutput[static_cast<size_t> (ch)]; - computeEnvelope (origLogMag, smoothHalfW, origEnv); - computeEnvelope (outLogMag, smoothHalfW, outEnv); + for (int s = 0; s < numSamples; ++s) + { + const float weight = psolaWeight[static_cast<size_t> (s)]; + if (weight > 1.0e-4f) + psola[static_cast<size_t> (s)] /= weight; + else + psola[static_cast<size_t> (s)] = out[static_cast<size_t> (s)]; + } - const float effectiveShiftSemitones = std::abs (12.0f * std::log2 (std::max (avgFormantRatio, 1.0e-4f))); - const float bassFocus = formantOnly && avgPitch > 0.0f - ? smoothstep01 ((195.0f - avgPitch) / 110.0f) - : 0.0f; - const float lowRegisterBoost = formantOnly && avgPitch > 0.0f - ? (1.0f + (previewFast ? 0.24f : 0.34f) * bassFocus) - : 1.0f; - const float voicedStrengthBias = formantOnly - ? (0.48f + 0.52f * voicedBlend) - : (0.28f + 0.72f * voicedBlend); - const float rawStrength = juce::jlimit (0.0f, formantOnly ? (previewFast ? 1.38f : 1.58f) : (previewFast ? 1.15f : 1.42f), - (0.76f + effectiveShiftSemitones * 0.20f) - * voicedStrengthBias * warpIntensity * lowRegisterBoost); - const float strength = (pos == 0) - ? rawStrength - : ((formantOnly ? 0.78f : 0.72f) * prevStrength - + (formantOnly ? 0.22f : 0.28f) * rawStrength); - prevStrength = strength; - minEffectiveStrength = std::min (minEffectiveStrength, strength); - maxEffectiveStrength = std::max (maxEffectiveStrength, strength); - appliedMinRatio = std::min (appliedMinRatio, avgFormantRatio); - appliedMaxRatio = std::max (appliedMaxRatio, avgFormantRatio); + if (addEnvelopeTrim) + { + std::vector<std::vector<float>> trimBuffer (1, psola); + const float* trimInput[1] = { originalInput[ch] }; + applyPitchEnvelopeAnchorMatch (trimBuffer, + trimInput, + 1, + numSamples, + sampleRate, + notes, + detectedPitchHz, + PitchOnlyRendererBranch::ModelCore, + renderQuality, + shouldCancel); + psola = std::move (trimBuffer.front()); + } - float energyBefore = 0.0f; - for (int i = 0; i < halfBins; ++i) - { - const float re = outFFTBuf[static_cast<size_t> (i * 2)]; - const float im = outFFTBuf[static_cast<size_t> (i * 2 + 1)]; - energyBefore += re * re + im * im; - } + for (int s = 0; s < numSamples; ++s) + { + const float mask = std::min (coreMask[static_cast<size_t> (s)], wetCap); + if (mask <= 0.0001f) + continue; - for (int i = 0; i < halfBins; ++i) - { - const float freqHz = static_cast<float> (i) * binHz; - const float lowAnchor = std::max (formantOnly ? 65.0f : 90.0f, avgPitch > 0.0f ? avgPitch * (formantOnly ? 0.78f : 0.95f) : (formantOnly ? 90.0f : 120.0f)); - const float lowFull = std::max (formantOnly ? 220.0f : 320.0f, avgPitch > 0.0f ? avgPitch * (formantOnly ? 1.65f : 2.15f) : (formantOnly ? 300.0f : 430.0f)); - const float lowBandBase = formantOnly - ? (0.58f + 0.12f * bassFocus) - : 0.30f; - const float lowBandLift = formantOnly - ? (0.42f - 0.10f * bassFocus) - : 0.70f; - const float lowBandWeight = lowBandBase + lowBandLift - * smoothstep01 ((freqHz - lowAnchor) / std::max (45.0f, lowFull - lowAnchor)); - const float detailProtect = smoothstep01 ((freqHz - 3400.0f) / 2200.0f); - const float airProtection = 1.0f - (formantOnly ? 0.16f : 0.28f) - * smoothstep01 ((freqHz - 5200.0f) / 2600.0f); - const float perBinWeight = juce::jlimit (0.0f, 1.0f, voicedStrengthBias * lowBandWeight * airProtection); + out[static_cast<size_t> (s)] = out[static_cast<size_t> (s)] * (1.0f - mask) + + psola[static_cast<size_t> (s)] * mask; + } + } - targetEnv[static_cast<size_t> (i)] = interpolateEnvelopeBin ( - origEnv, static_cast<float> (i) / avgFormantRatio); + stats.pitchMarkCount = totalPitchMarks; + stats.ran = true; + return true; +} - const float targetBlend = juce::jlimit (0.0f, 1.10f, - perBinWeight * (formantOnly ? (previewFast ? 1.06f : 1.10f) : 1.02f)); - const float blendedTargetEnv = outEnv[static_cast<size_t> (i)] - + (targetEnv[static_cast<size_t> (i)] - outEnv[static_cast<size_t> (i)]) - * targetBlend; - const float sourceEnv = outEnv[static_cast<size_t> (i)] + 1.0e-10f; - const float targetValue = blendedTargetEnv + 1.0e-10f; - totalEnvelopeDeltaBefore += std::abs (std::log (targetValue / sourceEnv)); +static PitchOnlyStageBStats applyPitchOnlyCoreSerialCorrection ( + std::vector<std::vector<float>>& output, + const float* const* originalInput, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const std::vector<float>& detectedPitchHz, + PitchResynthesizer::RenderQuality renderQuality, + PitchOnlyRendererBranch rendererBranch, + std::function<bool()> shouldCancel) +{ + PitchOnlyStageBStats stats; + stats.branchName = getPitchOnlyRendererBranchName (rendererBranch); + float signedShiftSum = 0.0f; + int signedShiftCount = 0; + for (const auto& note : notes) + { + if (! hasPitchStyleEdit (note)) + continue; + signedShiftSum += (note.correctedPitch - note.detectedPitch); + ++signedShiftCount; + } + const bool downwardShift = signedShiftCount > 0 && (signedShiftSum / static_cast<float> (signedShiftCount)) < 0.0f; + stats.wetCap = renderQuality == PitchResynthesizer::RenderQuality::PreviewFast + ? (downwardShift ? 0.40f : 0.34f) + : (downwardShift ? 0.62f : 0.52f); + float stageBWetScale = getPitchOnlyStageBWetScale (rendererBranch, downwardShift); + const float avgEditedNoteDurationSec = getAverageEditedNoteDurationSec (notes); + if ((rendererBranch == PitchOnlyRendererBranch::HybridReset + || rendererBranch == PitchOnlyRendererBranch::HybridStructural) + && ! downwardShift + && avgEditedNoteDurationSec > 0.0f + && avgEditedNoteDurationSec < 0.80f) + { + stageBWetScale *= 1.25f; + } + stats.wetCap = juce::jlimit (0.0f, 1.0f, stats.wetCap * stageBWetScale); - float gain = std::pow (targetValue / sourceEnv, - strength); - float minGain = juce::jmap (voicedBlend, 0.0f, 1.0f, - detailProtect > 0.25f ? 0.84f : 0.72f, - detailProtect > 0.25f ? 0.68f : (previewFast ? 0.56f : 0.46f)); - float maxGain = juce::jmap (voicedBlend, 0.0f, 1.0f, - detailProtect > 0.25f ? 1.20f : 1.55f, - detailProtect > 0.25f ? 1.54f : (previewFast ? 2.05f : 2.45f)); - if (formantOnly && bassFocus > 0.0f && freqHz < std::max (420.0f, avgPitch > 0.0f ? avgPitch * 3.1f : 420.0f)) - maxGain *= (1.0f + 0.16f * bassFocus); - gain = juce::jlimit (minGain, maxGain, gain); - totalEnvelopeDeltaAfter += std::abs (std::log (targetValue / (sourceEnv * gain))); - ++totalEnvelopeDeltaBins; - outFFTBuf[static_cast<size_t> (i * 2)] *= gain; - outFFTBuf[static_cast<size_t> (i * 2 + 1)] *= gain; - } + if (detectedPitchHz.empty()) + { + stats.failedClosed = true; + return stats; + } - float energyAfter = 0.0f; - for (int i = 0; i < halfBins; ++i) - { - const float re = outFFTBuf[static_cast<size_t> (i * 2)]; - const float im = outFFTBuf[static_cast<size_t> (i * 2 + 1)]; - energyAfter += re * re + im * im; - } - if (energyAfter > 1.0e-10f) - { - const float energyRatio = energyBefore / energyAfter; - const float scale = std::pow (energyRatio, formantOnly ? 0.24f : 0.24f); - for (int i = 0; i < halfBins; ++i) - { - outFFTBuf[static_cast<size_t> (i * 2)] *= scale; - outFFTBuf[static_cast<size_t> (i * 2 + 1)] *= scale; - } - } + const auto regions = derivePitchOnlyCoreRegions (originalInput, numChannels, numSamples, sampleRate, notes, detectedPitchHz); + stats.regionCount = static_cast<int> (regions.size()); + for (const auto& region : regions) + stats.longestCoreSamples = std::max (stats.longestCoreSamples, region.coreEndSample - region.coreStartSample); - for (int i = 0; i < halfBins; ++i) - { - const float freqHz = static_cast<float> (i) * binHz; - const float detailProtect = smoothstep01 ((freqHz - 3200.0f) / 2000.0f); - const float detailKeep = juce::jlimit (0.0f, 0.72f, - detailProtect * (0.20f - + 0.46f * (1.0f - voicedBlend) - + 0.18f * smoothstep01 ((0.028f - frameRms) / 0.018f))); - if (detailKeep <= 0.001f) - continue; - outFFTBuf[static_cast<size_t> (i * 2)] = - outFFTBuf[static_cast<size_t> (i * 2)] * (1.0f - detailKeep) - + preWarpFFTBuf[static_cast<size_t> (i * 2)] * detailKeep; - outFFTBuf[static_cast<size_t> (i * 2 + 1)] = - outFFTBuf[static_cast<size_t> (i * 2 + 1)] * (1.0f - detailKeep) - + preWarpFFTBuf[static_cast<size_t> (i * 2 + 1)] * detailKeep; - } + if (regions.empty()) + { + stats.failedClosed = true; + return stats; + } - fft.performRealOnlyInverseTransform (outFFTBuf.data()); + auto coreMask = buildPitchOnlyCoreMask (originalInput, numChannels, numSamples, sampleRate, notes, detectedPitchHz); + bool hasCore = false; + for (const auto mask : coreMask) + { + if (mask > 0.0001f) + { + hasCore = true; + break; + } + } + if (! hasCore) + { + stats.failedClosed = true; + return stats; + } - const float rawOriginalBlend = formantOnly - ? juce::jlimit (0.10f, 0.50f, - 0.10f - + (1.0f - voicedBlend) * 0.24f - + smoothstep01 ((0.032f - frameRms) / 0.020f) * 0.14f) - : 0.0f; - const float originalBlend = (pos == 0) - ? rawOriginalBlend - : (0.82f * prevOriginalBlend + 0.18f * rawOriginalBlend); - prevOriginalBlend = originalBlend; + if (stats.wetCap <= 0.0001f) + { + stats.failedClosed = true; + logPitchEditorFormant ("pitch-only core serial correction bypassed branch=" + stats.branchName + + " wetScale=" + juce::String (stageBWetScale, 3) + + " downward=" + juce::String (downwardShift ? "true" : "false")); + return stats; + } - for (int i = 0; i < fftSize; ++i) - { - const int idx = pos + i; - if (idx >= 0 && idx < numSamples) - { - const float w = hannWin[static_cast<size_t> (i)]; - const float processedSample = outFFTBuf[static_cast<size_t> (i)] * (1.0f - originalBlend) - + orig[idx] * originalBlend; - corrected[static_cast<size_t> (idx)] += processedSample * w; - winSum[static_cast<size_t> (idx)] += w * w; - } - } + if (rendererBranch == PitchOnlyRendererBranch::PsolaCore + || rendererBranch == PitchOnlyRendererBranch::ModelCore) + { + const bool addEnvelopeTrim = rendererBranch == PitchOnlyRendererBranch::ModelCore; + if (applyPitchOnlyCorePsolaCorrection (output, + originalInput, + numChannels, + numSamples, + sampleRate, + notes, + regions, + detectedPitchHz, + coreMask, + renderQuality, + addEnvelopeTrim, + stats, + shouldCancel)) + { + logPitchEditorFormant ("pitch-only core serial correction active branch=" + stats.branchName + + " wetCap=" + juce::String (stats.wetCap, 3) + + " regions=" + juce::String (stats.regionCount) + + " pitchMarks=" + juce::String (stats.pitchMarkCount) + + " longestCoreMs=" + juce::String (sampleRate > 0.0 + ? 1000.0 * static_cast<double> (stats.longestCoreSamples) / sampleRate : 0.0, 1)); + return stats; } + } + std::vector<std::vector<float>> corrected = output; + applyPitchEnvelopeAnchorMatch (corrected, originalInput, numChannels, numSamples, + sampleRate, notes, detectedPitchHz, + rendererBranch, + renderQuality, shouldCancel); + for (int ch = 0; ch < numChannels; ++ch) + { + auto& out = output[static_cast<size_t> (ch)]; + const auto& serialCorrected = corrected[static_cast<size_t> (ch)]; for (int s = 0; s < numSamples; ++s) { - if (winSum[static_cast<size_t> (s)] > 0.001f) - out[static_cast<size_t> (s)] = corrected[static_cast<size_t> (s)] - / winSum[static_cast<size_t> (s)]; + const float mask = std::min (coreMask[static_cast<size_t> (s)], stats.wetCap); + if (mask <= 0.0001f) + continue; + out[static_cast<size_t> (s)] = out[static_cast<size_t> (s)] * (1.0f - mask) + + serialCorrected[static_cast<size_t> (s)] * mask; } - } - - logPitchEditorFormant ("explicit formant warp blocks=" + juce::String (shiftedBlocks) - + " requestedRatio=[" + juce::String (requestedMinRatio, 3) + "," + juce::String (requestedMaxRatio, 3) + "]" - + " appliedRatio=[" + juce::String (appliedMinRatio == std::numeric_limits<float>::max() ? 1.0f : appliedMinRatio, 3) - + "," + juce::String (appliedMaxRatio, 3) + "]" - + " avgVoicedBlend=" + juce::String (shiftedBlocks > 0 ? avgVoicedBlendApplied / static_cast<float> (shiftedBlocks) : 0.0f, 3) - + " voicedBlendRange=[" + juce::String (minSmoothedVoicedBlend == std::numeric_limits<float>::max() ? 0.0f : minSmoothedVoicedBlend, 3) - + "," + juce::String (maxSmoothedVoicedBlend, 3) + "]" - + " strengthRange=[" + juce::String (minEffectiveStrength == std::numeric_limits<float>::max() ? 0.0f : minEffectiveStrength, 3) - + "," + juce::String (maxEffectiveStrength, 3) + "]" - + " warpIntensity=" + juce::String (warpIntensity, 3) - + " renderQuality=" + juce::String (previewFast ? "preview_fast" : "final_hq") - + " mode=" + juce::String (formantOnly ? "formant-only" : "mixed") - + " envDeltaBefore=" + juce::String (totalEnvelopeDeltaBins > 0 ? totalEnvelopeDeltaBefore / static_cast<float> (totalEnvelopeDeltaBins) : 0.0f, 4) - + " envDeltaAfter=" + juce::String (totalEnvelopeDeltaBins > 0 ? totalEnvelopeDeltaAfter / static_cast<float> (totalEnvelopeDeltaBins) : 0.0f, 4)); + } + + stats.ran = true; + logPitchEditorFormant ("pitch-only core serial correction active branch=" + stats.branchName + + " mode=envelope-anchor_only wetCap=" + + juce::String (stats.wetCap, 3) + + " regions=" + juce::String (stats.regionCount) + + " longestCoreMs=" + juce::String (sampleRate > 0.0 + ? 1000.0 * static_cast<double> (stats.longestCoreSamples) / sampleRate : 0.0, 1)); + return stats; } // Gaussian-smooth the per-frame pitch contour to remove YIN detection jitter. @@ -944,16 +5401,92 @@ std::vector<float> PitchResynthesizer::buildCorrectionCurve( return notes[a].startTime < notes[b].startTime; }); + auto frameIndexForTime = [&] (float timeSec) -> int + { + const int sample = static_cast<int> (std::round (static_cast<double> (timeSec) * sampleRate)); + return juce::jlimit (0, static_cast<int> (frames.size()) - 1, + (sample - firstFrameSample) / std::max (1, hopSize)); + }; + + auto averageFrameValue = [&] (int startFrame, int endFrame, bool confidence) -> float + { + if (frames.empty()) + return confidence ? 0.0f : -100.0f; + + startFrame = juce::jlimit (0, static_cast<int> (frames.size()) - 1, startFrame); + endFrame = juce::jlimit (startFrame, static_cast<int> (frames.size()) - 1, endFrame); + double sum = 0.0; + int count = 0; + for (int i = startFrame; i <= endFrame; ++i) + { + const auto& frame = frames[static_cast<size_t> (i)]; + sum += confidence ? static_cast<double> (frame.confidence) : static_cast<double> (frame.rmsDB); + ++count; + } + if (count <= 0) + return confidence ? 0.0f : -100.0f; + return static_cast<float> (sum / static_cast<double> (count)); + }; + + auto hasContinuousVoicedEntry = [&] (const PitchAnalyzer::PitchNote& note) -> bool + { + const auto entryKind = note.entryBoundaryKind.trim().toLowerCase(); + if (entryKind == "hard_word_like") + return false; + if (entryKind == "soft_legato" || entryKind == "internal_bend" || entryKind == "internal_vibrato") + return true; + + if (frames.empty()) + return false; + + const int startFrame = frameIndexForTime (note.startTime - 0.040f); + const int endFrame = frameIndexForTime (note.startTime + 0.020f); + if (endFrame <= startFrame) + return false; + + int voicedCount = 0; + int totalCount = 0; + int unvoicedNearEntry = 0; + float minRms = 100.0f; + float minConfidence = 1.0f; + for (int i = startFrame; i <= endFrame; ++i) + { + const auto& frame = frames[static_cast<size_t> (i)]; + const bool voiced = frame.voiced && frame.midiNote > 0.0f && frame.confidence >= 0.25f && frame.rmsDB > -60.0f; + voicedCount += voiced ? 1 : 0; + totalCount += 1; + minRms = std::min (minRms, frame.rmsDB); + minConfidence = std::min (minConfidence, frame.confidence); + + const float frameTime = frame.time; + if (std::abs (frameTime - note.startTime) <= 0.018f && ! voiced) + ++unvoicedNearEntry; + } + + const float voicedCoverage = totalCount > 0 ? static_cast<float> (voicedCount) / static_cast<float> (totalCount) : 0.0f; + const int leftStart = frameIndexForTime (note.startTime - 0.060f); + const int leftEnd = frameIndexForTime (note.startTime - 0.018f); + const int rightStart = frameIndexForTime (note.startTime + 0.010f); + const int rightEnd = frameIndexForTime (note.startTime + 0.045f); + const float leftRms = averageFrameValue (leftStart, leftEnd, false); + const float rightRms = averageFrameValue (rightStart, rightEnd, false); + const float leftConfidence = averageFrameValue (leftStart, leftEnd, true); + const float rightConfidence = averageFrameValue (rightStart, rightEnd, true); + const float referenceRms = std::max (leftRms, rightRms); + const float referenceConfidence = std::max (leftConfidence, rightConfidence); + const bool strongRmsDip = referenceRms - minRms >= 8.0f; + const bool strongConfidenceDip = referenceConfidence - minConfidence >= 0.22f; + const bool unvoicedBreak = unvoicedNearEntry >= 2; + + return voicedCoverage >= 0.65f && ! strongRmsDip && ! strongConfidenceDip && ! unvoicedBreak; + }; + for (size_t ni = 0; ni < noteOrder.size(); ++ni) { const auto& note = notes[noteOrder[ni]]; // Skip unedited notes entirely - bool isEdited = std::abs(note.correctedPitch - note.detectedPitch) > 0.01f - || std::abs(note.gain) > 0.01f - || std::abs(note.formantShift) > 0.01f - || note.driftCorrectionAmount > 0.01f - || std::abs(note.vibratoDepth - 1.0f) > 0.01f; + bool isEdited = hasPitchStyleEdit (note); if (!isEdited) continue; int startSample = static_cast<int>(note.startTime * sampleRate); @@ -966,11 +5499,37 @@ std::vector<float> PitchResynthesizer::buildCorrectionCurve( float driftCorrection = note.driftCorrectionAmount; float noteCenter = note.detectedPitch; // the note's average detected pitch - // Anticipation + release: ramp runs BEFORE note start and AFTER note end (RePitch-style). - // Default 40ms pre-roll: correction arrives at the note boundary fully applied. - // Default 60ms post-roll: correction eases back naturally after the note ends. - float effectiveTransIn = (note.transitionIn > 0.0f) ? note.transitionIn : 40.0f; - float effectiveTransOut = (note.transitionOut > 0.0f) ? note.transitionOut : 60.0f; + // Boundary ownership is intentionally narrow now: only the immediate + // left tail and immediate right head may be smoothed, and only with a + // tiny span. We no longer let default pitch-note spill chain across + // wider neighbor groups. + constexpr float kImmediateNeighborGapSec = 0.18f; + constexpr float kBoundarySmoothDefaultMs = 50.0f; + constexpr float kBoundarySmoothMaxMs = 80.0f; + + [[maybe_unused]] bool hasImmediatePrevNeighbor = false; + [[maybe_unused]] bool hasImmediateNextNeighbor = false; + if (ni > 0) + { + const auto& prevAny = notes[noteOrder[ni - 1]]; + const float gap = note.startTime - prevAny.endTime; + hasImmediatePrevNeighbor = gap <= kImmediateNeighborGapSec; + } + if (ni + 1 < noteOrder.size()) + { + const auto& nextAny = notes[noteOrder[ni + 1]]; + const float gap = nextAny.startTime - note.endTime; + hasImmediateNextNeighbor = gap <= kImmediateNeighborGapSec; + } + + const float requestedTransIn = (note.transitionIn > 0.0f) + ? juce::jlimit (0.0f, kBoundarySmoothMaxMs, note.transitionIn) + : 0.0f; + const float requestedTransOut = (note.transitionOut > 0.0f) + ? juce::jlimit (0.0f, kBoundarySmoothMaxMs, note.transitionOut) + : 0.0f; + const float effectiveTransIn = (requestedTransIn > 0.0f) ? requestedTransIn : kBoundarySmoothDefaultMs; + const float effectiveTransOut = (requestedTransOut > 0.0f) ? requestedTransOut : kBoundarySmoothDefaultMs; // --- Inter-note portamento --- // Find adjacent edited notes for smooth pitch glides at boundaries. @@ -1009,11 +5568,82 @@ std::vector<float> PitchResynthesizer::buildCorrectionCurve( // Anticipation + release: loop extends BEFORE note start (pre-roll) and AFTER note end // (post-roll). The ramp runs in these extended regions so the note body is always fully // corrected — no sudden onset or sudden cut at the note boundaries (RePitch-style). - int transInSamplesInt = static_cast<int>(effectiveTransIn * 0.001f * static_cast<float>(sampleRate)); + const bool upwardShift = shiftAmount >= 0.0f; + const float defaultLargeEntryHandoffMs = upwardShift + ? getEnvFloat ("OPENSTUDIO_PITCH_ENTRY_HANDOFF_UP_BODY_MS", 0.0f) + : getEnvFloat ("OPENSTUDIO_PITCH_ENTRY_HANDOFF_DOWN_BODY_MS", 40.0f); + const float largeEntryHandoffMs = juce::jlimit ( + 0.0f, 60.0f, getEnvFloat ("OPENSTUDIO_PITCH_ENTRY_HANDOFF_BODY_MS", defaultLargeEntryHandoffMs)); + const bool largeEntryPitchEdit = std::abs (shiftAmount) > 2.0f && largeEntryHandoffMs > 0.0f; + const auto entryKindForHandoff = note.entryBoundaryKind.trim().toLowerCase(); + const bool explicitContinuousEntry = entryKindForHandoff == "soft_legato" + || entryKindForHandoff == "internal_bend" + || entryKindForHandoff == "internal_vibrato"; + const bool continuousEntry = hasPrevNote || explicitContinuousEntry; + const bool entryPitchHandoff = continuousEntry || largeEntryPitchEdit; + const float entryDryPreserveMs = 0.0f; + const float entryPreMs = continuousEntry + ? (upwardShift ? 16.0f : 24.0f) + : 0.0f; + const float entryBodyMs = continuousEntry + ? (upwardShift ? 40.0f : 55.0f) + : (largeEntryPitchEdit ? largeEntryHandoffMs : 0.0f); + juce::ignoreUnused (hasContinuousVoicedEntry); + + const int standardTransInSamples = upwardShift + ? 0 + : static_cast<int> (std::round (effectiveTransIn * 0.001f * static_cast<float> (sampleRate))); + const int entryPreSamples = entryPitchHandoff + ? static_cast<int> (std::round (entryPreMs * 0.001f * static_cast<float> (sampleRate))) + : standardTransInSamples; + const int entryDryPreserveSamples = static_cast<int> (std::round (entryDryPreserveMs * 0.001f * static_cast<float> (sampleRate))); + const int entryBodySamples = entryPitchHandoff + ? std::max (1, static_cast<int> (std::round (entryBodyMs * 0.001f * static_cast<float> (sampleRate)))) + : 0; + const int entryHandoffStartSample = std::max (0, startSample - entryPreSamples); + const int entryHandoffEndSample = std::min (numSamples - 1, startSample + entryBodySamples); + + int transInSamplesInt = std::max (0, startSample - entryHandoffStartSample); int transOutSamplesInt = static_cast<int>(effectiveTransOut * 0.001f * static_cast<float>(sampleRate)); - int loopStart = std::max(0, startSample - transInSamplesInt); + int loopStart = entryHandoffStartSample; int loopEnd = std::min(numSamples - 1, endSample + transOutSamplesInt); + if (transInSamplesInt > 0) + { + lastRenderDiagnostics.immediateLeftNeighborUsed = true; + lastRenderDiagnostics.leftNeighborSamplesRendered += transInSamplesInt; + lastRenderDiagnostics.leftNeighborSmoothMs = std::max(lastRenderDiagnostics.leftNeighborSmoothMs, + static_cast<double>(effectiveTransIn)); + } + if (transOutSamplesInt > 0) + { + lastRenderDiagnostics.immediateRightNeighborUsed = true; + lastRenderDiagnostics.rightNeighborSamplesRendered += transOutSamplesInt; + lastRenderDiagnostics.rightNeighborSmoothMs = std::max(lastRenderDiagnostics.rightNeighborSmoothMs, + static_cast<double>(effectiveTransOut)); + } + + if (entryPitchHandoff) + { + lastRenderDiagnostics.noteHqEntryPitchHandoffUsed = true; + lastRenderDiagnostics.pitchOnlyEntryHandoffUsed = true; + if (lastRenderDiagnostics.noteHqEntryPitchHandoffStartSec <= 0.0) + lastRenderDiagnostics.noteHqEntryPitchHandoffStartSec = static_cast<double> (entryHandoffStartSample) / sampleRate; + else + lastRenderDiagnostics.noteHqEntryPitchHandoffStartSec = std::min ( + lastRenderDiagnostics.noteHqEntryPitchHandoffStartSec, + static_cast<double> (entryHandoffStartSample) / sampleRate); + lastRenderDiagnostics.noteHqEntryPitchHandoffEndSec = std::max ( + lastRenderDiagnostics.noteHqEntryPitchHandoffEndSec, + static_cast<double> (entryHandoffEndSample) / sampleRate); + lastRenderDiagnostics.noteHqEntryPitchHandoffPreMs = std::max ( + lastRenderDiagnostics.noteHqEntryPitchHandoffPreMs, + 1000.0 * static_cast<double> (std::max (0, startSample - entryHandoffStartSample)) / sampleRate); + lastRenderDiagnostics.noteHqEntryPitchHandoffBodyMs = std::max ( + lastRenderDiagnostics.noteHqEntryPitchHandoffBodyMs, + std::max (0.0, 1000.0 * static_cast<double> (std::max (0, entryHandoffEndSample - startSample - entryDryPreserveSamples)) / sampleRate)); + } + // Pre-compute drift/vibrato decomposition over note body (unchanged from before) // IIR filters only make sense over the note's voiced region, not the pre/post roll. std::vector<float> driftVec, vibratoVec; @@ -1067,16 +5697,18 @@ std::vector<float> PitchResynthesizer::buildCorrectionCurve( if (!inPostRoll && frame.rmsDB < -60.0f) continue; // --- Voiced/unvoiced classification --- - // Unvoiced content (consonants, sibilants, breaths) passes through unmodified - // within the note body and pre-roll only. - if (!inPostRoll) + // In the pre-roll region (before note start): skip unvoiced frames so the + // ramp only engages where there is actual voiced signal, keeping the + // pre-roll baseline at ratio=1.0 for non-vocal content. + // In the note body: always apply correction — skipping unvoiced frames here + // causes no-correction when the pitch detector is noisy or the voice is breathy. + if (!inPostRoll && s < startSample) { bool isUnvoiced = !frame.voiced || frame.midiNote <= 0.0f; - // Exception: short unvoiced gap within a voiced note = brief consonant ("t","d","k") if (isUnvoiced && isShortUnvoicedGap(frameIdx, frames, 6)) isUnvoiced = false; if (isUnvoiced) - continue; // ratio stays 1.0 — complete passthrough + continue; } // --- CONTOUR-PRESERVING PITCH CORRECTION --- @@ -1115,7 +5747,7 @@ std::vector<float> PitchResynthesizer::buildCorrectionCurve( float distFromEnd = static_cast<float>(s - endSample); float transitionBlend = 1.0f; // used by write guard below - if (distFromStart < 0.0f) + if (transInSamplesInt > 0 && distFromStart < 0.0f) { // --- Pre-roll --- // Blend from 0 → 1 as we approach note start. @@ -1148,6 +5780,22 @@ std::vector<float> PitchResynthesizer::buildCorrectionCurve( float postTarget = hasNextNote ? (framePitch + nextNoteShift) : framePitch; correctedMidi = postTarget + (noteEndCorrected - postTarget) * transitionBlend; } + else if (s < entryHandoffEndSample) + { + const float sourceAnchor = hasPrevNote ? (framePitch + prevNoteShift) : framePitch; + const int entryRampStartSample = largeEntryPitchEdit + ? std::min (entryHandoffEndSample, startSample + entryDryPreserveSamples) + : entryHandoffStartSample; + if (largeEntryPitchEdit && s < entryRampStartSample) + transitionBlend = 0.0f; + else + { + const float t = static_cast<float> (s - entryRampStartSample) + / static_cast<float> (std::max (1, entryHandoffEndSample - entryRampStartSample)); + transitionBlend = minimumJerk01 (t); + } + correctedMidi = sourceAnchor + (correctedMidi - sourceAnchor) * transitionBlend; + } // Accumulate weighted shift (in semitones) so overlapping pre/post-roll regions // from adjacent notes blend smoothly instead of hard-switching at blend=0.5. @@ -1165,6 +5813,12 @@ std::vector<float> PitchResynthesizer::buildCorrectionCurve( // ------------------------------------------------------------------------- // Final pass: convert accumulated weighted shifts → per-sample ratios. // Samples touched by no note keep ratios[s] = 1.0 (passthrough). + // + // Note: for equal-temperament MIDI, midiToHz(m+s)/midiToHz(m) = 2^(s/12) + // — independent of framePitch. So the ratio depends only on the averaged + // semitone shift. This also means samples in the note body still get the + // correct ratio even when pitch detection produced no voiced frame there + // (previously those samples stayed at ratio 1.0, silently dropping the edit). // ------------------------------------------------------------------------- for (int s = 0; s < numSamples; ++s) { @@ -1172,22 +5826,54 @@ std::vector<float> PitchResynthesizer::buildCorrectionCurve( if (totalBlend <= 0.0f) continue; // no correction applied here - int frameIdx = (s - firstFrameSample) / hopSize; - if (frameIdx < 0 || frameIdx >= static_cast<int>(frames.size())) - continue; + float avgShift = shiftAccum[static_cast<size_t>(s)] / totalBlend; + ratios[static_cast<size_t>(s)] = juce::jlimit(0.25f, 4.0f, + std::pow(2.0f, avgShift / 12.0f)); + } - float framePitch = (frameIdx < static_cast<int>(smoothedContour.size())) - ? smoothedContour[static_cast<size_t>(frameIdx)] - : 0.0f; - if (framePitch <= 0.0f) - continue; + if (shouldUseEngineV3TransientBypass (getPitchOnlyRendererBranch())) + { + const float maxBypassMs = juce::jlimit (6.0f, 30.0f, getEnvFloat ("OPENSTUDIO_ENGINE_V3_TRANSIENT_BYPASS_MS", 18.0f)); + const float fadeMs = juce::jlimit (3.0f, 12.0f, getEnvFloat ("OPENSTUDIO_ENGINE_V3_TRANSIENT_FADE_MS", 6.0f)); + const int maxBypassSamples = static_cast<int> (std::round (sampleRate * maxBypassMs * 0.001f)); + const int fadeSamples = std::max (1, static_cast<int> (std::round (sampleRate * fadeMs * 0.001f))); + + for (const auto& note : notes) + { + if (! hasPitchStyleEdit (note)) + continue; + + const int startSample = juce::jlimit (0, numSamples - 1, static_cast<int> (std::round (note.startTime * sampleRate))); + const int endSample = juce::jlimit (0, numSamples - 1, static_cast<int> (std::round (note.endTime * sampleRate))); + if (endSample <= startSample) + continue; + + int bypassEndSample = std::min (endSample, startSample + maxBypassSamples); + for (const auto& frame : frames) + { + const int frameSample = static_cast<int> (std::round (frame.time * sampleRate)); + if (frameSample < startSample) + continue; + if (frameSample > bypassEndSample) + break; + if (frame.voiced && frame.confidence >= 0.70f && frame.frequency > 40.0f) + { + bypassEndSample = std::max (startSample, std::min (bypassEndSample, frameSample)); + break; + } + } + + const int fadeEndSample = std::min (endSample, bypassEndSample + fadeSamples); + for (int s = startSample; s < bypassEndSample && s < numSamples; ++s) + ratios[static_cast<size_t> (s)] = 1.0f; - float avgShift = shiftAccum[static_cast<size_t>(s)] / totalBlend; - float correctedMidi = framePitch + avgShift; - float detectedHz = midiToHz(framePitch); - float correctedHz = midiToHz(correctedMidi); - if (detectedHz > 0.0f) - ratios[static_cast<size_t>(s)] = juce::jlimit(0.25f, 4.0f, correctedHz / detectedHz); + for (int s = bypassEndSample; s < fadeEndSample && s < numSamples; ++s) + { + const float dryToWet = smoothstep01 (static_cast<float> (s - bypassEndSample) + / static_cast<float> (std::max (1, fadeEndSample - bypassEndSample))); + ratios[static_cast<size_t> (s)] = 1.0f + (ratios[static_cast<size_t> (s)] - 1.0f) * dryToWet; + } + } } return ratios; @@ -1221,8 +5907,8 @@ std::vector<float> PitchResynthesizer::buildFormantCurve( if (std::abs (note.formantShift) < 0.01f) continue; - int startSample = juce::jlimit (0, numSamples - 1, static_cast<int> (note.startTime * sampleRate)); - int endSample = juce::jlimit (0, numSamples - 1, static_cast<int> (note.endTime * sampleRate)); + int startSample = juce::jlimit (0, numSamples - 1, static_cast<int> (getEffectiveNoteStartTime (note) * sampleRate)); + int endSample = juce::jlimit (0, numSamples - 1, static_cast<int> (getEffectiveNoteEndTime (note) * sampleRate)); float formantRatio = mapRequestedFormantRatio (std::pow (2.0f, note.formantShift / 12.0f)); @@ -1252,7 +5938,7 @@ std::vector<float> PitchResynthesizer::buildFormantCurve( } } - if (kPitchEditorFormantDebugLogs) + if (shouldEnablePitchEditorFormantDebugLogs()) { float minRatio = ratios.front(); float maxRatio = ratios.front(); @@ -1272,6 +5958,291 @@ std::vector<float> PitchResynthesizer::buildFormantCurve( return ratios; } +static const OwnPitchEngine::NoteIslandAnalysis* findEngineV3IslandForNote ( + const OwnPitchEngine::SharedAnalysis& analysis, + const PitchAnalyzer::PitchNote& note) +{ + for (const auto& island : analysis.islands) + { + for (const auto& islandNote : island.notes) + { + if (islandNote.id == note.id) + return &island; + } + } + + const int noteStartSample = static_cast<int> (std::round (note.startTime * analysis.sampleRate)); + const int noteEndSample = static_cast<int> (std::round (note.endTime * analysis.sampleRate)); + for (const auto& island : analysis.islands) + { + if (noteEndSample <= island.contextStartSample || noteStartSample >= island.contextEndSample) + continue; + return &island; + } + + return nullptr; +} + +[[maybe_unused]] static void applyEngineV3BoundaryZoneBlend ( + std::vector<std::vector<float>>& carrierOutput, + const std::vector<std::vector<float>>& boundaryOutput, + const float* const* input, + int numChannels, + int numSamples, + double sampleRate, + const std::vector<PitchAnalyzer::PitchNote>& notes, + const OwnPitchEngine::SharedAnalysis& analysis, + PitchResynthesizer::RenderDiagnostics& diagnostics) +{ + if (carrierOutput.size() != static_cast<size_t> (numChannels) + || boundaryOutput.size() != static_cast<size_t> (numChannels) + || input == nullptr + || sampleRate <= 0.0) + { + return; + } + + constexpr float kImmediateNeighborGapSec = 0.18f; + constexpr float kDefaultSmoothMs = 10.0f; + constexpr float kMaxSmoothMs = 15.0f; + const int shellSamplesDefault = static_cast<int> (std::round ( + sampleRate * juce::jlimit (5.0f, 12.0f, getEnvFloat ("OPENSTUDIO_ENGINE_V3_SHELL_MS", 8.0f)) * 0.001f)); + + std::vector<size_t> noteOrder (notes.size()); + std::iota (noteOrder.begin(), noteOrder.end(), 0); + std::sort (noteOrder.begin(), noteOrder.end(), [&] (size_t a, size_t b) { + return notes[a].startTime < notes[b].startTime; + }); + + for (size_t orderedIndex = 0; orderedIndex < noteOrder.size(); ++orderedIndex) + { + const auto& note = notes[noteOrder[orderedIndex]]; + if (! hasPitchStyleEdit (note)) + continue; + + const bool hasImmediatePrev = orderedIndex > 0 + && (note.startTime - notes[noteOrder[orderedIndex - 1]].endTime) <= kImmediateNeighborGapSec; + const bool hasImmediateNext = orderedIndex + 1 < noteOrder.size() + && (notes[noteOrder[orderedIndex + 1]].startTime - note.endTime) <= kImmediateNeighborGapSec; + + const float leftSmoothMs = hasImmediatePrev + ? juce::jlimit (5.0f, kMaxSmoothMs, note.transitionIn > 0.0f ? note.transitionIn : kDefaultSmoothMs) + : 0.0f; + const float rightSmoothMs = hasImmediateNext + ? juce::jlimit (5.0f, kMaxSmoothMs, note.transitionOut > 0.0f ? note.transitionOut : kDefaultSmoothMs) + : 0.0f; + + const int startSample = juce::jlimit (0, numSamples - 1, static_cast<int> (std::round (note.startTime * sampleRate))); + const int endSample = juce::jlimit (0, numSamples - 1, static_cast<int> (std::round (note.endTime * sampleRate))); + if (endSample <= startSample) + continue; + + const auto* island = findEngineV3IslandForNote (analysis, note); + const float detectedHz = island != nullptr && island->core.meanF0Hz > 40.0f + ? island->core.meanF0Hz + : std::max (55.0f, midiToHz (note.detectedPitch)); + const int periodSamples = juce::jlimit (16, static_cast<int> (0.03 * sampleRate), + static_cast<int> (std::round (sampleRate / std::max (55.0f, detectedHz)))); + + int entryCycleCount = 4; + int exitCycleCount = 4; + int entryAnchorSample = startSample; + int exitAnchorSample = endSample; + + if (island != nullptr && ! island->epochs.empty()) + { + const int noteStartLocal = juce::jlimit (0, static_cast<int> (island->monoSignal.size()) - 1, startSample - island->contextStartSample); + const int noteEndLocal = juce::jlimit (0, static_cast<int> (island->monoSignal.size()) - 1, endSample - island->contextStartSample); + const int searchRadius = juce::jlimit (periodSamples, static_cast<int> (0.03 * sampleRate), periodSamples * 6); + + for (size_t i = 0; i < island->epochs.size(); ++i) + { + const int epoch = island->epochs[i]; + if (epoch >= noteStartLocal && epoch <= noteStartLocal + searchRadius) + { + entryAnchorSample = island->contextStartSample + epoch; + if (i + 1 < island->epochs.size()) + { + const int nextEpoch = island->epochs[i + 1]; + entryCycleCount = juce::jlimit (2, 6, + static_cast<int> (std::round ((searchRadius * 0.35f) / std::max (16, nextEpoch - epoch)))); + } + break; + } + } + + for (int i = static_cast<int> (island->epochs.size()) - 1; i >= 0; --i) + { + const int epoch = island->epochs[static_cast<size_t> (i)]; + if (epoch <= noteEndLocal && epoch >= noteEndLocal - searchRadius) + { + exitAnchorSample = island->contextStartSample + epoch; + if (i > 0) + { + const int prevEpoch = island->epochs[static_cast<size_t> (i - 1)]; + exitCycleCount = juce::jlimit (2, 6, + static_cast<int> (std::round ((searchRadius * 0.35f) / std::max (16, epoch - prevEpoch)))); + } + break; + } + } + } + + const int entryCyclesSamples = juce::jlimit ( + static_cast<int> (std::round (0.008 * sampleRate)), + static_cast<int> (std::round (0.024 * sampleRate)), + periodSamples * juce::jlimit (2, 6, entryCycleCount)); + const int exitCyclesSamples = juce::jlimit ( + static_cast<int> (std::round (0.008 * sampleRate)), + static_cast<int> (std::round (0.024 * sampleRate)), + periodSamples * juce::jlimit (2, 6, exitCycleCount)); + + const int leftSmoothSamples = static_cast<int> (std::round (sampleRate * leftSmoothMs * 0.001f)); + const int rightSmoothSamples = static_cast<int> (std::round (sampleRate * rightSmoothMs * 0.001f)); + const int entryNeighborStart = std::max (0, startSample - leftSmoothSamples); + const int exitNeighborEnd = std::min (numSamples, endSample + rightSmoothSamples); + + const int entryShellEnd = juce::jlimit (startSample, endSample, + std::min (startSample + shellSamplesDefault, + std::max (startSample, entryAnchorSample - periodSamples / 2))); + const int entryBoundaryEnd = juce::jlimit (entryShellEnd, endSample, + std::max (entryShellEnd + entryCyclesSamples, + std::min (endSample, entryAnchorSample + (entryCycleCount * periodSamples) / 2))); + + const int exitShellStart = juce::jlimit (startSample, endSample, + std::max (endSample - shellSamplesDefault, + std::min (endSample, exitAnchorSample + periodSamples / 2))); + const int exitBoundaryStart = juce::jlimit (startSample, exitShellStart, + std::min (exitShellStart - exitCyclesSamples, + std::max (startSample, exitAnchorSample - (exitCycleCount * periodSamples) / 2))); + const bool upwardShift = (note.correctedPitch - note.detectedPitch) >= 0.0f; + const float bodyHarvestWet = juce::jlimit ( + 0.0f, + 0.35f, + getEnvFloat (upwardShift + ? "OPENSTUDIO_ENGINE_V3_BODY_HARVEST_WET_UP" + : "OPENSTUDIO_ENGINE_V3_BODY_HARVEST_WET_DOWN", + upwardShift ? 0.16f : 0.10f)); + const int bodyRampSamples = std::max ( + 1, + static_cast<int> (std::round (sampleRate + * juce::jlimit (0.008f, 0.040f, getEnvFloat ("OPENSTUDIO_ENGINE_V3_BODY_HARVEST_RAMP_SEC", 0.018f))))); + + diagnostics.v3TransitionPairUsed = diagnostics.v3TransitionPairUsed || hasImmediatePrev || hasImmediateNext; + diagnostics.firstVoicedCyclesEntryUsed = diagnostics.firstVoicedCyclesEntryUsed || (entryBoundaryEnd > entryShellEnd); + diagnostics.firstVoicedCyclesExitUsed = diagnostics.firstVoicedCyclesExitUsed || (exitShellStart > exitBoundaryStart); + diagnostics.v3FirstCyclesEntryCount = std::max (diagnostics.v3FirstCyclesEntryCount, entryCycleCount); + diagnostics.v3FirstCyclesExitCount = std::max (diagnostics.v3FirstCyclesExitCount, exitCycleCount); + diagnostics.v3EntryAnchorMs = std::max (diagnostics.v3EntryAnchorMs, + 1000.0 * static_cast<double> (std::max (0, entryBoundaryEnd - startSample)) / sampleRate); + diagnostics.v3ExitAnchorMs = std::max (diagnostics.v3ExitAnchorMs, + 1000.0 * static_cast<double> (std::max (0, endSample - exitBoundaryStart)) / sampleRate); + diagnostics.v3ShellDurationMs = std::max (diagnostics.v3ShellDurationMs, + 1000.0 * static_cast<double> (shellSamplesDefault) / sampleRate); + diagnostics.v3BodyDurationMs = std::max (diagnostics.v3BodyDurationMs, + 1000.0 * static_cast<double> (std::max (0, exitBoundaryStart - entryBoundaryEnd)) / sampleRate); + diagnostics.v3NeighborLeftOverlapMs = std::max (diagnostics.v3NeighborLeftOverlapMs, static_cast<double> (leftSmoothMs)); + diagnostics.v3NeighborRightOverlapMs = std::max (diagnostics.v3NeighborRightOverlapMs, static_cast<double> (rightSmoothMs)); + diagnostics.v3ResidualMix = std::max (diagnostics.v3ResidualMix, static_cast<double> (bodyHarvestWet)); + + for (int ch = 0; ch < numChannels; ++ch) + { + auto& carrier = carrierOutput[static_cast<size_t> (ch)]; + const auto& boundary = boundaryOutput[static_cast<size_t> (ch)]; + for (int s = entryNeighborStart; s < exitNeighborEnd; ++s) + { + float wCarrier = 1.0f; + float wOriginal = 0.0f; + float wBoundary = 0.0f; + + if (s < startSample) + { + const float t = static_cast<float> (s - entryNeighborStart) + / static_cast<float> (std::max (1, startSample - entryNeighborStart)); + wOriginal = smoothstep01 (t); + wCarrier = 1.0f - wOriginal; + } + else if (s < entryShellEnd) + { + wOriginal = 1.0f; + wCarrier = 0.0f; + } + else if (s < entryBoundaryEnd) + { + const int mid = entryShellEnd + std::max (1, (entryBoundaryEnd - entryShellEnd) / 2); + if (s < mid) + { + const float t = static_cast<float> (s - entryShellEnd) + / static_cast<float> (std::max (1, mid - entryShellEnd)); + wOriginal = 1.0f - smoothstep01 (t); + wBoundary = smoothstep01 (t); + wCarrier = 0.0f; + } + else + { + const float t = static_cast<float> (s - mid) + / static_cast<float> (std::max (1, entryBoundaryEnd - mid)); + wBoundary = 1.0f - smoothstep01 (t); + wCarrier = smoothstep01 (t); + wOriginal = 0.0f; + } + } + else if (s < exitBoundaryStart) + { + const float bodyIn = smoothstep01 ( + static_cast<float> (s - entryBoundaryEnd) + / static_cast<float> (std::max (1, bodyRampSamples))); + const float bodyOut = 1.0f - smoothstep01 ( + static_cast<float> (s - std::max (entryBoundaryEnd, exitBoundaryStart - bodyRampSamples)) + / static_cast<float> (std::max (1, bodyRampSamples))); + const float shapedWet = bodyHarvestWet * juce::jlimit (0.0f, 1.0f, std::min (bodyIn, bodyOut)); + if (shapedWet > 0.0f) + { + carrier[static_cast<size_t> (s)] = + carrier[static_cast<size_t> (s)] * (1.0f - shapedWet) + + boundary[static_cast<size_t> (s)] * shapedWet; + } + continue; + } + else if (s < exitShellStart) + { + const int mid = exitBoundaryStart + std::max (1, (exitShellStart - exitBoundaryStart) / 2); + if (s < mid) + { + const float t = static_cast<float> (s - exitBoundaryStart) + / static_cast<float> (std::max (1, mid - exitBoundaryStart)); + wCarrier = 1.0f - smoothstep01 (t); + wBoundary = smoothstep01 (t); + wOriginal = 0.0f; + } + else + { + const float t = static_cast<float> (s - mid) + / static_cast<float> (std::max (1, exitShellStart - mid)); + wBoundary = 1.0f - smoothstep01 (t); + wOriginal = smoothstep01 (t); + wCarrier = 0.0f; + } + } + else + { + const float t = static_cast<float> (s - exitShellStart) + / static_cast<float> (std::max (1, exitNeighborEnd - exitShellStart)); + wOriginal = 1.0f - smoothstep01 (t); + wCarrier = smoothstep01 (t); + wBoundary = 0.0f; + } + + const float sum = std::max (1.0e-5f, wCarrier + wOriginal + wBoundary); + carrier[static_cast<size_t> (s)] = + (carrier[static_cast<size_t> (s)] * wCarrier + + input[ch][s] * wOriginal + + boundary[static_cast<size_t> (s)] * wBoundary) / sum; + } + } + } +} + std::vector<std::vector<float>> PitchResynthesizer::processMultiChannel( const float* const* input, int numChannels, @@ -1284,7 +6255,27 @@ std::vector<std::vector<float>> PitchResynthesizer::processMultiChannel( RenderQuality renderQuality, std::function<bool()> shouldCancel) { - juce::ignoreUnused (engine); // Signalsmith is always used; engine param kept for API compat + juce::ignoreUnused (engine); + lastRenderDiagnostics = {}; + lastRenderDiagnostics.requestedRendererBranch = getRequestedPitchRendererBranchName(); + const auto pitchDirection = getPitchEditDirectionSummary (notes); + lastRenderDiagnostics.pitchDirection = pitchDirection.name; + lastRenderDiagnostics.downshiftFormantGuardAlpha = 0.0; + for (const auto& note : notes) + { + if (! hasPitchStyleEdit (note)) + continue; + if (note.entryBoundaryScore >= lastRenderDiagnostics.dominantEntryBoundaryScore) + { + lastRenderDiagnostics.dominantEntryBoundaryKind = note.entryBoundaryKind.isEmpty() ? "unknown" : note.entryBoundaryKind; + lastRenderDiagnostics.dominantEntryBoundaryScore = note.entryBoundaryScore; + } + if (note.exitBoundaryScore >= lastRenderDiagnostics.dominantExitBoundaryScore) + { + lastRenderDiagnostics.dominantExitBoundaryKind = note.exitBoundaryKind.isEmpty() ? "unknown" : note.exitBoundaryKind; + lastRenderDiagnostics.dominantExitBoundaryScore = note.exitBoundaryScore; + } + } enum class ProcessingMode { @@ -1315,6 +6306,9 @@ std::vector<std::vector<float>> PitchResynthesizer::processMultiChannel( // Build per-sample pitch ratio curve and formant ratio curve auto ratios = buildCorrectionCurve (numSamples, sampleRate, frames, notes, hopSize); auto formantRatios = buildFormantCurve (numSamples, sampleRate, notes, globalFormantSemitones); + lastRenderDiagnostics.explicitFormantRequested = hasExplicitFormant; + lastRenderDiagnostics.formantCurveUsed = ! formantRatios.empty(); + lastRenderDiagnostics.pitchOnlyFormantSuppressed = false; bool hasPitchShift = false; for (size_t i = 0; i < ratios.size() && ! hasPitchShift; i += 256) hasPitchShift = std::abs (ratios[i] - 1.0f) > 0.001f; @@ -1334,17 +6328,37 @@ std::vector<std::vector<float>> PitchResynthesizer::processMultiChannel( return "unknown"; }; + lastRenderDiagnostics.processingMode = processingModeName(); + if (processingMode == ProcessingMode::PitchOnly && pitchDirection.hasDownward) + lastRenderDiagnostics.downshiftFormantGuardUsed = true; + logPitchEditorFormant ("processMultiChannel samples=" + juce::String (numSamples) + " channels=" + juce::String (numChannels) + " globalFormantSt=" + juce::String (globalFormantSemitones, 3) + " formantCurve=" + juce::String (formantRatios.empty() ? "disabled" : "enabled") + " pitchShift=" + juce::String (hasPitchShift ? "true" : "false") + " mode=" + juce::String (processingModeName()) + + " direction=" + pitchDirection.name + + " pitchBranch=" + juce::String (getPitchOnlyRendererBranchName (getPitchOnlyRendererBranch())) + + " recoveryPath=" + juce::String (processingMode == ProcessingMode::PitchOnly + ? getPitchOnlyRecoveryPathName (getPitchOnlyRecoveryPath()) + : "not_applicable") + " renderQuality=" + juce::String (renderQuality == RenderQuality::PreviewFast ? "preview_fast" : "final_hq")); + const auto pitchRendererBranch = getPitchOnlyRendererBranch(); + const auto pitchOnlyRecoveryPath = getPitchOnlyRecoveryPath(); + lastRenderDiagnostics.pitchOnlyRecoveryPath = processingMode == ProcessingMode::PitchOnly + ? juce::String (getPitchOnlyRecoveryPathName (pitchOnlyRecoveryPath)) + : juce::String(); + lastRenderDiagnostics.pitchOnlyNeutralFormantUsed = processingMode == ProcessingMode::PitchOnly + && pitchOnlyRecoveryPath != PitchOnlyRecoveryPath::LegacyNatural; + const bool engineV3Branch = isEngineV3Branch (pitchRendererBranch); + const bool engineV3LpcTransfer = shouldUseEngineV3LpcTransfer (pitchRendererBranch); + const bool engineV3TransientBypass = shouldUseEngineV3TransientBypass (pitchRendererBranch); + // Build per-sample detected pitch in Hz only for explicit-formant processing. std::vector<float> detectedPitchHz; - if (processingMode != ProcessingMode::PitchOnly) + if (processingMode != ProcessingMode::PitchOnly || hasPitchShift) { detectedPitchHz.assign (static_cast<size_t> (numSamples), 0.0f); for (size_t fi = 0; fi < frames.size(); ++fi) @@ -1359,42 +6373,822 @@ std::vector<std::vector<float>> PitchResynthesizer::processMultiChannel( } std::vector<std::vector<float>> output (static_cast<size_t> (numChannels)); + struct AdaptiveSelectorBuildResult + { + std::vector<std::vector<float>> output; + OwnPitchEngine::RenderResult ownResult; + HybridStructuralBlendResult hybridBlend; + AdaptiveBoundaryCorrectionResult boundaryCorrection; + }; + + auto buildAdaptiveSelectorOutput = [&]() -> AdaptiveSelectorBuildResult + { + AdaptiveSelectorBuildResult result; + const bool adaptiveBoundaryCorrectionEnabled = false; + OwnPitchEngine ownEngine; + result.ownResult = ownEngine.renderPitchOnly ( + input, + numChannels, + numSamples, + sampleRate, + frames, + notes, + ratios, + toOwnEngineQuality (renderQuality), + shouldCancel); + auto legacyOutput = renderPitchOnlyIslands ( + input, numChannels, numSamples, sampleRate, + frames, notes, PitchOnlyRendererBranch::HybridReset, renderQuality, shouldCancel); + auto simpleOutput = copyInputChannels (input, numChannels, numSamples); + result.hybridBlend = blendHybridStructuralOutputs ( + input, + legacyOutput, + result.ownResult.output, + result.ownResult.analysis, + PitchOnlyRendererBranch::HybridStructural, + numChannels, + numSamples, + sampleRate, + notes); + result.output = composeAdaptiveSelectorOutput (result.hybridBlend.output, + simpleOutput, + numChannels, + numSamples, + sampleRate, + notes); + const float downBlendWeight = pitchDirection.hasDownward + ? juce::jlimit (0.0f, 0.42f, getEnvFloat ("OPENSTUDIO_PITCH_DOWNSHIFT_OWN_BLEND", 0.42f)) + : 0.24f; + applyAdaptiveDownwardOwnBlend (result.output, result.ownResult.output, numChannels, numSamples, sampleRate, notes, downBlendWeight); + if (pitchDirection.hasDownward) + lastRenderDiagnostics.downshiftFormantGuardUsed = true; + if (adaptiveBoundaryCorrectionEnabled) + { + result.boundaryCorrection = applyAdaptiveBoundaryCorrections ( + input, + result.output, + result.ownResult.analysis, + notes, + ratios, + numChannels, + numSamples, + sampleRate); + if (result.boundaryCorrection.used) + result.output = result.boundaryCorrection.output; + } + return result; + }; + + auto populateEngineV3BoundaryDiagnostics = [&]() + { + if (! engineV3Branch) + return; + + lastRenderDiagnostics.v3ContinuousRenderUsed = true; + lastRenderDiagnostics.v3FormantMode = engineV3LpcTransfer ? "lpc_transfer" : "native_preserve"; + lastRenderDiagnostics.v3ResidualMix = 0.0; + + constexpr float kImmediateNeighborGapSec = 0.18f; + const double defaultNeighborSmoothMs = 10.0; + std::vector<size_t> noteOrder (notes.size()); + std::iota (noteOrder.begin(), noteOrder.end(), 0); + std::sort (noteOrder.begin(), noteOrder.end(), [&] (size_t a, size_t b) { + return notes[a].startTime < notes[b].startTime; + }); + + for (size_t orderIndex = 0; orderIndex < noteOrder.size(); ++orderIndex) + { + const auto& note = notes[noteOrder[orderIndex]]; + if (! hasPitchStyleEdit (note)) + continue; + + const double noteDurationMs = std::max (0.0, 1000.0 * static_cast<double> (note.endTime - note.startTime)); + lastRenderDiagnostics.v3BodyDurationMs = std::max (lastRenderDiagnostics.v3BodyDurationMs, noteDurationMs); + + if (engineV3TransientBypass) + { + const double shellMs = juce::jlimit (6.0, 30.0, + static_cast<double> (getEnvFloat ("OPENSTUDIO_ENGINE_V3_TRANSIENT_BYPASS_MS", 18.0f))); + lastRenderDiagnostics.v3ShellDurationMs = std::max (lastRenderDiagnostics.v3ShellDurationMs, shellMs); + lastRenderDiagnostics.v3EntryAnchorMs = std::max (lastRenderDiagnostics.v3EntryAnchorMs, shellMs); + lastRenderDiagnostics.v3ExitAnchorMs = std::max (lastRenderDiagnostics.v3ExitAnchorMs, shellMs * 0.5); + const double periods = std::max (2.0, std::min (6.0, shellMs * std::max (1.0f, midiToHz (note.detectedPitch)) / 1000.0)); + lastRenderDiagnostics.v3FirstCyclesEntryCount = std::max (lastRenderDiagnostics.v3FirstCyclesEntryCount, static_cast<int> (std::round (periods))); + lastRenderDiagnostics.v3FirstCyclesExitCount = std::max (lastRenderDiagnostics.v3FirstCyclesExitCount, std::max (1, static_cast<int> (std::round (periods * 0.5)))); + } + + if (orderIndex > 0) + { + const auto& prev = notes[noteOrder[orderIndex - 1]]; + if ((note.startTime - prev.endTime) <= kImmediateNeighborGapSec) + { + lastRenderDiagnostics.v3TransitionPairUsed = true; + lastRenderDiagnostics.v3NeighborLeftOverlapMs = std::max ( + lastRenderDiagnostics.v3NeighborLeftOverlapMs, + note.transitionIn > 0.0f ? static_cast<double> (note.transitionIn) : defaultNeighborSmoothMs); + } + } + if (orderIndex + 1 < noteOrder.size()) + { + const auto& next = notes[noteOrder[orderIndex + 1]]; + if ((next.startTime - note.endTime) <= kImmediateNeighborGapSec) + { + lastRenderDiagnostics.v3TransitionPairUsed = true; + lastRenderDiagnostics.v3NeighborRightOverlapMs = std::max ( + lastRenderDiagnostics.v3NeighborRightOverlapMs, + note.transitionOut > 0.0f ? static_cast<double> (note.transitionOut) : defaultNeighborSmoothMs); + } + } + } + }; + switch (processingMode) { case ProcessingMode::PitchOnly: { if (shouldCancel && shouldCancel()) return output; - output = SignalsmithShifter::process (input, numChannels, numSamples, sampleRate, ratios); - logPitchEditorFormant ("using legacy pitch-only shifter path"); + if (pitchRendererBranch == PitchOnlyRendererBranch::VocalSourceFilterHq) + { + OwnPitchEngine ownEngine; + auto ownResult = ownEngine.renderVocalSourceFilterHq ( + input, + numChannels, + numSamples, + sampleRate, + frames, + notes, + ratios, + toOwnEngineQuality (renderQuality), + shouldCancel); + output = std::move (ownResult.output); + constexpr bool entryHybridEnabled = false; + constexpr bool coreHybridEnabled = false; + constexpr bool exitHybridEnabled = false; + const auto layerDumpDir = getPitchLayerDumpDirectory(); + if (layerDumpDir != juce::File()) + writePitchLayerDumpWav (layerDumpDir, "vsf_after_own_source_filter", output, sampleRate); + const bool hasLongEditedHybridNote = std::any_of ( + notes.begin(), + notes.end(), + [] (const PitchAnalyzer::PitchNote& note) + { + return hasPitchStyleEdit (note) + && static_cast<double> (note.endTime - note.startTime) >= 0.90; + }); + bool entryHybridUsed = false; + bool coreHybridUsed = false; + bool exitHybridUsed = false; + float maxCoreHybridAlpha = 0.0f; + if ((entryHybridEnabled || coreHybridEnabled || exitHybridEnabled) && hasLongEditedHybridNote) + { + auto adaptiveHybridBuild = buildAdaptiveSelectorOutput(); + if (layerDumpDir != juce::File()) + writePitchLayerDumpWav (layerDumpDir, "adaptive_hybrid_output", adaptiveHybridBuild.output, sampleRate); + const int entryFadeInSamples = std::max (1, static_cast<int> (std::round (0.006 * sampleRate))); + const int entryFadeOutSamples = std::max (1, static_cast<int> (std::round (0.030 * sampleRate))); + const int maxEntrySamples = std::max (1, static_cast<int> (std::round (0.095 * sampleRate))); + const int coreFadeSamples = std::max (1, static_cast<int> (std::round (0.040 * sampleRate))); + const int coreEntryProtectSamples = std::max (1, static_cast<int> (std::round (0.115 * sampleRate))); + const int coreExitProtectSamples = std::max (1, static_cast<int> (std::round (0.095 * sampleRate))); + const int exitFadeInSamples = std::max (1, static_cast<int> (std::round (0.030 * sampleRate))); + const int exitFadeOutSamples = std::max (1, static_cast<int> (std::round (0.010 * sampleRate))); + const int maxExitSamples = std::max (1, static_cast<int> (std::round (0.095 * sampleRate))); + for (const auto& note : notes) + { + if (! hasPitchStyleEdit (note)) + continue; + + const double noteDurationSec = static_cast<double> (note.endTime - note.startTime); + if (noteDurationSec < 0.90) + continue; + + const int noteStart = juce::jlimit (0, numSamples, static_cast<int> (std::round (note.startTime * sampleRate))); + const int noteEnd = juce::jlimit (noteStart, numSamples, static_cast<int> (std::round (note.endTime * sampleRate))); + if (noteEnd <= noteStart) + continue; + + if (entryHybridEnabled) + { + const int entryStart = noteStart; + const int entryEnd = juce::jlimit (entryStart, numSamples, std::min (noteEnd, entryStart + maxEntrySamples)); + if (entryEnd > entryStart) + { + for (int ch = 0; ch < numChannels; ++ch) + { + if (adaptiveHybridBuild.output.size() <= static_cast<size_t> (ch) + || output.size() <= static_cast<size_t> (ch)) + continue; + + auto& dst = output[static_cast<size_t> (ch)]; + const auto& adaptive = adaptiveHybridBuild.output[static_cast<size_t> (ch)]; + for (int s = entryStart; s < entryEnd; ++s) + { + if (s >= static_cast<int> (dst.size()) || s >= static_cast<int> (adaptive.size())) + continue; + + float wet = 1.0f; + if (s < entryStart + entryFadeInSamples) + { + const float t = static_cast<float> (s - entryStart) / static_cast<float> (entryFadeInSamples); + wet *= 0.5f - 0.5f * std::cos (juce::MathConstants<float>::pi * juce::jlimit (0.0f, 1.0f, t)); + } + if (s >= entryEnd - entryFadeOutSamples) + { + const float t = static_cast<float> (entryEnd - s) / static_cast<float> (entryFadeOutSamples); + wet *= 0.5f - 0.5f * std::cos (juce::MathConstants<float>::pi * juce::jlimit (0.0f, 1.0f, t)); + } + + dst[static_cast<size_t> (s)] = dst[static_cast<size_t> (s)] * (1.0f - wet) + + adaptive[static_cast<size_t> (s)] * wet; + } + } + entryHybridUsed = true; + } + } + + if (coreHybridEnabled) + { + const int coreStart = juce::jlimit (noteStart, noteEnd, noteStart + coreEntryProtectSamples); + const int coreEnd = juce::jlimit (coreStart, noteEnd, noteEnd - coreExitProtectSamples); + if (coreEnd > coreStart) + { + const bool downwardNote = note.correctedPitch < note.detectedPitch; + const float coreHybridAlpha = juce::jlimit ( + 0.0f, + 0.90f, + getEnvFloat ( + downwardNote ? "OPENSTUDIO_VSF_CORE_HYBRID_ALPHA_DOWN" : "OPENSTUDIO_VSF_CORE_HYBRID_ALPHA_UP", + downwardNote ? 0.65f : 0.55f)); + if (coreHybridAlpha > 1.0e-4f) + { + for (int ch = 0; ch < numChannels; ++ch) + { + if (adaptiveHybridBuild.output.size() <= static_cast<size_t> (ch) + || output.size() <= static_cast<size_t> (ch)) + continue; + + auto& dst = output[static_cast<size_t> (ch)]; + const auto& adaptive = adaptiveHybridBuild.output[static_cast<size_t> (ch)]; + for (int s = coreStart; s < coreEnd; ++s) + { + if (s >= static_cast<int> (dst.size()) || s >= static_cast<int> (adaptive.size())) + continue; + + float wet = coreHybridAlpha; + if (s < coreStart + coreFadeSamples) + { + const float t = static_cast<float> (s - coreStart) / static_cast<float> (coreFadeSamples); + wet *= 0.5f - 0.5f * std::cos (juce::MathConstants<float>::pi * juce::jlimit (0.0f, 1.0f, t)); + } + if (s >= coreEnd - coreFadeSamples) + { + const float t = static_cast<float> (coreEnd - s) / static_cast<float> (coreFadeSamples); + wet *= 0.5f - 0.5f * std::cos (juce::MathConstants<float>::pi * juce::jlimit (0.0f, 1.0f, t)); + } + + dst[static_cast<size_t> (s)] = dst[static_cast<size_t> (s)] * (1.0f - wet) + + adaptive[static_cast<size_t> (s)] * wet; + } + } + coreHybridUsed = true; + maxCoreHybridAlpha = std::max (maxCoreHybridAlpha, coreHybridAlpha); + } + } + } + + if (exitHybridEnabled) + { + const int exitEnd = noteEnd; + const int exitStart = juce::jlimit (noteStart, exitEnd, exitEnd - maxExitSamples); + if (exitEnd > exitStart) + { + for (int ch = 0; ch < numChannels; ++ch) + { + if (adaptiveHybridBuild.output.size() <= static_cast<size_t> (ch) + || output.size() <= static_cast<size_t> (ch)) + continue; + + auto& dst = output[static_cast<size_t> (ch)]; + const auto& adaptive = adaptiveHybridBuild.output[static_cast<size_t> (ch)]; + for (int s = exitStart; s < exitEnd; ++s) + { + if (s >= static_cast<int> (dst.size()) || s >= static_cast<int> (adaptive.size())) + continue; + + float wet = 1.0f; + if (s < exitStart + exitFadeInSamples) + { + const float t = static_cast<float> (s - exitStart) / static_cast<float> (exitFadeInSamples); + wet *= 0.5f - 0.5f * std::cos (juce::MathConstants<float>::pi * juce::jlimit (0.0f, 1.0f, t)); + } + if (s >= exitEnd - exitFadeOutSamples) + { + const float t = static_cast<float> (exitEnd - s) / static_cast<float> (exitFadeOutSamples); + wet *= 0.5f - 0.5f * std::cos (juce::MathConstants<float>::pi * juce::jlimit (0.0f, 1.0f, t)); + } + + dst[static_cast<size_t> (s)] = dst[static_cast<size_t> (s)] * (1.0f - wet) + + adaptive[static_cast<size_t> (s)] * wet; + } + } + exitHybridUsed = true; + } + } + } + } + lastRenderDiagnostics.pitchOnlyCoreTimbreCorrectionUsed = coreHybridUsed; + lastRenderDiagnostics.pitchOnlyCoreEnvelopeMix = maxCoreHybridAlpha; + lastRenderDiagnostics.pitchOnlyEntryHandoffUsed = entryHybridUsed; + lastRenderDiagnostics.pitchOnlyExitHandoffUsed = exitHybridUsed; + lastRenderDiagnostics.actualRendererBranch = juce::String (getPitchOnlyRendererBranchName (pitchRendererBranch)); + lastRenderDiagnostics.usedFallback = ownResult.usedFallback; + lastRenderDiagnostics.fallbackReason = ownResult.fallbackReason; + lastRenderDiagnostics.vocalSourceFilterUsed = ownResult.vocalSourceFilterUsed; + lastRenderDiagnostics.vocalSourceFilterVoicedCoverage = ownResult.vocalSourceFilterVoicedCoverage; + lastRenderDiagnostics.vocalSourceFilterResidualMix = ownResult.vocalSourceFilterResidualMix; + lastRenderDiagnostics.vocalSourceFilterFallbackUsed = ownResult.vocalSourceFilterFallbackUsed; + lastRenderDiagnostics.vocalSourceFilterFallbackReason = ownResult.vocalSourceFilterFallbackReason; + lastRenderDiagnostics.vocalSourceFilterEntryDryMs = ownResult.vocalSourceFilterEntryDryMs; + lastRenderDiagnostics.vocalSourceFilterExitDryMs = ownResult.vocalSourceFilterExitDryMs; + lastRenderDiagnostics.vocalSourceFilterResidualMixScale = ownResult.vocalSourceFilterResidualMixScale; + lastRenderDiagnostics.vocalSourceFilterEpochInterpolationUsed = ownResult.vocalSourceFilterEpochInterpolationUsed; + lastRenderDiagnostics.vocalSourceFilterEpochInterpolationStrength = ownResult.vocalSourceFilterEpochInterpolationStrength; + lastRenderDiagnostics.vocalSourceFilterGrainRadiusScale = ownResult.vocalSourceFilterGrainRadiusScale; + lastRenderDiagnostics.vocalSourceFilterUpPresenceTrimDb = ownResult.vocalSourceFilterUpPresenceTrimDb; + lastRenderDiagnostics.vocalSourceFilterUpPresenceHz = ownResult.vocalSourceFilterUpPresenceHz; + lastRenderDiagnostics.vocalSourceFilterDownNasalTrimDb = ownResult.vocalSourceFilterDownNasalTrimDb; + lastRenderDiagnostics.vocalSourceFilterDownNasalHz = ownResult.vocalSourceFilterDownNasalHz; + lastRenderDiagnostics.vocalSourceFilterDownBodyCompDb = ownResult.vocalSourceFilterDownBodyCompDb; + lastRenderDiagnostics.vocalSourceFilterDownBodyCompHz = ownResult.vocalSourceFilterDownBodyCompHz; + lastRenderDiagnostics.spectralEnvelopeCorrectionUsed = true; + if (layerDumpDir != juce::File()) + writePitchLayerDumpWav (layerDumpDir, "vsf_final_after_hybrid_blends", output, sampleRate); + logPitchEditorFormant ("using vocal source/filter pitch-only renderer" + + juce::String (" analysisMs=") + juce::String (ownResult.analysisMs, 2) + + " renderMs=" + juce::String (ownResult.renderMs, 2) + + " islands=" + juce::String (static_cast<int> (ownResult.analysis.islands.size())) + + " voicedCoverage=" + juce::String (ownResult.vocalSourceFilterVoicedCoverage, 3) + + " residualMix=" + juce::String (ownResult.vocalSourceFilterResidualMix, 3) + + " residualScale=" + juce::String (ownResult.vocalSourceFilterResidualMixScale, 3) + + " epochInterp=" + juce::String (ownResult.vocalSourceFilterEpochInterpolationUsed ? "true" : "false") + + " epochInterpStrength=" + juce::String (ownResult.vocalSourceFilterEpochInterpolationStrength, 3) + + " grainScale=" + juce::String (ownResult.vocalSourceFilterGrainRadiusScale, 3) + + " upPresenceTrimDb=" + juce::String (ownResult.vocalSourceFilterUpPresenceTrimDb, 2) + + " downNasalTrimDb=" + juce::String (ownResult.vocalSourceFilterDownNasalTrimDb, 2) + + " downBodyCompDb=" + juce::String (ownResult.vocalSourceFilterDownBodyCompDb, 2) + + " fallback=" + juce::String (ownResult.vocalSourceFilterFallbackUsed ? "true" : "false")); + } + else if (pitchOnlyRecoveryPath != PitchOnlyRecoveryPath::CurrentExperimental) + { + output = copyInputChannels (input, numChannels, numSamples); + lastRenderDiagnostics.actualRendererBranch = juce::String (getPitchOnlyRendererBranchName (pitchRendererBranch)); + lastRenderDiagnostics.usedFallback = false; + lastRenderDiagnostics.fallbackReason = {}; + lastRenderDiagnostics.spectralEnvelopeCorrectionUsed = false; + lastRenderDiagnostics.pitchOnlyCoreTimbreCorrectionUsed = false; + lastRenderDiagnostics.pitchOnlyCoreEnvelopeMix = 0.0; + lastRenderDiagnostics.pitchOnlyCoreRmsTrimDb = 0.0; + lastRenderDiagnostics.pitchOnlyCoreEnvelopeLifter = 0; + lastRenderDiagnostics.pitchOnlyEntryHandoffUsed = false; + lastRenderDiagnostics.pitchOnlyExitHandoffUsed = false; + logPitchEditorFormant ("using pitch-only recovery path=" + + juce::String (getPitchOnlyRecoveryPathName (pitchOnlyRecoveryPath)) + + " requestedBranch=" + juce::String (getPitchOnlyRendererBranchName (pitchRendererBranch))); + } + else if (engineV3Branch) + { + // Transient bypass: keep note attack region dry so the consonant + // onset passes through unshifted. + if (engineV3TransientBypass) + { + const float bypassMs = juce::jlimit (6.0f, 30.0f, + getEnvFloat ("OPENSTUDIO_ENGINE_V3_TRANSIENT_BYPASS_MS", 18.0f)); + const int bypassSamples = static_cast<int> (sampleRate * bypassMs * 0.001); + for (const auto& note : notes) + { + if (! hasPitchStyleEdit (note)) + continue; + const int noteStart = juce::jlimit (0, numSamples, + static_cast<int> (getEffectiveNoteStartTime (note) * sampleRate)); + const int bypassEnd = std::min (numSamples, noteStart + bypassSamples); + for (int s = noteStart; s < bypassEnd; ++s) + ratios[static_cast<size_t> (s)] = 1.0f; + } + } + + // Retired full-clip experimental branch; kept dry if an old + // diagnostic override reaches this block. + // The OwnPitchEngine boundary blend is intentionally skipped: the + // V3-1 probe showed it regressed note-body quality on hard clips. + // Formant residual is handled by the LPC post-pass below when + // engineV3LpcTransfer is true. + output = copyInputChannels (input, numChannels, numSamples); + lastRenderDiagnostics.actualRendererBranch = juce::String (getPitchOnlyRendererBranchName (pitchRendererBranch)); + lastRenderDiagnostics.transientBypassUsed = engineV3TransientBypass; + lastRenderDiagnostics.usedFallback = false; + lastRenderDiagnostics.fallbackReason = {}; + populateEngineV3BoundaryDiagnostics(); + logPitchEditorFormant ("engine-v3 full-clip processPitchOnlyBase" + + juce::String (" lpc=") + juce::String (engineV3LpcTransfer ? "true" : "false") + + " transient=" + juce::String (engineV3TransientBypass ? "true" : "false") + + " samples=" + juce::String (numSamples)); + } + else if (pitchRendererBranch == PitchOnlyRendererBranch::AdaptiveSelector) + { + auto adaptiveBuild = buildAdaptiveSelectorOutput(); + output = std::move (adaptiveBuild.output); + lastRenderDiagnostics.actualRendererBranch = juce::String (getPitchOnlyRendererBranchName (pitchRendererBranch)); + lastRenderDiagnostics.usedFallback = adaptiveBuild.ownResult.usedFallback; + lastRenderDiagnostics.fallbackReason = adaptiveBuild.ownResult.fallbackReason; + lastRenderDiagnostics.bridgeUsed = adaptiveBuild.hybridBlend.diagnostics.bridgeUsed; + lastRenderDiagnostics.bridgeFallbackUsed = adaptiveBuild.hybridBlend.diagnostics.bridgeFallbackUsed; + lastRenderDiagnostics.bridgeStartSec = static_cast<double> (adaptiveBuild.hybridBlend.diagnostics.bridgeStartSample) / sampleRate; + lastRenderDiagnostics.bridgeLengthMs = 1000.0 * static_cast<double> (adaptiveBuild.hybridBlend.diagnostics.bridgeLengthSamples) / sampleRate; + lastRenderDiagnostics.bridgeAlignmentLagSamples = adaptiveBuild.hybridBlend.diagnostics.bridgeAlignmentLagSamples; + lastRenderDiagnostics.bridgeCorrelationScore = adaptiveBuild.hybridBlend.diagnostics.bridgeCorrelationScore; + lastRenderDiagnostics.bridgeGainDeltaDb = adaptiveBuild.hybridBlend.diagnostics.bridgeGainDeltaDb; + lastRenderDiagnostics.bodyReplacementUsed = adaptiveBuild.hybridBlend.bodyReplacementDiagnostics.bodyReplacementUsed; + lastRenderDiagnostics.bodyReplacementFallbackUsed = adaptiveBuild.hybridBlend.bodyReplacementDiagnostics.bodyReplacementFallbackUsed; + lastRenderDiagnostics.entryLockStartSec = static_cast<double> (adaptiveBuild.hybridBlend.bodyReplacementDiagnostics.entryLockStartSample) / sampleRate; + lastRenderDiagnostics.entryLockLengthMs = 1000.0 * static_cast<double> (adaptiveBuild.hybridBlend.bodyReplacementDiagnostics.entryLockLengthSamples) / sampleRate; + lastRenderDiagnostics.exitLockStartSec = static_cast<double> (adaptiveBuild.hybridBlend.bodyReplacementDiagnostics.exitLockStartSample) / sampleRate; + lastRenderDiagnostics.renderedBodyStartSec = static_cast<double> (adaptiveBuild.hybridBlend.bodyReplacementDiagnostics.renderedBodyStartSample) / sampleRate; + lastRenderDiagnostics.renderedBodyEndSec = static_cast<double> (adaptiveBuild.hybridBlend.bodyReplacementDiagnostics.renderedBodyEndSample) / sampleRate; + lastRenderDiagnostics.islandNativeUsed = adaptiveBuild.hybridBlend.islandNativeDiagnostics.islandNativeUsed; + lastRenderDiagnostics.islandNativeFallbackUsed = adaptiveBuild.hybridBlend.islandNativeDiagnostics.islandNativeFallbackUsed; + lastRenderDiagnostics.islandRenderStartSec = static_cast<double> (adaptiveBuild.hybridBlend.islandNativeDiagnostics.islandRenderStartSample) / sampleRate; + lastRenderDiagnostics.islandRenderEndSec = static_cast<double> (adaptiveBuild.hybridBlend.islandNativeDiagnostics.islandRenderEndSample) / sampleRate; + lastRenderDiagnostics.transientMaskPeak = adaptiveBuild.hybridBlend.islandNativeDiagnostics.transientMaskPeak; + lastRenderDiagnostics.voicedCoreMaskPeak = adaptiveBuild.hybridBlend.islandNativeDiagnostics.voicedCoreMaskPeak; + lastRenderDiagnostics.spectralEnvelopeCorrectionUsed = adaptiveBuild.boundaryCorrection.spectralEnvelopeCorrectionUsed; + lastRenderDiagnostics.transientBypassUsed = adaptiveBuild.boundaryCorrection.transientBypassUsed; + lastRenderDiagnostics.residualCarryUsed = adaptiveBuild.boundaryCorrection.residualCarryUsed; + lastRenderDiagnostics.cepstralCutoffUsed = adaptiveBuild.boundaryCorrection.cepstralCutoffUsed; + lastRenderDiagnostics.engineV2FftSize = adaptiveBuild.boundaryCorrection.fftSizeUsed; + lastRenderDiagnostics.engineV2HopSize = adaptiveBuild.boundaryCorrection.hopSizeUsed; + logPitchEditorFormant ("using adaptive harvested selector" + + juce::String (" analysisMs=") + juce::String (adaptiveBuild.ownResult.analysisMs, 2) + + " renderMs=" + juce::String (adaptiveBuild.ownResult.renderMs, 2) + + " islands=" + juce::String (static_cast<int> (adaptiveBuild.ownResult.analysis.islands.size())) + + " epochs=" + juce::String (adaptiveBuild.ownResult.analysis.totalEpochCount) + + " boundaryCorrection=" + juce::String (adaptiveBuild.boundaryCorrection.used ? "true" : "false") + + " transientBypass=" + juce::String (adaptiveBuild.boundaryCorrection.transientBypassUsed ? "true" : "false") + + " residualCarry=" + juce::String (adaptiveBuild.boundaryCorrection.residualCarryUsed ? "true" : "false") + + " lifter=" + juce::String (adaptiveBuild.boundaryCorrection.cepstralCutoffUsed) + + " fft/hop=" + juce::String (adaptiveBuild.boundaryCorrection.fftSizeUsed) + "/" + juce::String (adaptiveBuild.boundaryCorrection.hopSizeUsed)); + } + else if (pitchRendererBranch == PitchOnlyRendererBranch::EngineV2Program) + { + auto adaptiveBuild = buildAdaptiveSelectorOutput(); + auto engineV2Render = renderEngineV2Program ( + input, + adaptiveBuild.output, + adaptiveBuild.ownResult.analysis, + notes, + ratios, + numChannels, + numSamples, + sampleRate); + output = std::move (engineV2Render.output); + const auto& engineV2Diagnostics = engineV2Render.diagnostics; + + lastRenderDiagnostics.actualRendererBranch = juce::String (getPitchOnlyRendererBranchName (pitchRendererBranch)); + lastRenderDiagnostics.usedFallback = adaptiveBuild.ownResult.usedFallback || engineV2Diagnostics.engineV2FallbackUsed; + lastRenderDiagnostics.fallbackReason = adaptiveBuild.ownResult.fallbackReason; + lastRenderDiagnostics.engineV2Used = engineV2Diagnostics.engineV2Used; + lastRenderDiagnostics.engineV2FallbackUsed = engineV2Diagnostics.engineV2FallbackUsed; + lastRenderDiagnostics.engineV2TransitionCount = engineV2Diagnostics.transitionCount; + lastRenderDiagnostics.engineV2TransitionStartSec = static_cast<double> (engineV2Diagnostics.firstTransitionStartSample) / sampleRate; + lastRenderDiagnostics.engineV2TransitionEndSec = static_cast<double> (engineV2Diagnostics.lastTransitionEndSample) / sampleRate; + lastRenderDiagnostics.engineV2HarmonicSupportPeak = engineV2Diagnostics.harmonicSupportPeak; + lastRenderDiagnostics.engineV2ResidualSupportPeak = engineV2Diagnostics.residualSupportPeak; + lastRenderDiagnostics.engineV2EnvelopeSupportPeak = engineV2Diagnostics.envelopeSupportPeak; + lastRenderDiagnostics.spectralEnvelopeCorrectionUsed = engineV2Render.spectralEnvelopeCorrectionUsed; + lastRenderDiagnostics.transientBypassUsed = engineV2Render.transientBypassUsed; + lastRenderDiagnostics.residualCarryUsed = engineV2Render.residualCarryUsed; + lastRenderDiagnostics.cepstralCutoffUsed = engineV2Render.cepstralCutoffUsed; + lastRenderDiagnostics.engineV2FftSize = engineV2Render.fftSizeUsed; + lastRenderDiagnostics.engineV2HopSize = engineV2Render.hopSizeUsed; + logPitchEditorFormant ("using engine-v2 scaffold" + + juce::String (" transitions=") + juce::String (engineV2Diagnostics.transitionCount) + + " harmonicPeak=" + juce::String (engineV2Diagnostics.harmonicSupportPeak, 3) + + " residualPeak=" + juce::String (engineV2Diagnostics.residualSupportPeak, 3) + + " envelopePeak=" + juce::String (engineV2Diagnostics.envelopeSupportPeak, 3) + + " transientBypass=" + juce::String (engineV2Render.transientBypassUsed ? "true" : "false") + + " residualCarry=" + juce::String (engineV2Render.residualCarryUsed ? "true" : "false") + + " lifter=" + juce::String (engineV2Render.cepstralCutoffUsed) + + " fft/hop=" + juce::String (engineV2Render.fftSizeUsed) + "/" + juce::String (engineV2Render.hopSizeUsed) + + " fallback=" + juce::String (engineV2Diagnostics.engineV2FallbackUsed ? "true" : "false") + + " analysisMs=" + juce::String (adaptiveBuild.ownResult.analysisMs, 2) + + " renderMs=" + juce::String (adaptiveBuild.ownResult.renderMs, 2)); + } + else if (pitchRendererBranch == PitchOnlyRendererBranch::HybridStructural + || pitchRendererBranch == PitchOnlyRendererBranch::IslandNative + || pitchRendererBranch == PitchOnlyRendererBranch::IslandNativePsola) + { + OwnPitchEngine ownEngine; + auto ownResult = ownEngine.renderPitchOnly ( + input, + numChannels, + numSamples, + sampleRate, + frames, + notes, + ratios, + toOwnEngineQuality (renderQuality), + shouldCancel); + auto legacyOutput = renderPitchOnlyIslands ( + input, numChannels, numSamples, sampleRate, + frames, notes, PitchOnlyRendererBranch::HybridReset, renderQuality, shouldCancel); + auto hybridBlend = blendHybridStructuralOutputs ( + input, + legacyOutput, + ownResult.output, + ownResult.analysis, + pitchRendererBranch, + numChannels, + numSamples, + sampleRate, + notes); + output = std::move (hybridBlend.output); + lastRenderDiagnostics.actualRendererBranch = juce::String (getPitchOnlyRendererBranchName (pitchRendererBranch)); + lastRenderDiagnostics.usedFallback = ownResult.usedFallback; + lastRenderDiagnostics.fallbackReason = ownResult.fallbackReason; + lastRenderDiagnostics.bridgeUsed = hybridBlend.diagnostics.bridgeUsed; + lastRenderDiagnostics.bridgeFallbackUsed = hybridBlend.diagnostics.bridgeFallbackUsed; + lastRenderDiagnostics.bridgeStartSec = static_cast<double> (hybridBlend.diagnostics.bridgeStartSample) / sampleRate; + lastRenderDiagnostics.bridgeLengthMs = 1000.0 * static_cast<double> (hybridBlend.diagnostics.bridgeLengthSamples) / sampleRate; + lastRenderDiagnostics.bridgeAlignmentLagSamples = hybridBlend.diagnostics.bridgeAlignmentLagSamples; + lastRenderDiagnostics.bridgeCorrelationScore = hybridBlend.diagnostics.bridgeCorrelationScore; + lastRenderDiagnostics.bridgeGainDeltaDb = hybridBlend.diagnostics.bridgeGainDeltaDb; + lastRenderDiagnostics.bodyReplacementUsed = hybridBlend.bodyReplacementDiagnostics.bodyReplacementUsed; + lastRenderDiagnostics.bodyReplacementFallbackUsed = hybridBlend.bodyReplacementDiagnostics.bodyReplacementFallbackUsed; + lastRenderDiagnostics.entryLockStartSec = static_cast<double> (hybridBlend.bodyReplacementDiagnostics.entryLockStartSample) / sampleRate; + lastRenderDiagnostics.entryLockLengthMs = 1000.0 * static_cast<double> (hybridBlend.bodyReplacementDiagnostics.entryLockLengthSamples) / sampleRate; + lastRenderDiagnostics.exitLockStartSec = static_cast<double> (hybridBlend.bodyReplacementDiagnostics.exitLockStartSample) / sampleRate; + lastRenderDiagnostics.renderedBodyStartSec = static_cast<double> (hybridBlend.bodyReplacementDiagnostics.renderedBodyStartSample) / sampleRate; + lastRenderDiagnostics.renderedBodyEndSec = static_cast<double> (hybridBlend.bodyReplacementDiagnostics.renderedBodyEndSample) / sampleRate; + lastRenderDiagnostics.islandNativeUsed = hybridBlend.islandNativeDiagnostics.islandNativeUsed; + lastRenderDiagnostics.islandNativeFallbackUsed = hybridBlend.islandNativeDiagnostics.islandNativeFallbackUsed; + lastRenderDiagnostics.islandRenderStartSec = static_cast<double> (hybridBlend.islandNativeDiagnostics.islandRenderStartSample) / sampleRate; + lastRenderDiagnostics.islandRenderEndSec = static_cast<double> (hybridBlend.islandNativeDiagnostics.islandRenderEndSample) / sampleRate; + lastRenderDiagnostics.transientMaskPeak = hybridBlend.islandNativeDiagnostics.transientMaskPeak; + lastRenderDiagnostics.voicedCoreMaskPeak = hybridBlend.islandNativeDiagnostics.voicedCoreMaskPeak; + logPitchEditorFormant ("using island-aware pitch-only renderer" + + juce::String (" analysisMs=") + juce::String (ownResult.analysisMs, 2) + + " renderMs=" + juce::String (ownResult.renderMs, 2) + + " islands=" + juce::String (static_cast<int> (ownResult.analysis.islands.size())) + + " epochs=" + juce::String (ownResult.analysis.totalEpochCount) + + " maxPartials=" + juce::String (ownResult.analysis.maxPartialCount) + + " bodyReplacement=" + juce::String (hybridBlend.bodyReplacementDiagnostics.bodyReplacementUsed ? "true" : "false") + + " bodyReplacementFallback=" + juce::String (hybridBlend.bodyReplacementDiagnostics.bodyReplacementFallbackUsed ? "true" : "false") + + " islandNative=" + juce::String (hybridBlend.islandNativeDiagnostics.islandNativeUsed ? "true" : "false") + + " islandNativeFallback=" + juce::String (hybridBlend.islandNativeDiagnostics.islandNativeFallbackUsed ? "true" : "false") + + " bridgeUsed=" + juce::String (hybridBlend.diagnostics.bridgeUsed ? "true" : "false") + + " bridgeFallback=" + juce::String (hybridBlend.diagnostics.bridgeFallbackUsed ? "true" : "false") + + " bridgeLag=" + juce::String (hybridBlend.diagnostics.bridgeAlignmentLagSamples) + + " bridgeScore=" + juce::String (hybridBlend.diagnostics.bridgeCorrelationScore, 3)); + } + else if (pitchRendererBranch == PitchOnlyRendererBranch::OwnEnginePitchOnly) + { + OwnPitchEngine ownEngine; + auto ownResult = ownEngine.renderPitchOnly ( + input, + numChannels, + numSamples, + sampleRate, + frames, + notes, + ratios, + toOwnEngineQuality (renderQuality), + shouldCancel); + output = std::move (ownResult.output); + lastRenderDiagnostics.actualRendererBranch = juce::String (getPitchOnlyRendererBranchName (pitchRendererBranch)); + lastRenderDiagnostics.usedFallback = ownResult.usedFallback; + lastRenderDiagnostics.fallbackReason = ownResult.fallbackReason; + logPitchEditorFormant ("using own-engine pitch-only renderer" + + juce::String (" analysisMs=") + juce::String (ownResult.analysisMs, 2) + + " renderMs=" + juce::String (ownResult.renderMs, 2) + + " islands=" + juce::String (static_cast<int> (ownResult.analysis.islands.size())) + + " epochs=" + juce::String (ownResult.analysis.totalEpochCount) + + " maxPartials=" + juce::String (ownResult.analysis.maxPartialCount)); + } + else if (pitchRendererBranch == PitchOnlyRendererBranch::SimpleCe33) + { + output = copyInputChannels (input, numChannels, numSamples); + lastRenderDiagnostics.actualRendererBranch = juce::String (getPitchOnlyRendererBranchName (pitchRendererBranch)); + logPitchEditorFormant ("using simple ce33-style pitch-only baseline path"); + } + else + { + output = renderPitchOnlyIslands (input, numChannels, numSamples, sampleRate, + frames, notes, pitchRendererBranch, renderQuality, shouldCancel); + lastRenderDiagnostics.actualRendererBranch = juce::String (getPitchOnlyRendererBranchName (pitchRendererBranch)); + logPitchEditorFormant ("using note-local pitch-only shifter path"); + } break; } case ProcessingMode::FormantOnly: { + if (pitchRendererBranch == PitchOnlyRendererBranch::OwnEngineFormantOnly) + { + OwnPitchEngine ownEngine; + const auto analysis = ownEngine.analyze ( + input, + numChannels, + numSamples, + sampleRate, + frames, + notes, + OwnPitchEngine::Mode::FormantOnly, + toOwnEngineQuality (renderQuality)); + lastRenderDiagnostics.actualRendererBranch = juce::String (getPitchOnlyRendererBranchName (pitchRendererBranch)); + logPitchEditorFormant ("own-engine shared analysis ready for formant-only render" + + juce::String (" islands=") + juce::String (static_cast<int> (analysis.islands.size())) + + " epochs=" + juce::String (analysis.totalEpochCount) + + " maxPartials=" + juce::String (analysis.maxPartialCount) + + " cacheHit=" + juce::String (analysis.cacheHit ? "true" : "false")); + } + else + { + lastRenderDiagnostics.actualRendererBranch = juce::String (getPitchOnlyRendererBranchName (pitchRendererBranch)); + } + if (engineV3Branch) + populateEngineV3BoundaryDiagnostics(); for (int ch = 0; ch < numChannels; ++ch) output[static_cast<size_t> (ch)].assign (input[ch], input[ch] + numSamples); - logPitchEditorFormant ("skipping Signalsmith pitch shifter for formant-only render"); + logPitchEditorFormant ("formant-only render unsupported; returning dry audio"); break; } case ProcessingMode::PitchPlusFormant: { if (shouldCancel && shouldCancel()) return output; - output = SignalsmithShifter::process (input, numChannels, numSamples, sampleRate, - ratios, formantRatios, detectedPitchHz); - logPitchEditorFormant ("using preserved-timbre pitch base with explicit formant overlay"); + if (pitchRendererBranch == PitchOnlyRendererBranch::OwnEnginePitchPlusFormant) + { + OwnPitchEngine ownEngine; + auto ownResult = ownEngine.renderPitchOnly ( + input, + numChannels, + numSamples, + sampleRate, + frames, + notes, + ratios, + toOwnEngineQuality (renderQuality), + shouldCancel); + output = std::move (ownResult.output); + lastRenderDiagnostics.actualRendererBranch = juce::String (getPitchOnlyRendererBranchName (pitchRendererBranch)); + lastRenderDiagnostics.usedFallback = ownResult.usedFallback; + lastRenderDiagnostics.fallbackReason = ownResult.fallbackReason; + logPitchEditorFormant ("using own-engine pitch base with explicit formant overlay" + + juce::String (" analysisMs=") + juce::String (ownResult.analysisMs, 2) + + " renderMs=" + juce::String (ownResult.renderMs, 2) + + " islands=" + juce::String (static_cast<int> (ownResult.analysis.islands.size()))); + } + else + { + output = copyInputChannels (input, numChannels, numSamples); + lastRenderDiagnostics.actualRendererBranch = juce::String (getPitchOnlyRendererBranchName (pitchRendererBranch)); + if (engineV3Branch) + populateEngineV3BoundaryDiagnostics(); + logPitchEditorFormant ("using preserved-timbre pitch base with explicit formant overlay"); + } break; } } + if (lastRenderDiagnostics.actualRendererBranch.isEmpty()) + lastRenderDiagnostics.actualRendererBranch = lastRenderDiagnostics.requestedRendererBranch; + + if (processingMode == ProcessingMode::PitchOnly + && pitchOnlyRecoveryPath == PitchOnlyRecoveryPath::CurrentExperimental + && pitchRendererBranch != PitchOnlyRendererBranch::VocalSourceFilterHq + && (pitchDirection.hasUpward || pitchDirection.hasDownward) + && ! output.empty()) + { + int lifter = 0; + int fftSize = 0; + int hop = 0; + float envelopeMixUsed = 0.0f; + float rmsTrimDb = 0.0f; + const float rmsTrimCapDb = pitchDirection.hasDownward && ! pitchDirection.hasUpward + ? juce::jlimit (0.0f, 1.5f, getEnvFloat ("OPENSTUDIO_PITCH_CORE_DOWN_RMS_TRIM_DB", 1.3f)) + : juce::jlimit (0.0f, 1.5f, getEnvFloat ("OPENSTUDIO_PITCH_CORE_UP_RMS_TRIM_DB", 1.5f)); + const bool rmsTrimUsed = applyPitchOnlyCoreRmsTrim ( + output, + input, + numChannels, + numSamples, + sampleRate, + notes, + detectedPitchHz, + rmsTrimCapDb, + pitchDirection.hasDownward && ! pitchDirection.hasUpward, + &rmsTrimDb); + const bool envelopeUsed = applyVoicedCoreEnvelopeTransferForPitchOnly ( + output, + input, + numChannels, + numSamples, + sampleRate, + frames, + notes, + detectedPitchHz, + 0.0f, + 0.05f, + 2.5f, + 38, + &lifter, + &fftSize, + &hop, + &envelopeMixUsed); + if (envelopeUsed || rmsTrimUsed) + { + if (envelopeUsed) + lastRenderDiagnostics.spectralEnvelopeCorrectionUsed = true; + lastRenderDiagnostics.pitchOnlyCoreTimbreCorrectionUsed = true; + lastRenderDiagnostics.pitchOnlyCoreEnvelopeMix = envelopeMixUsed; + lastRenderDiagnostics.pitchOnlyCoreRmsTrimDb = rmsTrimDb; + lastRenderDiagnostics.pitchOnlyCoreEnvelopeLifter = lifter; + lastRenderDiagnostics.cepstralCutoffUsed = std::max (lastRenderDiagnostics.cepstralCutoffUsed, lifter); + lastRenderDiagnostics.engineV2FftSize = std::max (lastRenderDiagnostics.engineV2FftSize, fftSize); + lastRenderDiagnostics.engineV2HopSize = std::max (lastRenderDiagnostics.engineV2HopSize, hop); + logPitchEditorFormant ("pitch-only core timbre correction" + + juce::String (" direction=") + pitchDirection.name + + " envelope=" + juce::String (envelopeUsed ? "true" : "false") + + " mix=" + juce::String (envelopeMixUsed, 3) + + " rmsTrimDb=" + juce::String (rmsTrimDb, 3) + + " lifter=" + juce::String (lifter) + + " fft/hop=" + juce::String (fftSize) + "/" + juce::String (hop)); + } + } + + const bool pitchOnlyCoreTextureEnabled = processingMode == ProcessingMode::PitchOnly + && pitchOnlyRecoveryPath == PitchOnlyRecoveryPath::NeutralFormantMinimal + && pitchRendererBranch != PitchOnlyRendererBranch::VocalSourceFilterHq + && (pitchDirection.hasUpward || pitchDirection.hasDownward) + && getEnvInt ("OPENSTUDIO_PITCH_CORE_TEXTURE_ENABLE", 1) != 0; + if (pitchOnlyCoreTextureEnabled && ! output.empty()) + { + float textureMixUsed = 0.0f; + const bool textureUsed = applyPitchOnlyCoreAperiodicTexture ( + output, + input, + numChannels, + numSamples, + sampleRate, + notes, + detectedPitchHz, + pitchDirection.hasDownward && ! pitchDirection.hasUpward, + &textureMixUsed); + + if (textureUsed) + { + lastRenderDiagnostics.pitchOnlyCoreTimbreCorrectionUsed = true; + lastRenderDiagnostics.residualCarryUsed = true; + logPitchEditorFormant ("pitch-only core aperiodic texture" + + juce::String (" direction=") + pitchDirection.name + + " mix=" + juce::String (textureMixUsed, 3)); + } + } + + const bool pitchOnlyCoreLevelEnabled = processingMode == ProcessingMode::PitchOnly + && pitchOnlyRecoveryPath == PitchOnlyRecoveryPath::NeutralFormantMinimal + && pitchRendererBranch != PitchOnlyRendererBranch::VocalSourceFilterHq + && (pitchDirection.hasUpward || pitchDirection.hasDownward) + && getEnvInt ("OPENSTUDIO_PITCH_CORE_LEVEL_ENABLE", 0) != 0; + if (pitchOnlyCoreLevelEnabled && ! output.empty()) + { + float coreLevelDb = 0.0f; + const bool coreLevelUsed = applyPitchOnlyCoreDirectionalLevel ( + output, + input, + numChannels, + numSamples, + sampleRate, + notes, + detectedPitchHz, + pitchDirection.hasDownward && ! pitchDirection.hasUpward, + &coreLevelDb); + + if (coreLevelUsed) + { + lastRenderDiagnostics.pitchOnlyCoreTimbreCorrectionUsed = true; + lastRenderDiagnostics.pitchOnlyCoreRmsTrimDb = coreLevelDb; + logPitchEditorFormant ("pitch-only core directional level" + + juce::String (" direction=") + pitchDirection.name + + " trimDb=" + juce::String (coreLevelDb, 3)); + } + } + const bool formantOnlyRender = processingMode == ProcessingMode::FormantOnly; - // Keep pitch-only edits on the old stable base path. - // The residual post-correction pass was previously disabled because it - // introduced artifacts; bringing it back is a likely source of the - // crackle/dropouts heard on plain note shifts. Let Signalsmith's base - // formant preservation handle pitch-only edits, and reserve the explicit - // warp stage for actual user-requested formant changes. + // Pitch-only now uses the stable note-local base render plus a strictly + // note-core serial correction inside renderPitchOnlyIslands(). Broad + // post-correction remains disabled here because it was the path that + // previously reintroduced crackle and shoulder damage. if (processingMode != ProcessingMode::PitchOnly) { logPitchEditorFormant ("applying explicit formant warp stage"); @@ -1407,9 +7201,27 @@ std::vector<std::vector<float>> PitchResynthesizer::processMultiChannel( renderQuality, shouldCancel); } - else if (hasPitchShift) + if (processingMode != ProcessingMode::PitchOnly) + { + applyReferenceResidualMatch (output, input, numChannels, numSamples, + sampleRate, notes, formantRatios, detectedPitchHz, + hasPitchShift, ! formantRatios.empty(), + renderQuality, shouldCancel); + } + + if (engineV3LpcTransfer && processingMode != ProcessingMode::FormantOnly && ! output.empty()) { - logPitchEditorFormant ("pitch-only render kept on preserved-timbre base path"); + LpcEnvelopeTransfer::Settings lpcSettings; + lpcSettings.lpcOrder = juce::jlimit (12, 20, getEnvInt ("OPENSTUDIO_ENGINE_V3_LPC_ORDER", 16)); + lpcSettings.fftOrder = juce::jlimit (8, 10, getEnvInt ("OPENSTUDIO_ENGINE_V3_LPC_FFT_ORDER", 9)); + lpcSettings.hopSize = std::max (1, (1 << lpcSettings.fftOrder) / 2); + lpcSettings.epsilonFloor = juce::jlimit (1.0e-5f, 1.0e-2f, getEnvFloat ("OPENSTUDIO_ENGINE_V3_LPC_EPSILON", 1.0e-3f)); + lpcSettings.maxCorrectionGain = juce::jlimit (1.2f, 3.0f, getEnvFloat ("OPENSTUDIO_ENGINE_V3_LPC_MAX_GAIN", 2.2f)); + if (LpcEnvelopeTransfer::applyToBuffer (output, input, numChannels, numSamples, lpcSettings)) + { + lastRenderDiagnostics.spectralEnvelopeCorrectionUsed = true; + lastRenderDiagnostics.v3FormantMode = "lpc_transfer"; + } } // Apply per-note gain adjustments (all channels) diff --git a/Source/PitchResynthesizer.h b/Source/PitchResynthesizer.h index b30ebd3..0bad878 100644 --- a/Source/PitchResynthesizer.h +++ b/Source/PitchResynthesizer.h @@ -9,19 +9,146 @@ * PitchResynthesizer — Offline pitch correction from graphical edits. * * Takes original audio + edited notes, produces corrected audio. - * Uses Signalsmith Stretch for pitch shifting — handles mono and stereo - * natively, with formant preservation and clean phase handling. + * Uses the Studio13 native VSF renderer for graphical offline pitch-only edits. * * Non-destructive: original audio is never modified. */ class PitchResynthesizer { public: + struct RenderDiagnostics + { + juce::String requestedRendererBranch; + juce::String actualRendererBranch; + juce::String processingMode; + juce::String pitchDirection; + juce::String pitchOnlyRecoveryPath; + bool formantCurveUsed = false; + bool explicitFormantRequested = false; + bool pitchOnlyFormantSuppressed = false; + bool pitchOnlyNeutralFormantUsed = false; + bool downshiftFormantGuardUsed = false; + double downshiftFormantGuardAlpha = 0.0; + juce::String dominantEntryBoundaryKind = "unknown"; + juce::String dominantExitBoundaryKind = "unknown"; + double dominantEntryBoundaryScore = 0.0; + double dominantExitBoundaryScore = 0.0; + bool usedFallback = false; + juce::String fallbackReason; + bool bridgeUsed = false; + bool bridgeFallbackUsed = false; + double bridgeStartSec = 0.0; + double bridgeLengthMs = 0.0; + int bridgeAlignmentLagSamples = 0; + float bridgeCorrelationScore = 0.0f; + float bridgeGainDeltaDb = 0.0f; + bool bodyReplacementUsed = false; + bool bodyReplacementFallbackUsed = false; + double entryLockStartSec = 0.0; + double entryLockLengthMs = 0.0; + double exitLockStartSec = 0.0; + double renderedBodyStartSec = 0.0; + double renderedBodyEndSec = 0.0; + bool islandNativeUsed = false; + bool islandNativeFallbackUsed = false; + double islandRenderStartSec = 0.0; + double islandRenderEndSec = 0.0; + float transientMaskPeak = 0.0f; + float voicedCoreMaskPeak = 0.0f; + bool hpssUsed = false; + bool hpssFallbackUsed = false; + float harmonicMaskPeak = 0.0f; + float aperiodicMaskPeak = 0.0f; + bool spectralEnvelopeCorrectionUsed = false; + bool pitchOnlyCoreTimbreCorrectionUsed = false; + double pitchOnlyCoreEnvelopeMix = 0.0; + double pitchOnlyCoreRmsTrimDb = 0.0; + int pitchOnlyCoreEnvelopeLifter = 0; + bool pitchOnlyEntryHandoffUsed = false; + bool pitchOnlyExitHandoffUsed = false; + bool vocalSourceFilterUsed = false; + double vocalSourceFilterVoicedCoverage = 0.0; + double vocalSourceFilterResidualMix = 0.0; + bool vocalSourceFilterFallbackUsed = false; + juce::String vocalSourceFilterFallbackReason; + double vocalSourceFilterEntryDryMs = 0.0; + double vocalSourceFilterExitDryMs = 0.0; + double vocalSourceFilterResidualMixScale = 1.0; + bool vocalSourceFilterEpochInterpolationUsed = false; + double vocalSourceFilterEpochInterpolationStrength = 0.0; + double vocalSourceFilterGrainRadiusScale = 1.0; + double vocalSourceFilterUpPresenceTrimDb = 0.0; + double vocalSourceFilterUpPresenceHz = 0.0; + double vocalSourceFilterDownNasalTrimDb = 0.0; + double vocalSourceFilterDownNasalHz = 0.0; + double vocalSourceFilterDownBodyCompDb = 0.0; + double vocalSourceFilterDownBodyCompHz = 0.0; + bool wsolaUsed = false; + bool wsolaFallbackUsed = false; + int wsolaEntryLagSamples = 0; + int wsolaExitLagSamples = 0; + float wsolaCorrelationScore = 0.0f; + bool phaseLockUsed = false; + bool phaseLockFallbackUsed = false; + bool phaseAlignedEntry = false; + bool phaseAlignedExit = false; + int phasePeakCount = 0; + bool transitionHqUsed = false; + bool transitionHqFallbackUsed = false; + double transitionStartSec = 0.0; + double transitionEndSec = 0.0; + float transitionTransientPeak = 0.0f; + float transitionVoicedCorePeak = 0.0f; + float transitionResidualPeak = 0.0f; + bool transitionEnvelopeCorrectionUsed = false; + bool engineV2Used = false; + bool engineV2FallbackUsed = false; + int engineV2TransitionCount = 0; + double engineV2TransitionStartSec = 0.0; + double engineV2TransitionEndSec = 0.0; + float engineV2HarmonicSupportPeak = 0.0f; + float engineV2ResidualSupportPeak = 0.0f; + float engineV2EnvelopeSupportPeak = 0.0f; + bool transientBypassUsed = false; + bool residualCarryUsed = false; + int cepstralCutoffUsed = 0; + int engineV2FftSize = 0; + int engineV2HopSize = 0; + bool immediateLeftNeighborUsed = false; + bool immediateRightNeighborUsed = false; + int leftNeighborSamplesRendered = 0; + int rightNeighborSamplesRendered = 0; + double leftNeighborSmoothMs = 0.0; + double rightNeighborSmoothMs = 0.0; + bool nonImmediateNeighborTouched = false; + double entryAlignmentOffsetMs = 0.0; + double exitAlignmentOffsetMs = 0.0; + bool firstVoicedCyclesEntryUsed = false; + bool firstVoicedCyclesExitUsed = false; + bool v3TransitionPairUsed = false; + bool v3ContinuousRenderUsed = false; + double v3EntryAnchorMs = 0.0; + double v3ExitAnchorMs = 0.0; + int v3FirstCyclesEntryCount = 0; + int v3FirstCyclesExitCount = 0; + double v3ShellDurationMs = 0.0; + double v3BodyDurationMs = 0.0; + double v3ResidualMix = 0.0; + juce::String v3FormantMode; + double v3NeighborLeftOverlapMs = 0.0; + double v3NeighborRightOverlapMs = 0.0; + bool noteHqEntryPitchHandoffUsed = false; + double noteHqEntryPitchHandoffStartSec = 0.0; + double noteHqEntryPitchHandoffEndSec = 0.0; + double noteHqEntryPitchHandoffPreMs = 0.0; + double noteHqEntryPitchHandoffBodyMs = 0.0; + double noteHqEntryPitchSlopeJumpStPerSec = 0.0; + bool noteHqEntryPitchAccelerationLimited = false; + }; + enum class PitchEngine { - Signalsmith, // Signalsmith Stretch — default, high quality, stereo-native - WorldVocoder, // WORLD fallback (mono only, kept for compatibility) - PhaseVocoder // Basic phase vocoder fallback + NativeVsf }; enum class RenderQuality @@ -43,7 +170,7 @@ class PitchResynthesizer double sampleRate, const std::vector<PitchAnalyzer::PitchFrame>& frames, const std::vector<PitchAnalyzer::PitchNote>& notes, - PitchEngine engine = PitchEngine::Signalsmith, + PitchEngine engine = PitchEngine::NativeVsf, float globalFormantSemitones = 0.0f, RenderQuality renderQuality = RenderQuality::FinalHQ, std::function<bool()> shouldCancel = {}); @@ -67,9 +194,13 @@ class PitchResynthesizer const std::vector<PitchAnalyzer::PitchNote>& notes, float globalFormantSemitones = 0.0f); + static juce::String getRequestedPitchRendererBranchName(); + const RenderDiagnostics& getLastRenderDiagnostics() const noexcept { return lastRenderDiagnostics; } + /** No-op — kept for API compatibility with AudioEngine. */ void clearCache() {} private: + RenderDiagnostics lastRenderDiagnostics; JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (PitchResynthesizer) }; diff --git a/Source/PlaybackEngine.cpp b/Source/PlaybackEngine.cpp index a43ba73..cc24f82 100644 --- a/Source/PlaybackEngine.cpp +++ b/Source/PlaybackEngine.cpp @@ -1,4 +1,5 @@ #include "PlaybackEngine.h" +#include <algorithm> #include <cmath> namespace @@ -25,6 +26,101 @@ static float peakForBuffer(const juce::AudioBuffer<float>& buffer, int numSample } return peak; } + +static float getPitchOnlyPreviewTonalityLimitHz (bool downwardShift) +{ + const auto specificName = downwardShift + ? "OPENSTUDIO_PITCH_STAGEA_TONALITY_LIMIT_HZ_DOWN" + : "OPENSTUDIO_PITCH_STAGEA_TONALITY_LIMIT_HZ_UP"; + const auto specificValue = juce::SystemStats::getEnvironmentVariable (specificName, {}).trim(); + if (specificValue.isNotEmpty()) + return juce::jlimit (0.0f, 20000.0f, specificValue.getFloatValue()); + + const auto value = juce::SystemStats::getEnvironmentVariable ("OPENSTUDIO_PITCH_STAGEA_TONALITY_LIMIT_HZ", {}).trim(); + if (value.isEmpty()) + return downwardShift ? 2600.0f : 1050.0f; + + return juce::jlimit (0.0f, 20000.0f, value.getFloatValue()); +} + +static float sampleLoopBufferLocal (const juce::AudioBuffer<float>& buffer, + int channel, + double position, + int crossfadeSamples) +{ + const int numChannels = buffer.getNumChannels(); + const int numSamples = buffer.getNumSamples(); + if (numChannels <= 0 || numSamples <= 1) + return 0.0f; + + const int safeChannel = juce::jlimit (0, numChannels - 1, channel); + auto wrapPosition = std::fmod (position, static_cast<double> (numSamples)); + if (wrapPosition < 0.0) + wrapPosition += static_cast<double> (numSamples); + + const auto sampleAt = [&buffer, safeChannel, numSamples] (double pos) + { + auto wrapped = std::fmod (pos, static_cast<double> (numSamples)); + if (wrapped < 0.0) + wrapped += static_cast<double> (numSamples); + const int index0 = juce::jlimit (0, numSamples - 1, static_cast<int> (std::floor (wrapped))); + const int index1 = (index0 + 1) % numSamples; + const float frac = static_cast<float> (wrapped - static_cast<double> (index0)); + const float s0 = buffer.getSample (safeChannel, index0); + const float s1 = buffer.getSample (safeChannel, index1); + return s0 + (s1 - s0) * frac; + }; + + float value = sampleAt (wrapPosition); + const int safeCrossfade = juce::jlimit (0, numSamples / 3, crossfadeSamples); + if (safeCrossfade <= 1) + return value; + + const double crossfadeStart = static_cast<double> (numSamples - safeCrossfade); + if (wrapPosition >= crossfadeStart) + { + const float blend = static_cast<float> ((wrapPosition - crossfadeStart) + / static_cast<double> (safeCrossfade)); + const float wrappedValue = sampleAt (wrapPosition - static_cast<double> (numSamples)); + value = value * (1.0f - blend) + wrappedValue * blend; + } + + return value; +} + +static float sampleBufferCubic (const juce::AudioBuffer<float>& buffer, + int channel, + int availableSamples, + double position) +{ + if (availableSamples <= 0 || buffer.getNumChannels() <= 0) + return 0.0f; + + const int safeChannel = juce::jlimit (0, buffer.getNumChannels() - 1, channel); + if (availableSamples == 1) + return buffer.getSample (safeChannel, 0); + + const double clampedPosition = juce::jlimit (0.0, + static_cast<double> (availableSamples - 1), + position); + const int i1 = juce::jlimit (0, availableSamples - 1, + static_cast<int> (std::floor (clampedPosition))); + const int i0 = juce::jlimit (0, availableSamples - 1, i1 - 1); + const int i2 = juce::jlimit (0, availableSamples - 1, i1 + 1); + const int i3 = juce::jlimit (0, availableSamples - 1, i1 + 2); + const float t = static_cast<float> (clampedPosition - static_cast<double> (i1)); + const float t2 = t * t; + const float t3 = t2 * t; + const float y0 = buffer.getSample (safeChannel, i0); + const float y1 = buffer.getSample (safeChannel, i1); + const float y2 = buffer.getSample (safeChannel, i2); + const float y3 = buffer.getSample (safeChannel, i3); + + return 0.5f * ((2.0f * y1) + + (-y0 + y2) * t + + (2.0f * y0 - 5.0f * y1 + 4.0f * y2 - y3) * t2 + + (-y0 + 3.0f * y1 - 3.0f * y2 + y3) * t3); +} } float PlaybackEngine::applyFadeCurve(float t, int curveType) @@ -42,6 +138,11 @@ float PlaybackEngine::applyFadeCurve(float t, int curveType) PlaybackEngine::PlaybackEngine() { formatManager.registerBasicFormats(); + + // Pre-allocate pitch-preview channel-pointer vectors (max stereo = 2 channels). + // Avoids heap allocation inside fillTrackBuffer for every pitch-previewed clip. + pitchPreviewInPtrs.resize (2); + pitchPreviewOutPtrs.resize (2); } PlaybackEngine::~PlaybackEngine() @@ -191,9 +292,22 @@ void PlaybackEngine::addClip(const juce::File& audioFile, double startTime, doub // Pre-load the reader on the message thread so audio thread never does disk I/O preloadReader(effectiveFile); + if (clipId.isNotEmpty()) + { + auto segmentIt = renderedPreviewSegments.find (clipId); + if (segmentIt != renderedPreviewSegments.end()) + { + for (const auto& segment : segmentIt->second) + { + if (segment.audioFile.existsAsFile()) + preloadReader (segment.audioFile); + } + } + } ClipInfo clip(effectiveFile, startTime, duration, trackId, effectiveOffset, volumeDB, fadeIn, fadeOut); clip.clipId = clipId; + clip.envelopeKey = trackId + "::" + clipId; // Pre-compute to avoid string alloc on audio thread clip.originalAudioFile = sourceAudioFile.existsAsFile() ? sourceAudioFile : audioFile; clip.originalOffset = sourceOffset >= 0.0 ? sourceOffset : offset; clips.push_back(clip); @@ -255,7 +369,17 @@ void PlaybackEngine::replaceClipAudioFile(const juce::String& clipId, const juce // Clear any active pitch preview — the corrected audio is now baked // into the file, so the real-time PitchShifter must not double-shift. clipPitchPreviews.erase(clipId); + if (pitchScrubPreview.clipId == clipId) + { + pitchScrubPreview = {}; + pitchScrubPreviewStatus = {}; + pitchScrubStretcherPrepared = false; + } renderedPreviewSegments.erase(clipId); + auto& renderedGeneration = renderedPreviewSegmentGenerations[clipId]; + ++renderedGeneration; + if (renderedGeneration <= 0) + renderedGeneration = 1; deferredClipSwaps.erase(clipId); // Remember the corrected file so it survives clearAllClips + re-add cycles. if (restoringOriginal) @@ -315,52 +439,244 @@ int PlaybackEngine::commitAllDeferredClipAudioFiles() return committed; } -void PlaybackEngine::setClipRenderedPreviewSegment(const juce::String& clipId, +bool PlaybackEngine::setClipRenderedPreviewSegment(const juce::String& clipId, const juce::File& audioFile, double startSec, - double endSec) + double endSec, + double fileOffsetSec, + int generation) { if (!audioFile.existsAsFile()) { juce::Logger::writeToLog("PlaybackEngine::setClipRenderedPreviewSegment: file does not exist: " + audioFile.getFullPathName()); - return; + return false; } juce::ScopedLock sl(lock); - preloadReader(audioFile); - auto& segments = renderedPreviewSegments[clipId]; - bool replaced = false; - for (auto& segment : segments) + if (pitchCorrectedFiles.find(clipId) != pitchCorrectedFiles.end()) { - if (std::abs(segment.startSec - startSec) < 0.001 && std::abs(segment.endSec - endSec) < 0.001) + juce::Logger::writeToLog("PlaybackEngine: Rejected rendered preview segment for corrected-source clip " + clipId); + return false; + } + + if (generation > 0) + { + const auto genIt = renderedPreviewSegmentGenerations.find(clipId); + const int currentGeneration = genIt != renderedPreviewSegmentGenerations.end() ? genIt->second : 0; + if (currentGeneration != generation) { - segment.audioFile = audioFile; - replaced = true; - break; + juce::Logger::writeToLog("PlaybackEngine: Rejected stale rendered preview segment for clip " + clipId + + " generation=" + juce::String(generation) + + " current=" + juce::String(currentGeneration)); + return false; } } - if (!replaced) - segments.push_back({ audioFile, startSec, endSec }); + + preloadReader(audioFile); + auto& segments = renderedPreviewSegments[clipId]; + segments.erase(std::remove_if(segments.begin(), segments.end(), + [startSec, endSec](const RenderedPreviewSegment& segment) + { + const bool sameWindow = std::abs(segment.startSec - startSec) < 0.001 + && std::abs(segment.endSec - endSec) < 0.001; + const bool overlaps = !(segment.endSec <= startSec || segment.startSec >= endSec); + return sameWindow || overlaps; + }), + segments.end()); + segments.push_back({ audioFile, startSec, endSec, fileOffsetSec }); std::sort(segments.begin(), segments.end(), [] (const RenderedPreviewSegment& a, const RenderedPreviewSegment& b) { return a.startSec < b.startSec; }); juce::Logger::writeToLog("PlaybackEngine: Set rendered preview segment for clip " + clipId + " [" + juce::String(startSec, 3) + ", " + juce::String(endSec, 3) + "]" + + " fileOffset=" + juce::String(fileOffsetSec, 3) + + " generation=" + juce::String(generation) + " -> " + audioFile.getFullPathName()); + return true; +} + +void PlaybackEngine::beginRenderedPreviewSegmentGeneration(const juce::String& clipId, int generation) +{ + juce::ScopedLock sl(lock); + renderedPreviewSegments.erase(clipId); + renderedPreviewSegmentGenerations[clipId] = generation; + juce::Logger::writeToLog("PlaybackEngine: Began rendered preview generation for clip " + clipId + + " generation=" + juce::String(generation)); +} + +void PlaybackEngine::invalidateRenderedPreviewSegments(const juce::String& clipId) +{ + juce::ScopedLock sl(lock); + renderedPreviewSegments.erase(clipId); + auto& generation = renderedPreviewSegmentGenerations[clipId]; + ++generation; + if (generation <= 0) + generation = 1; + juce::Logger::writeToLog("PlaybackEngine: Invalidated rendered preview segments for clip " + clipId + + " generation=" + juce::String(generation)); } void PlaybackEngine::clearClipRenderedPreviewSegments(const juce::String& clipId) { juce::ScopedLock sl(lock); renderedPreviewSegments.erase(clipId); - juce::Logger::writeToLog("PlaybackEngine: Cleared rendered preview segments for clip " + clipId); + auto& generation = renderedPreviewSegmentGenerations[clipId]; + ++generation; + if (generation <= 0) + generation = 1; + juce::Logger::writeToLog("PlaybackEngine: Cleared rendered preview segments for clip " + clipId + + " generation=" + juce::String(generation)); +} + +void PlaybackEngine::clearAllPitchPreviewRoutes(const juce::String& clipId) +{ + juce::ScopedLock sl(lock); + const bool clearAll = clipId.isEmpty(); + if (clearAll) + { + clipPitchPreviews.clear(); + renderedPreviewSegments.clear(); + for (auto& [id, generation] : renderedPreviewSegmentGenerations) + { + juce::ignoreUnused(id); + ++generation; + if (generation <= 0) + generation = 1; + } + pitchScrubPreview = {}; + pitchScrubPreviewStatus = {}; + pitchScrubStretcherPrepared = false; + juce::Logger::writeToLog("PlaybackEngine: Hard-cleared all pitch preview routes"); + return; + } + + clipPitchPreviews.erase(clipId); + renderedPreviewSegments.erase(clipId); + auto& generation = renderedPreviewSegmentGenerations[clipId]; + ++generation; + if (generation <= 0) + generation = 1; + + if (pitchScrubPreview.clipId == clipId) + { + pitchScrubPreview = {}; + pitchScrubPreviewStatus = {}; + pitchScrubStretcherPrepared = false; + } + + juce::Logger::writeToLog("PlaybackEngine: Hard-cleared pitch preview routes for clip " + clipId + + " generation=" + juce::String(generation)); +} + +int PlaybackEngine::clearPitchPreviewRoutesForCorrectedSources() +{ + juce::ScopedLock sl(lock); + int cleared = 0; + for (const auto& [clipId, file] : pitchCorrectedFiles) + { + juce::ignoreUnused(file); + const bool hadLive = clipPitchPreviews.erase(clipId) > 0; + const bool hadRendered = renderedPreviewSegments.erase(clipId) > 0; + bool hadScrub = false; + if (pitchScrubPreview.clipId == clipId) + { + pitchScrubPreview = {}; + pitchScrubPreviewStatus = {}; + pitchScrubStretcherPrepared = false; + hadScrub = true; + } + + if (hadLive || hadRendered || hadScrub) + { + auto& generation = renderedPreviewSegmentGenerations[clipId]; + ++generation; + if (generation <= 0) + generation = 1; + ++cleared; + } + } + + if (cleared > 0) + juce::Logger::writeToLog("PlaybackEngine: Hard-cleared pitch preview routes for " + + juce::String(cleared) + " corrected-source clip(s)"); + return cleared; +} + +std::map<juce::String, std::vector<PlaybackEngine::RenderedPreviewSegment>> PlaybackEngine::getRenderedPreviewSegmentSnapshot() const +{ + juce::ScopedLock sl(lock); + return renderedPreviewSegments; +} + +PlaybackEngine::ClipPlaybackSourceStatus PlaybackEngine::getClipPlaybackSourceAtTime(const juce::String& trackId, + const juce::String& clipId, + double projectTimeSec) const +{ + juce::ScopedLock sl(lock); + ClipPlaybackSourceStatus status; + for (const auto& clip : clips) + { + if (clip.trackId != trackId || clip.clipId != clipId || !clip.isActive) + continue; + + const double clipEndTime = clip.startTime + clip.duration; + if (projectTimeSec < clip.startTime || projectTimeSec >= clipEndTime) + continue; + + status.clipFound = true; + status.clipTime = projectTimeSec - clip.startTime; + status.audioFile = clip.audioFile.getFullPathName(); + status.playbackOffset = clip.offset + status.clipTime; + status.sourceType = "original"; + + auto segmentIt = renderedPreviewSegments.find(clipId); + if (segmentIt != renderedPreviewSegments.end()) + { + for (const auto& segment : segmentIt->second) + { + if (status.clipTime >= segment.startSec && status.clipTime < segment.endSec) + { + status.renderedSegmentActiveAtTime = true; + status.sourceType = "rendered_segment"; + status.audioFile = segment.audioFile.getFullPathName(); + status.playbackOffset = segment.fileOffsetSec + (status.clipTime - segment.startSec); + return status; + } + } + } + + auto correctedIt = pitchCorrectedFiles.find(clipId); + if (correctedIt != pitchCorrectedFiles.end() + && correctedIt->second.existsAsFile() + && clip.audioFile == correctedIt->second) + { + status.correctedSourceActiveAtTime = true; + status.sourceType = "corrected_source"; + status.audioFile = correctedIt->second.getFullPathName(); + } + + return status; + } + + return status; } void PlaybackEngine::clearPitchCorrectionFile(const juce::String& clipId) { juce::ScopedLock sl(lock); pitchCorrectedFiles.erase(clipId); + clipPitchPreviews.erase(clipId); renderedPreviewSegments.erase(clipId); + auto& generation = renderedPreviewSegmentGenerations[clipId]; + ++generation; + if (generation <= 0) + generation = 1; + if (pitchScrubPreview.clipId == clipId) + { + pitchScrubPreview = {}; + pitchScrubPreviewStatus = {}; + pitchScrubStretcherPrepared = false; + } deferredClipSwaps.erase(clipId); juce::Logger::writeToLog("PlaybackEngine: Cleared pitch correction file for clip " + clipId); } @@ -460,6 +776,13 @@ void PlaybackEngine::setClipPitchPreview (const juce::String& clipId, // has the old pitch shift baked in, and the preview would add the new ratio // on top — always increasing pitch regardless of direction. auto corrIt = pitchCorrectedFiles.find (clipId); + if (corrIt != pitchCorrectedFiles.end() && ! preview.allowReplacingCorrectedSource) + { + juce::Logger::writeToLog ("PlaybackEngine: Rejected pitch preview for corrected-source clip " + clipId + + " because the request did not opt into a new interactive preview generation"); + return; + } + for (auto& clip : clips) { if (clip.clipId == clipId && clip.originalAudioFile.existsAsFile()) @@ -521,6 +844,267 @@ bool PlaybackEngine::hasClipPitchPreview (const juce::String& clipId) const return clipPitchPreviews.find (clipId) != clipPitchPreviews.end(); } +void PlaybackEngine::setPitchScrubPreview (const PitchScrubPreviewData& preview) +{ + juce::ScopedLock sl (lock); + pitchScrubPreview = preview; + pitchScrubPreview.active = preview.loopBuffer.getNumSamples() > 8 + && preview.loopBuffer.getNumChannels() > 0 + && preview.pitchRatio > 0.0f; + pitchScrubPreview.readPosition = 0.0; + pitchScrubPreview.currentGain = 0.0f; + pitchScrubPreview.targetGain = juce::jlimit (0.0f, 2.0f, preview.gain); + pitchScrubPreview.releasePending = false; + pitchScrubPreview.lastPeak = 0.0f; + pitchScrubPreview.lastRenderWallTimeMs = 0.0; + pitchScrubPreview.mixedCallbackCount = 0; + pitchScrubPreview.mixedSampleCount = 0; + pitchScrubPreviewStatus.renderMethod = "formant_preserving_stretch"; + pitchScrubStretcherPrepared = false; + pitchScrubPreview.loopCrossfadeSamples = juce::jlimit (8, + std::max (8, preview.loopBuffer.getNumSamples() / 3), + preview.loopCrossfadeSamples > 0 + ? preview.loopCrossfadeSamples + : std::max (16, preview.loopBuffer.getNumSamples() / 8)); + + pitchScrubPreviewStatus = {}; + pitchScrubPreviewStatus.active = pitchScrubPreview.active; + pitchScrubPreviewStatus.previewArmed = pitchScrubPreview.active; + pitchScrubPreviewStatus.trackId = pitchScrubPreview.trackId; + pitchScrubPreviewStatus.clipId = pitchScrubPreview.clipId; + pitchScrubPreviewStatus.pitchRatio = pitchScrubPreview.pitchRatio; + pitchScrubPreviewStatus.basePitchHz = pitchScrubPreview.basePitchHz; + pitchScrubPreviewStatus.currentGain = pitchScrubPreview.currentGain; + pitchScrubPreviewStatus.targetGain = pitchScrubPreview.targetGain; + pitchScrubPreviewStatus.repeatStability = pitchScrubPreview.repeatStability; + pitchScrubPreviewStatus.loopDurationMs = 1000.0 * std::max (0.0, pitchScrubPreview.loopEndSec - pitchScrubPreview.loopStartSec); + pitchScrubPreviewStatus.renderMethod = "formant_preserving_stretch"; + + const int preloadSamples = juce::jmax (512, preview.loopBuffer.getNumSamples()); + pitchScrubInputBuffer.setSize (juce::jmax (1, preview.loopBuffer.getNumChannels()), preloadSamples, false, true, true); + pitchScrubOutputBuffer.setSize (juce::jmax (1, preview.loopBuffer.getNumChannels()), preloadSamples, false, true, true); + + logAudioPlayback ("setPitchScrubPreview clip=" + preview.clipId + + " track=" + preview.trackId + + " samples=" + juce::String (preview.loopBuffer.getNumSamples()) + + " channels=" + juce::String (preview.loopBuffer.getNumChannels()) + + " pitchRatio=" + juce::String (preview.pitchRatio, 3) + + " basePitchHz=" + juce::String (preview.basePitchHz, 2) + + " active=" + juce::String (pitchScrubPreview.active ? "true" : "false")); +} + +bool PlaybackEngine::updatePitchScrubPreview (const juce::String& clipId, float pitchRatio) +{ + juce::ScopedLock sl (lock); + if ((! pitchScrubPreview.active && ! pitchScrubPreview.releasePending) || pitchScrubPreview.clipId != clipId) + return false; + + pitchScrubPreview.pitchRatio = juce::jlimit (0.25f, 4.0f, pitchRatio); + pitchScrubPreviewStatus.pitchRatio = pitchScrubPreview.pitchRatio; + pitchScrubPreviewStatus.renderMethod = "formant_preserving_stretch"; + return true; +} + +void PlaybackEngine::clearPitchScrubPreview (const juce::String& clipId) +{ + juce::ScopedLock sl (lock); + if (pitchScrubPreview.clipId == clipId && (pitchScrubPreview.active || pitchScrubPreview.releasePending)) + { + pitchScrubPreview.releasePending = true; + pitchScrubPreview.targetGain = 0.0f; + pitchScrubPreviewStatus.releasePending = true; + logAudioPlayback ("clearPitchScrubPreview clip=" + clipId); + } +} + +bool PlaybackEngine::hasPitchScrubPreview (const juce::String& clipId) const +{ + const juce::ScopedLock sl (lock); + return (pitchScrubPreview.active || pitchScrubPreview.releasePending) && pitchScrubPreview.clipId == clipId; +} + +#if defined(_MSC_VER) + #pragma warning(push) + #pragma warning(disable: 4244 4267 4305 4456) +#endif +void PlaybackEngine::renderPitchScrubPreview (juce::AudioBuffer<float>& buffer, double sampleRate) +{ + const juce::ScopedTryLock sl (lock); + if (! sl.isLocked()) + return; + + if ((! pitchScrubPreview.active && ! pitchScrubPreview.releasePending) + || pitchScrubPreview.loopBuffer.getNumSamples() <= 8 + || pitchScrubPreview.loopBuffer.getNumChannels() <= 0 + || sampleRate <= 0.0) + return; + + const auto playbackRatio = (pitchScrubPreview.sourceSampleRate > 0.0) + ? (pitchScrubPreview.sourceSampleRate / sampleRate) + : 1.0; + const double readIncrement = playbackRatio; + const int loopChannels = pitchScrubPreview.loopBuffer.getNumChannels(); + const int outputChannels = buffer.getNumChannels(); + const int numSamples = buffer.getNumSamples(); + if (pitchScrubInputBuffer.getNumChannels() < loopChannels + || pitchScrubInputBuffer.getNumSamples() < numSamples) + { + pitchScrubInputBuffer.setSize (loopChannels, numSamples, false, true, true); + } + if (pitchScrubOutputBuffer.getNumChannels() < loopChannels + || pitchScrubOutputBuffer.getNumSamples() < numSamples) + { + pitchScrubOutputBuffer.setSize (loopChannels, numSamples, false, true, true); + } + pitchScrubInputBuffer.clear (0, numSamples); + pitchScrubOutputBuffer.clear (0, numSamples); + + const float startStep = pitchScrubPreview.targetGain + / static_cast<float> (std::max (1, static_cast<int> (std::round (sampleRate * pitchScrubPreview.startRampMs * 0.001)))); + const float stopStep = std::max (pitchScrubPreview.gain, 0.001f) + / static_cast<float> (std::max (1, static_cast<int> (std::round (sampleRate * pitchScrubPreview.stopRampMs * 0.001)))); + + for (int i = 0; i < numSamples; ++i) + { + for (int ch = 0; ch < loopChannels; ++ch) + { + pitchScrubInputBuffer.setSample (ch, i, sampleLoopBufferLocal (pitchScrubPreview.loopBuffer, + ch, + pitchScrubPreview.readPosition, + pitchScrubPreview.loopCrossfadeSamples)); + } + + pitchScrubPreview.readPosition += readIncrement; + const double loopLength = static_cast<double> (pitchScrubPreview.loopBuffer.getNumSamples()); + if (loopLength > 0.0 && pitchScrubPreview.readPosition >= loopLength) + pitchScrubPreview.readPosition = std::fmod (pitchScrubPreview.readPosition, loopLength); + } + + if (! pitchScrubStretcherPrepared) + { + pitchScrubStretcher.presetCheaper (loopChannels, static_cast<float> (sampleRate)); + pitchScrubStretcherPrepared = true; + } + + const float pitchRatio = juce::jlimit (0.25f, 4.0f, pitchScrubPreview.pitchRatio); + const float tonalityLimitNorm = static_cast<float> ( + sampleRate > 0.0 + ? getPitchOnlyPreviewTonalityLimitHz (pitchRatio < 1.0f) / sampleRate + : 0.0); + pitchScrubStretcher.setTransposeFactor (pitchRatio, tonalityLimitNorm); + pitchScrubStretcher.setFormantFactor (1.0f, true); + + if (static_cast<int> (pitchPreviewInPtrs.size()) < loopChannels) + { + pitchPreviewInPtrs.resize (static_cast<size_t> (loopChannels)); + pitchPreviewOutPtrs.resize (static_cast<size_t> (loopChannels)); + } + for (int ch = 0; ch < loopChannels; ++ch) + { + pitchPreviewInPtrs[static_cast<size_t> (ch)] = pitchScrubInputBuffer.getReadPointer (ch); + pitchPreviewOutPtrs[static_cast<size_t> (ch)] = pitchScrubOutputBuffer.getWritePointer (ch); + } + pitchScrubStretcher.process (pitchPreviewInPtrs, numSamples, pitchPreviewOutPtrs, numSamples); + + float peak = 0.0f; + for (int i = 0; i < numSamples; ++i) + { + if (pitchScrubPreview.releasePending) + pitchScrubPreview.currentGain = std::max (0.0f, pitchScrubPreview.currentGain - stopStep); + else + pitchScrubPreview.currentGain = std::min (pitchScrubPreview.targetGain, pitchScrubPreview.currentGain + startStep); + + for (int ch = 0; ch < outputChannels; ++ch) + { + const int loopChannel = juce::jmin (ch, loopChannels - 1); + const float sample = pitchScrubOutputBuffer.getSample (loopChannel, i) * pitchScrubPreview.currentGain; + buffer.addSample (ch, i, sample); + peak = juce::jmax (peak, std::abs (sample)); + } + } + + pitchScrubPreview.lastPeak = peak; + pitchScrubPreview.lastRenderWallTimeMs = juce::Time::getMillisecondCounterHiRes(); + pitchScrubPreview.mixedCallbackCount += 1; + pitchScrubPreview.mixedSampleCount += numSamples; + pitchScrubPreview.firstCallbackServiced = true; + + pitchScrubPreviewStatus.active = pitchScrubPreview.active; + pitchScrubPreviewStatus.releasePending = pitchScrubPreview.releasePending; + pitchScrubPreviewStatus.audible = pitchScrubPreview.currentGain > 0.0001f && peak > 1.0e-4f; + pitchScrubPreview.firstDragAudible = pitchScrubPreview.firstDragAudible || pitchScrubPreviewStatus.audible; + pitchScrubPreviewStatus.previewArmed = pitchScrubPreview.active || pitchScrubPreview.releasePending; + pitchScrubPreviewStatus.firstCallbackServiced = pitchScrubPreview.firstCallbackServiced; + pitchScrubPreviewStatus.firstDragAudible = pitchScrubPreview.firstDragAudible; + pitchScrubPreviewStatus.trackId = pitchScrubPreview.trackId; + pitchScrubPreviewStatus.clipId = pitchScrubPreview.clipId; + pitchScrubPreviewStatus.pitchRatio = pitchScrubPreview.pitchRatio; + pitchScrubPreviewStatus.basePitchHz = pitchScrubPreview.basePitchHz; + pitchScrubPreviewStatus.currentGain = pitchScrubPreview.currentGain; + pitchScrubPreviewStatus.targetGain = pitchScrubPreview.targetGain; + pitchScrubPreviewStatus.repeatStability = pitchScrubPreview.repeatStability; + pitchScrubPreviewStatus.lastPeak = peak; + pitchScrubPreviewStatus.loopDurationMs = 1000.0 * std::max (0.0, pitchScrubPreview.loopEndSec - pitchScrubPreview.loopStartSec); + pitchScrubPreviewStatus.lastRenderWallTimeMs = pitchScrubPreview.lastRenderWallTimeMs; + pitchScrubPreviewStatus.mixedCallbackCount = pitchScrubPreview.mixedCallbackCount; + pitchScrubPreviewStatus.mixedSampleCount = pitchScrubPreview.mixedSampleCount; + pitchScrubPreviewStatus.renderMethod = "formant_preserving_stretch"; + + if (pitchScrubPreview.releasePending && pitchScrubPreview.currentGain <= 1.0e-5f) + { + pitchScrubPreview.active = false; + pitchScrubPreview.releasePending = false; + pitchScrubPreview.currentGain = 0.0f; + pitchScrubPreview.targetGain = 0.0f; + pitchScrubPreviewStatus.active = false; + pitchScrubPreviewStatus.releasePending = false; + pitchScrubPreviewStatus.previewArmed = false; + pitchScrubStretcherPrepared = false; + } +} +#if defined(_MSC_VER) + #pragma warning(pop) +#endif + +PlaybackEngine::PitchScrubPreviewStatus PlaybackEngine::getPitchScrubPreviewStatus (const juce::String& clipId) const +{ + const juce::ScopedLock sl (lock); + if (clipId.isNotEmpty() && pitchScrubPreviewStatus.clipId != clipId) + return {}; + return pitchScrubPreviewStatus; +} + +PlaybackEngine::PitchPreviewRoutingStatus PlaybackEngine::getPitchPreviewRoutingStatus (const juce::String& clipId) const +{ + const juce::ScopedLock sl (lock); + PitchPreviewRoutingStatus status; + const bool queryAll = clipId.isEmpty(); + status.scrubPreviewActive = (pitchScrubPreview.active || pitchScrubPreview.releasePending) + && (queryAll || pitchScrubPreview.clipId == clipId); + status.clipLivePreviewActive = queryAll + ? ! clipPitchPreviews.empty() + : clipPitchPreviews.find (clipId) != clipPitchPreviews.end(); + status.renderedSegmentActive = queryAll + ? ! renderedPreviewSegments.empty() + : renderedPreviewSegments.find (clipId) != renderedPreviewSegments.end(); + status.correctedSourceActive = queryAll + ? ! pitchCorrectedFiles.empty() + : pitchCorrectedFiles.find (clipId) != pitchCorrectedFiles.end(); + + if (status.renderedSegmentActive) + status.monitorMode = "rendered_segment"; + else if (status.scrubPreviewActive) + status.monitorMode = "scrub"; + else if (status.clipLivePreviewActive) + status.monitorMode = "clip_live_preview"; + else if (status.correctedSourceActive) + status.monitorMode = "corrected_source"; + else + status.monitorMode = "none"; + + return status; +} + float PlaybackEngine::lookupPitchRatio (const std::vector<PitchCorrectionSegment>& segments, double timeInClip) { // Binary search could be used for large segment lists, but linear is fine for typical note counts @@ -532,6 +1116,10 @@ float PlaybackEngine::lookupPitchRatio (const std::vector<PitchCorrectionSegment return 1.0f; // No correction at this time position } +#if defined(_MSC_VER) + #pragma warning(push) + #pragma warning(disable: 4244 4267 4305 4456 4702) +#endif void PlaybackEngine::fillTrackBuffer(const juce::String& trackId, juce::AudioBuffer<float>& buffer, double currentTime, @@ -575,6 +1163,264 @@ void PlaybackEngine::fillTrackBuffer(const juce::String& trackId, continue; // Clip not active in this window ++overlappingClipCount; + { + const int clipOutputStart = juce::jlimit (0, numSamples, + static_cast<int> (std::ceil ((clip.startTime - currentTime) * sampleRate - 1.0e-9))); + const int clipOutputEnd = juce::jlimit (clipOutputStart, numSamples, + static_cast<int> (std::ceil ((clipEndTime - currentTime) * sampleRate - 1.0e-9))); + if (clipOutputEnd <= clipOutputStart) + continue; + + const std::vector<RenderedPreviewSegment>* renderedSegmentsForClip = nullptr; + if (clip.clipId.isNotEmpty()) + { + auto segmentIt = renderedPreviewSegments.find (clip.clipId); + if (segmentIt != renderedPreviewSegments.end() && ! segmentIt->second.empty()) + renderedSegmentsForClip = &segmentIt->second; + } + + auto& chunkBoundaries = reusableChunkBoundaries; + chunkBoundaries.clear(); + const auto requiredBoundaryCapacity = renderedSegmentsForClip != nullptr ? renderedSegmentsForClip->size() * 2 + 2 : 2; + if (chunkBoundaries.capacity() < requiredBoundaryCapacity) + { + chunkBoundaries.reserve (requiredBoundaryCapacity); + chunkBoundaryReserveCount.fetch_add (1, std::memory_order_relaxed); + } + chunkBoundaries.push_back (clipOutputStart); + chunkBoundaries.push_back (clipOutputEnd); + if (renderedSegmentsForClip != nullptr) + { + for (const auto& segment : *renderedSegmentsForClip) + { + const int segmentStart = juce::jlimit (clipOutputStart, clipOutputEnd, + static_cast<int> (std::ceil ((clip.startTime + segment.startSec - currentTime) * sampleRate - 1.0e-9))); + const int segmentEnd = juce::jlimit (clipOutputStart, clipOutputEnd, + static_cast<int> (std::ceil ((clip.startTime + segment.endSec - currentTime) * sampleRate - 1.0e-9))); + if (segmentStart > clipOutputStart && segmentStart < clipOutputEnd) + chunkBoundaries.push_back (segmentStart); + if (segmentEnd > clipOutputStart && segmentEnd < clipOutputEnd) + chunkBoundaries.push_back (segmentEnd); + } + } + std::sort (chunkBoundaries.begin(), chunkBoundaries.end()); + chunkBoundaries.erase (std::unique (chunkBoundaries.begin(), chunkBoundaries.end()), chunkBoundaries.end()); + + const float clipGain = juce::Decibels::decibelsToGain (static_cast<float> (clip.volumeDB)); + const std::vector<GainEnvelopePoint>* envPoints = nullptr; + if (clip.clipId.isNotEmpty()) + { + auto envIt = gainEnvelopes.find (clip.envelopeKey); + if (envIt != gainEnvelopes.end() && ! envIt->second.empty()) + envPoints = &envIt->second; + } + + auto mixChunk = [&] (int outputStart, int requestedOutputSamples, + const juce::File& playbackFile, double playbackOffset, + bool usingRenderedPreviewSegment, bool usingCorrectedSource) + { + if (requestedOutputSamples <= 0) + return false; + + auto* reader = getCachedReader (playbackFile); + if (reader == nullptr) + { + const int missingReaders = missingReaderCount.fetch_add (1, std::memory_order_relaxed) + 1; + logAudioPlayback ("fillTrackBuffer missingReader track=" + trackId + + " clipId=" + clip.clipId + + " file=" + playbackFile.getFullPathName() + + " currentTime=" + juce::String (currentTime, 3) + + " missingReaderCount=" + juce::String (missingReaders)); + return false; + } + + const double fileSampleRate = reader->sampleRate; + const double ratio = fileSampleRate / sampleRate; + double exactFileStart = juce::jmax (0.0, playbackOffset) * fileSampleRate; + const double roundedFileStart = std::round (exactFileStart); + if (std::abs (exactFileStart - roundedFileStart) < 1.0e-6) + exactFileStart = roundedFileStart; + + const juce::int64 fileStartSample = static_cast<juce::int64> (std::floor (exactFileStart)); + const juce::int64 readStartSample = fileStartSample > 0 ? fileStartSample - 1 : fileStartSample; + const int readStartOffset = static_cast<int> (fileStartSample - readStartSample); + const double fileStartFraction = exactFileStart - static_cast<double> (fileStartSample); + const double bufferStartPosition = static_cast<double> (readStartOffset) + fileStartFraction; + int outputSamples = requestedOutputSamples; + int fileSamplesToRead = static_cast<int> (std::ceil (bufferStartPosition + outputSamples * ratio)) + 3; + const juce::int64 fileSamplesAvailable = reader->lengthInSamples - readStartSample; + if (fileSamplesAvailable <= 0) + return false; + if (fileSamplesAvailable < fileSamplesToRead) + { + fileSamplesToRead = static_cast<int> (fileSamplesAvailable); + outputSamples = static_cast<int> (std::floor ( + (static_cast<double> (fileSamplesToRead - 1) - bufferStartPosition) / ratio)); + } + if (outputSamples <= 0 || fileSamplesToRead <= 0) + return false; + + const int readerChannels = static_cast<int> (reader->numChannels); + if (reusableFileBuffer.getNumChannels() < readerChannels + || reusableFileBuffer.getNumSamples() < fileSamplesToRead) + { + reusableFileBuffer.setSize (juce::jmax (readerChannels, reusableFileBuffer.getNumChannels()), + juce::jmax (fileSamplesToRead, reusableFileBuffer.getNumSamples())); + fileBufferResizeCount.fetch_add (1, std::memory_order_relaxed); + } + reusableFileBuffer.clear (0, fileSamplesToRead); + reader->read (&reusableFileBuffer, 0, fileSamplesToRead, readStartSample, true, true); + + const bool allowLivePitchPreviewForChunk = ! usingRenderedPreviewSegment && ! usingCorrectedSource; + const double chunkClipStart = currentTime + (static_cast<double> (outputStart) / sampleRate) - clip.startTime; + + if (clip.clipId.isNotEmpty()) + { + auto previewIt = clipPitchPreviews.find (clip.clipId); + if (previewIt != clipPitchPreviews.end() && previewIt->second != nullptr) + { + juce::ScopedLock clipSl (previewIt->second->clipLock); + auto& preview = *previewIt->second; + const auto& previewData = preview.previewData; + const double blockMidTime = chunkClipStart + (outputSamples * 0.5 / sampleRate); + const bool withinPreviewWindow = blockMidTime >= previewData.previewStartSec + && blockMidTime <= previewData.previewEndSec; + const float pitchRatio = lookupPitchRatio (previewData.pitchSegments, blockMidTime); + const bool pitchPreviewActive = allowLivePitchPreviewForChunk + && withinPreviewWindow + && std::abs (pitchRatio - 1.0f) > 0.001f; + + if (! pitchPreviewActive) + { + preview.lastPlaybackTime = -1.0; + } + else + { + if (! preview.prepared) + { + preview.stretcher.presetCheaper (readerChannels, static_cast<float> (fileSampleRate)); + preview.prepared = true; + } + if (preview.lastPlaybackTime < 0.0 + || std::abs (chunkClipStart - preview.lastPlaybackTime) > 0.1) + { + preview.stretcher.presetCheaper (readerChannels, static_cast<float> (fileSampleRate)); + } + preview.lastPlaybackTime = chunkClipStart + (outputSamples / sampleRate); + + const float tonalityLimitNorm = static_cast<float> ( + fileSampleRate > 0.0 + ? getPitchOnlyPreviewTonalityLimitHz (pitchRatio < 1.0f) / fileSampleRate + : 0.0); + preview.stretcher.setTransposeFactor (pitchRatio, tonalityLimitNorm); + preview.stretcher.setFormantFactor (1.0f, true); + + if (pitchShiftWorkBuffer.getNumSamples() < fileSamplesToRead) + { + pitchShiftWorkBuffer.setSize (readerChannels, fileSamplesToRead); + pitchShiftWorkBufferResizeCount.fetch_add (1, std::memory_order_relaxed); + } + for (int ch = 0; ch < readerChannels; ++ch) + { + pitchPreviewInPtrs[static_cast<size_t> (ch)] = reusableFileBuffer.getReadPointer (ch); + pitchPreviewOutPtrs[static_cast<size_t> (ch)] = pitchShiftWorkBuffer.getWritePointer (ch); + } + preview.stretcher.process (pitchPreviewInPtrs, fileSamplesToRead, pitchPreviewOutPtrs, fileSamplesToRead); + for (int ch = 0; ch < readerChannels; ++ch) + reusableFileBuffer.copyFrom (ch, 0, pitchShiftWorkBuffer, ch, 0, fileSamplesToRead); + } + } + } + + const int outChannels = buffer.getNumChannels(); + const int channelsToProcess = std::min (outChannels, readerChannels); + for (int i = 0; i < outputSamples; ++i) + { + const double filePos = bufferStartPosition + i * ratio; + const double sampleTimeInClip = chunkClipStart + (i / sampleRate); + float fadeGain = 1.0f; + if (clip.fadeIn > 0.0 && sampleTimeInClip < clip.fadeIn) + fadeGain *= applyFadeCurve (static_cast<float> (sampleTimeInClip / clip.fadeIn), clip.fadeInCurve); + const double timeFromEnd = clip.duration - sampleTimeInClip; + if (clip.fadeOut > 0.0 && timeFromEnd < clip.fadeOut) + fadeGain *= applyFadeCurve (static_cast<float> (timeFromEnd / clip.fadeOut), clip.fadeOutCurve); + const float envGain = envPoints ? interpolateGainEnvelope (*envPoints, sampleTimeInClip) : 1.0f; + const float totalGain = clipGain * fadeGain * envGain; + + for (int ch = 0; ch < channelsToProcess; ++ch) + { + const float sourceSample = sampleBufferCubic (reusableFileBuffer, ch, fileSamplesToRead, filePos); + buffer.addSample (ch, outputStart + i, sourceSample * totalGain); + } + } + + if (shouldLogDetailed) + { + logAudioPlayback ("fillTrackBuffer chunk track=" + trackId + + " clipId=" + clip.clipId + + " out=[" + juce::String (outputStart) + "," + juce::String (outputStart + outputSamples) + "]" + + " clipStart=" + juce::String (chunkClipStart, 4) + + " sourceType=" + juce::String (usingRenderedPreviewSegment ? "rendered_segment" + : (usingCorrectedSource ? "corrected_source" : "original")) + + " fileOffset=" + juce::String (playbackOffset, 4) + + " src=shared_cubic_fractional"); + } + return true; + }; + + bool mixedAnyChunk = false; + for (size_t boundaryIndex = 0; boundaryIndex + 1 < chunkBoundaries.size(); ++boundaryIndex) + { + const int chunkStart = chunkBoundaries[boundaryIndex]; + const int chunkEnd = chunkBoundaries[boundaryIndex + 1]; + if (chunkEnd <= chunkStart) + continue; + + const double chunkClipMid = currentTime + + ((static_cast<double> (chunkStart + chunkEnd) * 0.5) / sampleRate) + - clip.startTime; + const RenderedPreviewSegment* activeSegment = nullptr; + if (renderedSegmentsForClip != nullptr) + { + for (const auto& segment : *renderedSegmentsForClip) + { + if (chunkClipMid >= segment.startSec && chunkClipMid < segment.endSec) + { + activeSegment = &segment; + break; + } + } + } + + const double chunkClipStart = currentTime + (static_cast<double> (chunkStart) / sampleRate) - clip.startTime; + juce::File playbackFile = clip.audioFile; + double playbackOffset = clip.offset + chunkClipStart; + bool usingRenderedPreviewSegment = false; + bool usingCorrectedSource = false; + + if (activeSegment != nullptr) + { + playbackFile = activeSegment->audioFile; + playbackOffset = activeSegment->fileOffsetSec + (chunkClipStart - activeSegment->startSec); + usingRenderedPreviewSegment = true; + } + else if (clip.clipId.isNotEmpty()) + { + auto correctedIt = pitchCorrectedFiles.find (clip.clipId); + usingCorrectedSource = correctedIt != pitchCorrectedFiles.end() + && correctedIt->second.existsAsFile() + && playbackFile == correctedIt->second; + } + + mixedAnyChunk = mixChunk (chunkStart, chunkEnd - chunkStart, playbackFile, playbackOffset, + usingRenderedPreviewSegment, usingCorrectedSource) + || mixedAnyChunk; + } + if (mixedAnyChunk) + ++mixedClipCount; + } + continue; + // Calculate read position within clip double offsetInClip = currentTime - clip.startTime; if (offsetInClip < 0) @@ -601,9 +1447,10 @@ void PlaybackEngine::fillTrackBuffer(const juce::String& trackId, { playbackFile = segment.audioFile; usingRenderedPreviewSegment = true; - // Preview segment files are rendered from their own local zero, - // so map the clip-relative time back into that segment range. - playbackOffset = juce::jmax(0.0, offsetInClip - segment.startSec); + // Segment override files can either be local-zero window renders + // or full-clip renders carrying only a covered region. fileOffsetSec + // maps the clip-relative playhead back into the override file. + playbackOffset = juce::jmax(0.0, offsetInClip - segment.startSec + segment.fileOffsetSec); break; } } @@ -741,24 +1588,30 @@ void PlaybackEngine::fillTrackBuffer(const juce::String& trackId, } preview.lastPlaybackTime = clipTime + (fileSamplesToRead / fileSampleRate); - preview.stretcher.setTransposeFactor (pitchRatio); - // Match the old stable preview behavior: ordinary pitch edits - // preserve timbre by compensating the pitch-induced formant shift. - preview.stretcher.setFormantFactor (juce::jlimit (0.25f, 4.0f, 1.0f / pitchRatio), true); + const float tonalityLimitNorm = static_cast<float> ( + fileSampleRate > 0.0 + ? getPitchOnlyPreviewTonalityLimitHz (pitchRatio < 1.0f) / fileSampleRate + : 0.0); + preview.stretcher.setTransposeFactor (pitchRatio, tonalityLimitNorm); + // Keep the legacy live fallback in the same timbre family as the + // note-local renderer: preserve formants directly instead of using + // pitch-ratio compensation, which brightens upward edits and darkens + // downward edits by construction. + preview.stretcher.setFormantFactor (1.0f, true); // Ensure pitch shift work buffer is large enough if (pitchShiftWorkBuffer.getNumSamples() < fileSamplesToRead) pitchShiftWorkBuffer.setSize (readerChannels, fileSamplesToRead); - std::vector<const float*> inPtrs (static_cast<size_t> (readerChannels)); - std::vector<float*> outPtrs (static_cast<size_t> (readerChannels)); + // Use pre-allocated pointer vectors — avoids heap alloc per clip per callback. + // readerChannels is always 1 or 2; pitchPreviewInPtrs/OutPtrs are sized to 2. for (int ch = 0; ch < readerChannels; ++ch) { - inPtrs[static_cast<size_t> (ch)] = reusableFileBuffer.getReadPointer (ch); - outPtrs[static_cast<size_t> (ch)] = pitchShiftWorkBuffer.getWritePointer (ch); + pitchPreviewInPtrs[static_cast<size_t> (ch)] = reusableFileBuffer.getReadPointer (ch); + pitchPreviewOutPtrs[static_cast<size_t> (ch)] = pitchShiftWorkBuffer.getWritePointer (ch); } - preview.stretcher.process (inPtrs, fileSamplesToRead, outPtrs, fileSamplesToRead); + preview.stretcher.process (pitchPreviewInPtrs, fileSamplesToRead, pitchPreviewOutPtrs, fileSamplesToRead); for (int ch = 0; ch < readerChannels; ++ch) reusableFileBuffer.copyFrom (ch, 0, pitchShiftWorkBuffer, ch, 0, fileSamplesToRead); @@ -775,8 +1628,7 @@ void PlaybackEngine::fillTrackBuffer(const juce::String& trackId, const std::vector<GainEnvelopePoint>* envPoints = nullptr; if (clip.clipId.isNotEmpty()) { - juce::String envKey = clip.trackId + "::" + clip.clipId; - auto envIt = gainEnvelopes.find(envKey); + auto envIt = gainEnvelopes.find(clip.envelopeKey); if (envIt != gainEnvelopes.end() && !envIt->second.empty()) envPoints = &envIt->second; } @@ -879,3 +1731,6 @@ void PlaybackEngine::fillTrackBuffer(const juce::String& trackId, + (overlappingClipCount > 0 && mixedClipCount == 0 ? " WARNING_noMixedClips" : "")); } } +#if defined(_MSC_VER) + #pragma warning(pop) +#endif diff --git a/Source/PlaybackEngine.h b/Source/PlaybackEngine.h index cdb6a50..d70cb26 100644 --- a/Source/PlaybackEngine.h +++ b/Source/PlaybackEngine.h @@ -1,7 +1,16 @@ #pragma once #include <JuceHeader.h> + +#if defined(_MSC_VER) + #pragma warning(push) + #pragma warning(disable: 4244 4267 4305 4456) +#endif #include "signalsmith-stretch.h" +#if defined(_MSC_VER) + #pragma warning(pop) +#endif + #include <memory> #include <vector> #include <map> @@ -41,6 +50,7 @@ class PlaybackEngine int fadeOutCurve; // Fade out curve type: 0=linear, 1=equal_power, 2=s_curve, 3=log, 4=exp juce::String trackId; // Which track this clip belongs to juce::String clipId; // Unique clip ID for envelope lookup + juce::String envelopeKey; // Pre-computed "trackId::clipId" key — avoids string alloc in audio thread bool isActive; // Whether clip is currently loaded ClipInfo(const juce::File& file, double start, double dur, const juce::String& track, double off = 0.0, @@ -73,13 +83,33 @@ class PlaybackEngine juce::File audioFile; double startSec = 0.0; double endSec = 0.0; + double fileOffsetSec = 0.0; + }; + + struct ClipPlaybackSourceStatus + { + bool clipFound = false; + bool renderedSegmentActiveAtTime = false; + bool correctedSourceActiveAtTime = false; + juce::String sourceType = "none"; + juce::String audioFile; + double clipTime = 0.0; + double playbackOffset = 0.0; }; - void setClipRenderedPreviewSegment(const juce::String& clipId, + bool setClipRenderedPreviewSegment(const juce::String& clipId, const juce::File& audioFile, double startSec, - double endSec); + double endSec, + double fileOffsetSec = 0.0, + int generation = 0); + void beginRenderedPreviewSegmentGeneration(const juce::String& clipId, int generation); + void invalidateRenderedPreviewSegments(const juce::String& clipId); void clearClipRenderedPreviewSegments(const juce::String& clipId); + std::map<juce::String, std::vector<RenderedPreviewSegment>> getRenderedPreviewSegmentSnapshot() const; + ClipPlaybackSourceStatus getClipPlaybackSourceAtTime(const juce::String& trackId, + const juce::String& clipId, + double projectTimeSec) const; // Clip gain envelope management void setClipGainEnvelope(const juce::String& trackId, const juce::String& clipId, @@ -109,6 +139,67 @@ class PlaybackEngine float globalFormantSemitones = 0.0f; double previewStartSec = 0.0; double previewEndSec = std::numeric_limits<double>::max(); + bool allowReplacingCorrectedSource = false; + }; + + struct PitchScrubPreviewData + { + juce::String trackId; + juce::String clipId; + juce::AudioBuffer<float> loopBuffer; + double sourceSampleRate = 0.0; + double loopStartSec = 0.0; + double loopEndSec = 0.0; + float basePitchHz = 0.0f; + float pitchRatio = 1.0f; + bool active = false; + double readPosition = 0.0; + int loopCrossfadeSamples = 0; + float gain = 1.0f; + float currentGain = 0.0f; + float targetGain = 1.0f; + float repeatStability = 0.0f; + double startRampMs = 7.5; + double stopRampMs = 14.0; + bool releasePending = false; + bool firstCallbackServiced = false; + bool firstDragAudible = false; + float lastPeak = 0.0f; + double lastRenderWallTimeMs = 0.0; + int mixedCallbackCount = 0; + juce::int64 mixedSampleCount = 0; + }; + + struct PitchScrubPreviewStatus + { + bool active = false; + bool releasePending = false; + bool audible = false; + bool previewArmed = false; + bool firstCallbackServiced = false; + bool firstDragAudible = false; + juce::String trackId; + juce::String clipId; + float pitchRatio = 1.0f; + float basePitchHz = 0.0f; + float currentGain = 0.0f; + float targetGain = 0.0f; + float repeatStability = 0.0f; + float lastPeak = 0.0f; + double loopDurationMs = 0.0; + double lastRenderWallTimeMs = 0.0; + int mixedCallbackCount = 0; + juce::int64 mixedSampleCount = 0; + juce::String renderMethod; + }; + + struct PitchPreviewRoutingStatus + { + bool scrubPreviewActive = false; + bool clipLivePreviewActive = false; + bool renderedSegmentActive = false; + bool correctedSourceActive = false; + juce::String monitorMode; }; // Set a pitch correction map for a clip (enables real-time preview) @@ -117,10 +208,20 @@ class PlaybackEngine // Clear pitch preview for a clip (disables real-time preview) void clearClipPitchPreview (const juce::String& clipId); + void clearAllPitchPreviewRoutes (const juce::String& clipId); + int clearPitchPreviewRoutesForCorrectedSources(); // Check if a clip has an active pitch preview bool hasClipPitchPreview (const juce::String& clipId) const; + void setPitchScrubPreview (const PitchScrubPreviewData& preview); + bool updatePitchScrubPreview (const juce::String& clipId, float pitchRatio); + void clearPitchScrubPreview (const juce::String& clipId); + bool hasPitchScrubPreview (const juce::String& clipId) const; + void renderPitchScrubPreview (juce::AudioBuffer<float>& buffer, double sampleRate); + PitchScrubPreviewStatus getPitchScrubPreviewStatus (const juce::String& clipId = {}) const; + PitchPreviewRoutingStatus getPitchPreviewRoutingStatus (const juce::String& clipId = {}) const; + // Utility int getNumClips() const { return (int)clips.size(); } int getNumClipsForTrack(const juce::String& trackId) const; @@ -129,6 +230,10 @@ class PlaybackEngine int getLastOverlappingClipCount() const { return lastOverlappingClipCount.load(std::memory_order_relaxed); } int getLastMixedClipCount() const { return lastMixedClipCount.load(std::memory_order_relaxed); } float getLastTrackPlaybackPeak() const { return lastTrackPlaybackPeak.load(std::memory_order_relaxed); } + int getFileBufferResizeCount() const { return fileBufferResizeCount.load(std::memory_order_relaxed); } + int getPitchShiftWorkBufferResizeCount() const { return pitchShiftWorkBufferResizeCount.load(std::memory_order_relaxed); } + int getRenderResampleScratchResizeCount() const { return renderResampleScratchResizeCount.load(std::memory_order_relaxed); } + int getChunkBoundaryReserveCount() const { return chunkBoundaryReserveCount.load(std::memory_order_relaxed); } // Thread-safe snapshot of all clips (for offline rendering) std::vector<ClipInfo> getClipSnapshot() const; @@ -143,7 +248,7 @@ class PlaybackEngine std::vector<ClipInfo> clips; std::map<juce::String, std::unique_ptr<juce::AudioFormatReader>> readers; juce::AudioFormatManager formatManager; - juce::CriticalSection lock; + mutable juce::CriticalSection lock; // Clip gain envelopes: key = "trackId::clipId", value = sorted envelope points std::map<juce::String, std::vector<GainEnvelopePoint>> gainEnvelopes; @@ -153,6 +258,8 @@ class PlaybackEngine // Pre-allocated file read buffer (avoids heap alloc on audio thread) juce::AudioBuffer<float> reusableFileBuffer; + juce::AudioBuffer<float> renderResampleScratch; + std::vector<int> reusableChunkBoundaries; // Get cached audio format reader (audio-thread safe — never creates readers) juce::AudioFormatReader* getCachedReader(const juce::File& file); @@ -191,10 +298,21 @@ class PlaybackEngine // Keyed by clipId — only clips with active pitch preview have entries std::map<juce::String, std::unique_ptr<ClipPitchPreviewState>> clipPitchPreviews; + PitchScrubPreviewData pitchScrubPreview; + PitchScrubPreviewStatus pitchScrubPreviewStatus; + signalsmith::stretch::SignalsmithStretch<float> pitchScrubStretcher; + bool pitchScrubStretcherPrepared = false; + juce::AudioBuffer<float> pitchScrubInputBuffer; + juce::AudioBuffer<float> pitchScrubOutputBuffer; // Pre-allocated buffer for pitch-shifted audio (avoids heap alloc on audio thread) juce::AudioBuffer<float> pitchShiftWorkBuffer; + // Pre-allocated channel-pointer vectors for pitch-preview Signalsmith calls. + // Audio files are at most stereo, so size 2 covers all cases. + std::vector<const float*> pitchPreviewInPtrs; + std::vector<float*> pitchPreviewOutPtrs; + // Look up pitch ratio from correction segments at a given clip-relative time static float lookupPitchRatio (const std::vector<PitchCorrectionSegment>& segments, double timeInClip); @@ -211,12 +329,17 @@ class PlaybackEngine }; std::map<juce::String, std::vector<RenderedPreviewSegment>> renderedPreviewSegments; + std::map<juce::String, int> renderedPreviewSegmentGenerations; std::map<juce::String, DeferredClipSwap> deferredClipSwaps; std::atomic<int> tryLockFailureCount { 0 }; std::atomic<int> missingReaderCount { 0 }; std::atomic<int> lastOverlappingClipCount { 0 }; std::atomic<int> lastMixedClipCount { 0 }; std::atomic<float> lastTrackPlaybackPeak { 0.0f }; + std::atomic<int> fileBufferResizeCount { 0 }; + std::atomic<int> pitchShiftWorkBufferResizeCount { 0 }; + std::atomic<int> renderResampleScratchResizeCount { 0 }; + std::atomic<int> chunkBoundaryReserveCount { 0 }; JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(PlaybackEngine) }; diff --git a/Source/PolyResynthesizer.h b/Source/PolyResynthesizer.h index 0f26494..f81a1be 100644 --- a/Source/PolyResynthesizer.h +++ b/Source/PolyResynthesizer.h @@ -9,8 +9,8 @@ * * The old SMS-based polyphonic resynthesis has been removed. * This stub provides the API surface that AudioEngine uses so it compiles. - * Polyphonic pitch correction will be re-implemented with Signalsmith Stretch - * as a future improvement (per pitch_corrector_feat_plan.md). + * Polyphonic pitch correction will be re-implemented as a future improvement + * (per pitch_corrector_feat_plan.md). */ class PolyResynthesizer { diff --git a/Source/S13PitchCorrector.cpp b/Source/S13PitchCorrector.cpp index 11f5182..587860b 100644 --- a/Source/S13PitchCorrector.cpp +++ b/Source/S13PitchCorrector.cpp @@ -28,6 +28,13 @@ void S13PitchCorrector::prepareToPlay(double sampleRate, int samplesPerBlock) detector.setSensitivity(sensitivity.load()); setLatencySamples (stretcher.outputLatency()); + + // Pre-allocate per-block scratch buffers so processBlock never heap-allocates. + // samplesPerBlock is the maximum; actual numSamples will always be <= this. + dryBuffer.setSize (2, samplesPerBlock, false, true, false); + stretchOutputBuf.setSize (2, samplesPerBlock, false, true, false); + inPtrs.resize (2); + outPtrs.resize (2); } void S13PitchCorrector::releaseResources() @@ -51,12 +58,12 @@ void S13PitchCorrector::processBlock(juce::AudioBuffer<float>& buffer, juce::Mid detector.setMaxFrequency(maxFreqParam.load(std::memory_order_relaxed)); detector.setSensitivity(sensitivity.load(std::memory_order_relaxed)); - // Save dry signal for mix - juce::AudioBuffer<float> dryBuffer; + // Save dry signal for mix (copyFrom into pre-allocated member — no heap alloc) float mixVal = mix.load(std::memory_order_relaxed); if (mixVal < 0.999f) { - dryBuffer.makeCopyOf(buffer); + for (int ch = 0; ch < numChannels; ++ch) + dryBuffer.copyFrom (ch, 0, buffer, ch, 0, numSamples); } // Pitch detection on mono sum (L channel or mono mix) @@ -81,29 +88,23 @@ void S13PitchCorrector::processBlock(juce::AudioBuffer<float>& buffer, juce::Mid ratio = juce::jlimit(0.25f, 4.0f, ratio); } - // Apply pitch shift via Signalsmith Stretch (real-time, native stereo) + // Apply pitch shift via Signalsmith Stretch (real-time, native stereo). + // Use pre-allocated inPtrs/outPtrs/stretchOutputBuf to avoid heap allocation. stretcher.setTransposeFactor (ratio); stretcher.setFormantBase (detectedHz > 0.0f ? detectedHz : 0.0f); // help formant estimation stretcher.setFormantFactor (1.0f, true); // preserve formants via library's exact freq map + for (int ch = 0; ch < numChannels; ++ch) { - std::vector<const float*> inPtrs (static_cast<size_t> (numChannels)); - std::vector<float*> outPtrs (static_cast<size_t> (numChannels)); - std::vector<std::vector<float>> tempOut (static_cast<size_t> (numChannels), - std::vector<float> (static_cast<size_t> (numSamples))); - - for (int ch = 0; ch < numChannels; ++ch) - { - inPtrs[static_cast<size_t> (ch)] = buffer.getReadPointer (ch); - outPtrs[static_cast<size_t> (ch)] = tempOut[static_cast<size_t> (ch)].data(); - } + inPtrs[static_cast<size_t> (ch)] = buffer.getReadPointer (ch); + outPtrs[static_cast<size_t> (ch)] = stretchOutputBuf.getWritePointer (ch); + } - stretcher.process (inPtrs, numSamples, outPtrs, numSamples); + stretcher.process (inPtrs, numSamples, outPtrs, numSamples); - for (int ch = 0; ch < numChannels; ++ch) - std::memcpy (buffer.getWritePointer (ch), tempOut[static_cast<size_t> (ch)].data(), - static_cast<size_t> (numSamples) * sizeof (float)); - } + for (int ch = 0; ch < numChannels; ++ch) + std::memcpy (buffer.getWritePointer (ch), stretchOutputBuf.getReadPointer (ch), + static_cast<size_t> (numSamples) * sizeof (float)); // Apply dry/wet mix if (mixVal < 0.999f) @@ -119,14 +120,16 @@ void S13PitchCorrector::processBlock(juce::AudioBuffer<float>& buffer, juce::Mid } } - // Store pitch history for UI + // Store pitch history for UI — lock-free, audio thread is sole writer. + // Write the frame first, then publish the new index with release semantics so + // the UI thread (which reads with acquire) is guaranteed to see the frame data. { float detMidi = detectedHz > 0.0f ? hzToMidi(detectedHz) : 0.0f; float corMidi = correctedHz > 0.0f ? hzToMidi(correctedHz) : 0.0f; - const std::lock_guard<std::mutex> lock(pitchHistoryMutex); - pitchHistory[static_cast<size_t>(pitchHistoryWritePos)] = { detMidi, corMidi, conf }; - pitchHistoryWritePos = (pitchHistoryWritePos + 1) % maxPitchHistory; + const int writePos = pitchHistoryWritePos.load (std::memory_order_relaxed); + pitchHistory[static_cast<size_t> (writePos)] = { detMidi, corMidi, conf }; + pitchHistoryWritePos.store ((writePos + 1) % maxPitchHistory, std::memory_order_release); } // MIDI output generation @@ -200,7 +203,9 @@ S13PitchCorrector::PitchData S13PitchCorrector::getCurrentPitchData() const std::vector<S13PitchCorrector::PitchHistoryFrame> S13PitchCorrector::getPitchHistory(int numFrames) const { - const std::lock_guard<std::mutex> lock(pitchHistoryMutex); + // Acquire the current write position with acquire semantics so all frame + // data written before this store (release) is visible to this thread. + const int wp = pitchHistoryWritePos.load (std::memory_order_acquire); int count = std::min(numFrames, maxPitchHistory); std::vector<PitchHistoryFrame> result; @@ -208,7 +213,7 @@ std::vector<S13PitchCorrector::PitchHistoryFrame> S13PitchCorrector::getPitchHis for (int i = 0; i < count; ++i) { - int idx = (pitchHistoryWritePos - count + i + maxPitchHistory) % maxPitchHistory; + int idx = (wp - count + i + maxPitchHistory) % maxPitchHistory; result.push_back(pitchHistory[static_cast<size_t>(idx)]); } return result; diff --git a/Source/S13PitchCorrector.h b/Source/S13PitchCorrector.h index 2e1faa7..2e5faf4 100644 --- a/Source/S13PitchCorrector.h +++ b/Source/S13PitchCorrector.h @@ -5,7 +5,6 @@ #include "PitchMapper.h" #include "signalsmith-stretch.h" #include <atomic> -#include <mutex> #include <vector> /** @@ -99,11 +98,19 @@ class S13PitchCorrector : public juce::AudioProcessor float lastDetectedHz = 0.0f; float lastCorrectedHz = 0.0f; - // Pitch history for UI + // Pre-allocated buffers/vectors for processBlock — sized in prepareToPlay, + // reused every callback to avoid heap allocation on the audio thread. + juce::AudioBuffer<float> dryBuffer; // dry copy for wet/mix blend + std::vector<const float*> inPtrs; // Signalsmith input channel pointers + std::vector<float*> outPtrs; // Signalsmith output channel pointers + juce::AudioBuffer<float> stretchOutputBuf; // Signalsmith output staging buffer + + // Pitch history for UI — lock-free single-writer (audio thread) / single-reader (UI). + // Audio thread writes at writePos then increments with release semantics. + // UI thread loads writePos with acquire, then reads older indices without a lock. static constexpr int maxPitchHistory = 512; std::vector<PitchHistoryFrame> pitchHistory; - int pitchHistoryWritePos = 0; - mutable std::mutex pitchHistoryMutex; + std::atomic<int> pitchHistoryWritePos { 0 }; // MIDI output state int currentMidiNote = -1; // Currently sounding note (-1 = none) diff --git a/Source/ScriptEngine.cpp b/Source/ScriptEngine.cpp index 4386fdc..a9751ea 100644 --- a/Source/ScriptEngine.cpp +++ b/Source/ScriptEngine.cpp @@ -56,7 +56,7 @@ static void pushVar(lua_State* L, const juce::var& v) else if (v.isBool()) lua_pushboolean(L, (bool)v ? 1 : 0); else if (v.isInt() || v.isInt64()) - lua_pushinteger(L, (lua_Integer)(int64_t)v); + lua_pushinteger(L, static_cast<lua_Integer> (static_cast<juce::int64> (v))); else if (v.isDouble()) lua_pushnumber(L, (double)v); else if (v.isString()) diff --git a/Source/SignalsmithShifter.cpp b/Source/SignalsmithShifter.cpp deleted file mode 100644 index 94b208a..0000000 --- a/Source/SignalsmithShifter.cpp +++ /dev/null @@ -1,292 +0,0 @@ -#include "SignalsmithShifter.h" - -// Signalsmith Stretch — MIT license, header-only -// https://github.com/Signalsmith-Audio/signalsmith-stretch -#include "signalsmith-stretch.h" - -#include <juce_core/juce_core.h> -#include <cmath> -#include <algorithm> -#include <numeric> -#include <limits> - -#if JUCE_DEBUG -static constexpr bool kPitchEditorFormantDebugLogs = true; -#else -static constexpr bool kPitchEditorFormantDebugLogs = false; -#endif - -static void logPitchEditorFormant(const juce::String& message) -{ - if (kPitchEditorFormantDebugLogs) - juce::Logger::writeToLog ("[pitchEditor.formant] " + message); -} - -std::vector<std::vector<float>> SignalsmithShifter::process ( - const float* const* input, - int numChannels, - int numSamples, - double sampleRate, - const std::vector<float>& ratios, - const std::vector<float>& formantRatios, - const std::vector<float>& detectedPitchHz) -{ - // Passthrough guard - if (numSamples <= 0 || numChannels <= 0) - { - std::vector<std::vector<float>> result (static_cast<size_t> (numChannels)); - for (int ch = 0; ch < numChannels; ++ch) - result[static_cast<size_t> (ch)].assign (input[ch], input[ch] + numSamples); - return result; - } - - // Check if anything actually needs shifting (pitch OR formant) - bool hasShift = false; - for (int s = 0; s < numSamples && ! hasShift; s += 512) - { - if (static_cast<size_t> (s) < ratios.size() - && std::abs (ratios[static_cast<size_t> (s)] - 1.0f) > 1e-4f) - hasShift = true; - } - - // Also check formant ratios — formant-only edits must still be processed - bool hasFormant = false; - if (! formantRatios.empty()) - { - for (int s = 0; s < numSamples && ! hasFormant; s += 512) - { - if (static_cast<size_t> (s) < formantRatios.size() - && std::abs (formantRatios[static_cast<size_t> (s)] - 1.0f) > 1e-4f) - hasFormant = true; - } - } - - if (! hasShift && ! hasFormant) - { - // No pitch or formant edits — return input unchanged - std::vector<std::vector<float>> result (static_cast<size_t> (numChannels)); - for (int ch = 0; ch < numChannels; ++ch) - result[static_cast<size_t> (ch)].assign (input[ch], input[ch] + numSamples); - return result; - } - - const bool useLegacyPitchOnlyPath = ! hasFormant; - - // ------------------------------------------------------------------------- - // Configure Signalsmith Stretch - // Pitch-only uses the old stable presetDefault() path from 609c5cb. - // Explicit formant work keeps the newer custom path and pitch-base guidance. - // ------------------------------------------------------------------------- - signalsmith::stretch::SignalsmithStretch<float> stretcher; - - if (useLegacyPitchOnlyPath) - { - stretcher.presetDefault (numChannels, static_cast<float> (sampleRate)); - } - else - { - // Offline quality preset: 120ms analysis window (same as presetDefault), 10ms hop. - // This newer path is kept only for explicit formant rendering. - int blockSamples = static_cast<int> (sampleRate * 0.12); - int intervalSamples = static_cast<int> (sampleRate * 0.01); - stretcher.configure (numChannels, blockSamples, intervalSamples); - } - - const int blockSize = stretcher.intervalSamples(); // process this many samples per block - const int outputLatency = stretcher.outputLatency(); - - // We need numSamples of corrected output. - // Allocate: numSamples + outputLatency (to hold both the main pass and the flush). - const int totalOutSamples = numSamples + outputLatency; - - std::vector<std::vector<float>> outputBuf ( - static_cast<size_t> (numChannels), - std::vector<float> (static_cast<size_t> (totalOutSamples), 0.0f)); - - // Per-channel pointer vectors reused across blocks - std::vector<const float*> inPtrs (static_cast<size_t> (numChannels)); - std::vector<float*> outPtrs (static_cast<size_t> (numChannels)); - float minAvgRatio = std::numeric_limits<float>::max(); - float maxAvgRatio = 0.0f; - float minAvgFormant = std::numeric_limits<float>::max(); - float maxAvgFormant = 0.0f; - float minTargetFormant = std::numeric_limits<float>::max(); - float maxTargetFormant = 0.0f; - float minLibraryFormant = std::numeric_limits<float>::max(); - float maxLibraryFormant = 0.0f; - int blocksWithPitchBase = 0; - - // ------------------------------------------------------------------------- - // Process input in blocks of blockSize, updating pitch ratio per block - // ------------------------------------------------------------------------- - int pos = 0; - while (pos < numSamples) - { - int thisBlock = std::min (blockSize, numSamples - pos); - - // Average ratio for this block (ratios[] is per-sample) - float sumRatio = 0.0f; - int counted = 0; - for (int s = pos; s < pos + thisBlock; ++s) - { - if (static_cast<size_t> (s) < ratios.size()) - { - sumRatio += ratios[static_cast<size_t> (s)]; - ++counted; - } - } - float avgRatio = (counted > 0) ? sumRatio / static_cast<float> (counted) : 1.0f; - avgRatio = juce::jlimit (0.25f, 4.0f, avgRatio); - minAvgRatio = std::min (minAvgRatio, avgRatio); - maxAvgRatio = std::max (maxAvgRatio, avgRatio); - - stretcher.setTransposeFactor (avgRatio); - - // Provide detected fundamental frequency so the library's formant - // envelope estimation uses the correct smoothing width. - // Without this, it uses a "VERY rough" auto-detection that often - // fails, especially on downward-shifted audio with denser harmonics. - if (! useLegacyPitchOnlyPath && ! detectedPitchHz.empty()) - { - float sumPitch = 0.0f; - int cntPitch = 0; - for (int s = pos; s < pos + thisBlock; ++s) - { - if (static_cast<size_t> (s) < detectedPitchHz.size() - && detectedPitchHz[static_cast<size_t> (s)] > 0.0f) - { - sumPitch += detectedPitchHz[static_cast<size_t> (s)]; - ++cntPitch; - } - } - if (cntPitch > 0) - { - stretcher.setFormantBase (sumPitch / static_cast<float> (cntPitch)); - ++blocksWithPitchBase; - } - else - stretcher.setFormantBase (0); // fall back to auto-detect - } - - // Formant handling — two-pass approach: - // Pass 1 (here): Signalsmith's built-in compensatePitch=true does - // coarse formant preservation using its spectral envelope estimator. - // Pass 2 (PitchResynthesizer post-correction): log-domain envelope - // matching catches residual formant drift the library missed. - if (! useLegacyPitchOnlyPath) - { - float sumFormant = 0.0f; - int cntFormant = 0; - for (int s = pos; s < pos + thisBlock; ++s) - { - if (static_cast<size_t> (s) < formantRatios.size()) - { - sumFormant += formantRatios[static_cast<size_t> (s)]; - ++cntFormant; - } - } - float avgFormant = (cntFormant > 0) ? sumFormant / static_cast<float> (cntFormant) : 1.0f; - avgFormant = juce::jlimit (0.25f, 4.0f, avgFormant); - const float targetFormant = juce::jlimit (0.25f, 4.0f, avgFormant / avgRatio); - const float libraryFormant = juce::jlimit (0.35f, 3.5f, targetFormant); - minAvgFormant = std::min (minAvgFormant, avgFormant); - maxAvgFormant = std::max (maxAvgFormant, avgFormant); - minTargetFormant = std::min (minTargetFormant, targetFormant); - maxTargetFormant = std::max (maxTargetFormant, targetFormant); - minLibraryFormant = std::min (minLibraryFormant, libraryFormant); - maxLibraryFormant = std::max (maxLibraryFormant, libraryFormant); - stretcher.setFormantFactor (libraryFormant, true); - } - else - { - const float preservedFormant = juce::jlimit (0.25f, 4.0f, 1.0f / avgRatio); - minTargetFormant = std::min (minTargetFormant, preservedFormant); - maxTargetFormant = std::max (maxTargetFormant, preservedFormant); - minLibraryFormant = std::min (minLibraryFormant, preservedFormant); - maxLibraryFormant = std::max (maxLibraryFormant, preservedFormant); - stretcher.setFormantFactor (preservedFormant); - } - - // Set input and output pointers for this block - for (int ch = 0; ch < numChannels; ++ch) - { - inPtrs[static_cast<size_t> (ch)] = input[ch] + pos; - outPtrs[static_cast<size_t> (ch)] = outputBuf[static_cast<size_t> (ch)].data() + pos; - } - - stretcher.process (inPtrs, thisBlock, outPtrs, thisBlock); - pos += thisBlock; - } - - // ------------------------------------------------------------------------- - // Flush the remaining buffered samples (latency compensation). - // After processing all numSamples of input, the stretcher still holds - // outputLatency worth of unread processed samples. Flushing with silence - // pushes them out. - // ------------------------------------------------------------------------- - { - for (int ch = 0; ch < numChannels; ++ch) - outPtrs[static_cast<size_t> (ch)] = outputBuf[static_cast<size_t> (ch)].data() + numSamples; - - stretcher.flush (outPtrs, outputLatency); - } - - // ------------------------------------------------------------------------- - // Assemble final output: skip the first outputLatency samples (latency fill), - // take samples [outputLatency .. outputLatency + numSamples - 1]. - // ------------------------------------------------------------------------- - std::vector<std::vector<float>> result (static_cast<size_t> (numChannels)); - for (int ch = 0; ch < numChannels; ++ch) - { - auto& out = outputBuf[static_cast<size_t> (ch)]; - auto& res = result[static_cast<size_t> (ch)]; - res.resize (static_cast<size_t> (numSamples)); - - int srcStart = std::min (outputLatency, numSamples); - int srcAvail = totalOutSamples - srcStart; // how many valid samples follow - int toCopy = std::min (numSamples, srcAvail); - - if (toCopy > 0) - std::copy (out.begin() + srcStart, - out.begin() + srcStart + toCopy, - res.begin()); - - // If latency > numSamples (extremely short clips), the rest stays zero. - } - - { - float minFormant = 1.0f, maxFormant = 1.0f; - for (const auto& f : formantRatios) { minFormant = std::min (minFormant, f); maxFormant = std::max (maxFormant, f); } - juce::Logger::writeToLog ("SignalsmithShifter: processed " + juce::String (numSamples) - + " samples, " + juce::String (numChannels) + " ch, " - + "latency=" + juce::String (outputLatency) - + " formantRange=[" + juce::String (minFormant, 3) + "," + juce::String (maxFormant, 3) + "]" - + " mode=" + juce::String (useLegacyPitchOnlyPath ? "pitch_only_legacy" : "explicit_formant")); - if (! useLegacyPitchOnlyPath) - { - logPitchEditorFormant ("Signalsmith blocks=" + juce::String ((numSamples + blockSize - 1) / blockSize) - + " avgPitchRange=[" + juce::String (minAvgRatio, 3) + "," + juce::String (maxAvgRatio, 3) + "]" - + " avgFormantRange=[" + juce::String (minAvgFormant == std::numeric_limits<float>::max() ? 1.0f : minAvgFormant, 3) - + "," + juce::String (maxAvgFormant, 3) + "]" - + " targetFormantRange=[" + juce::String (minTargetFormant == std::numeric_limits<float>::max() ? 1.0f : minTargetFormant, 3) - + "," + juce::String (maxTargetFormant, 3) + "]" - + " libraryFormantRange=[" + juce::String (minLibraryFormant == std::numeric_limits<float>::max() ? 1.0f : minLibraryFormant, 3) - + "," + juce::String (maxLibraryFormant, 3) + "]" - + " pitchBaseBlocks=" + juce::String (blocksWithPitchBase) - + " latency=" + juce::String (outputLatency) - + " copiedSamples=" + juce::String (numSamples)); - } - else - { - logPitchEditorFormant ("Signalsmith pitch-only legacy blocks=" + juce::String ((numSamples + blockSize - 1) / blockSize) - + " avgPitchRange=[" + juce::String (minAvgRatio, 3) + "," + juce::String (maxAvgRatio, 3) + "]" - + " preservedFormantRange=[" + juce::String (minTargetFormant == std::numeric_limits<float>::max() ? 1.0f : minTargetFormant, 3) - + "," + juce::String (maxTargetFormant, 3) + "]" - + " pitchBaseBlocks=" + juce::String (blocksWithPitchBase) - + " latency=" + juce::String (outputLatency) - + " copiedSamples=" + juce::String (numSamples)); - } - } - - return result; -} diff --git a/Source/SignalsmithShifter.h b/Source/SignalsmithShifter.h deleted file mode 100644 index dffb6c4..0000000 --- a/Source/SignalsmithShifter.h +++ /dev/null @@ -1,45 +0,0 @@ -#pragma once - -#include <vector> - -/** - * SignalsmithShifter — wrapper around Signalsmith Stretch for offline pitch correction. - * - * Handles: - * - Multi-channel audio natively (stereo, mono — no mid-side hack needed) - * - Per-block varying pitch ratio (block size = intervalSamples, ~30ms at 44100Hz) - * - Per-block formant compensation (keeps formants at original position) - * - Latency compensation (output is exactly numSamples, time-aligned with input) - * - * Usage: - * auto output = SignalsmithShifter::process(input, 2, numSamples, 44100, - * ratios, formantRatios); - */ -class SignalsmithShifter -{ -public: - /** - * Process audio with per-sample pitch ratios. - * - * @param input Array of numChannels pointers, each numSamples floats - * @param numChannels Number of channels (1 = mono, 2 = stereo, etc.) - * @param numSamples Number of samples per channel - * @param sampleRate Sample rate in Hz - * @param ratios Per-sample pitch ratio (1.0 = no change, 1.122 = +2 semitones) - * @param formantRatios Per-sample formant ratio. Empty = preserve formants automatically. - * 1.0 = keep at original position, 1.5 = brighter, 0.7 = darker. - * @param detectedPitchHz Per-sample detected fundamental frequency in Hz. - * Empty = let the library auto-detect (less accurate). - * Providing this dramatically improves formant preservation - * because it controls the spectral envelope smoothing width. - * @return Vector of numChannels vectors, each exactly numSamples floats. - */ - static std::vector<std::vector<float>> process ( - const float* const* input, - int numChannels, - int numSamples, - double sampleRate, - const std::vector<float>& ratios, - const std::vector<float>& formantRatios = {}, - const std::vector<float>& detectedPitchHz = {}); -}; diff --git a/Source/StemSeparator.cpp b/Source/StemSeparator.cpp index a5851c1..53ae519 100644 --- a/Source/StemSeparator.cpp +++ b/Source/StemSeparator.cpp @@ -1,13 +1,25 @@ #include "StemSeparator.h" +#if JUCE_MAC || JUCE_LINUX + #include <sys/stat.h> +#endif + namespace { constexpr auto kStemModelName = "BS-Roformer-SW.ckpt"; +constexpr auto kPinnedMusicGenerationModelId = "acestep-v15-xl-turbo"; +constexpr auto kPinnedMusicGenerationModelRepoId = "ACE-Step/acestep-v15-xl-turbo"; +constexpr auto kPinnedMusicGenerationSharedRepoId = "ACE-Step/Ace-Step1.5"; constexpr auto kPythonHelpUrl = "https://www.python.org/downloads/"; constexpr auto kInstallSourceDownloadedRuntime = "downloadedRuntime"; constexpr auto kInstallSourceExternalPython = "externalPython"; constexpr auto kBuildRuntimeModeDownloadedRuntime = "downloaded-runtime"; constexpr auto kBuildRuntimeModeUnbundledDev = "unbundled-dev"; +#if JUCE_WINDOWS +constexpr auto kSupportedExternalPythonRange = "Python 3.11"; +#else +constexpr auto kSupportedExternalPythonRange = "Python 3.10 through 3.12"; +#endif constexpr double kInstallerOutputTimeoutMs = 20000.0; constexpr double kInstallerMonitorPollMs = 150.0; constexpr double kInstallerHeartbeatMs = 1000.0; @@ -78,6 +90,12 @@ bool isInstallerTerminalFailureCode (const juce::String& errorCode) || errorCode == "installer_output_timeout"; } +bool hasNativeMusicProfile (const StemSeparator::AiToolsStatus& status) +{ + return status.musicGenerationAvailableProfiles.isEmpty() + || status.musicGenerationAvailableProfiles.contains("native-xl-turbo"); +} + juce::String sanitiseArchiveEntryName (juce::String name) { name = name.replaceCharacter('\\', '/').trim(); @@ -141,15 +159,26 @@ juce::File StemSeparator::getUserDataRoot() const const auto localAppData = juce::SystemStats::getEnvironmentVariable("LOCALAPPDATA", {}); if (localAppData.isNotEmpty()) return juce::File(localAppData).getChildFile("OpenStudio"); + return juce::File::getSpecialLocation(juce::File::userApplicationDataDirectory) + .getChildFile("OpenStudio"); #elif JUCE_MAC return juce::File::getSpecialLocation(juce::File::userHomeDirectory) .getChildFile("Library") .getChildFile("Application Support") .getChildFile("OpenStudio"); - #endif - + #elif JUCE_LINUX + // XDG Base Directory spec: use $XDG_DATA_HOME or the default ~/.local/share + const auto xdgDataHome = juce::SystemStats::getEnvironmentVariable("XDG_DATA_HOME", {}); + if (xdgDataHome.isNotEmpty()) + return juce::File(xdgDataHome).getChildFile("OpenStudio"); + return juce::File::getSpecialLocation(juce::File::userHomeDirectory) + .getChildFile(".local") + .getChildFile("share") + .getChildFile("OpenStudio"); + #else return juce::File::getSpecialLocation(juce::File::userApplicationDataDirectory) .getChildFile("OpenStudio"); + #endif } juce::File StemSeparator::getUserRuntimeRoot() const @@ -162,6 +191,14 @@ juce::File StemSeparator::getUserModelsDir() const return getUserDataRoot().getChildFile("models"); } +juce::File StemSeparator::getMusicGenerationCheckpointRoot() const +{ + return juce::File::getSpecialLocation(juce::File::userHomeDirectory) + .getChildFile(".cache") + .getChildFile("ace-step") + .getChildFile("checkpoints"); +} + juce::File StemSeparator::getAiToolsInstallLogFile() const { return getUserDataRoot().getChildFile("logs").getChildFile("AiToolsInstall.log"); @@ -202,6 +239,12 @@ juce::String StemSeparator::getAiRuntimeArchitectureKey() const #else return "x64"; #endif +#elif JUCE_LINUX + #if defined(__aarch64__) || defined(__arm64__) + return "arm64"; + #else + return "x64"; + #endif #else return {}; #endif @@ -210,24 +253,77 @@ juce::String StemSeparator::getAiRuntimeArchitectureKey() const juce::File StemSeparator::findSystemPython() const { #if JUCE_WINDOWS + const auto isSupportedPythonVersion = [] (juce::String versionText) + { + versionText = versionText.trim(); + if (versionText.startsWithIgnoreCase("Python ")) + versionText = versionText.fromFirstOccurrenceOf("Python ", false, false).trim(); + + auto parts = juce::StringArray::fromTokens(versionText, ".", ""); + if (parts.size() < 2) + return false; + + const auto major = parts[0].getIntValue(); + const auto minor = parts[1].retainCharacters("0123456789").getIntValue(); + return major == 3 && minor >= 10 && minor <= 12; + }; + + const auto isSupportedPythonExecutable = [&] (const juce::File& candidate) + { + if (! candidate.existsAsFile()) + return false; + + juce::ChildProcess versionProbe; + if (! versionProbe.start(quoteCommandPart(candidate.getFullPathName()) + " --version") + || ! versionProbe.waitForProcessToFinish(5000)) + return false; + + return isSupportedPythonVersion(versionProbe.readAllProcessOutput()); + }; + + const auto resolvePyLauncherPython = [&] (const juce::String& selector) -> juce::File + { + juce::ChildProcess pyLauncher; + const auto command = "py -" + selector + " -c \"import sys; print(sys.executable)\""; + if (! pyLauncher.start(command) || ! pyLauncher.waitForProcessToFinish(5000)) + return {}; + + const auto output = pyLauncher.readAllProcessOutput().trim(); + if (output.isEmpty()) + return {}; + + juce::File systemPython(output); + if (isSupportedPythonExecutable(systemPython)) + return systemPython; + + return {}; + }; + + for (const auto& selector : { juce::String("3.12"), juce::String("3.11"), juce::String("3.10") }) + if (auto launcherPython = resolvePyLauncherPython(selector); launcherPython.existsAsFile()) + return launcherPython; + juce::ChildProcess where; if (where.start("where python") && where.waitForProcessToFinish(3000)) { auto output = where.readAllProcessOutput().trim(); if (output.isNotEmpty()) { - auto firstLine = output.upToFirstOccurrenceOf("\n", false, false).trim(); - juce::File systemPython(firstLine); - if (systemPython.existsAsFile()) - return systemPython; + juce::StringArray candidates; + candidates.addLines(output); + for (auto candidatePath : candidates) + { + juce::File systemPython(candidatePath.trim()); + if (isSupportedPythonExecutable(systemPython)) + return systemPython; + } } } - - juce::ChildProcess pyLauncher; - if (pyLauncher.start("py -3 -c \"import sys; print(sys.executable)\"") - && pyLauncher.waitForProcessToFinish(5000)) +#elif JUCE_MAC + juce::ChildProcess which; + if (which.start("which python3") && which.waitForProcessToFinish(3000)) { - auto output = pyLauncher.readAllProcessOutput().trim(); + auto output = which.readAllProcessOutput().trim(); if (output.isNotEmpty()) { juce::File systemPython(output); @@ -235,7 +331,14 @@ juce::File StemSeparator::findSystemPython() const return systemPython; } } -#elif JUCE_MAC + + for (const auto& path : { "/opt/homebrew/bin/python3", "/usr/local/bin/python3", "/usr/bin/python3" }) + { + juce::File systemPy(path); + if (systemPy.existsAsFile()) + return systemPy; + } +#elif JUCE_LINUX juce::ChildProcess which; if (which.start("which python3") && which.waitForProcessToFinish(3000)) { @@ -248,16 +351,12 @@ juce::File StemSeparator::findSystemPython() const } } - for (const auto& path : { "/opt/homebrew/bin/python3", "/usr/local/bin/python3", "/usr/bin/python3" }) + for (const auto& path : { "/usr/bin/python3", "/usr/local/bin/python3", "/opt/homebrew/bin/python3" }) { juce::File systemPy(path); if (systemPy.existsAsFile()) return systemPy; } -#elif JUCE_LINUX - juce::File systemPy("/usr/bin/python3"); - if (systemPy.existsAsFile()) - return systemPy; #endif return {}; @@ -417,25 +516,59 @@ StemSeparator::RuntimeCapabilities StemSeparator::probeRuntimeCapabilities (cons RuntimeCapabilities capabilities; if (! python.existsAsFile()) + { + appendAiToolsLogLine (makeAiLogEvent ("host", "probe", "probe_skipped", "", + [] (juce::DynamicObject& o) { o.setProperty ("reason", "python executable not found"); })); return capabilities; + } const auto probeScript = findRuntimeProbeScript(); if (! probeScript.existsAsFile()) + { + appendAiToolsLogLine (makeAiLogEvent ("host", "probe", "probe_skipped", "", + [&] (juce::DynamicObject& o) { o.setProperty ("reason", "probe script not found"); })); return capabilities; + } juce::ChildProcess probe; - const auto command = quoteCommandPart(python.getFullPathName()) - + " " + quoteCommandPart(probeScript.getFullPathName()) - + " --models-dir " + quoteCommandPart(modelsDir.getFullPathName()) - + " --model " + quoteCommandPart(modelName) - + " --acceleration-mode " + quoteCommandPart(accelerationMode); + const auto command = quoteCommandPart (python.getFullPathName()) + + " " + quoteCommandPart (probeScript.getFullPathName()) + + " --models-dir " + quoteCommandPart (modelsDir.getFullPathName()) + + " --model " + quoteCommandPart (modelName) + + " --music-gen-checkpoint-root " + quoteCommandPart (getMusicGenerationCheckpointRoot().getFullPathName()) + + " --music-gen-model " + quoteCommandPart (kPinnedMusicGenerationModelId) + + " --acceleration-mode " + quoteCommandPart (accelerationMode); + + if (! probe.start (command)) + { + appendAiToolsLogLine (makeAiLogEvent ("host", "probe", "probe_failed", "", + [&] (juce::DynamicObject& o) + { + o.setProperty ("reason", "child process failed to start"); + o.setProperty ("python", python.getFullPathName()); + })); + return capabilities; + } - if (! probe.start(command) || ! probe.waitForProcessToFinish(15000)) + if (! probe.waitForProcessToFinish (30000)) + { + appendAiToolsLogLine (makeAiLogEvent ("host", "probe", "probe_failed", "", + [] (juce::DynamicObject& o) { o.setProperty ("reason", "probe timed out after 30 s"); })); return capabilities; + } auto output = probe.readAllProcessOutput().trim(); if (probe.getExitCode() != 0 || output.isEmpty()) + { + appendAiToolsLogLine (makeAiLogEvent ("host", "probe", "probe_failed", "", + [&] (juce::DynamicObject& o) + { + o.setProperty ("reason", "non-zero exit code or empty output"); + o.setProperty ("exitCode", static_cast<int> (probe.getExitCode())); + o.setProperty ("outputSnippet", output.substring (0, 300)); + })); return capabilities; + } const auto lastLine = output.fromLastOccurrenceOf("\n", false, false).trim(); const auto json = juce::JSON::parse(lastLine.isNotEmpty() ? lastLine : output); @@ -452,6 +585,22 @@ StemSeparator::RuntimeCapabilities StemSeparator::probeRuntimeCapabilities (cons capabilities.runtimeVersion = obj->getProperty("runtimeVersion").toString(); capabilities.modelVersion = obj->getProperty("modelVersion").toString(); capabilities.restartRequired = static_cast<bool>(obj->getProperty("restartRequired")); + capabilities.aceStepVersion = obj->getProperty("aceStepVersion").toString(); + capabilities.musicGenerationReady = static_cast<bool>(obj->getProperty("musicGenerationReady")); + capabilities.musicGenerationLayoutValid = static_cast<bool>(obj->getProperty("musicGenerationLayoutValid")); + capabilities.musicGenerationModelId = obj->getProperty("musicGenerationModelId").toString(); + capabilities.musicGenerationModelRepoId = obj->getProperty("musicGenerationModelRepoId").toString(); + capabilities.musicGenerationSharedRepoId = obj->getProperty("musicGenerationSharedRepoId").toString(); + capabilities.musicGenerationCheckpointRoot = obj->getProperty("musicGenerationCheckpointRoot").toString(); + capabilities.musicGenerationStatusMessage = obj->getProperty("musicGenerationStatusMessage").toString(); + capabilities.musicGenerationFailureCode = obj->getProperty("musicGenerationFailureCode").toString(); + capabilities.musicGenerationPerformanceReady = static_cast<bool>(obj->getProperty("musicGenerationPerformanceReady")); + capabilities.musicGenerationPerformanceStatusMessage = obj->getProperty("musicGenerationPerformanceStatusMessage").toString(); + capabilities.musicGenerationRuntimeProfiles = obj->getProperty("musicGenerationRuntimeProfiles"); + capabilities.musicGenerationAvailableProfiles = varToStringArray(obj->getProperty("musicGenerationAvailableProfiles")); + capabilities.musicGenerationUnavailableProfiles = obj->getProperty("musicGenerationUnavailableProfiles"); + capabilities.musicGenerationDefaultProfile = obj->getProperty("musicGenerationDefaultProfile").toString(); + capabilities.musicGenerationWarmSessionCapable = static_cast<bool>(obj->getProperty("musicGenerationWarmSessionCapable")); } if (capabilities.supportedBackends.isEmpty()) @@ -509,7 +658,11 @@ StemSeparator::AiToolsStatus StemSeparator::buildAiToolsStatus (const juce::File { status.state = "ready"; status.progress = 1.0f; - status.stepLabel = "AI tools are ready."; + const auto musicGenerationFullyReady = status.musicGenerationReady + && status.musicGenerationLayoutValid + && status.musicGenerationPerformanceReady + && hasNativeMusicProfile(status); + status.stepLabel = musicGenerationFullyReady ? "AI tools are ready." : "Stem separation is ready."; status.stepIndex = 0; status.stepCount = 0; status.bytesDownloaded = 0; @@ -522,9 +675,19 @@ StemSeparator::AiToolsStatus StemSeparator::buildAiToolsStatus (const juce::File status.statusWarningCode.clear(); status.terminalReason.clear(); status.lastPhase = "ready"; - status.message = "AI tools are ready."; + status.message = musicGenerationFullyReady + ? "AI tools are ready." + : (! hasNativeMusicProfile(status) + ? "Stem separation is ready, but the Native XL Turbo ACE-Step profile is still missing required music-generation assets." + : (status.musicGenerationPerformanceStatusMessage.isNotEmpty() + ? status.musicGenerationPerformanceStatusMessage + : (status.musicGenerationStatusMessage.isNotEmpty() + ? status.musicGenerationStatusMessage + : "Stem separation is ready, but music generation still needs a compatible ACE-Step runtime bridge."))); status.activityLines.clear(); - status.activityLines.add(status.message); + status.activityLines.add(status.stepLabel); + if (status.message != status.stepLabel) + status.activityLines.add(status.message); return status; } @@ -570,7 +733,7 @@ StemSeparator::AiToolsStatus StemSeparator::buildAiToolsStatus (const juce::File { status.state = "pythonMissing"; status.progress = 0.0f; - status.message = "Python 3.10 through 3.13 is required for this dev build before AI tools can be installed."; + status.message = kSupportedExternalPythonRange + juce::String(" is required for this dev build before AI tools can be installed."); status.error.clear(); status.errorCode = "python_missing"; status.lastPhase = "pythonMissing"; @@ -628,6 +791,11 @@ StemSeparator::AiToolsStatus StemSeparator::buildInitialAiToolsStatus() const status.supportedBackends.add("cpu"); status.selectedBackend = "cpu"; status.modelVersion = kStemModelName; + status.musicGenerationModelId = kPinnedMusicGenerationModelId; + status.musicGenerationModelRepoId = kPinnedMusicGenerationModelRepoId; + status.musicGenerationSharedRepoId = kPinnedMusicGenerationSharedRepoId; + status.musicGenerationCheckpointRoot = getMusicGenerationCheckpointRoot().getFullPathName(); + status.musicGenerationLayoutValid = false; status.fallbackAttempted = false; return status; } @@ -669,6 +837,14 @@ StemSeparator::AiToolsStatus StemSeparator::getCachedAiToolsStatusSnapshot() con status.selectedBackend = "cpu"; if (status.modelVersion.isEmpty()) status.modelVersion = kStemModelName; + if (status.musicGenerationModelId.isEmpty()) + status.musicGenerationModelId = kPinnedMusicGenerationModelId; + if (status.musicGenerationModelRepoId.isEmpty()) + status.musicGenerationModelRepoId = kPinnedMusicGenerationModelRepoId; + if (status.musicGenerationSharedRepoId.isEmpty()) + status.musicGenerationSharedRepoId = kPinnedMusicGenerationSharedRepoId; + if (status.musicGenerationCheckpointRoot.isEmpty()) + status.musicGenerationCheckpointRoot = getMusicGenerationCheckpointRoot().getFullPathName(); return status; } @@ -758,6 +934,26 @@ void StemSeparator::scheduleStatusRefresh() refreshedStatus.runtimeVersion = runtimeCapabilities.runtimeVersion; refreshedStatus.modelVersion = runtimeCapabilities.modelVersion; refreshedStatus.restartRequired = runtimeCapabilities.restartRequired; + refreshedStatus.aceStepVersion = runtimeCapabilities.aceStepVersion; + refreshedStatus.musicGenerationReady = runtimeCapabilities.musicGenerationReady; + refreshedStatus.musicGenerationLayoutValid = runtimeCapabilities.musicGenerationLayoutValid; + refreshedStatus.musicGenerationModelId = runtimeCapabilities.musicGenerationModelId; + refreshedStatus.musicGenerationModelRepoId = runtimeCapabilities.musicGenerationModelRepoId; + refreshedStatus.musicGenerationSharedRepoId = runtimeCapabilities.musicGenerationSharedRepoId; + refreshedStatus.musicGenerationCheckpointRoot = runtimeCapabilities.musicGenerationCheckpointRoot; + refreshedStatus.musicGenerationPerformanceReady = runtimeCapabilities.musicGenerationPerformanceReady; + refreshedStatus.musicGenerationPerformanceStatusMessage = runtimeCapabilities.musicGenerationPerformanceStatusMessage; + refreshedStatus.musicGenerationRuntimeProfiles = runtimeCapabilities.musicGenerationRuntimeProfiles; + refreshedStatus.musicGenerationAvailableProfiles = runtimeCapabilities.musicGenerationAvailableProfiles; + refreshedStatus.musicGenerationUnavailableProfiles = runtimeCapabilities.musicGenerationUnavailableProfiles; + refreshedStatus.musicGenerationDefaultProfile = runtimeCapabilities.musicGenerationDefaultProfile; + refreshedStatus.musicGenerationWarmSessionCapable = runtimeCapabilities.musicGenerationWarmSessionCapable; + const auto musicGenerationFullyReady = refreshedStatus.musicGenerationReady + && refreshedStatus.musicGenerationLayoutValid + && refreshedStatus.musicGenerationPerformanceReady + && hasNativeMusicProfile(refreshedStatus); + + const auto isReconciliationRefresh = previousStatus.statusWarningCode == "reconciling_install_state"; if (isInstallerTerminalFailureCode(previousStatus.errorCode)) { @@ -777,7 +973,7 @@ void StemSeparator::scheduleStatusRefresh() })); } - if (previousStatus.state == "error" && refreshedStatus.available) + if (previousStatus.state == "error" && refreshedStatus.available && musicGenerationFullyReady) { appendAiToolsLogLine(makeAiLogEvent("host", "refresh", @@ -791,7 +987,8 @@ void StemSeparator::scheduleStatusRefresh() })); } - if (! previousStatus.available && refreshedStatus.available && previousStatus.installSessionId.isNotEmpty()) + if (! previousStatus.available && refreshedStatus.available && musicGenerationFullyReady + && previousStatus.installSessionId.isNotEmpty()) { appendAiToolsLogLine(makeAiLogEvent("host", "refresh", @@ -805,6 +1002,40 @@ void StemSeparator::scheduleStatusRefresh() })); } + if (isReconciliationRefresh && ! refreshedStatus.available) + { + appendAiToolsLogLine(makeAiLogEvent("host", + "refresh", + "installer_reconciliation_failed", + previousStatus.installSessionId, + [&] (juce::DynamicObject& obj) + { + obj.setProperty("runtimeCandidate", previousStatus.runtimeCandidate); + obj.setProperty("previousErrorCode", previousStatus.errorCode); + obj.setProperty("refreshedState", refreshedStatus.state); + obj.setProperty("runtimeInstalled", refreshedStatus.runtimeInstalled); + obj.setProperty("modelInstalled", refreshedStatus.modelInstalled); + })); + + refreshedStatus.state = "error"; + refreshedStatus.progress = 0.0f; + refreshedStatus.available = false; + refreshedStatus.installInProgress = false; + refreshedStatus.errorCode = previousStatus.errorCode.isNotEmpty() + ? previousStatus.errorCode + : juce::String("installer_exited_incomplete"); + refreshedStatus.message = "OpenStudio could not confirm that AI tools finished installing."; + refreshedStatus.error = previousStatus.error.isNotEmpty() + ? previousStatus.error + : juce::String("The AI tools installer exited before it reported a ready state."); + refreshedStatus.lastPhase = previousStatus.lastPhase.isNotEmpty() + ? previousStatus.lastPhase + : refreshedStatus.lastPhase; + refreshedStatus.terminalReason = refreshedStatus.errorCode; + refreshedStatus.statusWarning.clear(); + refreshedStatus.statusWarningCode.clear(); + } + { const juce::ScopedLock lock (aiToolsStatusLock); lastAiToolsStatus = refreshedStatus; @@ -1011,6 +1242,48 @@ bool StemSeparator::extractRuntimeArchive (const juce::File& archiveFile, return false; } + #if JUCE_MAC || JUCE_LINUX + { + // juce::ZipFile::uncompressTo does not restore Unix execute bits from the zip's file + // attributes. On Unix-like platforms this means the extracted Python binary has no +x permission and + // posix_spawn() refuses to run it (EACCES), causing probeRuntimeCapabilities() to + // return all-false defaults. Restore +x on every regular file under bin/ and lib/, + // and on the Python binary itself (which may live elsewhere in the archive layout). + auto restoreExecuteBit = [] (const juce::File& dir) + { + if (! dir.isDirectory()) + return; + for (const auto& entry : juce::RangedDirectoryIterator (dir, false, "*", juce::File::findFiles)) + { + const auto path = entry.getFile().getFullPathName(); + struct ::stat st {}; + if (::stat (path.toUTF8(), &st) == 0) + ::chmod (path.toUTF8(), st.st_mode | S_IXUSR | S_IXGRP | S_IXOTH); + } + }; + restoreExecuteBit (destinationRoot.getChildFile ("bin")); + restoreExecuteBit (destinationRoot.getChildFile ("lib")); + + const auto py = findPythonInRuntimeRoot (destinationRoot); + if (py.existsAsFile()) + { + struct ::stat st {}; + if (::stat (py.getFullPathName().toUTF8(), &st) == 0) + ::chmod (py.getFullPathName().toUTF8(), st.st_mode | S_IXUSR | S_IXGRP | S_IXOTH); + } + + #if JUCE_MAC + // macOS attaches com.apple.quarantine to all files extracted from a downloaded zip. + // Gatekeeper blocks execution of quarantined binaries. Strip the attribute recursively. + // xattr is a system tool always present on macOS; failure here is non-fatal because + // the chmod step above is the primary fix. + juce::ChildProcess xattr; + xattr.start ("xattr -rd com.apple.quarantine " + quoteCommandPart (destinationRoot.getFullPathName())); + xattr.waitForProcessToFinish (10000); + #endif + } + #endif + extractionRoot.deleteRecursively(); return true; } @@ -1024,6 +1297,9 @@ juce::var StemSeparator::aiToolsStatusToVar(const AiToolsStatus& status) juce::Array<juce::var> activityLines; for (const auto& line : status.activityLines) activityLines.add(line); + juce::Array<juce::var> availableProfiles; + for (const auto& profile : status.musicGenerationAvailableProfiles) + availableProfiles.add(profile); obj->setProperty("state", status.state); obj->setProperty("progress", static_cast<double>(status.progress)); obj->setProperty("stepIndex", status.stepIndex); @@ -1056,6 +1332,19 @@ juce::var StemSeparator::aiToolsStatusToVar(const AiToolsStatus& status) obj->setProperty("selectedBackend", status.selectedBackend); obj->setProperty("runtimeVersion", status.runtimeVersion); obj->setProperty("modelVersion", status.modelVersion); + obj->setProperty("aceStepVersion", status.aceStepVersion); + obj->setProperty("musicGenerationModelId", status.musicGenerationModelId); + obj->setProperty("musicGenerationModelRepoId", status.musicGenerationModelRepoId); + obj->setProperty("musicGenerationSharedRepoId", status.musicGenerationSharedRepoId); + obj->setProperty("musicGenerationCheckpointRoot", status.musicGenerationCheckpointRoot); + obj->setProperty("musicGenerationStatusMessage", status.musicGenerationStatusMessage); + obj->setProperty("musicGenerationFailureCode", status.musicGenerationFailureCode); + obj->setProperty("musicGenerationPerformanceStatusMessage", status.musicGenerationPerformanceStatusMessage); + obj->setProperty("musicGenerationRuntimeProfiles", status.musicGenerationRuntimeProfiles); + obj->setProperty("musicGenerationAvailableProfiles", availableProfiles); + obj->setProperty("musicGenerationUnavailableProfiles", status.musicGenerationUnavailableProfiles); + obj->setProperty("musicGenerationDefaultProfile", status.musicGenerationDefaultProfile); + obj->setProperty("musicGenerationWarmSessionCapable", status.musicGenerationWarmSessionCapable); obj->setProperty("verificationMode", status.verificationMode); obj->setProperty("runtimeCandidate", status.runtimeCandidate); obj->setProperty("backendRequested", status.backendRequested); @@ -1064,6 +1353,9 @@ juce::var StemSeparator::aiToolsStatusToVar(const AiToolsStatus& status) obj->setProperty("terminalReason", status.terminalReason); obj->setProperty("fallbackAttempted", status.fallbackAttempted); obj->setProperty("restartRequired", status.restartRequired); + obj->setProperty("musicGenerationReady", status.musicGenerationReady); + obj->setProperty("musicGenerationLayoutValid", status.musicGenerationLayoutValid); + obj->setProperty("musicGenerationPerformanceReady", status.musicGenerationPerformanceReady); return juce::var(obj.release()); } @@ -1088,15 +1380,19 @@ juce::var StemSeparator::refreshAiToolsStatus() return aiToolsStatusToVar(getCachedAiToolsStatusSnapshot()); } -juce::var StemSeparator::installAiTools() +juce::var StemSeparator::installAiTools (bool userConfirmedDownload) { if (! aiToolsInstallWorkInProgress.load()) scheduleStatusRefresh(); auto cachedStatus = getCachedAiToolsStatusSnapshot(); + const bool musicGenerationFullyReady = cachedStatus.musicGenerationReady + && cachedStatus.musicGenerationLayoutValid + && cachedStatus.musicGenerationPerformanceReady + && hasNativeMusicProfile(cachedStatus); auto result = std::make_unique<juce::DynamicObject>(); - if (cachedStatus.available) + if (cachedStatus.available && musicGenerationFullyReady) { result->setProperty("started", false); result->setProperty("message", "AI tools are already installed."); @@ -1112,6 +1408,15 @@ juce::var StemSeparator::installAiTools() return juce::var(result.release()); } + if (! userConfirmedDownload) + { + result->setProperty("started", false); + result->setProperty("error", "AI tools setup requires explicit confirmation before downloading runtime or model files."); + result->setProperty("message", "Open the AI Tools setup window and confirm the download to continue."); + result->setProperty("status", aiToolsStatusToVar(cachedStatus)); + return juce::var(result.release()); + } + const auto devFallbackEnabled = isExternalPythonFallbackEnabled(); aiToolsCancelRequested = false; aiToolsInstallWorkInProgress = true; @@ -1142,6 +1447,7 @@ juce::var StemSeparator::installAiTools() const auto logFile = getAiToolsInstallLogFile(); const auto runtimeRoot = getUserRuntimeRoot(); const auto modelsDir = getUserModelsDir(); + const auto musicGenerationCheckpointRoot = getMusicGenerationCheckpointRoot(); const auto downloadsDir = getAiRuntimeDownloadsDir(); const auto manifestUrl = getAiRuntimeManifestUrl().trim(); const auto sessionId = juce::Uuid().toString(); @@ -1269,6 +1575,7 @@ juce::var StemSeparator::installAiTools() appendAiToolsLogLine("systemPython=" + systemPython.getFullPathName()); appendAiToolsLogLine("runtimeRoot=" + runtimeRoot.getFullPathName()); appendAiToolsLogLine("modelsDir=" + modelsDir.getFullPathName()); + appendAiToolsLogLine("musicGenerationCheckpointRoot=" + musicGenerationCheckpointRoot.getFullPathName()); if (! installerScript.existsAsFile()) { @@ -1293,7 +1600,7 @@ juce::var StemSeparator::installAiTools() if (! systemPython.existsAsFile()) { finishWithStatus("pythonMissing", 0.0f, - "Python 3.10 through 3.13 is required for this dev build before AI Tools can be installed.", + juce::String(kSupportedExternalPythonRange) + " is required for this dev build before AI Tools can be installed.", {}, "python_missing", false, @@ -1487,8 +1794,48 @@ juce::var StemSeparator::installAiTools() runtimeCandidates.add(buildCandidateFromNode("windows-legacy", "Windows runtime", "Using legacy flat Windows runtime manifest entry.", platformNode)); } #else - if (! platformNode.isVoid() && ! platformNode.isUndefined()) - runtimeCandidates.add(buildCandidateFromNode(getAiRuntimePlatformKey(), getAiRuntimePlatformKey() + " runtime", "Using platform runtime manifest entry.", platformNode)); + if (auto* linuxPlatformObject = platformNode.getDynamicObject()) + { + const auto architectureKey = getAiRuntimeArchitectureKey(); + const auto architectureNode = linuxPlatformObject->getProperty(architectureKey); + if (! architectureNode.isVoid() && ! architectureNode.isUndefined()) + { + appendAiToolsLogLine("runtimeArchitecture=" + architectureKey); + runtimeCandidates.add(buildCandidateFromNode("linux-" + architectureKey, + "Linux " + architectureKey, + "Selected by current Linux architecture.", + architectureNode)); + } + else if (getPropertyString(platformNode, "url").isNotEmpty()) + { + runtimeCandidates.add(buildCandidateFromNode("linux-legacy", + "Linux runtime", + "Using legacy flat Linux runtime manifest entry.", + platformNode)); + } + else if (! linuxPlatformObject->getProperty("x64").isVoid() + || ! linuxPlatformObject->getProperty("arm64").isVoid()) + { + finishWithStatus("error", 0.05f, + "AI Tools are not available for this Linux machine yet.", + "The published AI runtime metadata does not include a Linux " + architectureKey + " runtime for this release.", + "runtime_platform_unsupported", + false, + false, + false, + false, + kInstallSourceDownloadedRuntime, + kBuildRuntimeModeDownloadedRuntime); + return; + } + } + else if (! platformNode.isVoid() && ! platformNode.isUndefined()) + { + runtimeCandidates.add(buildCandidateFromNode("linux-legacy", + "Linux runtime", + "Using legacy flat Linux runtime manifest entry.", + platformNode)); + } #endif } } @@ -1772,6 +2119,8 @@ juce::var StemSeparator::installAiTools() + " --runtime-root " + quoteCommandPart(runtimeRoot.getFullPathName()) + " --models-dir " + quoteCommandPart(modelsDir.getFullPathName()) + " --model " + quoteCommandPart(kStemModelName) + + " --music-gen-model " + quoteCommandPart(kPinnedMusicGenerationModelId) + + " --music-gen-checkpoint-root " + quoteCommandPart(musicGenerationCheckpointRoot.getFullPathName()) + " --log-path " + quoteCommandPart(logFile.getFullPathName()) + launchMode; if (selectedBackendRequested.isNotEmpty()) @@ -1794,7 +2143,7 @@ juce::var StemSeparator::installAiTools() obj.setProperty("command", cmd); })); - auto nextInstallProcess = std::make_unique<juce::ChildProcess>(); + auto nextInstallProcess = std::make_shared<juce::ChildProcess>(); if (! nextInstallProcess->start(cmd)) { finishWithStatus("error", 0.0f, @@ -1813,6 +2162,7 @@ juce::var StemSeparator::installAiTools() { const juce::ScopedLock lock (aiToolsStatusLock); installOutputBuffer.clear(); + installLogReadOffset = logFile.existsAsFile() ? logFile.getSize() : 0; installDiagnosticLines.clear(); installCommandLine = cmd; installRuntimePythonPath = launcherPython.getFullPathName(); @@ -1825,6 +2175,9 @@ juce::var StemSeparator::installAiTools() installFirstOutputTimeMs = 0.0; installLastOutputTimeMs = 0.0; installLastHeartbeatTimeMs = installLaunchTimeMs; + installLastStructuredStatusTimeMs = 0.0; + installLastDownloadProgressTimeMs = 0.0; + installLastBytesDownloaded = 0; installSawTerminalStatus = false; installFallbackAttempted = fallbackAttempted; installOutputTimeoutLogged = false; @@ -1873,12 +2226,42 @@ juce::var StemSeparator::installAiTools() void StemSeparator::pollInstallProgress() { bool shouldRefreshAfterExit = false; + std::shared_ptr<juce::ChildProcess> process; + juce::int64 logReadOffset = 0; + + { + const juce::ScopedLock lock (aiToolsStatusLock); + process = installProcess; + logReadOffset = installLogReadOffset; + } + + if (! process) + return; + + juce::String newlyReadOutput; + bool sawNewOutput = false; + const auto logFile = getAiToolsInstallLogFile(); + if (logFile.existsAsFile()) + { + const auto logFileSize = logFile.getSize(); + if (logFileSize < logReadOffset) + logReadOffset = 0; + + juce::FileInputStream input(logFile); + if (input.openedOk() && input.setPosition(logReadOffset)) + { + newlyReadOutput = input.readEntireStreamAsString(); + logReadOffset = input.getPosition(); + } + } { const juce::ScopedLock lock (aiToolsStatusLock); - if (! installProcess) + if (installProcess != process) return; + installLogReadOffset = logReadOffset; + auto noteInstallerOutput = [&] { const auto nowMs = juce::Time::getMillisecondCounterHiRes(); @@ -1899,19 +2282,49 @@ void StemSeparator::pollInstallProgress() lastAiToolsStatus.statusWarningCode.clear(); }; - char buffer[4096]; - while (installProcess->isRunning()) + if (newlyReadOutput.isNotEmpty()) { - auto bytesRead = installProcess->readProcessOutput(buffer, sizeof(buffer) - 1); - if (bytesRead <= 0) - break; + installOutputBuffer += newlyReadOutput; - buffer[bytesRead] = '\0'; - installOutputBuffer += juce::String::fromUTF8(buffer, static_cast<int>(bytesRead)); - noteInstallerOutput(); + juce::StringArray addedLines; + addedLines.addLines(newlyReadOutput); + for (const auto& addedLine : addedLines) + { + const auto trimmed = addedLine.trim(); + if (trimmed.isEmpty()) + continue; + + if (! trimmed.startsWith("{")) + { + sawNewOutput = true; + break; + } + + const auto json = juce::JSON::parse(trimmed); + if (! json.isObject()) + continue; + + if (const auto* obj = json.getDynamicObject(); obj != nullptr) + { + if (obj->hasProperty("state")) + { + sawNewOutput = true; + break; + } + + if (obj->getProperty("component").toString() == "installer") + { + sawNewOutput = true; + break; + } + } + } } - if (installProcess->isRunning() + if (sawNewOutput) + noteInstallerOutput(); + + if (process->isRunning() && installFirstOutputTimeMs <= 0.0 && juce::Time::getMillisecondCounterHiRes() - installLaunchTimeMs >= kInstallerOutputTimeoutMs && ! installOutputTimeoutLogged) @@ -1933,23 +2346,38 @@ void StemSeparator::pollInstallProgress() } const auto nowMs = juce::Time::getMillisecondCounterHiRes(); - if (installProcess->isRunning() && nowMs - installLastHeartbeatTimeMs >= kInstallerHeartbeatMs) + if (process->isRunning() && nowMs - installLastHeartbeatTimeMs >= kInstallerHeartbeatMs) { lastAiToolsStatus.elapsedMs = static_cast<juce::int64>(nowMs - installLaunchTimeMs); installLastHeartbeatTimeMs = nowMs; - } - if (! installProcess->isRunning()) - { - for (;;) + if (installFirstOutputTimeMs > 0.0 + && nowMs - installLastOutputTimeMs >= kInstallerOutputTimeoutMs) { - auto bytesRead = installProcess->readProcessOutput(buffer, sizeof(buffer) - 1); - if (bytesRead <= 0) - break; + const auto hasRecentStructuredStatus = installLastStructuredStatusTimeMs > 0.0 + && nowMs - installLastStructuredStatusTimeMs < kInstallerOutputTimeoutMs; + const auto hasRecentDownloadProgress = installLastObservedPhase == "downloading_model" + && installLastDownloadProgressTimeMs > 0.0 + && nowMs - installLastDownloadProgressTimeMs < kInstallerOutputTimeoutMs; - buffer[bytesRead] = '\0'; - installOutputBuffer += juce::String::fromUTF8(buffer, static_cast<int>(bytesRead)); - noteInstallerOutput(); + if (installLastObservedPhase == "downloading_model") + { + if (! hasRecentStructuredStatus && ! hasRecentDownloadProgress) + { + lastAiToolsStatus.statusWarning = "The model download appears stalled. OpenStudio has not seen a fresh heartbeat or byte progress update recently."; + lastAiToolsStatus.statusWarningCode = "model_download_stalled"; + } + else + { + lastAiToolsStatus.statusWarning.clear(); + lastAiToolsStatus.statusWarningCode.clear(); + } + } + else if (! hasRecentStructuredStatus) + { + lastAiToolsStatus.statusWarning = "The AI installer is still working, but the current step has not produced a fresh progress line yet."; + lastAiToolsStatus.statusWarningCode = "installer_silent"; + } } } @@ -1961,42 +2389,74 @@ void StemSeparator::pollInstallProgress() if (line.startsWith("{")) { - lastAiToolsStatus = parseInstallJsonLine(line); - installLastObservedPhase = lastAiToolsStatus.lastPhase.isNotEmpty() ? lastAiToolsStatus.lastPhase - : lastAiToolsStatus.state; - if (isAiToolsTerminalState(lastAiToolsStatus.state)) - installSawTerminalStatus = true; + const auto json = juce::JSON::parse(line); + if (json.isObject()) + { + if (const auto* obj = json.getDynamicObject(); obj != nullptr && obj->hasProperty("state")) + { + const auto statusNowMs = juce::Time::getMillisecondCounterHiRes(); + lastAiToolsStatus = parseInstallJsonLine(line); + installLastObservedPhase = lastAiToolsStatus.lastPhase.isNotEmpty() ? lastAiToolsStatus.lastPhase + : lastAiToolsStatus.state; + installLastStructuredStatusTimeMs = statusNowMs; + if (lastAiToolsStatus.state == "downloading_model" + && lastAiToolsStatus.bytesDownloaded > installLastBytesDownloaded) + { + installLastDownloadProgressTimeMs = statusNowMs; + installLastBytesDownloaded = lastAiToolsStatus.bytesDownloaded; + } + else if (lastAiToolsStatus.state != "downloading_model") + { + installLastBytesDownloaded = 0; + installLastDownloadProgressTimeMs = statusNowMs; + } + if (isAiToolsTerminalState(lastAiToolsStatus.state)) + installSawTerminalStatus = true; + } + } } else if (line.isNotEmpty()) { installDiagnosticLines.add(line); while (installDiagnosticLines.size() > 8) installDiagnosticLines.remove(0); - appendAiToolsLogLine("[installer] " + line); lastAiToolsStatus.activityLines.add(line); while (lastAiToolsStatus.activityLines.size() > 12) lastAiToolsStatus.activityLines.remove(0); } } - if (! installProcess->isRunning()) + if (! process->isRunning()) { - const auto exitCode = installProcess->getExitCode(); + const auto exitCode = process->getExitCode(); const auto finalLine = installOutputBuffer.trim(); if (finalLine.startsWith("{")) { - lastAiToolsStatus = parseInstallJsonLine(finalLine); - installLastObservedPhase = lastAiToolsStatus.lastPhase.isNotEmpty() ? lastAiToolsStatus.lastPhase - : lastAiToolsStatus.state; - if (isAiToolsTerminalState(lastAiToolsStatus.state)) - installSawTerminalStatus = true; + const auto json = juce::JSON::parse(finalLine); + if (json.isObject()) + { + if (const auto* obj = json.getDynamicObject(); obj != nullptr && obj->hasProperty("state")) + { + lastAiToolsStatus = parseInstallJsonLine(finalLine); + installLastObservedPhase = lastAiToolsStatus.lastPhase.isNotEmpty() ? lastAiToolsStatus.lastPhase + : lastAiToolsStatus.state; + installLastStructuredStatusTimeMs = juce::Time::getMillisecondCounterHiRes(); + if (lastAiToolsStatus.state == "downloading_model" + && lastAiToolsStatus.bytesDownloaded > installLastBytesDownloaded) + { + installLastDownloadProgressTimeMs = installLastStructuredStatusTimeMs; + installLastBytesDownloaded = lastAiToolsStatus.bytesDownloaded; + } + if (isAiToolsTerminalState(lastAiToolsStatus.state)) + installSawTerminalStatus = true; + } + } } else if (finalLine.isNotEmpty()) { installDiagnosticLines.add(finalLine); while (installDiagnosticLines.size() > 8) installDiagnosticLines.remove(0); - appendAiToolsLogLine("[installer] " + finalLine); } if (exitCode != 0 && lastAiToolsStatus.state != "error" && lastAiToolsStatus.state != "cancelled") @@ -2039,14 +2499,19 @@ void StemSeparator::pollInstallProgress() obj.setProperty("sawTerminalStatus", installSawTerminalStatus); })); - lastAiToolsStatus.state = "error"; - lastAiToolsStatus.progress = 0.0f; + lastAiToolsStatus.state = "checking"; + lastAiToolsStatus.progress = 0.98f; lastAiToolsStatus.available = false; lastAiToolsStatus.errorCode = "installer_exited_incomplete"; - lastAiToolsStatus.message = "OpenStudio could not confirm that AI tools finished installing."; + lastAiToolsStatus.message = "Verifying whether AI tools finished after a delayed installer response..."; lastAiToolsStatus.error = "The AI tools installer exited before it reported a ready state."; lastAiToolsStatus.lastPhase = installLastObservedPhase.isNotEmpty() ? installLastObservedPhase : juce::String("installer_process"); - lastAiToolsStatus.terminalReason = "installer_exited_incomplete"; + lastAiToolsStatus.terminalReason.clear(); + lastAiToolsStatus.statusWarning = "OpenStudio temporarily lost contact with the AI installer. It is now checking the installed runtime before showing a result."; + lastAiToolsStatus.statusWarningCode = "reconciling_install_state"; + lastAiToolsStatus.activityLines.add(lastAiToolsStatus.message); + while (lastAiToolsStatus.activityLines.size() > 12) + lastAiToolsStatus.activityLines.remove(0); if (lastDiagnostics.isNotEmpty()) lastAiToolsStatus.error += " Last output: " + lastDiagnostics; @@ -2061,6 +2526,7 @@ void StemSeparator::pollInstallProgress() aiToolsInstallWorkInProgress = false; installProcess.reset(); installOutputBuffer.clear(); + installLogReadOffset = 0; installDiagnosticLines.clear(); installCommandLine.clear(); installRuntimePythonPath.clear(); @@ -2073,6 +2539,9 @@ void StemSeparator::pollInstallProgress() installFirstOutputTimeMs = 0.0; installLastOutputTimeMs = 0.0; installLastHeartbeatTimeMs = 0.0; + installLastStructuredStatusTimeMs = 0.0; + installLastDownloadProgressTimeMs = 0.0; + installLastBytesDownloaded = 0; installSawTerminalStatus = false; installFallbackAttempted = false; installOutputTimeoutLogged = false; @@ -2150,6 +2619,32 @@ StemSeparator::AiToolsStatus StemSeparator::parseInstallJsonLine(const juce::Str status.runtimeVersion = json["runtimeVersion"].toString(); if (json.hasProperty("modelVersion")) status.modelVersion = json["modelVersion"].toString(); + if (json.hasProperty("aceStepVersion")) + status.aceStepVersion = json["aceStepVersion"].toString(); + if (json.hasProperty("musicGenerationModelId")) + status.musicGenerationModelId = json["musicGenerationModelId"].toString(); + if (json.hasProperty("musicGenerationModelRepoId")) + status.musicGenerationModelRepoId = json["musicGenerationModelRepoId"].toString(); + if (json.hasProperty("musicGenerationSharedRepoId")) + status.musicGenerationSharedRepoId = json["musicGenerationSharedRepoId"].toString(); + if (json.hasProperty("musicGenerationCheckpointRoot")) + status.musicGenerationCheckpointRoot = json["musicGenerationCheckpointRoot"].toString(); + if (json.hasProperty("musicGenerationStatusMessage")) + status.musicGenerationStatusMessage = json["musicGenerationStatusMessage"].toString(); + if (json.hasProperty("musicGenerationFailureCode")) + status.musicGenerationFailureCode = json["musicGenerationFailureCode"].toString(); + if (json.hasProperty("musicGenerationPerformanceStatusMessage")) + status.musicGenerationPerformanceStatusMessage = json["musicGenerationPerformanceStatusMessage"].toString(); + if (json.hasProperty("musicGenerationRuntimeProfiles")) + status.musicGenerationRuntimeProfiles = json["musicGenerationRuntimeProfiles"]; + if (json.hasProperty("musicGenerationAvailableProfiles")) + status.musicGenerationAvailableProfiles = varToStringArray(json["musicGenerationAvailableProfiles"]); + if (json.hasProperty("musicGenerationUnavailableProfiles")) + status.musicGenerationUnavailableProfiles = json["musicGenerationUnavailableProfiles"]; + if (json.hasProperty("musicGenerationDefaultProfile")) + status.musicGenerationDefaultProfile = json["musicGenerationDefaultProfile"].toString(); + if (json.hasProperty("musicGenerationWarmSessionCapable")) + status.musicGenerationWarmSessionCapable = static_cast<bool>(json["musicGenerationWarmSessionCapable"]); if (json.hasProperty("verificationMode")) status.verificationMode = json["verificationMode"].toString(); if (json.hasProperty("runtimeCandidate")) @@ -2166,6 +2661,12 @@ StemSeparator::AiToolsStatus StemSeparator::parseInstallJsonLine(const juce::Str status.fallbackAttempted = static_cast<bool>(json["fallbackAttempted"]); if (json.hasProperty("restartRequired")) status.restartRequired = static_cast<bool>(json["restartRequired"]); + if (json.hasProperty("musicGenerationReady")) + status.musicGenerationReady = static_cast<bool>(json["musicGenerationReady"]); + if (json.hasProperty("musicGenerationLayoutValid")) + status.musicGenerationLayoutValid = static_cast<bool>(json["musicGenerationLayoutValid"]); + if (json.hasProperty("musicGenerationPerformanceReady")) + status.musicGenerationPerformanceReady = static_cast<bool>(json["musicGenerationPerformanceReady"]); if (status.lastPhase.isEmpty() && status.state.isNotEmpty()) status.lastPhase = status.state; @@ -2180,6 +2681,14 @@ StemSeparator::AiToolsStatus StemSeparator::parseInstallJsonLine(const juce::Str if (status.buildRuntimeMode.isEmpty()) status.buildRuntimeMode = isExternalPythonFallbackEnabled() ? kBuildRuntimeModeUnbundledDev : kBuildRuntimeModeDownloadedRuntime; + if (status.musicGenerationModelId.isEmpty()) + status.musicGenerationModelId = kPinnedMusicGenerationModelId; + if (status.musicGenerationModelRepoId.isEmpty()) + status.musicGenerationModelRepoId = kPinnedMusicGenerationModelRepoId; + if (status.musicGenerationSharedRepoId.isEmpty()) + status.musicGenerationSharedRepoId = kPinnedMusicGenerationSharedRepoId; + if (status.musicGenerationCheckpointRoot.isEmpty()) + status.musicGenerationCheckpointRoot = getMusicGenerationCheckpointRoot().getFullPathName(); return status; } @@ -2205,7 +2714,7 @@ bool StemSeparator::startSeparation(const juce::File& inputFile, { juce::String errorMessage = status.message; if (status.state == "pythonMissing") - errorMessage = "Python 3.10 through 3.13 is required before installing AI Tools in this dev build."; + errorMessage = juce::String(kSupportedExternalPythonRange) + " is required before installing AI Tools in this dev build."; else if (status.state == "runtimeMissing" || status.state == "modelMissing") errorMessage = "Install AI Tools before using stem separation."; else if (status.state == "error" && status.error.isNotEmpty()) @@ -2380,6 +2889,7 @@ void StemSeparator::cancelAiToolsInstall() } installProcess.reset(); installOutputBuffer.clear(); + installLogReadOffset = 0; installDiagnosticLines.clear(); installCommandLine.clear(); installRuntimePythonPath.clear(); @@ -2392,6 +2902,9 @@ void StemSeparator::cancelAiToolsInstall() installFirstOutputTimeMs = 0.0; installLastOutputTimeMs = 0.0; installLastHeartbeatTimeMs = 0.0; + installLastStructuredStatusTimeMs = 0.0; + installLastDownloadProgressTimeMs = 0.0; + installLastBytesDownloaded = 0; installSawTerminalStatus = false; installFallbackAttempted = false; installOutputTimeoutLogged = false; @@ -2421,6 +2934,76 @@ void StemSeparator::cancelAiToolsInstall() }); } +juce::var StemSeparator::resetAiTools() +{ + cancelAiToolsInstall(); + cancel(); + + const auto runtimeRoot = getUserRuntimeRoot(); + const auto modelsDir = getUserModelsDir(); + const auto checkpointRoot = getMusicGenerationCheckpointRoot(); + const auto hubCacheDir = checkpointRoot.getParentDirectory().getChildFile(".hub-cache"); + const auto logFile = getAiToolsInstallLogFile(); + const auto downloadsDir = getAiRuntimeDownloadsDir(); + + juce::StringArray deletedPaths; + juce::StringArray failedPaths; + + const auto deletePath = [&] (const juce::File& path) + { + if (! path.exists()) + return; + + const auto deleted = path.isDirectory() ? path.deleteRecursively() : path.deleteFile(); + if (deleted) + deletedPaths.add(path.getFullPathName()); + else + failedPaths.add(path.getFullPathName()); + }; + + deletePath(runtimeRoot); + deletePath(modelsDir); + deletePath(checkpointRoot); + deletePath(hubCacheDir); + deletePath(downloadsDir); + deletePath(logFile); + + const auto logsDir = logFile.getParentDirectory(); + if (logsDir.isDirectory() && logsDir.findChildFiles(juce::File::findFilesAndDirectories, false).isEmpty()) + logsDir.deleteRecursively(); + + const auto checkpointParent = checkpointRoot.getParentDirectory(); + if (checkpointParent.isDirectory() && checkpointParent.findChildFiles(juce::File::findFilesAndDirectories, false).isEmpty()) + checkpointParent.deleteRecursively(); + + { + const juce::ScopedLock lock (aiToolsStatusLock); + lastAiToolsStatus = buildInitialAiToolsStatus(); + initialStatusPrepared = true; + } + + scheduleStatusRefresh(); + + auto result = std::make_unique<juce::DynamicObject>(); + const auto success = failedPaths.isEmpty(); + result->setProperty("success", success); + result->setProperty( + "message", + success + ? "AI tools files were removed. You can install from a clean state now." + : "Some AI tools files could not be removed. Close any processes using them and retry."); + if (! success) + result->setProperty("error", "Failed to delete: " + failedPaths.joinIntoString(", ")); + result->setProperty("status", aiToolsStatusToVar(getCachedAiToolsStatusSnapshot())); + + juce::Array<juce::var> deletedVars; + for (const auto& path : deletedPaths) + deletedVars.add(path); + result->setProperty("deletedPaths", juce::var(deletedVars)); + + return juce::var(result.release()); +} + bool StemSeparator::isRunning() const { return childProcess && childProcess->isRunning(); diff --git a/Source/StemSeparator.h b/Source/StemSeparator.h index f6e9fc3..037f09b 100644 --- a/Source/StemSeparator.h +++ b/Source/StemSeparator.h @@ -55,6 +55,19 @@ class StemSeparator juce::String selectedBackend { "cpu" }; juce::String runtimeVersion; juce::String modelVersion { "BS-Roformer-SW.ckpt" }; + juce::String aceStepVersion; + juce::String musicGenerationModelId; + juce::String musicGenerationModelRepoId; + juce::String musicGenerationSharedRepoId; + juce::String musicGenerationCheckpointRoot; + juce::String musicGenerationStatusMessage; + juce::String musicGenerationFailureCode; + juce::String musicGenerationPerformanceStatusMessage; + juce::var musicGenerationRuntimeProfiles; + juce::StringArray musicGenerationAvailableProfiles; + juce::var musicGenerationUnavailableProfiles; + juce::String musicGenerationDefaultProfile; + bool musicGenerationWarmSessionCapable = false; juce::String verificationMode; juce::String runtimeCandidate; juce::String backendRequested; @@ -63,6 +76,9 @@ class StemSeparator juce::String terminalReason; bool fallbackAttempted = false; bool restartRequired = false; + bool musicGenerationReady = false; + bool musicGenerationLayoutValid = false; + bool musicGenerationPerformanceReady = true; }; /** Check if Python environment and audio-separator are available. */ @@ -74,8 +90,11 @@ class StemSeparator /** Schedule a background AI tools status refresh and return the current cached status. */ juce::var refreshAiToolsStatus(); - /** Start installing the optional AI tools into the user's app-data directory. */ - juce::var installAiTools(); + /** Start installing the optional AI tools into the user's app-data directory after explicit user confirmation. */ + juce::var installAiTools (bool userConfirmedDownload); + + /** Remove installed AI runtime files, local models, and pinned ACE-Step caches. */ + juce::var resetAiTools(); /** Cancel a running AI tools installation. */ void cancelAiToolsInstall(); @@ -137,6 +156,9 @@ class StemSeparator /** Get the preferred models directory under user app-data. */ juce::File getUserModelsDir() const; + /** Get the pinned ACE-Step checkpoint root used for music generation. */ + juce::File getMusicGenerationCheckpointRoot() const; + /** Find the prepared user-runtime Python executable. */ juce::File findPython() const; @@ -189,6 +211,22 @@ class StemSeparator juce::String runtimeVersion; juce::String modelVersion { "BS-Roformer-SW.ckpt" }; bool restartRequired = false; + juce::String aceStepVersion; + bool musicGenerationReady = false; + bool musicGenerationLayoutValid = false; + juce::String musicGenerationModelId; + juce::String musicGenerationModelRepoId; + juce::String musicGenerationSharedRepoId; + juce::String musicGenerationCheckpointRoot; + juce::String musicGenerationStatusMessage; + juce::String musicGenerationFailureCode; + bool musicGenerationPerformanceReady = true; + juce::String musicGenerationPerformanceStatusMessage; + juce::var musicGenerationRuntimeProfiles; + juce::StringArray musicGenerationAvailableProfiles; + juce::var musicGenerationUnavailableProfiles; + juce::String musicGenerationDefaultProfile; + bool musicGenerationWarmSessionCapable = false; }; /** Probe runtime capabilities via the helper script. */ @@ -256,9 +294,10 @@ class StemSeparator static juce::var aiToolsStatusToVar (const AiToolsStatus& status); std::unique_ptr<juce::ChildProcess> childProcess; - std::unique_ptr<juce::ChildProcess> installProcess; + std::shared_ptr<juce::ChildProcess> installProcess; juce::String outputBuffer; // Accumulated stdout from child juce::String installOutputBuffer; + juce::int64 installLogReadOffset = 0; juce::StringArray installDiagnosticLines; juce::String installCommandLine; juce::String installRuntimePythonPath; @@ -271,6 +310,9 @@ class StemSeparator double installFirstOutputTimeMs = 0.0; double installLastOutputTimeMs = 0.0; double installLastHeartbeatTimeMs = 0.0; + double installLastStructuredStatusTimeMs = 0.0; + double installLastDownloadProgressTimeMs = 0.0; + juce::int64 installLastBytesDownloaded = 0; bool installSawTerminalStatus = false; bool installFallbackAttempted = false; bool installOutputTimeoutLogged = false; diff --git a/Source/TrackProcessor.cpp b/Source/TrackProcessor.cpp index afe5b81..bef65d6 100644 --- a/Source/TrackProcessor.cpp +++ b/Source/TrackProcessor.cpp @@ -99,11 +99,51 @@ static void normalizeMonoLikeBufferToDualMono(juce::AudioBuffer<float>& buffer, buffer.copyFrom(0, 0, buffer, 1, 0, numSamples); } +static float backendWidthToPercent(float backendWidth) +{ + return juce::jlimit(0.0f, 200.0f, (juce::jlimit(-1.0f, 1.0f, backendWidth) + 1.0f) * 100.0f); +} + +static float widthPercentToBackend(float widthPercent) +{ + return juce::jlimit(-1.0f, 1.0f, (juce::jlimit(0.0f, 200.0f, widthPercent) / 100.0f) - 1.0f); +} + +static void applyStereoWidthToBuffer(juce::AudioBuffer<float>& buffer, + int bufferChannels, + int numSamples, + float widthPercent) +{ + if (bufferChannels < 2 || std::abs(widthPercent - 100.0f) <= 0.01f) + return; + + const float widthFactor = widthPercent / 100.0f; + float* left = buffer.getWritePointer(0); + float* right = buffer.getWritePointer(1); + + for (int sample = 0; sample < numSamples; ++sample) + { + const float mid = (left[sample] + right[sample]) * 0.5f; + const float side = (left[sample] - right[sample]) * 0.5f; + left[sample] = mid + side * widthFactor; + right[sample] = mid - side * widthFactor; + } +} + TrackProcessor::TrackProcessor() : AudioProcessor (BusesProperties() .withInput ("Input", juce::AudioChannelSet::stereo(), true) .withOutput ("Output", juce::AudioChannelSet::stereo(), true)) { + widthAutomation.setDefaultValue(widthPercentToBackend(stereoWidth.load(std::memory_order_relaxed))); + preFXVolumeAutomation.setDefaultValue(0.0f); + preFXPanAutomation.setDefaultValue(0.0f); + preFXWidthAutomation.setDefaultValue(0.0f); + trimVolumeAutomation.setDefaultValue(0.0f); + muteAutomation.setDefaultValue(0.0f); + std::atomic_store_explicit(&pluginAutomationSnapshot, + std::make_shared<const PluginAutomationRouteSnapshot>(), + std::memory_order_release); publishRealtimeStateSnapshots(); } @@ -111,6 +151,188 @@ TrackProcessor::~TrackProcessor() { } +std::optional<std::pair<int, int>> TrackProcessor::parsePluginAutomationParameterId(const juce::String& parameterId) +{ + if (!parameterId.startsWith("plugin_")) + return std::nullopt; + + auto suffix = parameterId.substring(7); + auto parts = juce::StringArray::fromTokens(suffix, "_", ""); + if (parts.size() != 2) + return std::nullopt; + + const int fxIndex = parts[0].getIntValue(); + const int paramIndex = parts[1].getIntValue(); + if (fxIndex < 0 || paramIndex < 0) + return std::nullopt; + + return std::make_pair(fxIndex, paramIndex); +} + +std::shared_ptr<TrackProcessor::PluginAutomationRoute> TrackProcessor::findPluginAutomationRoute(const juce::String& parameterId) const +{ + auto snapshot = std::atomic_load_explicit(&pluginAutomationSnapshot, std::memory_order_acquire); + if (!snapshot) + return nullptr; + + for (const auto& route : *snapshot) + { + if (route && route->parameterId == parameterId) + return route; + } + + return nullptr; +} + +std::shared_ptr<TrackProcessor::PluginAutomationRoute> TrackProcessor::getOrCreatePluginAutomationRoute(const juce::String& parameterId) +{ + if (auto existing = findPluginAutomationRoute(parameterId)) + return existing; + + auto parsed = parsePluginAutomationParameterId(parameterId); + if (!parsed.has_value()) + return nullptr; + + const auto [fxIndex, paramIndex] = *parsed; + auto route = std::make_shared<PluginAutomationRoute>(); + route->parameterId = parameterId; + route->fxIndex = fxIndex; + route->paramIndex = paramIndex; + + const bool hasInputFx = fxIndex >= 0 && fxIndex < getNumInputFX(); + const bool hasTrackFx = fxIndex >= 0 && fxIndex < getNumTrackFX(); + if (!hasInputFx && !hasTrackFx) + return nullptr; + + // The current frontend lane id does not encode whether a plugin lives in the + // input or track chain, so keep resolution stable once the lane is created. + route->isInputFX = hasInputFx; + + const juce::ScopedLock sl(pluginAutomationRouteLock); + if (auto existing = findPluginAutomationRoute(parameterId)) + return existing; + + auto snapshot = std::atomic_load_explicit(&pluginAutomationSnapshot, std::memory_order_acquire); + auto nextSnapshot = std::make_shared<PluginAutomationRouteSnapshot>(); + if (snapshot) + *nextSnapshot = *snapshot; + nextSnapshot->push_back(route); + std::atomic_store_explicit(&pluginAutomationSnapshot, + std::static_pointer_cast<const PluginAutomationRouteSnapshot>(nextSnapshot), + std::memory_order_release); + return route; +} + +std::optional<TrackProcessor::AutomationTarget> TrackProcessor::resolveAutomationTarget(const juce::String& parameterId, + bool createIfNeeded) +{ + AutomationTarget target; + if (parameterId == "volume") + { + target.kind = AutomationTarget::Kind::Volume; + target.list = &volumeAutomation; + return target; + } + if (parameterId == "pan") + { + target.kind = AutomationTarget::Kind::Pan; + target.list = &panAutomation; + return target; + } + if (parameterId == "width") + { + target.kind = AutomationTarget::Kind::Width; + target.list = &widthAutomation; + return target; + } + if (parameterId == "volume_prefx") + { + target.kind = AutomationTarget::Kind::PreFXVolume; + target.list = &preFXVolumeAutomation; + return target; + } + if (parameterId == "pan_prefx") + { + target.kind = AutomationTarget::Kind::PreFXPan; + target.list = &preFXPanAutomation; + return target; + } + if (parameterId == "width_prefx") + { + target.kind = AutomationTarget::Kind::PreFXWidth; + target.list = &preFXWidthAutomation; + return target; + } + if (parameterId == "trim_volume") + { + target.kind = AutomationTarget::Kind::TrimVolume; + target.list = &trimVolumeAutomation; + return target; + } + if (parameterId == "mute") + { + target.kind = AutomationTarget::Kind::Mute; + target.list = &muteAutomation; + return target; + } + + auto route = createIfNeeded ? getOrCreatePluginAutomationRoute(parameterId) : findPluginAutomationRoute(parameterId); + if (!route) + return std::nullopt; + + target.kind = AutomationTarget::Kind::PluginParameter; + target.list = route->automation.get(); + target.isInputFX = route->isInputFX; + target.fxIndex = route->fxIndex; + target.paramIndex = route->paramIndex; + return target; +} + +float TrackProcessor::getAutomationDefaultValue(const AutomationTarget& target) const +{ + switch (target.kind) + { + case AutomationTarget::Kind::Volume: + return getVolume(); + case AutomationTarget::Kind::Pan: + return getPan(); + case AutomationTarget::Kind::Width: + return widthPercentToBackend(getStereoWidth()); + case AutomationTarget::Kind::PreFXVolume: + case AutomationTarget::Kind::PreFXPan: + case AutomationTarget::Kind::PreFXWidth: + case AutomationTarget::Kind::TrimVolume: + return 0.0f; + case AutomationTarget::Kind::Mute: + return getMute() ? 1.0f : 0.0f; + case AutomationTarget::Kind::PluginParameter: + { + const juce::AudioProcessor* processor = nullptr; + if (target.isInputFX) + { + if (target.fxIndex >= 0 && target.fxIndex < getNumInputFX()) + processor = getInputFXProcessor(target.fxIndex); + } + else + { + if (target.fxIndex >= 0 && target.fxIndex < getNumTrackFX()) + processor = getTrackFXProcessor(target.fxIndex); + } + + if (processor == nullptr) + return 0.0f; + + const auto& params = processor->getParameters(); + if (target.paramIndex < 0 || target.paramIndex >= params.size() || params[target.paramIndex] == nullptr) + return 0.0f; + + return params[target.paramIndex]->getValue(); + } + default: + return 0.0f; + } +} + void TrackProcessor::publishRealtimeStateSnapshots() { auto inputSnapshot = std::make_shared<const ProcessorSnapshot>(inputFXPlugins.begin(), inputFXPlugins.end()); @@ -134,6 +356,51 @@ void TrackProcessor::publishRealtimeStateSnapshots() std::atomic_store_explicit(&realtimeInstrumentSnapshot, instrumentSnapshot, std::memory_order_release); } +void TrackProcessor::applyPluginAutomationForProcessor(juce::AudioProcessor* proc, + bool isInputFX, + int fxIndex, + double blockTimeSeconds) +{ + if (proc == nullptr) + return; + + auto snapshot = std::atomic_load_explicit(&pluginAutomationSnapshot, std::memory_order_acquire); + if (!snapshot || snapshot->empty()) + return; + + auto& params = proc->getParameters(); + if (params.isEmpty()) + return; + + for (const auto& route : *snapshot) + { + if (!route + || route->isInputFX != isInputFX + || route->fxIndex != fxIndex + || route->automation == nullptr + || route->paramIndex < 0 + || route->paramIndex >= params.size()) + { + continue; + } + + auto* param = params[route->paramIndex]; + if (param == nullptr) + continue; + + const float automatedValue = route->automation->shouldPlayback() + ? route->automation->eval(blockTimeSeconds) + : route->automation->getDefaultValue(); + const float clampedValue = juce::jlimit(0.0f, 1.0f, automatedValue); + const float lastValue = route->lastAppliedValue.load(std::memory_order_relaxed); + if (std::isfinite(lastValue) && std::abs(lastValue - clampedValue) <= 1.0e-6f) + continue; + + param->setValue(clampedValue); + route->lastAppliedValue.store(clampedValue, std::memory_order_relaxed); + } +} + const juce::String TrackProcessor::getName() const { return "Track Processor"; @@ -354,7 +621,13 @@ bool TrackProcessor::tryProcessBlock(juce::AudioBuffer<float>& buffer, juce::Mid void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juce::MidiBuffer& midiMessages) { - const double trackProcessStartMs = juce::Time::getMillisecondCounterHiRes(); + // Only time the track when ARA diagnostics are enabled and an ARA plugin is active. + // QueryPerformanceCounter is cheap but not free — at 32-sample blocks this fires + // 1500×/sec, so we avoid it for non-ARA tracks (e.g. Amplitube, S13 FX). + const bool isARATrack = kEnableARADebugDiagnostics + && araController != nullptr + && araController->isActive(); + const double trackProcessStartMs = isARATrack ? juce::Time::getMillisecondCounterHiRes() : 0.0; juce::ScopedNoDenormals noDenormals; auto totalNumInputChannels = getTotalNumInputChannels(); auto totalNumOutputChannels = getTotalNumOutputChannels(); @@ -369,11 +642,11 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc auto sidechainSnapshot = std::atomic_load_explicit(&realtimeSidechainSnapshot, std::memory_order_acquire); auto sendSnapshot = std::atomic_load_explicit(&realtimeSendSnapshot, std::memory_order_acquire); const bool instrumentForceFloat = instrumentForceFloatOverride.load(std::memory_order_acquire); - const double blockStartTimeSeconds = blockSampleRate > 0.0 ? (blockStartSample / blockSampleRate) : 0.0; + const double blockTimeSeconds = this->blockStartTimeSeconds; if (araController != nullptr && araController->isActive()) araController->updateTransportDebugState(araTransportPlayingDebugState.load(std::memory_order_acquire), - blockStartTimeSeconds); + blockTimeSeconds); // Safety: only clear channels that actually exist in the buffer int bufferChannels = buffer.getNumChannels(); @@ -381,7 +654,7 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc buffer.clear (i, 0, buffer.getNumSamples()); // Apply mute first - if muted, silence and return - if (isMuted.load()) + if (isMuted.load() && !ignoreStaticMuteDuringProcessing.load(std::memory_order_relaxed)) { buffer.clear(); currentRMS = 0.0f; @@ -459,9 +732,12 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc } // Channel-safe FX processing helper - auto safeProcessFX = [&](juce::AudioProcessor* proc, bool forceFloat) + auto safeProcessFX = [&](juce::AudioProcessor* proc, bool forceFloat, bool isInputFXChain, int fxIndex) { - const double envelopeStartMs = juce::Time::getMillisecondCounterHiRes(); + applyPluginAutomationForProcessor(proc, isInputFXChain, fxIndex, blockTimeSeconds); + + // Compute isARAProcessor first so we can gate expensive QPC calls on it. + // For non-ARA plugins (Amplitube, S13 FX, etc.) all timing overhead is skipped. int pluginChannels = juce::jmax(proc->getTotalNumInputChannels(), proc->getTotalNumOutputChannels()); const bool isARAProcessor = araController != nullptr @@ -470,6 +746,7 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc && trackFXSnapshot && araFXIndex < static_cast<int>(trackFXSnapshot->size()) && (*trackFXSnapshot)[static_cast<size_t>(araFXIndex)].get() == proc; + const double envelopeStartMs = isARAProcessor ? juce::Time::getMillisecondCounterHiRes() : 0.0; const bool useDoublePrecision = processingPrecisionMode == ProcessingPrecisionMode::Hybrid64 && !forceFloat @@ -538,10 +815,10 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc if (!useDoublePrecision && pluginChannels == bufferChannels) { - const double processStartMs = juce::Time::getMillisecondCounterHiRes(); - preProcessMs = processStartMs - envelopeStartMs; + const double processStartMs = isARAProcessor ? juce::Time::getMillisecondCounterHiRes() : 0.0; + preProcessMs = isARAProcessor ? (processStartMs - envelopeStartMs) : 0.0; proc->processBlock(buffer, midiMessages); - processDurationMs = juce::Time::getMillisecondCounterHiRes() - processStartMs; + processDurationMs = isARAProcessor ? (juce::Time::getMillisecondCounterHiRes() - processStartMs) : 0.0; // Mono plugin on stereo track: duplicate processed output to all channels // (matches Reaper behaviour — avoids dry right channel when plugin is mono out) int outCh = proc->getTotalNumOutputChannels(); @@ -550,7 +827,7 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc for (int ch = outCh; ch < bufferChannels; ++ch) buffer.copyFrom (ch, 0, buffer, 0, 0, numSamps); } - logARAProcessDuration(juce::Time::getMillisecondCounterHiRes() - (processStartMs + processDurationMs)); + logARAProcessDuration(isARAProcessor ? (juce::Time::getMillisecondCounterHiRes() - (processStartMs + processDurationMs)) : 0.0); } else { @@ -576,10 +853,10 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc channelPtrs[ch] = fxProcessBufferDouble.getWritePointer(ch); juce::AudioBuffer<double> pluginBuffer(channelPtrs, expandedCh, numSamps); - const double processStartMs = juce::Time::getMillisecondCounterHiRes(); - preProcessMs = processStartMs - envelopeStartMs; + const double processStartMs = isARAProcessor ? juce::Time::getMillisecondCounterHiRes() : 0.0; + preProcessMs = isARAProcessor ? (processStartMs - envelopeStartMs) : 0.0; proc->processBlock(pluginBuffer, midiMessages); - processDurationMs = juce::Time::getMillisecondCounterHiRes() - processStartMs; + processDurationMs = isARAProcessor ? (juce::Time::getMillisecondCounterHiRes() - processStartMs) : 0.0; if (expandedCh == 1 && bufferChannels > 1) { @@ -601,7 +878,7 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc dest[sample] = static_cast<float>(src[sample]); } } - logARAProcessDuration(juce::Time::getMillisecondCounterHiRes() - (processStartMs + processDurationMs)); + logARAProcessDuration(isARAProcessor ? (juce::Time::getMillisecondCounterHiRes() - (processStartMs + processDurationMs)) : 0.0); } else { @@ -627,10 +904,10 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc channelPtrs[ch] = fxProcessBuffer.getWritePointer(ch); juce::AudioBuffer<float> pluginBuffer(channelPtrs, expandedCh, numSamps); - const double processStartMs = juce::Time::getMillisecondCounterHiRes(); - preProcessMs = processStartMs - envelopeStartMs; + const double processStartMs = isARAProcessor ? juce::Time::getMillisecondCounterHiRes() : 0.0; + preProcessMs = isARAProcessor ? (processStartMs - envelopeStartMs) : 0.0; proc->processBlock(pluginBuffer, midiMessages); - processDurationMs = juce::Time::getMillisecondCounterHiRes() - processStartMs; + processDurationMs = isARAProcessor ? (juce::Time::getMillisecondCounterHiRes() - processStartMs) : 0.0; if (expandedCh == 1 && bufferChannels > 1) { @@ -642,11 +919,65 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc for (int ch = 0; ch < bufferChannels; ++ch) buffer.copyFrom(ch, 0, pluginBuffer, ch < expandedCh ? ch : 0, 0, numSamps); } - logARAProcessDuration(juce::Time::getMillisecondCounterHiRes() - (processStartMs + processDurationMs)); + logARAProcessDuration(isARAProcessor ? (juce::Time::getMillisecondCounterHiRes() - (processStartMs + processDurationMs)) : 0.0); } } }; + // ===== PRE-FX AUTOMATION ===== + const int numSamps = buffer.getNumSamples(); + const double processingSampleRate = juce::jmax(1.0, getSampleRate()); + const float staticPreFXVolDb = 0.0f; + const float staticPreFXPan = 0.0f; + const float staticPreFXWidth = 100.0f; + const bool preFXVolAutoActive = preFXVolumeAutomation.shouldPlayback() && preFXVolumeAutomation.getNumPoints() > 0; + const bool preFXPanAutoActive = preFXPanAutomation.shouldPlayback() && preFXPanAutomation.getNumPoints() > 0; + const bool preFXWidthAutoActive = preFXWidthAutomation.shouldPlayback() && preFXWidthAutomation.getNumPoints() > 0; + + if (preFXVolAutoActive || preFXPanAutoActive) + { + for (int i = 0; i < numSamps; ++i) + { + const double timeSeconds = blockTimeSeconds + (static_cast<double>(i) / processingSampleRate); + float volDb = preFXVolAutoActive ? preFXVolumeAutomation.eval(timeSeconds) : staticPreFXVolDb; + float pan = preFXPanAutoActive ? preFXPanAutomation.eval(timeSeconds) : staticPreFXPan; + volDb = juce::jlimit(-60.0f, 12.0f, volDb); + pan = juce::jlimit(-1.0f, 1.0f, pan); + + float leftGain = 1.0f; + float rightGain = 1.0f; + computePanLawGains(panLaw, pan, juce::Decibels::decibelsToGain(volDb), leftGain, rightGain); + + if (bufferChannels >= 1) + buffer.setSample(0, i, buffer.getSample(0, i) * leftGain); + if (bufferChannels >= 2) + buffer.setSample(1, i, buffer.getSample(1, i) * rightGain); + } + } + + if (bufferChannels >= 2) + { + if (preFXWidthAutoActive) + { + float* left = buffer.getWritePointer(0); + float* right = buffer.getWritePointer(1); + for (int i = 0; i < numSamps; ++i) + { + const double timeSeconds = blockTimeSeconds + (static_cast<double>(i) / processingSampleRate); + const float widthPercent = backendWidthToPercent(preFXWidthAutomation.eval(timeSeconds)); + const float widthFactor = widthPercent / 100.0f; + const float mid = (left[i] + right[i]) * 0.5f; + const float side = (left[i] - right[i]) * 0.5f; + left[i] = mid + side * widthFactor; + right[i] = mid - side * widthFactor; + } + } + else + { + applyStereoWidthToBuffer(buffer, bufferChannels, numSamps, staticPreFXWidth); + } + } + // Channel strip EQ (processed before plugin FX chains) if (channelStripEQEnabled) { @@ -666,14 +997,14 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc && inputFXPrecisionOverrideSnapshot->count(pluginIndex) > 0 && inputFXPrecisionOverrideSnapshot->at(pluginIndex); if (plugin && !bypassed) - safeProcessFX(plugin.get(), forceFloat); + safeProcessFX(plugin.get(), forceFloat, true, pluginIndex); } } // Instrument processing lives between input FX and track FX so that // instrument output can be post-processed by normal track FX. if (currentTrackType == TrackType::Instrument && instrumentSnapshot) - safeProcessFX(instrumentSnapshot.get(), instrumentForceFloat); + safeProcessFX(instrumentSnapshot.get(), instrumentForceFloat, false, -1); // Process through track FX chain (with sidechain support) for (int fxIdx = 0; trackFXSnapshot && fxIdx < (int)trackFXSnapshot->size(); ++fxIdx) @@ -705,6 +1036,8 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc if (hasSidechain) { + applyPluginAutomationForProcessor(proc, false, fxIdx, blockTimeSeconds); + // Sidechain path: expand buffer to include sidechain channels after // the main stereo channels. The plugin's second input bus receives // the sidechain audio. @@ -817,11 +1150,11 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc else { // No sidechain — use normal channel-safe processing - safeProcessFX(proc, forceFloat); + safeProcessFX(proc, forceFloat, false, fxIdx); } } - const double trackProcessDurationMs = juce::Time::getMillisecondCounterHiRes() - trackProcessStartMs; + const double trackProcessDurationMs = isARATrack ? (juce::Time::getMillisecondCounterHiRes() - trackProcessStartMs) : 0.0; if (kEnableARADebugDiagnostics && trackProcessDurationMs > 10.0 && araController != nullptr @@ -831,7 +1164,7 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc + " callback=" + juce::String(static_cast<juce::int64>(currentARAProcessDebugInfo.callbackCounter)) + " firstCallbackAfterTransportStart=" + juce::String(currentARAProcessDebugInfo.firstCallbackAfterTransportStart ? "true" : "false") + " totalTrackMs=" + juce::String(trackProcessDurationMs, 2) - + " blockStartSeconds=" + juce::String(blockStartTimeSeconds, 3) + + " blockStartSeconds=" + juce::String(blockTimeSeconds, 3) + " numSamples=" + juce::String(buffer.getNumSamples())); } @@ -886,19 +1219,29 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc buffer.getNumSamples()); } - // ===== STEREO WIDTH (M/S processing) ===== - float widthVal = stereoWidth.load(std::memory_order_relaxed); - if (bufferChannels >= 2 && std::abs(widthVal - 100.0f) > 0.01f) + // ===== POST-FX WIDTH ===== + const bool widthAutoActive = widthAutomation.shouldPlayback() && widthAutomation.getNumPoints() > 0; + if (bufferChannels >= 2) { - float w = widthVal / 100.0f; // 0.0 = mono, 1.0 = normal, 2.0 = extra wide - float* L = buffer.getWritePointer(0); - float* R = buffer.getWritePointer(1); - for (int i = 0; i < buffer.getNumSamples(); ++i) + if (widthAutoActive) { - float mid = (L[i] + R[i]) * 0.5f; - float side = (L[i] - R[i]) * 0.5f; - L[i] = mid + side * w; - R[i] = mid - side * w; + float* left = buffer.getWritePointer(0); + float* right = buffer.getWritePointer(1); + for (int i = 0; i < buffer.getNumSamples(); ++i) + { + const double timeSeconds = blockTimeSeconds + (static_cast<double>(i) / processingSampleRate); + const float widthPercent = backendWidthToPercent(widthAutomation.eval(timeSeconds)); + const float widthFactor = widthPercent / 100.0f; + const float mid = (left[i] + right[i]) * 0.5f; + const float side = (left[i] - right[i]) * 0.5f; + left[i] = mid + side * widthFactor; + right[i] = mid - side * widthFactor; + } + } + else + { + applyStereoWidthToBuffer(buffer, bufferChannels, buffer.getNumSamples(), + stereoWidth.load(std::memory_order_relaxed)); } } @@ -916,29 +1259,23 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc preFaderBuffer.copyFrom(ch, 0, buffer, ch, 0, pfSamples); } - // ===== AUTOMATION-AWARE GAIN APPLICATION ===== - int numSamps = buffer.getNumSamples(); + // ===== AUTOMATION-AWARE FADER/TAIL APPLICATION ===== bool volAutoActive = volumeAutomation.shouldPlayback() && volumeAutomation.getNumPoints() > 0; bool panAutoActive = panAutomation.shouldPlayback() && panAutomation.getNumPoints() > 0; + bool trimAutoActive = trimVolumeAutomation.shouldPlayback() && trimVolumeAutomation.getNumPoints() > 0; + bool muteAutoActive = muteAutomation.shouldPlayback() && muteAutomation.getNumPoints() > 0; if (volAutoActive || panAutoActive) { - // Ensure pre-allocated automation buffer is large enough - if (automationGainBuffer.getNumSamples() < numSamps) - automationGainBuffer.setSize(2, numSamps, false, false, true); - - // Evaluate volume automation (dB values) per sample, or use static fader value float staticVolDB = trackVolumeDB.load(std::memory_order_relaxed); float staticPan = trackPan.load(std::memory_order_relaxed); for (int i = 0; i < numSamps; ++i) { - double samplePos = blockStartSample + static_cast<double>(i); - - float volDB = volAutoActive ? volumeAutomation.eval(samplePos) : staticVolDB; - float pan = panAutoActive ? panAutomation.eval(samplePos) : staticPan; + const double timeSeconds = blockTimeSeconds + (static_cast<double>(i) / processingSampleRate); + float volDB = volAutoActive ? volumeAutomation.eval(timeSeconds) : staticVolDB; + float pan = panAutoActive ? panAutomation.eval(timeSeconds) : staticPan; - // Clamp to safe ranges volDB = juce::jlimit(-60.0f, 12.0f, volDB); pan = juce::jlimit(-1.0f, 1.0f, pan); @@ -956,7 +1293,6 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc } else { - // No automation active — use pre-computed cached gains (original fast path) float leftGain = cachedPanL.load(std::memory_order_relaxed); float rightGain = cachedPanR.load(std::memory_order_relaxed); @@ -966,6 +1302,33 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc buffer.applyGain(1, 0, numSamps, rightGain); } + if (trimAutoActive) + { + for (int i = 0; i < numSamps; ++i) + { + const double timeSeconds = blockTimeSeconds + (static_cast<double>(i) / processingSampleRate); + const float trimDb = juce::jlimit(-60.0f, 12.0f, trimVolumeAutomation.eval(timeSeconds)); + const float trimGain = juce::Decibels::decibelsToGain(trimDb); + if (bufferChannels >= 1) + buffer.setSample(0, i, buffer.getSample(0, i) * trimGain); + if (bufferChannels >= 2) + buffer.setSample(1, i, buffer.getSample(1, i) * trimGain); + } + } + + if (muteAutoActive) + { + for (int i = 0; i < numSamps; ++i) + { + const double timeSeconds = blockTimeSeconds + (static_cast<double>(i) / processingSampleRate); + const bool muted = muteAutomation.eval(timeSeconds) > 0.5f; + if (!muted) + continue; + for (int ch = 0; ch < bufferChannels; ++ch) + buffer.setSample(ch, i, 0.0f); + } + } + // ---- REAPER-style peak metering with decimation ---- // getMagnitude() uses FloatVectorOperations::findMinAndMax (SIMD, no sqrt), // which is much cheaper than getRMSLevel() (which computes sqrt per channel). @@ -988,14 +1351,6 @@ void TrackProcessor::processBlockInternal (juce::AudioBuffer<float>& buffer, juc meterSampleCount = 0; } - if (realtimeFallbackBuffer.getNumChannels() < bufferChannels - || realtimeFallbackBuffer.getNumSamples() < buffer.getNumSamples()) - { - realtimeFallbackBuffer.setSize(bufferChannels, buffer.getNumSamples(), false, false, true); - } - - for (int ch = 0; ch < bufferChannels; ++ch) - realtimeFallbackBuffer.copyFrom(ch, 0, buffer, ch, 0, buffer.getNumSamples()); } bool TrackProcessor::hasEditor() const @@ -1231,6 +1586,13 @@ juce::AudioProcessor* TrackProcessor::getInputFXProcessor(int index) return nullptr; } +const juce::AudioProcessor* TrackProcessor::getInputFXProcessor(int index) const +{ + if (index >= 0 && index < (int)inputFXPlugins.size()) + return inputFXPlugins[index].get(); + return nullptr; +} + juce::AudioProcessor* TrackProcessor::getTrackFXProcessor(int index) { if (index >= 0 && index < (int)trackFXPlugins.size()) @@ -1238,6 +1600,13 @@ juce::AudioProcessor* TrackProcessor::getTrackFXProcessor(int index) return nullptr; } +const juce::AudioProcessor* TrackProcessor::getTrackFXProcessor(int index) const +{ + if (index >= 0 && index < (int)trackFXPlugins.size()) + return trackFXPlugins[index].get(); + return nullptr; +} + std::shared_ptr<const std::vector<std::shared_ptr<juce::AudioProcessor>>> TrackProcessor::getInputFXSnapshot() const { return std::atomic_load_explicit(&realtimeInputFXSnapshot, std::memory_order_acquire); @@ -1613,14 +1982,14 @@ void TrackProcessor::markActiveMIDINoteState(const juce::MidiMessage& message) } void TrackProcessor::appendScheduledMIDIToBuffer(juce::MidiBuffer& destination, - double blockStartTimeSeconds, + double blockTimeSeconds, int numSamples, double sampleRate) const { auto clips = std::atomic_load_explicit(&scheduledMIDIClips, std::memory_order_acquire); if (!clips || clips->empty() || sampleRate <= 0.0) return; - const double blockEndTimeSeconds = blockStartTimeSeconds + (static_cast<double>(numSamples) / sampleRate); + const double blockEndTimeSeconds = blockTimeSeconds + (static_cast<double>(numSamples) / sampleRate); for (const auto& clip : *clips) { @@ -1628,16 +1997,16 @@ void TrackProcessor::appendScheduledMIDIToBuffer(juce::MidiBuffer& destination, continue; const double clipEndTime = clip.startTime + clip.duration; - if (clipEndTime <= blockStartTimeSeconds || clip.startTime >= blockEndTimeSeconds) + if (clipEndTime <= blockTimeSeconds || clip.startTime >= blockEndTimeSeconds) continue; for (const auto& event : clip.events) { const double absoluteEventTime = clip.startTime + event.timestampSeconds; - if (absoluteEventTime < blockStartTimeSeconds || absoluteEventTime >= blockEndTimeSeconds) + if (absoluteEventTime < blockTimeSeconds || absoluteEventTime >= blockEndTimeSeconds) continue; - int sampleOffset = static_cast<int>(std::floor((absoluteEventTime - blockStartTimeSeconds) * sampleRate)); + int sampleOffset = static_cast<int>(std::floor((absoluteEventTime - blockTimeSeconds) * sampleRate)); sampleOffset = juce::jlimit(0, juce::jmax(0, numSamples - 1), sampleOffset); destination.addEvent(event.message, sampleOffset); } @@ -1665,26 +2034,26 @@ bool TrackProcessor::hasQueuedMIDI() const return midiQueueReadIndex.load(std::memory_order_acquire) != midiQueueWriteIndex.load(std::memory_order_acquire); } -bool TrackProcessor::hasScheduledMIDIInBlock(double blockStartTimeSeconds, int numSamples, double sampleRate) const +bool TrackProcessor::hasScheduledMIDIInBlock(double blockTimeSeconds, int numSamples, double sampleRate) const { auto clips = std::atomic_load_explicit(&scheduledMIDIClips, std::memory_order_acquire); if (!clips || clips->empty() || sampleRate <= 0.0) return false; - const double blockEndTimeSeconds = blockStartTimeSeconds + (static_cast<double>(numSamples) / sampleRate); + const double blockEndTimeSeconds = blockTimeSeconds + (static_cast<double>(numSamples) / sampleRate); for (const auto& clip : *clips) { if (clip.events.empty()) continue; const double clipEndTime = clip.startTime + clip.duration; - if (clipEndTime <= blockStartTimeSeconds || clip.startTime >= blockEndTimeSeconds) + if (clipEndTime <= blockTimeSeconds || clip.startTime >= blockEndTimeSeconds) continue; for (const auto& event : clip.events) { const double absoluteEventTime = clip.startTime + event.timestampSeconds; - if (absoluteEventTime >= blockStartTimeSeconds && absoluteEventTime < blockEndTimeSeconds) + if (absoluteEventTime >= blockTimeSeconds && absoluteEventTime < blockEndTimeSeconds) return true; } } @@ -1692,13 +2061,13 @@ bool TrackProcessor::hasScheduledMIDIInBlock(double blockStartTimeSeconds, int n return false; } -void TrackProcessor::buildMidiBuffer(juce::MidiBuffer& destination, double blockStartTimeSeconds, +void TrackProcessor::buildMidiBuffer(juce::MidiBuffer& destination, double blockTimeSeconds, int numSamples, double sampleRate, bool playing) { destination.clear(); if (playing) - appendScheduledMIDIToBuffer(destination, blockStartTimeSeconds, numSamples, sampleRate); + appendScheduledMIDIToBuffer(destination, blockTimeSeconds, numSamples, sampleRate); appendQueuedMIDIToBuffer(destination, numSamples); @@ -1714,7 +2083,7 @@ void TrackProcessor::buildMidiBuffer(juce::MidiBuffer& destination, double block markActiveMIDINoteState(metadata.getMessage()); } -bool TrackProcessor::needsProcessing(double blockStartTimeSeconds, int numSamples, +bool TrackProcessor::needsProcessing(double blockTimeSeconds, int numSamples, double sampleRate, bool playing) const { // Instrument tracks must always be processed so they can respond to @@ -1726,7 +2095,7 @@ bool TrackProcessor::needsProcessing(double blockStartTimeSeconds, int numSample if (hasQueuedMIDI()) return true; - if (playing && hasScheduledMIDIInBlock(blockStartTimeSeconds, numSamples, sampleRate)) + if (playing && hasScheduledMIDIInBlock(blockTimeSeconds, numSamples, sampleRate)) return true; return false; diff --git a/Source/TrackProcessor.h b/Source/TrackProcessor.h index 9d2db0e..9c56aab 100644 --- a/Source/TrackProcessor.h +++ b/Source/TrackProcessor.h @@ -6,6 +6,8 @@ #include "ARAHostController.h" #include <map> #include <array> +#include <limits> +#include <optional> // Track type enumeration enum class TrackType @@ -64,6 +66,28 @@ class TrackProcessor : public juce::AudioProcessor bool phaseInvert = false; }; + struct AutomationTarget + { + enum class Kind + { + Volume, + Pan, + Width, + PreFXVolume, + PreFXPan, + PreFXWidth, + TrimVolume, + Mute, + PluginParameter + }; + + Kind kind = Kind::Volume; + AutomationList* list = nullptr; + bool isInputFX = false; + int fxIndex = -1; + int paramIndex = -1; + }; + TrackProcessor(); ~TrackProcessor() override; @@ -127,7 +151,9 @@ class TrackProcessor : public juce::AudioProcessor int getNumInputFX() const; int getNumTrackFX() const; juce::AudioProcessor* getInputFXProcessor(int index); + const juce::AudioProcessor* getInputFXProcessor(int index) const; juce::AudioProcessor* getTrackFXProcessor(int index); + const juce::AudioProcessor* getTrackFXProcessor(int index) const; std::shared_ptr<const std::vector<std::shared_ptr<juce::AudioProcessor>>> getInputFXSnapshot() const; std::shared_ptr<const std::vector<std::shared_ptr<juce::AudioProcessor>>> getTrackFXSnapshot() const; std::shared_ptr<const std::map<int, bool>> getInputFXBypassSnapshot() const; @@ -203,9 +229,9 @@ class TrackProcessor : public juce::AudioProcessor // MIDI intake / scheduling bool enqueueMidiMessage(const juce::MidiMessage& message, int sampleOffset = 0); void setScheduledMIDIClips(std::vector<ScheduledMIDIClip> clips); - void buildMidiBuffer(juce::MidiBuffer& destination, double blockStartTimeSeconds, + void buildMidiBuffer(juce::MidiBuffer& destination, double blockTimeSeconds, int numSamples, double sampleRate, bool playing); - bool needsProcessing(double blockStartTimeSeconds, int numSamples, double sampleRate, bool playing) const; + bool needsProcessing(double blockTimeSeconds, int numSamples, double sampleRate, bool playing) const; void queueAllNotesOff(); std::vector<juce::String> getSidechainSourceSnapshot() const; std::vector<RealtimeSendInfo> getRealtimeSendSnapshot() const; @@ -271,22 +297,33 @@ class TrackProcessor : public juce::AudioProcessor juce::String getMIDIOutputDeviceName() const { return midiOutputDeviceName; } void sendMIDIToOutput(const juce::MidiBuffer& buffer); - // Automation (Phase 1.1) - // Each track has automation for volume and pan. Plugin param automation - // uses the paramId "plugin-{index}-param-{paramIndex}" key in AudioEngine's - // per-track automation map — TrackProcessor only handles volume + pan. + // Automation AutomationList& getVolumeAutomation() { return volumeAutomation; } AutomationList& getPanAutomation() { return panAutomation; } + AutomationList& getWidthAutomation() { return widthAutomation; } + AutomationList& getPreFXVolumeAutomation() { return preFXVolumeAutomation; } + AutomationList& getPreFXPanAutomation() { return preFXPanAutomation; } + AutomationList& getPreFXWidthAutomation() { return preFXWidthAutomation; } + AutomationList& getTrimVolumeAutomation() { return trimVolumeAutomation; } + AutomationList& getMuteAutomation() { return muteAutomation; } const AutomationList& getVolumeAutomation() const { return volumeAutomation; } const AutomationList& getPanAutomation() const { return panAutomation; } + const AutomationList& getWidthAutomation() const { return widthAutomation; } + const AutomationList& getPreFXVolumeAutomation() const { return preFXVolumeAutomation; } + const AutomationList& getPreFXPanAutomation() const { return preFXPanAutomation; } + const AutomationList& getPreFXWidthAutomation() const { return preFXWidthAutomation; } + const AutomationList& getTrimVolumeAutomation() const { return trimVolumeAutomation; } + const AutomationList& getMuteAutomation() const { return muteAutomation; } + std::optional<AutomationTarget> resolveAutomationTarget(const juce::String& parameterId, bool createIfNeeded); + float getAutomationDefaultValue(const AutomationTarget& target) const; // Set the current timeline position for this block (called by AudioEngine // before processBlock so automation knows where it is on the timeline). - void setCurrentBlockPosition(double samplePosition, double sRate) + void setCurrentBlockPosition(double timeSeconds) { - blockStartSample = samplePosition; - blockSampleRate = sRate; + blockStartTimeSeconds = timeSeconds; } + void setIgnoreStaticMuteForProcessing(bool ignore) { ignoreStaticMuteDuringProcessing.store(ignore, std::memory_order_relaxed); } bool tryProcessBlock(juce::AudioBuffer<float>&, juce::MidiBuffer&); @@ -315,8 +352,24 @@ class TrackProcessor : public juce::AudioProcessor void shutdownARA(); private: + struct PluginAutomationRoute + { + juce::String parameterId; + bool isInputFX = false; + int fxIndex = -1; + int paramIndex = -1; + std::shared_ptr<AutomationList> automation = std::make_shared<AutomationList>(); + std::atomic<float> lastAppliedValue { std::numeric_limits<float>::quiet_NaN() }; + }; + + using PluginAutomationRouteSnapshot = std::vector<std::shared_ptr<PluginAutomationRoute>>; + void processBlockInternal(juce::AudioBuffer<float>&, juce::MidiBuffer&); void publishRealtimeStateSnapshots(); + std::shared_ptr<PluginAutomationRoute> getOrCreatePluginAutomationRoute(const juce::String& parameterId); + std::shared_ptr<PluginAutomationRoute> findPluginAutomationRoute(const juce::String& parameterId) const; + static std::optional<std::pair<int, int>> parsePluginAutomationParameterId(const juce::String& parameterId); + void applyPluginAutomationForProcessor(juce::AudioProcessor* proc, bool isInputFX, int fxIndex, double blockTimeSeconds); // Current peak level (was named currentRMS but now holds peak — kept as-is // to avoid changing the public getRMSLevel() / resetRMS() API used by AudioEngine). @@ -405,14 +458,22 @@ class TrackProcessor : public juce::AudioProcessor std::shared_ptr<juce::AudioPluginInstance> instrumentPlugin; juce::MidiBuffer midiBuffer; // For MIDI event storage - // Automation (Phase 1.1) + // Automation AutomationList volumeAutomation; AutomationList panAutomation; - double blockStartSample { 0.0 }; // Set by AudioEngine before processBlock - double blockSampleRate { 44100.0 }; + AutomationList widthAutomation; + AutomationList preFXVolumeAutomation; + AutomationList preFXPanAutomation; + AutomationList preFXWidthAutomation; + AutomationList trimVolumeAutomation; + AutomationList muteAutomation; + double blockStartTimeSeconds { 0.0 }; + std::atomic<bool> ignoreStaticMuteDuringProcessing { false }; // Pre-allocated buffer for per-sample automation gain (avoids alloc on audio thread) juce::AudioBuffer<float> automationGainBuffer; + juce::CriticalSection pluginAutomationRouteLock; + std::shared_ptr<const PluginAutomationRouteSnapshot> pluginAutomationSnapshot; // Plugin Delay Compensation (PDC) juce::dsp::DelayLine<float> pdcDelayLine { 96000 }; // max 2 seconds at 48kHz @@ -504,11 +565,11 @@ class TrackProcessor : public juce::AudioProcessor ProcessingPrecisionMode processingPrecisionMode { ProcessingPrecisionMode::Float32 }; void markActiveMIDINoteState(const juce::MidiMessage& message); - void appendScheduledMIDIToBuffer(juce::MidiBuffer& destination, double blockStartTimeSeconds, + void appendScheduledMIDIToBuffer(juce::MidiBuffer& destination, double blockTimeSeconds, int numSamples, double sampleRate) const; void appendQueuedMIDIToBuffer(juce::MidiBuffer& destination, int numSamples); bool hasQueuedMIDI() const; - bool hasScheduledMIDIInBlock(double blockStartTimeSeconds, int numSamples, double sampleRate) const; + bool hasScheduledMIDIInBlock(double blockTimeSeconds, int numSamples, double sampleRate) const; // ARA Plugin Hosting (Phase 9) mutable juce::CriticalSection araStatusLock; diff --git a/WORKFLOWS.md b/WORKFLOWS.md index ad24e65..a506550 100644 --- a/WORKFLOWS.md +++ b/WORKFLOWS.md @@ -28,7 +28,7 @@ cmake --build build --config Debug ### First Time Setup ```bash -python build.py dev +python build.py dev --run ``` ### Daily Workflow diff --git a/build.py b/build.py index d3e2ea6..6b2fd79 100644 --- a/build.py +++ b/build.py @@ -6,11 +6,15 @@ import signal import atexit import time +import urllib.request +import urllib.error # Track background processes for cleanup vite_process = None cpp_process = None +VITE_DEV_URL = "http://localhost:5173" + def kill_process_tree(pid): """Kill a process and all its child processes (Windows: taskkill /T)""" if platform.system() == "Windows": @@ -24,7 +28,7 @@ def kill_process_tree(pid): pass def kill_orphaned_openstudio(): - """Kill any leftover OpenStudio.exe and its WebView2 child processes from previous runs""" + """Kill any leftover OpenStudio and its child processes from previous runs""" if platform.system() == "Windows": result = subprocess.run( ["tasklist", "/FI", "IMAGENAME eq OpenStudio.exe", "/FO", "CSV", "/NH"], @@ -38,6 +42,9 @@ def kill_orphaned_openstudio(): print(f" Killing orphaned OpenStudio.exe (PID {pid}) and its child processes...") kill_process_tree(int(pid)) time.sleep(0.5) + elif platform.system() == "Linux": + subprocess.run(["pkill", "-x", "OpenStudio"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def cleanup(): """Kill background processes on exit (including child process trees)""" @@ -68,6 +75,35 @@ def run_command(command, cwd=None, shell=True): print(f"Error running command: {command}") sys.exit(1) +def get_npm_executable(): + return "npm.cmd" if platform.system() == "Windows" else "npm" + +def wait_for_http_server(url, timeout_seconds=30, poll_interval=0.5): + """Wait until an HTTP server responds successfully.""" + deadline = time.time() + timeout_seconds + last_error = None + + while time.time() < deadline: + global vite_process + if vite_process and vite_process.poll() is not None: + print("Vite dev server exited before becoming ready.") + return False + + try: + with urllib.request.urlopen(url, timeout=1) as response: + status = getattr(response, "status", 200) + if 200 <= status < 500: + print(f"Vite dev server is ready at {url}") + return True + except (urllib.error.URLError, OSError) as exc: + last_error = exc + time.sleep(poll_interval) + + print(f"Timed out waiting for Vite dev server at {url}") + if last_error is not None: + print(f"Last connection error: {last_error}") + return False + def build_frontend(mode="dev"): frontend_dir = os.path.join(os.getcwd(), "frontend") print("--- Building Frontend ---") @@ -82,6 +118,17 @@ def build_frontend(mode="dev"): elif mode == "dev": print("Frontend dependencies ready.") +def get_cpp_exe_path(config="Debug"): + """Return the platform-appropriate path to the built C++ executable.""" + if platform.system() == "Windows": + return os.path.join("build", "OpenStudio_artefacts", config, "OpenStudio.exe") + elif platform.system() == "Darwin": + return os.path.join("build", "OpenStudio_artefacts", config, + "OpenStudio.app", "Contents", "MacOS", "OpenStudio") + else: # Linux + return os.path.join("build", "OpenStudio_artefacts", config, "OpenStudio") + + def build_backend(mode="debug"): print("--- Building Backend ---") build_dir = os.path.join(os.getcwd(), "build") @@ -89,12 +136,15 @@ def build_backend(mode="debug"): os.makedirs(build_dir) config_type = "Debug" if mode == "debug" else "Release" - - # Configure CMake - cmd = f"cmake -B \"{build_dir}\" -DCMAKE_BUILD_TYPE={config_type}" + + # Configure CMake. On single-config generators (Linux/macOS Make/Ninja) + # CMAKE_BUILD_TYPE must be set at configure time, not just at build time. + cmd = f"cmake -B \"{build_dir}\"" + if platform.system() != "Windows": + cmd += f" -DCMAKE_BUILD_TYPE={config_type}" if mode == "debug": cmd += " -DJUCE_DEBUG=ON" - + run_command(cmd) # Build @@ -106,24 +156,26 @@ def start_vite_server(): frontend_dir = os.path.join(os.getcwd(), "frontend") print("\n--- Starting Vite Dev Server ---") vite_process = subprocess.Popen( - ["npm", "run", "dev"], + [get_npm_executable(), "run", "dev"], cwd=frontend_dir, - shell=True + shell=False ) - # Give Vite time to start - print("Waiting for Vite to start...") - time.sleep(3) + print(f"Waiting for Vite to start on {VITE_DEV_URL}...") + if not wait_for_http_server(VITE_DEV_URL): + cleanup() + print("Unable to confirm the Vite dev server is reachable, so the native app was not launched.") + sys.exit(1) return vite_process def run_cpp_app(): """Run the C++ executable""" global cpp_process - exe_path = os.path.join("build", "OpenStudio_artefacts", "Debug", "OpenStudio.exe") + exe_path = get_cpp_exe_path("Debug") if not os.path.exists(exe_path): print(f"ERROR: Executable not found at {exe_path}") print("Run 'python build.py dev' first to build the app.") sys.exit(1) - + print(f"\n--- Launching {exe_path} ---") cpp_process = subprocess.Popen([exe_path]) @@ -162,7 +214,7 @@ def main(): build_frontend("prod") build_backend("release") print("\nProduction Build Complete.") - print("Run: build/OpenStudio_artefacts/Release/OpenStudio.exe") + print(f"Run: {get_cpp_exe_path('Release')}") if __name__ == "__main__": main() diff --git a/docs/ace-step-integration-challenges.md b/docs/ace-step-integration-challenges.md new file mode 100644 index 0000000..5fd14c3 --- /dev/null +++ b/docs/ace-step-integration-challenges.md @@ -0,0 +1,207 @@ +# What It Took to Bring ACE-Step Music Generation Into OpenStudio + +Adding ACE-Step music generation to OpenStudio looked, at first, like a straightforward integration task. We had a known-good workflow, we had the model files, and we had a clear product goal: let a user type a prompt and lyrics, press generate, and get a coherent song directly inside the app. No separate graph editor, no external desktop runtime, no manual model wiring. + +The hard part was that the visible workflow was only the surface of the system. + +ACE-Step is not just one model file and one inference call. It is a graph of model loading, text and lyric conditioning, latent creation, sampling, VAE audio decoding, runtime memory management, and file export. Two systems can appear to run the same graph and still produce very different audio if they differ in dtype selection, executor semantics, CUDA builds, decode strategy, cache lifecycle, or even the exact shape assumptions used while loading a checkpoint. + +This document is a short write-up of the main challenges we hit, why they were deceptive, and what ultimately made the integration work. + +## The First Trap: Same Settings Did Not Mean Same Sound + +The first symptom was simple and frustrating: OpenStudio generated audio, but it did not sound like the reference render. It was audible, but it was not musically coherent in the same way. There were too many artifacts, too much random noise, and the result felt degraded even when prompt, lyrics, seed, model selection, sampler settings, and duration all appeared to match. + +That is the kind of bug that tempts you to stare at UI parameters forever. Is the BPM being passed as a number or a string? Is `3/4` becoming `3`? Is the seed staying `-1` or being normalized to `0`? Is CFG being mapped to the right sampler field? + +Those checks mattered, but they were not enough. We eventually confirmed the visible graph was broadly right: + +- ACE 1.5 XL Turbo diffusion model +- Qwen 0.6B and 4B ACE text encoders +- ACE 1.5 VAE +- AuraFlow shift set to `3` +- `KSampler` using `euler`, `simple`, `steps=8`, `cfg=1`, `denoise=1` +- text conditioning through `TextEncodeAceStepAudio1.5` +- full `VAEDecodeAudio` path + +The missing piece was that a graph is not only node names and widget values. A graph is also the executor, runtime context, memory policy, tensor dtype policy, cache behavior, and node implementation version. + +## Checkpoint Shape Mismatches Were Real Bugs, But Not The Whole Bug + +Early failures were more obvious. The runner crashed while loading ACE model weights with long lists of tensor shape mismatches. The model expected dimensions like `2048`, while the checkpoint contained decoder tensors shaped around `2560`. + +That pointed to an ACE 1.5 split-shape issue. The encoder and decoder do not use the same hidden sizes in the XL checkpoint. A simpler config path treated them as if they shared dimensions, which made weight loading impossible or partially wrong. + +The fix was to teach the packaged runtime to detect and preserve separate encoder and decoder dimensions: + +- encoder hidden/intermediate sizes +- decoder hidden/intermediate sizes +- encoder and decoder attention head counts +- key/value head counts +- the 2048-to-2560 conditioning projection + +That fixed the obvious state-dict loading failures. It did not, by itself, fix the sound quality. This was the first big lesson: a crash fix can get the model to run, while the model can still be running under subtly wrong runtime semantics. + +## Decode Was A Quality Boundary, Not Just A Final Step + +Another suspicious symptom showed up near the end of long renders. Generation would appear to make progress, then stall around audio decode, sometimes eventually failing with CUDA out-of-memory. Earlier experiments allowed fallback to tiled VAE decode. Tiling is a valid low-memory strategy in many image/audio pipelines, but it is also a quality-sensitive change. + +For this specific goal, silently falling back was the wrong product behavior. If the reference path uses full VAE decode and OpenStudio silently returns tiled decode, then OpenStudio may "succeed" while producing a degraded song. That is worse than a clear failure, because it hides the real mismatch behind a bad output. + +So we changed the default expectation: + +- full decode is the quality path +- no silent tiled fallback in normal generation +- if full decode runs out of memory, fail clearly +- do not return a lower-quality result while pretending generation succeeded + +That made failures more honest. It also helped narrow the problem: if full decode succeeds, the sound should be judged; if it cannot succeed, we need memory work rather than audio post-processing guesses. + +## Direct Node Calls Were Not Equivalent To Real Graph Execution + +The most important implementation change was moving away from manually calling individual nodes. + +The direct runner originally assembled the right pieces, but it still called into nodes like an approximation: + +1. load models +2. call text encode +3. call sampler +4. call VAE decode +5. save WAV + +That sounds equivalent to graph execution, but it was not. The executor wraps node execution in a particular inference context, manages object and output caches, handles node output conventions, and applies memory lifecycle behavior. It also normalizes modern node APIs differently from older tuple-returning node APIs. + +This difference produced several bugs: + +- an in-place update error from tensors created under inference mode but later mutated outside the expected context +- invalid assumptions about node output shape +- missing executor cache arguments +- memory pressure that did not match the reference path + +The final working direction was to make OpenStudio own a packaged ACE runtime and execute a generated prompt graph through the executor semantics, rather than approximating the graph with manual calls. + +OpenStudio now builds the ACE prompt graph with fixed node IDs and known-good values, runs it through the packaged executor, extracts the decoded audio object from the executor output cache, and writes the requested WAV output without extra normalization, compression, or resampling. + +## The Output Was Wrapped One More Time Than Expected + +One of the final bugs was small but revealing. After switching to executor-based graph execution, the graph ran, but OpenStudio reported: + +```text +OpenStudio ACE graph produced an invalid decoded audio object. +``` + +The decode node was not actually producing invalid audio. Our extraction code was wrong. + +The direct-call path expected: + +```python +audio = {"waveform": ..., "sample_rate": ...} +``` + +The executor cache stored the single-output node result one layer deeper: + +```python +[{"waveform": ..., "sample_rate": ...}] +``` + +So OpenStudio was validating a list as if it were the audio dict. The fix was a small `unwrap_executor_output()` helper that unwraps single-item list/tuple layers before validation. That is a tiny code change, but it represents the broader theme: executor semantics are not optional details. They are part of the runtime contract. + +## Runtime Version Differences Mattered + +Another challenge was that the working reference environment was not identical to OpenStudio's initial runtime. Python version, PyTorch CUDA build, runtime package versions, and node implementation details all differed. + +Some examples: + +- different Python versions +- different CUDA wheel builds +- different packaged node code +- different latent dtype behavior +- different model-management behavior + +One concrete mismatch was latent dtype creation. The ACE 1.5 latent node needs to use the runtime's intermediate dtype, not a hardcoded default that happens to work in one environment. If latents are created at a different dtype or device policy, sampling can still complete but produce different numerical behavior. + +The eventual answer was not to require users to install the reference environment. That would be fragile and hostile to the product experience. Instead, OpenStudio vendors the required open-source runtime files under an OpenStudio-owned package boundary and runs its own packaged ACE graph. + +That gives us a stable product story: + +- users do not need a separate graph app installed +- OpenStudio controls the runtime files it ships +- the installer controls Python dependencies and models +- generation can work offline after setup + +## Offline Behavior Needed To Be Explicit + +A local AI feature should not unexpectedly reach the internet during generation after setup is complete. The first install or AI setup can download packages and model checkpoints, but a normal generation run should use local assets only. + +To make that clear, the runner validates the required local assets first: + +- diffusion model +- text encoders +- VAE + +Only after validation does it enable offline environment guards: + +```text +HF_HUB_OFFLINE=1 +TRANSFORMERS_OFFLINE=1 +HF_DATASETS_OFFLINE=1 +``` + +If a model file is missing, OpenStudio should tell the user to run AI setup while connected to the internet. It should not attempt a surprise download from the generation path. + +This matters for trust. A musician should be able to install the AI tools once, go offline, and still generate music as long as the needed files are already present. + +## Packaging Was Part Of The Feature + +The integration was not done when the model produced sound on one development machine. It also had to be packaged correctly. + +That meant: + +- copying `openstudio_ace_runner.py` into the app artifacts +- copying `openstudio_ace_backend` into the app artifacts +- keeping third-party license files +- removing stale debug/parity scripts from reused build directories +- pruning unused API-node runtime files that were not part of ACE generation +- updating Linux and macOS dependency manifests +- making sure Windows, Linux, and macOS install paths all prepare enough Python dependencies + +The vendored runtime is large, but most of it is source code, not model weights. The truly large files are the ACE checkpoints, which live in the user's model/cache area and are downloaded by setup. The runtime package exists so OpenStudio can execute the graph consistently without depending on another app being installed. + +## What We Learned + +The biggest lesson is that AI workflow integration is often not a "call the model" task. It is a runtime reproduction task. + +For ACE-Step, correctness depended on several layers lining up at once: + +- checkpoint shape detection +- text-conditioning semantics +- latent dtype/device policy +- sampler settings +- executor context +- cache and output conventions +- VAE decode strategy +- memory behavior +- packaging and install dependencies +- offline asset validation + +When any one of those is wrong, the failure mode may not be a clean crash. It may be worse: a song that technically renders but sounds wrong. + +That is why this took longer than simply copying visible workflow values. The visible graph told us what should happen. The runtime determined what actually happened. + +## The Current Shape + +OpenStudio now owns the ACE generation path: + +- it builds a known ACE graph internally +- it executes through the packaged OpenStudio ACE runtime +- it uses full VAE decode as the quality path +- it writes WAV output directly +- it validates local model files before generation +- it can run offline after setup +- it does not require users to install a separate graph runtime + +There is still future cleanup and hardening we can do. The vendored runtime can probably be reduced further with careful import and generation tests. Linux and macOS need real-machine validation, not just manifest and import checks from Windows. Long renders still depend on available VRAM, and full decode should fail clearly if the machine cannot handle the requested duration. + +But the important architectural turn has happened: OpenStudio is no longer approximating ACE-Step from the outside. It owns a packaged graph execution path and treats the runtime semantics as part of the feature. + +That is what finally made the generated song behave like a song. diff --git a/docs/audio_signal_chain_qa.md b/docs/audio_signal_chain_qa.md new file mode 100644 index 0000000..8051126 --- /dev/null +++ b/docs/audio_signal_chain_qa.md @@ -0,0 +1,32 @@ +# Audio Signal Chain QA + +Pitch, render, and playback artifact work must start with the signal chain before any DSP tuning. + +## Chain References + +- Live playback: clip route resolution -> `PlaybackEngine` read/mix -> `TrackProcessor` -> sends/sidechain -> master/monitoring FX -> gain/pan/mono -> meters/spectrum -> audio device. +- Offline render/export: `renderProject` -> render `PlaybackEngine` snapshot -> `TrackProcessor` -> master FX/gain -> writer. + +If a click/noise is present in an exported render, debug the shared render/playback path first. Do not attribute it to WebView IPC, audio-device underruns, or live-only preview routing until the render chain is clean. + +## Render Chain Debug Packet + +Set `OPENSTUDIO_AUDIO_CHAIN_DEBUG=1` before rendering to emit a debug packet next to the rendered file, or set `OPENSTUDIO_AUDIO_CHAIN_DEBUG_DIR` to choose a folder. The packet includes: + +- `render_chain_report.json` +- `playback_output.wav` +- `track_post_processing.wav` +- `master_pre_fx.wav` +- `master_post_fx.wav` +- `writer_input.wav` + +The report records per-block route/source details and peak/RMS/high-derivative/non-finite stats. Use `OPENSTUDIO_AUDIO_CHAIN_DEBUG_MAX_SEC` to cap the captured duration; the default is `12` seconds. + +## First Dirty Stage Rule + +- Dirty at playback: inspect source routing, stale pitch-preview state, reader/sample-rate conversion, chunking, and hot-path allocation counters. +- Dirty after track/master processing: inspect no-FX bypass, processor reset/state, default EQ/gain, and denormal handling. +- Dirty only at writer output: inspect write alignment, format conversion, dither, and block/tail handling. +- Dirty only in live playback after clean render: inspect callback duration/deadline counters, spectrum lock misses, callback resizes, IPC/timer pressure, and transport preview cleanup. + +Do not call a render-noise fix done until the first dirty stage is identified and the fixed output passes spectrogram/high-derivative checks plus user audition. diff --git a/docs/engine_v3_fullclip_prototype_20260417.md b/docs/engine_v3_fullclip_prototype_20260417.md new file mode 100644 index 0000000..972c87b --- /dev/null +++ b/docs/engine_v3_fullclip_prototype_20260417.md @@ -0,0 +1,97 @@ +# Engine-V3 Full-Clip Prototype Status (2026-04-17) + +## What Landed +- Added new renderer branches: + - `engine_v3_fullclip` + - `engine_v3_fullclip_lpc` + - `engine_v3_fullclip_lpc_transient` +- Added engine-v3 diagnostics plumbing through native + frontend regression results. +- Added continuous full-clip HQ output path for engine-v3 in `AudioEngine`. +- Added first LPC spectral envelope transfer helper in `LpcEnvelopeTransfer`. +- Hardened the current Signalsmith pitch-only path for better baseline formant configuration: + - `setFormantBase(avgDetectedHz)` per block + - inverse-ratio formant factor by default + +## Important Bug Fixed During Bring-Up +- The first engine-v3 full-clip implementation created a corrected full buffer and then overwrote it with the original `clipBuffer` before writing the output file. +- This made the branch look active in diagnostics while exporting source-identical audio. +- That overwrite bug is now fixed. + +## Current Truth +- `engine_v3_fullclip` now produces a real full-clip processed output. +- `engine_v3_fullclip_lpc` no longer collapses into NaN/silence after stability hardening, but it is still not keepable. +- The current engine-v3 prototype does **not** yet beat `pitch_only_adaptive_selector` on the canonical `pitchOrg +4` truth case. + +## First Canonical Results +### Plain continuous carrier +- Run: + - `D:\test projects\os tests\runs\20260417_035915_engine_v3_pitchOrg_plus4_fullclip_plain_r2fresh` +- Summary: + - `engine_v3_fullclip` + - sane audio output + - note mel `9.963` + - note envelope `1.229` + - entry mel `7.867` + - exit mel `9.306` + - onset artifact `2.43` +- Verdict: + - proves the continuous full-clip carrier path is live + - still substantially worse than the kept adaptive baseline on the target problem + +### LPC formant pass +- Run: + - `D:\test projects\os tests\runs\20260417_035915_engine_v3_pitchOrg_plus4_fullclip_lpc_r3fresh` +- Summary: + - `engine_v3_fullclip_lpc` + - numerically stable after guard fixes + - still extremely poor perceptual/regression quality + - note mel `48.943` + - note envelope `6.362` + - entry mel `49.398` + - exit mel `48.759` +- Verdict: + - current LPC transfer implementation is not viable yet + - it remains prototype-only and must not be promoted + +## What This Means +- The structural continuity change is now real and benchmarkable. +- The first formant-transfer implementation is still wrong for the target vocal edit quality. +- Engine-v3 remains a prototype branch, not a product-ready replacement. + +## Boundary-Zone Follow-Up +- Added a true boundary-zone blend on top of the continuous carrier: + - original shell at the boundary + - own-engine first-voiced-cycles patch + - continuous carrier for the stable body +- Canonical runs: + - `D:\test projects\os tests\runs\20260417_040833_engine_v3_pitchOrg_plus4_fullclip_plain_r3boundary` + - `D:\test projects\os tests\runs\20260417_040833_engine_v3_pitchTest_plus4_fullclip_plain_r1boundary` +- Diagnostics confirm the boundary slice is engaging: + - `firstVoicedCyclesEntryUsed=true` + - `firstVoicedCyclesExitUsed=true` + - `v3ContinuousRenderUsed=true` +- Verdict: + - this is the first real boundary-zone ownership prototype + - it still does not clear the quality gate on the canonical `+4` cases + +## Safer Formant/Body Harvest Follow-Up +- Added a low-wet own-engine body-color harvest inside the edited note body only. +- Canonical runs: + - `D:\test projects\os tests\runs\20260417_101035_engine_v3_pitchOrg_plus4_fullclip_plain_r4bodyharvest` + - `D:\test projects\os tests\runs\20260417_101035_engine_v3_pitchTest_plus4_fullclip_r2bodyharvest` +- Verdict: + - this bounded formant/body-color attempt did not materially move the canonical truth metrics + - it is not the missing win + +## Immediate Next Work +1. Keep `pitch_only_adaptive_selector` as the shipping baseline. +2. Treat `engine_v3_fullclip` as the active engine-v3 prototype. +3. Keep `engine_v3_fullclip_lpc` and `_transient` as experimental-only until they beat the baseline on: + - `pitchOrg +4` + - `pitchTestOrg +4` + - `pitchOrg -4` + - `pitchTestOrg -4` +4. Next engine-v3 DSP work should focus on: + - first-voiced-cycles boundary ownership + - safer formant transfer + - immediate-neighbor-only smoothing validation diff --git a/docs/macos-audio-fixes.md b/docs/macos-audio-fixes.md new file mode 100644 index 0000000..1233108 --- /dev/null +++ b/docs/macos-audio-fixes.md @@ -0,0 +1,209 @@ +# macOS Audio Fixes — Root Cause Analysis & Implementation Notes + +**Date:** 2026-04-13 +**Affects:** macOS arm64 (Apple Silicon), all macOS releases +**Files changed:** +- `Source/StemSeparator.cpp` +- `CMakeLists.txt` + +--- + +## Background + +Two separate bugs were found during macOS testing with an Audient ID14 USB audio interface. Both trace back to the same root condition: **the app was distributed without notarization**, triggering macOS Gatekeeper quarantine on first launch. The quarantine created a cascade of failures in two independent subsystems. + +The user had to run `xattr -dr com.apple.quarantine /Applications/OpenStudio.app` manually before the app would open. This is the key diagnostic signal that connected both bugs. + +--- + +## Bug 1 — AI Tools Installation Fails on macOS arm64 + +### Symptom + +Install log shows the download, verification, and extraction all succeeding, then: + +```json +{ + "phase": "probe", + "event": "runtime_probe_finished", + "baseRuntimeReady": false, + "runtimeReady": false, + "selectedBackend": "cpu", + "supportedBackends": "" +} +``` + +`supportedBackends: ""` (empty string, not even `"cpu"`) is the diagnostic fingerprint — it means `probeRuntimeCapabilities()` returned a **default-initialized struct**, i.e., it exited before parsing any JSON from the probe process. + +### Root Cause Trace + +#### Step 1 — Why `supportedBackends` is empty + +`StemSeparator::probeRuntimeCapabilities()` has several early-return paths that return a default `RuntimeCapabilities{}` struct before reaching the JSON parse loop. The log call at the end (`capabilities.supportedBackends.joinIntoString(",")`) then produces an empty string because the array was never populated. The "add cpu if empty" safety net only runs after a successful parse — not on early returns. + +#### Step 2 — Which early-return fired + +The Python binary path check passes (the extracted Python file exists on disk — confirmed by the `findPythonInRuntimeRoot` check at line 1674 succeeding). The failure is at the subprocess launch: + +```cpp +if (! probe.start(command) || ! probe.waitForProcessToFinish(30000)) + return capabilities; +``` + +`juce::ChildProcess::start()` uses `posix_spawn()` internally on macOS. `posix_spawn()` returns `EACCES` and the call returns `false` when the target binary is not executable. + +#### Step 3 — Why the binary is not executable + +`juce::ZipFile::uncompressTo()` does not restore Unix file permission bits from the zip's external file attributes. The Python binary inside the AI runtime archive has execute permission in the zip metadata, but after extraction the file mode is set without the `x` bit — typically `-rw-r--r--` instead of `-rwxr-xr-x`. + +This is a known limitation of JUCE's zip extraction: it writes file content correctly but ignores the Unix permission field stored in the zip's local file headers. + +#### Step 4 — The quarantine layer on top + +Even if the execute bit were somehow present, macOS applies the `com.apple.quarantine` extended attribute to all files extracted from a downloaded zip archive (because the zip itself was downloaded from the internet and carries the quarantine attribute). Gatekeeper blocks execution of quarantined binaries launched by a child process. + +The user's `xattr -dr` on the app bundle only cleared the `.app` itself. Files extracted later to `~/Library/Application Support/OpenStudio/stem-runtime/` were quarantined separately and untouched by that command. + +Both problems apply simultaneously on a fresh install: +- No execute bit → `posix_spawn()` fails with `EACCES` +- Quarantine attribute → Gatekeeper blocks even if execute bit is present + +### Fix + +**File:** `Source/StemSeparator.cpp` — `extractRuntimeArchive()` + +After the successful `copyDirectoryTo(destinationRoot)` call and before cleanup, a `#if JUCE_MAC` block is inserted that performs two post-extraction steps: + +**Step A — Restore execute bits using POSIX `chmod()`** + +A lambda (`restoreExecuteBit`) iterates every regular file under `bin/` and `lib/` using `juce::RangedDirectoryIterator`, reads current permissions with `::stat()`, and ORs in `S_IXUSR | S_IXGRP | S_IXOTH` via `::chmod()`. A separate block does the same for the Python binary itself (which may live outside `bin/` depending on the archive layout). + +`chmod()` is used directly instead of spawning a subprocess because: +- It avoids the overhead of a child process +- It avoids the circular problem of needing a working Python to fix Python's own permissions +- `<sys/stat.h>` is always available on macOS/Linux + +**Step B — Strip the quarantine attribute using `xattr -rd`** + +`xattr` is a macOS system tool that always exists at a known path. It is invoked via `juce::ChildProcess` with `xattr -rd com.apple.quarantine <destinationRoot>`. The `-r` flag makes it recursive across the entire runtime directory. Failure is treated as non-fatal — the chmod step (Step A) is the primary fix; the quarantine strip is defense-in-depth. + +There is no single POSIX API equivalent to a recursive `removexattr` over a directory tree, so the subprocess approach is necessary for this step. + +**Additional changes in `probeRuntimeCapabilities()`:** + +- All early-return paths now emit a structured JSON log line via `appendAiToolsLogLine()` explaining which check failed (python not found, script not found, process start failure, timeout, bad exit code). Previously these returned silently, making the failure impossible to diagnose from the log alone. +- Probe subprocess timeout increased from 15 000 ms to 30 000 ms. Even after quarantine is stripped, macOS Gatekeeper performs a one-time scan of newly executable binaries. On slow or heavily loaded systems this scan can exceed 15 seconds. + +### Platform Impact + +The chmod/xattr block is wrapped in `#if JUCE_MAC` and does not compile on Windows or Linux. The `#include <sys/stat.h>` is wrapped in `#if JUCE_MAC || JUCE_LINUX` — it is a standard POSIX header on Linux and a harmless addition there. The timeout and logging changes are cross-platform but additive only (longer timeout, extra log lines on failure). + +--- + +## Bug 2 — Input Monitoring Produces No Audio (No Meter Movement) + +### Symptom + +- Audient ID14 connected via USB, inputs visible in the device selector inside the app +- Track added, armed for recording, FX loaded +- No audio heard through monitoring +- No movement in the channel strip peak meters at all + +### Why This Is Not a Driver Issue + +The Audient ID14 is a class-compliant USB audio device. macOS has native Core Audio support for it with no third-party driver required. Other DAWs working with the same device confirms the hardware and driver layer are fine. + +### Root Cause Trace + +#### Step 1 — Enumeration vs. capture + +The app can list the interface's inputs (channel count, names) because device enumeration uses `AudioObjectGetPropertyData` with `kAudioDevicePropertyStreamConfiguration`, which does **not** require microphone permission. + +Actual audio capture — reading samples from `inputChannelData` in the audio callback — goes through a different code path that macOS gates behind the privacy permission system. + +#### Step 2 — macOS treats all audio input as "microphone" + +On macOS, **the "Microphone" privacy permission covers every audio input source** — built-in microphone, USB audio interfaces, Thunderbolt interfaces, HDMI audio return, everything. It is not specific to the physical built-in microphone. The OS does not distinguish between a laptop mic and a professional audio interface at the privacy layer. + +When microphone permission is denied or not yet requested: +- `numInputChannels` in the audio callback may still be > 0 (the device is open) +- But all values in `inputChannelData` are zero — silence +- The monitoring path in `AudioEngine::audioDeviceIOCallbackWithContext()` faithfully copies those zeros into the track buffer +- The track FX chain processes zeros, the meter computes 0.0 RMS, the channel strip shows nothing + +#### Step 3 — The permission was never requested + +The `NSMicrophoneUsageDescription` key was missing from the app's `Info.plist`. This key is required for macOS to allow the app to use audio input at all. When the key is absent: + +- macOS never shows a permission dialog +- The app's permission state stays at "not determined" (or is silently treated as denied) +- Core Audio provides zero input data on every callback + +This key must be set in the generated `Info.plist` at build time. In a JUCE CMake project, it is controlled through the `juce_add_gui_app()` call. + +**The existing `juce_add_gui_app()` call in `CMakeLists.txt`:** + +```cmake +juce_add_gui_app(OpenStudio + PRODUCT_NAME "OpenStudio" + VERSION ${PROJECT_VERSION} + ICON_BIG "${CMAKE_CURRENT_SOURCE_DIR}/assets/icon-256x256.png" + ICON_SMALL "${CMAKE_CURRENT_SOURCE_DIR}/assets/icon-16x16.png" + # MICROPHONE_PERMISSION_ENABLED and MICROPHONE_PERMISSION_TEXT were absent +) +``` + +#### Step 4 — The quarantine connection + +The app being quarantined on first launch is related but not the direct cause. The direct cause is the missing plist key. The quarantine connection is: + +- A quarantined app on its first launch may have its entitlement and permission requests suppressed or fail silently by Gatekeeper +- If the key had been present but the first launch was quarantined, macOS might have failed to record the "not determined" state correctly +- After the user ran `xattr -dr` on the app, subsequent launches no longer trigger Gatekeeper — but the permission was already never requested, so audio input still returns zeros + +### Fix + +**File:** `CMakeLists.txt` — `juce_add_gui_app()` call + +Two properties added: + +```cmake +MICROPHONE_PERMISSION_ENABLED TRUE +MICROPHONE_PERMISSION_TEXT "OpenStudio needs access to your audio input to record from microphones and audio interfaces." +``` + +`MICROPHONE_PERMISSION_ENABLED TRUE` tells JUCE's CMake module to emit `NSMicrophoneUsageDescription` into the generated `Info.plist`. `MICROPHONE_PERMISSION_TEXT` is the string value for that key — it is shown to the user in the macOS permission dialog and in System Settings → Privacy → Microphone. + +On the first launch of the rebuilt app, macOS will display a one-time permission dialog. After the user grants access, Core Audio provides real input data and monitoring works. + +### Platform Impact + +`MICROPHONE_PERMISSION_ENABLED` and `MICROPHONE_PERMISSION_TEXT` are Apple-platform-only properties in JUCE's CMake module. The module does not act on them for Windows or Linux builds — they are silently ignored. This change has zero effect on Windows and Linux builds. + +### User-Facing Behavior Change + +**New installs:** The permission dialog appears on first launch. Expected and correct. + +**Existing installs (users who had the old build):** Their stored permission state is "not determined" (the key was never in the plist, so macOS never asked). On first launch of the updated build, macOS will show the dialog. They grant access, and monitoring works from that point forward. + +**If a user previously explicitly denied access** (unlikely given the permission dialog was never shown, but possible through System Settings): They need to enable it manually at System Settings → Privacy & Security → Microphone. + +To reset a specific installation's stored permission state from the terminal: + +```bash +tccutil reset Microphone <bundle-identifier> +# e.g.: tccutil reset Microphone in.openstudio.app +``` + +--- + +## Summary of All Changes + +| File | Change | Reason | +|---|---|---| +| `Source/StemSeparator.cpp` | `#include <sys/stat.h>` under `#if JUCE_MAC \|\| JUCE_LINUX` | Required for POSIX `stat()` and `chmod()` | +| `Source/StemSeparator.cpp` | Post-extraction `chmod +x` block in `extractRuntimeArchive()` | `juce::ZipFile::uncompressTo()` does not restore Unix execute bits; Python binary extracted without `+x` causes `posix_spawn()` to return `EACCES` | +| `Source/StemSeparator.cpp` | Post-extraction `xattr -rd com.apple.quarantine` in `extractRuntimeArchive()` | macOS applies quarantine to all files extracted from a downloaded zip; Gatekeeper blocks execution of quarantined binaries | +| `Source/StemSeparator.cpp` | Diagnostic log lines on all early returns in `probeRuntimeCapabilities()` | All early returns were previously silent; log now records which specific check failed | +| `Source/StemSeparator.cpp` | Probe timeout 15 000 ms → 30 000 ms | First-time Gatekeeper scan of a newly executable binary can exceed 15 s on loaded systems | +| `CMakeLists.txt` | `MICROPHONE_PERMISSION_ENABLED TRUE` + `MICROPHONE_PERMISSION_TEXT` in `juce_add_gui_app()` | Without `NSMicrophoneUsageDescription` in `Info.plist`, macOS never prompts for audio input permission and Core Audio silently returns zeros for all input | diff --git a/docs/pitch_editor_engine_v2_implementation_plan.md b/docs/pitch_editor_engine_v2_implementation_plan.md new file mode 100644 index 0000000..2378d7d --- /dev/null +++ b/docs/pitch_editor_engine_v2_implementation_plan.md @@ -0,0 +1,473 @@ +# Pitch Editor Engine V2 Implementation Guide + +Status note on 2026-04-17: + +- the engine-v2 implementation work is complete enough for comparison, but it is no longer the recommended active recovery path +- root-cause research, ML benchmark, and engine-v3 feasibility are now recorded in: + - [pitch_root_cause_research_20260417.md](c:/Users/srvds/Documents/Codes/Studio13-v3/docs/pitch_root_cause_research_20260417.md) +- current repo truth: + - `pitch_only_adaptive_selector` remains the kept working baseline + - local ML restoration is blocked because no materially stronger note-local restorer is available in this environment + - the first `engine-v3` feasibility probe returned `stop` + +## Goal +Move the pitch editor to a 2-tier workflow: + +- `Tier 1`: instant drag monitoring while the user is moving a note +- `Tier 2`: debounced HQ note/island render that becomes the authoritative playback result for the changed region + +This program is built on top of the current repo truth: + +- kept live branch: `pitch_only_adaptive_selector` +- active experimental path: `pitch_only_engine_v2_program` +- note-change stutter and formant drift are still unresolved +- code must stay in place for user audition before cleanup + +## Locked Product Behavior +- Use `2` tiers, not 1 or 3 +- Drag monitoring starts immediately while the note is moving +- HQ note render starts after roughly `300 ms` of inactivity +- Transport playback does not use preview-quality audio for committed edits +- Changed regions wait for HQ-ready cache audio before the new edit becomes authoritative +- Experimental code is not removed until after user listening validation + +## Implemented Foundation +### Current v1 infrastructure +- Added `note_hq` as a first-class pitch correction render mode +- Auto-apply now targets note-local HQ renders instead of staged playhead segments +- Drag lifecycle now starts and stops interactive pitch monitoring explicitly +- Playback region replacement keeps the last valid rendered region until a newer overlapping HQ region is ready +- Regression harness accepts `note_hq` render jobs +- Added a dedicated RAM scrub-preview path: + - backend entrypoints: + - `startPitchScrubPreview` + - `updatePitchScrubPreview` + - `stopPitchScrubPreview` + - native playback mixes a RAM loop monitor voice directly from `PlaybackEngine` + - scrub loops are extracted from the stable interior of the selected note, not the onset + - stereo clips are supported by deriving loop bounds from mono analysis and extracting matching multichannel audio +- Added the first real `engine-v2` audio implementation on top of the scaffold: + - Signalsmith-based voiced-core renderer in the transition window + - cepstral envelope restoration on voiced-support frames + - spectral-flatness-driven transient/unvoiced bypass mask + - residual carry reinjection from shared own-engine analysis + - transition compositor that mixes: + - original transient shell + - voiced core + - adaptive-selector base + - residual carry + +### Current limitations +- The scrub voice is now benchmarked through a dedicated stopped-transport scrub harness, but it still needs real user audition in the editor UI +- `engine-v2` sound work is now implemented, but the first full audible pass regresses both primary `+4` truth clips badly +- The current `engine-v2` transition compositor is still too broad and is dragging the stable note body off target +- The audible product issues remain unresolved: + - stutter on note change + - formant/timbre drift on note change + +## Architecture +### Tier 1: drag monitoring +- Start on note drag begin for pitch-affecting gestures +- Use note-local pitch preview derived from the selected note window +- Clear immediately on drag end +- This tier is intentionally fast and temporary + +### Tier 2: HQ note cache +- Trigger after debounce +- Render only the changed note/island region plus transition shoulders +- Store the result as a playback override region +- Replace overlapping cached regions atomically when a newer HQ render finishes + +### Transport behavior +- While a new HQ note render is pending, transport keeps using the last valid audio for that region +- Once the new HQ note render completes, playback swaps to the new override region +- No preview-quality renderer is used for committed transport playback + +## Next Engine-V2 Audio Steps +1. Tighten `engine-v2` engagement to the transition core instead of the wider note window +2. Lower wet exposure so adaptive-selector remains the dominant carrier outside the transition nucleus +3. Keep cepstral envelope restoration only where voiced support is strong and stable +4. Keep transient/unvoiced bypass mandatory at the transition edges +5. Rebalance residual carry so it restores breathiness without shifting core pitch/body +6. Re-run the truth cases against: + - `CTRL-SHIP` + - `CTRL-R6` + - `pitch_only_adaptive_selector` + - current `pitch_only_engine_v2_program` + +## Current Benchmark Snapshot +### Dedicated scrub preview +- Implemented and build-clean +- Native/backend path is active +- Dedicated scrub regression now passes with the natural-segment path on stopped transport: + - run: `20260416_200227_pitchOrg_scrub_preview_r8` + - result: + - `scrubPreviewAudible=true` + - `scrubPreviewFirstDragAudible=true` + - start latency `26.9 ms` + - stop latency `26.8 ms` + - loop duration `240.0 ms` + - base pitch `365.17 Hz` + - repeat stability `0.4699` + - last peak `0.1123` +- The render-quality harness still does not score drag feel, so scrub regression remains a separate check +- Multi-scenario scrub suite now exists: + - runner: `tools/run-ui-pitch-scrub-suite.ps1` + - richer runs: + - `20260416_231821_pitchOrg_scrub_suite_richer_r1` + - `20260416_234516_pitchTest_scrub_suite_richer_r1` + - current measured state: + - first drag audible: `true` + - repeated-drag suite case audible: `true` + - after-transport-cycle suite case audible: `true` + - true multi-note fixtures now exist: + - `tests/fixtures/pitch-regression/example_pitchOrg_scrub_multinote.json` + - `tests/fixtures/pitch-regression/example_pitchTest_scrub_multinote.json` + - multi-note scrub run: `20260416_235640_pitchTest_scrub_suite_multinote_r3` + - selection-change is now exercised end-to-end on the multi-note scrub fixture +- Important honesty note: + - scrub preview is now structurally present and richer-benchmarked across the current canonical local scrub scenarios + - `H1` is complete for the current canonical local fixture corpus + - the suite does not yet score perceived “breaking/tearing” quality directly + - the user-facing sound still needs listening validation in the editor + +### Boundary / transient / formant harnesses +- Boundary suite exists and runs: + - runner: `tools/run-ui-pitch-boundary-suite.ps1` +- Manifest-driven regression suite runner now exists: + - `tools/run-ui-pitch-regression-suite.ps1` +- Transient and formant wrappers now exist on top of it: + - `tools/run-ui-pitch-transient-suite.ps1` + - `tools/run-ui-pitch-formant-suite.ps1` +- Richer manifests added: + - `tests/fixtures/pitch-regression/suites/transient_richer_suite.json` + - `tests/fixtures/pitch-regression/suites/formant_richer_suite.json` +- Richer benchmark runs: + - transient: `20260416_231844_transient_suite_richer_r1` + - formant: `20260416_233426_formant_suite_richer_r1` +- Important honesty note: + - `H3` and `H4` are now complete for the current canonical local fixture corpus + - they still do not represent a broad real-world vocal corpus, but they are no longer smoke-only placeholders + +## Remaining Iterations +- Mandatory iterations still remaining: `0` +- Conditional phase-locking iterations still remaining: `+2` +- Why `0` remains: + - `S1-S3` are effectively in + - `H2` boundary harness is in + - `H1`, `H3`, and `H4` now have richer canonical-suite coverage and are treated as closed for the current local fixture corpus + - the adaptive-carrier tuning track is closed for now at its current best profile + - `A7` is complete and flat + - the engine-v2 challenger is now frozen after the narrowed `r8` close-out pass + - there is no remaining mandatory harness close-out work in the current local fixture corpus + +## What We Learned After Close-Out +- engine-v2 did not fail because of missing plumbing anymore +- it failed because the remaining error is centered on: + - transition ownership and boundary timing drift + - mixed transient and first-voiced-cycle handling + - formant preservation that is too weak and too local for hard transitions +- those findings came from the bounded root-cause pass on 2026-04-17, not from another speculative renderer loop +- that means this document should now be read as implementation history and available scaffolding, not as the primary next-step plan + +### `engine-v2` first audible implementation +- `pitchOrg +4` + - branch: `pitch_only_engine_v2_program` + - run: `20260416_124252_pitchOrg_plus4_note_hq_engine_v2_program_impl_r2` + - result: + - note mel `9.653` + - env `1.782` + - note/body/core cents `-18.72 / -18.72 / -37.23` + - entry mel `10.430` + - exit mel `11.219` + - onset artifact `3.23` + - `spectralEnvelopeCorrectionUsed=true` + - `engineV2Used=true` +- `pitchTestOrg +4` + - branch: `pitch_only_engine_v2_program` + - run: `20260416_124622_pitchTestOrg_plus4_note_hq_engine_v2_program_impl_r3` + - result: + - note mel `11.303` + - env `1.425` + - note/body/core cents `-17.94 / -17.94 / -36.45` + - entry mel `8.389` + - exit mel `10.274` + - onset artifact `0.85` + - `spectralEnvelopeCorrectionUsed=true` + - `engineV2Used=true` +- Verdict: + - infrastructure is real and benchmarkable + - first audible pass is not keepable + - code remains in place for user test and further tuning + +### Latest tuning snapshot +- `B1-B3` engine-v2 challenger close-out + - runs: + - `20260416_230357_enginev2_narrow_pitchOrg_plus4_r8` + - `20260416_230555_enginev2_narrow_pitchTest_plus4_r8` + - result: + - `pitchOrg +4` + - note mel `6.608` + - env `1.009` + - entry `7.620` + - exit `7.112` + - onset artifact `1.390` + - formant drift `0.395` + - `pitchTestOrg +4` + - note mel `3.143` + - env `0.498` + - entry `7.017` + - exit `1.887` + - onset artifact `4.000` + - formant drift `0.062` + - verdict: + - the narrowed/drier engine-v2 pass still loses clearly to the adaptive correction carrier on the hard clip + - easy-clip onset improves, but entry timbre still worsens + - hard-clip entry and onset remain much worse than the adaptive carrier + - freeze the challenger and stop spending main budget here + +- `A7` STFT correction-layer sweep + - runs: + - `20260416_224905_adaptive_stft_tune_r10_1024o8` + - `20260416_224905_adaptive_stft_tune_r10_2048o8` + - `20260416_224905_adaptive_stft_tune_r10_2048o4` + - result: + - all three profiles were effectively identical on both smoke cases + - `pitchOrg +4` stayed at about: + - note mel `6.525` + - env `1.022` + - entry `6.885` + - exit `7.112` + - onset artifact `2.257` + - formant drift `0.374` + - `pitchTestOrg +4` stayed at about: + - note mel `2.828` + - env `0.469` + - entry `1.703` + - exit `1.887` + - onset artifact `1.688` + - formant drift `0.050` + - verdict: + - STFT size / hop is not the dominant lever in the current adaptive correction layer + - `A7` should be treated as complete and flat + +- `pitchOrg +4` + - branch: `pitch_only_adaptive_selector` + - run: `20260416_223535_adaptive_residual_tune_r9_dry` + - result: + - note mel `6.525` + - env `1.022` + - entry mel `6.885` + - exit mel `7.112` + - onset artifact `2.257` + - boundary timing error `8.417 ms` + - formant body harmonic drift `0.374` + - `spectralEnvelopeCorrectionUsed=true` +- `pitchTestOrg +4` + - branch: `pitch_only_adaptive_selector` + - run: `20260416_223535_adaptive_residual_tune_r9_dry` + - result: + - note mel `2.828` + - env `0.469` + - entry mel `1.703` + - exit mel `1.887` + - onset artifact `1.688` + - boundary timing error `2.229 ms` + - formant body harmonic drift `0.050` + - `spectralEnvelopeCorrectionUsed=true` +- Verdict on the current best adaptive-carrier correction pass: + - `r9 dry` is the current best adaptive correction profile + - `A5` cepstral strength/lifter sweeps were effectively flat on the smoke suite + - `A6` residual carry sweep favored the driest profile, so the default correction path now keeps residual reinjection off + - it is still not fully keepable: + - `pitchOrg +4` onset artifact remains materially worse than plain adaptive + - `pitchTestOrg +4` onset and exit remain somewhat worse than plain adaptive even though the gap is much smaller + - the next tuning priority should now be: + - `A7` STFT correction-layer sweep + - then challenger close-out `B1-B4` + +- `pitchOrg +4` + - branch: `pitch_only_adaptive_selector` + - run: `20260416_213810_adaptive_boundary_tune_r7_onsetcleanup` + - result: + - note mel `6.526` + - env `1.023` + - entry mel `6.890` + - exit mel `7.112` + - onset artifact `2.261` + - boundary timing error `8.396 ms` + - formant body harmonic drift `0.374` + - `spectralEnvelopeCorrectionUsed=true` +- `pitchTestOrg +4` + - branch: `pitch_only_adaptive_selector` + - run: `20260416_213810_adaptive_boundary_tune_r7_onsetcleanup` + - result: + - note mel `2.828` + - env `0.470` + - entry mel `1.707` + - exit mel `1.887` + - onset artifact `1.714` + - boundary timing error `2.250 ms` + - formant body harmonic drift `0.050` + - `spectralEnvelopeCorrectionUsed=true` +- Verdict on the current best adaptive-carrier correction pass: + - `r7` is the best adaptive correction profile so far + - it meaningfully improves the hard clip: + - note mel is now close to the plain adaptive baseline + - exit damage is far lower than earlier correction passes + - it is still not fully keepable: + - `pitchOrg +4` onset artifact remains materially worse than plain adaptive + - `pitchTestOrg +4` onset artifact and exit are still worse than plain adaptive even though they are much closer now + - the next tuning priority should now be: + - `A5` cepstral lifter retune with richer formant fixtures + - `A6` residual carry rebalance + - then decide whether one more adaptive boundary cleanup pass is justified before more engine-v2 work + +- `pitchOrg +4` + - branch: `pitch_only_adaptive_selector` + - run: `20260416_212441_adaptive_boundary_tune_r5_compromise` + - result: + - note mel `6.533` + - env `1.039` + - entry mel `7.462` + - exit mel `7.044` + - onset artifact `2.261` + - boundary timing error `8.396 ms` + - formant body harmonic drift `0.373` + - `spectralEnvelopeCorrectionUsed=true` +- `pitchTestOrg +4` + - branch: `pitch_only_adaptive_selector` + - run: `20260416_212441_adaptive_boundary_tune_r5_compromise` + - result: + - note mel `2.935` + - env `0.487` + - entry mel `1.833` + - exit mel `3.665` + - onset artifact `1.748` + - boundary timing error `2.250 ms` + - formant body harmonic drift `0.040` + - `spectralEnvelopeCorrectionUsed=true` +- Verdict on the current best adaptive-carrier correction pass: + - `r5` is the best adaptive correction profile so far + - it is still not keepable: + - `pitchOrg +4` note-start artifact is still materially worse than the plain adaptive baseline + - `pitchTestOrg +4` exit quality is improved but still much worse than baseline + - the next tuning priority should now be: + - `A4` entry timing compensation + - `A5` cepstral lifter retune + - then a narrower `A3` crossfade-law cleanup if the start remains rough in listening + +- `pitchOrg +4` + - branch: `pitch_only_adaptive_selector` + - run: `20260416_211448_adaptive_boundary_tune_r3` + - result: + - note mel `6.537` + - env `1.040` + - entry mel `7.435` + - exit mel `7.028` + - onset artifact `2.383` + - boundary timing error `8.396 ms` + - formant body harmonic drift `0.373` + - `spectralEnvelopeCorrectionUsed=true` +- `pitchTestOrg +4` + - branch: `pitch_only_adaptive_selector` + - run: `20260416_211448_adaptive_boundary_tune_r3` + - result: + - note mel `2.960` + - env `0.493` + - entry mel `1.876` + - exit mel `4.067` + - onset artifact `1.540` + - boundary timing error `2.250 ms` + - formant body harmonic drift `0.039` + - `spectralEnvelopeCorrectionUsed=true` +- Verdict on the second adaptive-carrier correction pass: + - this pass is better than `r2`, especially on `pitchTestOrg +4` + - it is still not keepable: + - `pitchOrg +4` onset artifact remains materially worse than the pre-correction adaptive baseline + - `pitchTestOrg +4` exit quality and boundary timing are still much worse than baseline even after the improvement + - the next tuning priority remains: + - `A3` transient handoff crossfade sweep + - `A4` entry timing compensation + - `A5` cepstral lifter retune + +- `pitchOrg +4` + - branch: `pitch_only_adaptive_selector` + - run: `20260416_210838_adaptive_boundary_tune_r2` + - result: + - note mel `6.548` + - env `1.044` + - entry mel `7.383` + - exit mel `7.007` + - onset artifact `3.259` + - boundary timing error `8.417 ms` + - formant body harmonic drift `0.372` + - `spectralEnvelopeCorrectionUsed=true` +- `pitchTestOrg +4` + - branch: `pitch_only_adaptive_selector` + - run: `20260416_210838_adaptive_boundary_tune_r2` + - result: + - note mel `3.029` + - env `0.511` + - entry mel `1.993` + - exit mel `5.269` + - onset artifact `0.824` + - boundary timing error `5.458 ms` + - formant body harmonic drift `0.038` + - `spectralEnvelopeCorrectionUsed=true` +- Verdict on the first adaptive-carrier correction pass: + - this was a real engaged run, not a stale-binary repeat + - it is still not keepable: + - `pitchOrg +4` improved note/body similarity slightly, but onset artifact got much worse + - `pitchTestOrg +4` improved onset/formant proxy, but exit quality and boundary timing got much worse + - the current tuning priority should move to: + - `A3` transient handoff crossfade sweep + - `A4` entry timing compensation + - `A5` cepstral lifter retune + +- `pitchOrg +4` + - branch: `pitch_only_engine_v2_program` + - run: `20260416_152736_pitchOrg_plus4_note_hq_engine_v2_tune_r7` + - result: + - note mel `7.314` + - env `1.347` + - entry mel `7.396` + - exit mel `7.027` + - onset artifact `1.388` + - boundary timing error `0.792 ms` + - formant body harmonic drift `0.415` +- `pitchTestOrg +4` + - branch: `pitch_only_engine_v2_program` + - run: `20260416_152736_pitchTestOrg_plus4_note_hq_engine_v2_tune_r7` + - result: + - note mel `3.540` + - env `0.521` + - entry mel `7.513` + - exit mel `1.632` + - onset artifact `4.013` + - onset delay `1.479 ms` + - formant body harmonic drift `0.069` +- Comparison baseline: + - adaptive still wins the hard case clearly: + - `pitchOrg +4`: note mel `7.085`, entry mel `7.078`, onset artifact `1.803` + - `pitchTestOrg +4`: note mel `2.810`, entry mel `1.530`, onset artifact `0.705` + +## Validation Rules +- Keep comparing on: + - `pitchOrg.wav -> pitchOrg+4s.wav` + - `pitchTestOrg.wav -> pitchTestOrg+4s.wav` +- Run guard cases only after a real `+4` win +- Track: + - note mel + - note env RMSE + - note/body/core cents + - entry and exit mel/env + - onset artifact + - harmonic-envelope drift + - render latency for note-local HQ jobs + +## Cleanup Rule +- Mark new paths as `Active Test` or `User Pending` +- Do not remove experimental code until: + - benchmark failure is recorded + - and the user has listened to it or explicitly approved cleanup diff --git a/docs/pitch_editor_research_summary.md b/docs/pitch_editor_research_summary.md new file mode 100644 index 0000000..66aef33 --- /dev/null +++ b/docs/pitch_editor_research_summary.md @@ -0,0 +1,4523 @@ +# Pitch Editor Research Decision Log + +Canonical status board: [pitch_recovery_master_map.md](c:/Users/srvds/Documents/Codes/Studio13-v3/docs/pitch_recovery_master_map.md) + +Decision rule from 2026-04-14 onward: +- keep lifecycle state, queue order, and harvestable traits in the master map +- keep this file as the chronological experiment ledger with exact runs, metrics, and keep/reject decisions + +### 2026-04-28: Entry Contour-Handoff Correction + +Purpose: +- test the hypothesis that the remaining start artifact is caused by a pitch-trajectory discontinuity rather than only an audio splice. +- keep render context before the note without mutating the previous word or delaying hard step edits. + +Implementation: +- added note-HQ entry pitch-handoff diagnostics through C++, the frontend bridge, and regression summaries. +- changed the native pitch-ratio curve so explicit continuous/internal transitions can use a minimum-jerk pitch handoff. +- kept hard/unknown entries on the legacy-compatible rule: pre-note render context is allowed, but the ratio reaches the target by `note.startTime` and audible dry/wet ownership stays with the entry bridge. +- added F0 slope/acceleration reporting to the reference harness; the hard gate is applied only when an actual pitch handoff is active. + +Decision: +- keep the diagnostic and explicit-continuous handoff support. +- do not use a body-delayed pitch ramp for hard/unknown entries. Trial runs with a 20-40 ms hard-entry pitch ramp regressed `pitchOrg +4` onset artifact to `1.615` and then entry lag to `19-25 ms`; the correct hard-entry behavior is render pre-roll plus dry-protected commit, not delayed pitch onset. + +Validation: +- `D:\test projects\os tests\runs\20260428_115125_entry_contour_handoff_plus4_final3`: passed, onset artifact `1.501`, exit-next `2.156`, pre-bridge residual `-172.205 dB`. +- `D:\test projects\os tests\runs\20260428_115314_entry_contour_handoff_minus4_final`: passed, onset artifact `1.187`, exit-next `0.207`, downshift harmonic drift `0.339`, spectral envelope correction enabled. +- `D:\test projects\os tests\runs\20260428_115719_entry_contour_handoff_two_adjacent_plus4_r2`: passed, one edit island for two edited notes, with internal pitch handoff diagnostics and no doubled dry/wet ownership. + +### 2026-04-28: Emergency Word-Grouping Repair + +Purpose: +- fix the regression where broad `wordGroupId` ownership caused one clicked note to move unrelated notes. +- prevent phrase-first segmentation from collapsing separate words into oversized editable regions. + +Implementation: +- restored normal UI ownership: click/drag/edit operations affect only explicitly selected notes. +- removed the word-group hull overlay and "Word (n)" inspector behavior. +- kept `wordGroupId` as assistive metadata for diagnostics/render grouping, not as automatic selection or drag ownership. +- restored conservative analyzer merging: short `40 ms` merge gap plus pitch-distance guard, instead of merging every short gap bridged by phrase context. +- allowed strong acoustic `hard_word_like` candidates to split default analyzer regions, while pitch hysteresis/vibrato candidates remain non-destructive diagnostics. +- harness metrics now separate hard acoustic splits from destructive pitch-corner/pitch-jump failures and report expected-region overhang to catch collapsed words. + +Decision: +- keep this repair immediately; product default must be "the note I clicked is the note I move." +- whole-word movement should require explicit multi-selection or a future dedicated group-edit mode, not implicit analyzer grouping. +- verification: + - analyzer: `D:\test projects\os tests\runs\20260428_110455_emergency_word_group_repair_analysis_pitchOrg`, `noteCount=5`, `wordGroupCount=5`, destructive corner/pitch-jump splits `0/0`, hard acoustic splits `2`, max expected overhang `0.459`. + - UI ownership: `pitchEditorSingleNoteOwnership.test.ts` passed; shared `wordGroupId` no longer expands selection, drag updates, or selected pitch moves. + - audio: `pitchOrg +4` and `pitchOrg -4` note-HQ runs passed; `-4` harmonic drift measured `0.343`, onset artifacts measured `1.48` / `1.598`, exit-next artifacts `2.31` / `0.091`. + - adjacent selected notes: `D:\test projects\os tests\runs\20260428_110807_emergency_repair_two_adjacent_plus4` reported one note-HQ edit island for two selected edits, confirming the double-voice ownership fix remains active. + +### 2026-04-28: Phrase-First Vibrato-Safe Word Detection + +Purpose: +- fix continued over-fragmentation where one sung word in a single breath is split by vibrato, bend, or melisma movement. +- make word/phrase regions the primary editable units while preserving pitch contour diagnostics. + +Implementation: +- removed the destructive running-average pitch-jump split from default segmentation. +- short voiced detector dropouts are bridged up to `80 ms`; hard automatic cuts require long unvoiced gaps or sustained energy-break evidence. +- sustained pitch deviations are exported as `pitch_hysteresis_*` boundary candidates, never destructive splits. +- vibrato-like periodic reversals are marked as `internal_vibrato` diagnostics and remain inside the same editable word/phrase. +- close fragments can merge across short non-hard gaps regardless of pitch distance, because pitch difference alone is not a word boundary. +- superseded by the emergency repair: the canvas no longer selects/highlights `wordGroupId` as the primary object, and dragging one internal fragment no longer moves the whole group. + +Decision: +- keep this as a product-model/analyzer correction, not a renderer experiment. +- corrected product rule: continuous breath, vibrato, bend, and melisma may stay related by metadata, but visible editing remains note-first unless the user explicitly selects multiple notes. +- destructive pitch-corner and pitch-jump splitting stays research-only/diagnostic by default. + +Validation: +- native build and frontend build passed after the change. +- analyzer run `D:\test projects\os tests\runs\20260428_040921_phrase_first_word_detection_pitchOrg` passed with `noteCount=3`, `wordGroupCount=3`, `destructivePitchJumpSplitCount=0`, `destructiveCornerSplitCount=0`, edited-word overlap `1.000`, and max fragments `1`. +- primary note-HQ runs: + - `D:\test projects\os tests\runs\20260428_040951_phrase_first_pitchOrg_plus4` + - `D:\test projects\os tests\runs\20260428_041117_phrase_first_pitchOrg_minus4` +- adjacent-fragment diagnostic `D:\test projects\os tests\runs\20260428_041244_phrase_first_two_adjacent_plus4` confirmed one raw note-HQ edit island for two supplied fragments (`noteHqEditIslandCount=1`, `noteHqEditedNoteCount=2`). + +### 2026-04-28: Word Grouping And Edit-Island Correction + +Purpose: +- fix the regression where a single sung word is broken into several editable pieces. +- fix adjacent selected notes sounding doubled by treating them as one render island. + +Implementation: +- pitch-corner detections are now `boundaryCandidates` by default, not automatic note splits. +- destructive corner splitting is available only with `OPENSTUDIO_ANALYZER_APPLY_CORNER_SPLITS=1`. +- analyzer notes now carry `wordGroupId`; close voiced fragments without a hard acoustic boundary are grouped as one editable word/phrase. +- superseded by the emergency repair: pitch move and scale correction affect explicitly selected notes only. +- note-HQ request building coalesces adjacent edited notes into edit islands; only island entry/exit get bridge ownership, while internal boundaries stay on the continuous pitch curve. +- backend commit ranges merge adjacent edited notes into island ownership and no longer average diagnostic pitch ratios with `sqrt(previous * current)`. + +Decision: +- keep this as a product-model/signal-chain correction, not a new renderer experiment. +- product rule: word-like editing is the default; internal micro-note editing requires explicit manual split. +- corner detection remains useful as an analyzer hint, but not as a default edit seam. + +Validation: +- native build and frontend build passed after the change. +- analyzer run `D:\test projects\os tests\runs\20260428_022002_word_group_analysis_pitchOrg_r2` passed with `destructiveCornerSplitCount=0` and min expected word-group overlap `0.928`. +- primary note-HQ runs: + - `D:\test projects\os tests\runs\20260428_022028_word_group_island_plus4` + - `D:\test projects\os tests\runs\20260428_022028_word_group_island_minus4` +- adjacent-fragment diagnostic `D:\test projects\os tests\runs\20260428_023520_word_group_two_adjacent_plus4_r4` confirmed one raw note-HQ edit island for two edited fragments (`noteHqEditIslandCount=1`, `noteHqEditedNoteCount=2`). + +### 2026-04-27: Segmentation-Corner And Voiced-Core Timbre Correction + +Purpose: +- address the remaining stutter as a possible wrong-boundary problem, not only a compositor problem. +- reduce downshift formant drift with an envelope transfer tied to the original voiced vowel core. + +Implementation: +- note segmentation now detects conservative pitch-curve corners from the smoothed MIDI contour. +- a corner split requires supporting vocal evidence: energy dip, confidence dip, nearby unvoiced/noise, or strong pitch prominence. +- vibrato-like periodic reversals are suppressed so normal sustained vibrato does not become many small notes. +- analyzed notes now expose `entryBoundaryKind`, `exitBoundaryKind`, reason, and score. +- note-HQ uses those boundary kinds when choosing bridge ownership: hard word-like boundaries get shorter audible bridges, while soft legato boundaries may use wider phrase smoothing. +- downward pitch-only note-HQ now applies voiced-core spectral envelope transfer on edited note bodies after the native directional renderer. + +Decision: +- keep this as a signal-chain and segmentation correction, not a new renderer-family experiment. +- product rule: previous/next word bodies stay dry across hard boundaries; continuous sustain/legato may share transition smoothing, but neighboring note bodies are not fully retuned. +- acceptance remains the primary `pitchOrg +4/-4` note-HQ gates plus the richer formant/transient/boundary suites. + +Measured runs: +- `pitchOrg +4`: `D:\test projects\os tests\runs\20260428_013135_seg_corner_timbre_plus4` + - body/core pitch error `0.00 / 0.00 cents` + - entry lag `-0.125 ms`, onset artifact `1.479`, onset derivative `1.087` + - protected pre-bridge residual `-172.823 dB` + - exit-next artifact `2.307` + - harmonic drift `0.379` +- first `pitchOrg -4` aggressive envelope-transfer attempt: `D:\test projects\os tests\runs\20260428_013344_seg_corner_timbre_minus4` + - rejected because harmonic drift regressed to `0.615 > 0.360`. + - lesson: envelope transfer must be a bounded support-weighted correction, not a replacement of the native downshift body. +- final `pitchOrg -4`: `D:\test projects\os tests\runs\20260428_013650_seg_corner_timbre_minus4_mix005` + - body/core pitch error `0.00 / +11.90 cents` + - `spectralEnvelopeCorrectionUsed=true` + - entry lag `+0.771 ms`, onset artifact `1.598`, onset derivative `1.598` + - protected pre-bridge residual `-240.000 dB` + - exit-next artifact `0.091` + - harmonic drift `0.343`, low/mid/high deltas `-1.083 / -2.067 / -0.053 dB` +- analyzer diagnostic run: `D:\test projects\os tests\runs\20260428_013900_seg_corner_boundary_analysis` + - detected `11` notes over the 4 s `pitchOrg` clip, with `2` corner-boundary notes reported. + - boundary kind counts were `hard_word_like=22`, confirming diagnostics are serialized; further product tuning should use manual vocal-boundary references before making corner splits more aggressive. + +### 2026-04-27: Entry Bridge Fix For Final Note-HQ Apply + +Purpose: +- fix the remaining edited-note entry stutter without moving the artifact back into the previous word. +- keep phrase/effective render context, but make audible entry ownership bridge-aware rather than forcing either a hard dry edge or a broad wet left shoulder. + +Implementation: +- final note-HQ compositing now chooses a direction-aware entry bridge. +- upward edits use a tight in-body bridge because the pre-note bridge regressed the upward onset metrics. +- downward edits may start audible bridge ownership up to `24 ms` before `note.startTime`; on the canonical case the bridge starts at `0.876009s`, then lands back on body timing by the end of the entry window. +- downshift bridge applies a local wet-read delay (`-21.995 ms` on `pitchOrg -4`), a bounded envelope correction (`+1.7 dB`), and a short dry transient preservation window (`10 ms`). +- diagnostics now include `noteHqEntryBridgeStartSec`, `noteHqEntryBridgeEndSec`, `noteHqEntryBridgeWetLagMs`, `noteHqEntryBridgeEnvelopeGainDb`, `noteHqEntryBridgeUsed`, and `noteHqEntryTransientDryPreservedMs`. +- the harness now protects only the pre-bridge tail, not the whole pre-body region, and writes entry-bridge audition WAVs. + +Measured runs: +- `pitchOrg +4`: `tmp_pitch_runs/20260428_004839_entry_bridge_v15_plus4` + - body/core pitch error `0.00 / 0.00 cents` + - entry bridge `0.900000s -> 0.916009s`, lag `0.0 ms`, gain `0.0 dB` + - entry lag `-0.125 ms` + - onset artifact `1.479`, onset derivative `1.087` + - protected pre-bridge residual `-172.823 dB` + - exit-next artifact `2.307` + - harmonic drift `0.379` +- `pitchOrg -4`: `tmp_pitch_runs/20260428_004708_entry_bridge_v15_minus4` + - body/core pitch error `0.00 / +11.90 cents` + - entry bridge `0.876009s -> 0.980000s`, lag `-21.995 ms`, gain `+1.7 dB` + - entry lag `+0.771 ms` + - onset artifact `1.598`, onset derivative `1.598` + - protected pre-bridge residual `-240.000 dB` + - exit-next artifact `0.091` + - harmonic drift `0.344` +- export parity: + - `tmp_pitch_runs/20260428_005155_entry_bridge_v15_plus4_export` + - `tmp_pitch_runs/20260428_005458_entry_bridge_v15_minus4_export` + - source-vs-export dry-tail residual is still informational because the mixer/export path is full-file, but preview/export body pitch and note-HQ product parity passed. + +Decision: +- keep this as a signal-chain/compositor correctness fix, not a reopened renderer-family experiment. +- product rule: final apply may use a tiny bounded pre-note bridge for best sound, but anything before `noteHqEntryBridgeStartSec` must remain original/dry. +- expected perceived match after this pass: about `91-94%` for `+4` and `89-92%` for `-4` versus the provided samples on this fixture. + +### 2026-04-27: Final Pre-Body Dry Ownership Fix For Note-HQ Apply + +Purpose: +- fix the remaining previous-word stutter without regressing the edited-note exit/next-note handoff. +- separate renderer context from audible commit ownership. + +Root cause: +- the renderer needs left-shoulder context for phase/envelope history, but committing that left shoulder as wet audio moved the stitching artifact backward into the previous word. +- the existing `preCommitArtifactScore` could miss this because it looked at a narrow boundary point rather than auditing the whole previous-word tail. + +Implementation: +- note-HQ still renders with phrase/effective context. +- final dry-protected compositing now starts audible wet ownership at `note.startTime`, not `effectiveStartTime`. +- `[effectiveStartTime, note.startTime)` remains original/dry. +- entry blends dry-to-wet inside the edited note body over `12 ms`. +- exit keeps the previous fix: wet-to-dry starts at `note.endTime - 12 ms` and releases through `effectiveEndTime`. +- diagnostics now include context range, audible commit range, pre-body dry-protected samples, entry fade ms, and exit lead-in ms. +- the harness now has a full pre-body ownership audit: + - `preBodyTailOriginalResidualDb` over `[noteStart - 80 ms, noteStart)`. + - `candidateActiveDifferenceStartSec`. + - `preBodyTailArtifactScore`. + - audition WAVs: `orig_pre_body_tail.wav`, `cand_pre_body_tail.wav`, `diff_pre_body_tail.wav`. +- the ownership audit uses `noteHqAudibleCommitStartSec`, not analysis-only body/core windows. + +Measured runs: +- `pitchOrg +4`: `tmp_pitch_runs/20260427_213815_pre_body_dry_v3_plus4` + - body/core pitch error `0.00 / 0.00 cents` + - pre-body residual `-170.878 dB` + - active difference start `0.900063s` + - onset artifact `1.706` + - exit-next artifact `2.307` + - harmonic drift `0.379` +- `pitchOrg -4`: `tmp_pitch_runs/20260427_213940_pre_body_dry_v3_minus4` + - body/core pitch error `0.00 / +11.90 cents` + - pre-body residual `-169.515 dB` + - active difference start `0.900063s` + - onset artifact `2.721` + - exit-next artifact `0.091` + - harmonic drift `0.352` +- export parity: + - `tmp_pitch_runs/20260427_214538_pre_body_dry_v4_plus4_export` + - `tmp_pitch_runs/20260427_214814_pre_body_dry_v4_minus4_export` + - note: source-vs-export dry residual is informational only because the mixer/export path changes the full file from time zero; export parity is judged against the note-HQ product. +- richer formant suite: + - `tmp_pitch_runs/pre_body_dry_v2_formant_richer/20260427_212751_pre_body_dry_v2_formant_richer` +- richer transient suite: + - `tmp_pitch_runs/pre_body_dry_v4_transient_richer/20260427_215055_pre_body_dry_v4_transient_richer` +- boundary suite: + - `tmp_pitch_runs/pre_body_dry_v2_boundary/20260427_212152_pre_body_dry_v2_boundary` + - primary real `pitchOrg` cases stay under the exit-next gate; the synthetic shortened-end stress case still warns with a high exit-next artifact. + +Decision: +- keep this as a signal-chain/compositor correctness fix, not a renderer-family experiment. +- product rule: render context may extend before the edited note, but final committed audio before `note.startTime` must remain original/dry. +- expected perceived match after this pass: about `89-92%` for `+4` and `87-90%` for `-4` versus the provided samples, with the remaining gap mostly from timbre/body-envelope match rather than word-break stitching. + +### 2026-04-27: Pitch-Only Signal-Chain Fix, Note-HQ Slicing Fix, And Transition-Shoulder Commit + +Purpose: +- fix the reported pitch-note change failures as signal-chain bugs: + - timbre getting thinner/fatter when pitch moves + - stutter/word-break just before or after the edited note +- stop accepting low-confidence note-HQ fallback silently +- make the regression harness catch formant/spectrogram drift on the real candidate slice + +Root causes found: +- several `SignalsmithShifter::process(...)` pitch-only callsites passed `detectedPitchHz` as the sixth argument, which is actually `formantRatios`; that could turn F0 Hz values into huge unintended formant factors. +- `note_hq` regression slicing treated every candidate like a window-local render, even when the app wrote a full-clip/phrase result; this let formant and boundary metrics inspect the wrong slice. +- note-HQ apply request construction dropped the note's own transition shoulders when there was no immediate neighbor, so the dry patch still committed only the body and could hard-switch at word edges. + +Implementation: +- added explicit `SignalsmithShifter` pitch-only entrypoints: + - `processPitchOnlyBase(..., ratios, detectedPitchHz)` + - `processPitchOnlyCe33Base(..., ratios)` +- routed pitch-only detected-F0 renders through `process(..., ratios, {}, detectedPitchHz)`, keeping explicit formant rendering on the non-empty `formantRatios` path only. +- changed note-HQ pitch-only final render to require phrase/full-context offline HQ unless `OPENSTUDIO_PITCH_ALLOW_NOTE_HQ_NATIVE_FALLBACK=1` is set. +- expanded note-HQ commit ownership to include effective transition shoulders; for the canonical `0.900s-1.550s` body the final diagnostic runs commit `0.860s-1.610s`. +- added preview segment entry/exit crossfade to avoid hard-switching cached chunks against the source clip. +- fixed the formant proxy peak scorer to order selected broad peaks by frequency before treating them as F1/F2 proxies. + +Measured runs: +- production HQ-required check: + - `20260427_135602_after_fix_pitchOrg_plus4_hq_required_missing_runtime` + - expected failure in this checkout: bundled Rubber Band executable exits with Windows status `0xC0000135` because dependent runtime DLLs are missing. + - result is now a hard failure, not a silent native fallback. +- follow-up runtime fix: + - added `tools/rubberband/sndfile.dll` from the bundled Python `libsndfile_x64.dll`, plus the matching VC runtime DLLs. + - `rubberband.exe --version` and `rubberband-r3.exe --version` now report `4.0.0`. + - app-path result `20260427_145800_rubberband_runtime_fixed_pitchOrg_plus4` used `rubberband_hq_phrase_hq` with `phraseHqExternalUsed=true` and `pitchRenderBackendVersion=4.0.0`. + - quality status is still not promoted: `20260427_145847_rubberband_runtime_fixed_pitchOrg_plus4` failed the strict formant gate at mid-band delta `+7.866 dB > 7.000 dB` and boundary timing `38.54 ms`. +- final debug-native `pitchOrg +4`: + - run: `20260427_135220_after_fix_pitchOrg_plus4_native_override_final` + - body/core pitch error: `0.00 / 0.00 cents` + - note mel/env: `6.804 dB / 1.258` + - formant body harmonic drift: `0.378` + - low/mid/high deltas: `-2.46 / +0.53 / -3.59 dB` + - core F1/F2 proxy drift: `-46.9 / +23.4 Hz` + - boundary timing error: `8.42 ms` + - onset artifact score: `1.77` + - spectrogram assets: `D:\test projects\os tests\runs\20260427_135220_after_fix_pitchOrg_plus4_native_override_final\spectrogram` +- final debug-native `pitchOrg -4`: + - run: `20260427_135413_after_fix_pitchOrg_minus4_native_override_final` + - body/core pitch error: `0.00 / +11.90 cents` + - note mel/env: `6.557 dB / 1.086` + - formant body harmonic drift: `0.469` + - low/mid/high deltas: `-0.41 / +0.83 / +1.12 dB` + - core F1/F2 proxy drift: `+46.9 / -70.3 Hz` + - boundary timing error: `25.38 ms` + - onset artifact score: `0.94` + - spectrogram assets: `D:\test projects\os tests\runs\20260427_135413_after_fix_pitchOrg_minus4_native_override_final\spectrogram` +- richer suites: + - formant richer suite passed all `6` cases: `20260427_135901_after_fix_formant_richer_native_override` + - transient richer suite passed all `8` cases: `20260427_141431_after_fix_transient_richer_native_override` + - export/preview parity passed for `pitchOrg +4`: `20260427_143420_after_fix_export_preview_parity_plus4_native_override` +- boundary suite: + - run: `20260427_142907_after_fix_boundary_pitchOrg_plus4_native_override` + - `start_earlier` and `start_later` passed. + - synthetic `end_earlier` failed the new hard gate: boundary timing `38.938 ms > 32 ms`. + - decision: keep this as a known strict stress failure, not a pass. + +Decision: +- this is a correctness fix to the existing signal chain and harness, not a reopened renderer-family experiment. +- keep `pitch_only_adaptive_selector` as the native diagnostic fallback. +- production note-HQ needs the external HQ backend/runtime fixed before claiming final HQ parity. + +### 2026-04-17: Root-Cause Research + ML Benchmark + Engine-v3 Feasibility + +Purpose: +- stop doing blind renderer experiments +- turn the remaining two product issues into evidence-backed causes +- quickly decide whether local ML restoration or a clean-sheet `engine-v3` branch is actually credible + +New tooling: +- `tools/pitch_root_cause_research.py` +- `tools/run-pitch-root-cause-research.ps1` +- `tools/run-pitch-ml-benchmark.ps1` +- `tools/pitch_engine_v3_feasibility.py` +- `tools/run-pitch-engine-v3-feasibility.ps1` + +Primary outputs: +- root-cause: + - `20260417_012542_pitch_root_cause_research` + - summary doc: + - `D:\test projects\os tests\runs\20260417_012542_pitch_root_cause_research\pitch_root_cause_research.md` +- ML benchmark: + - `20260417_003355_pitch_ml_benchmark` +- engine-v3 feasibility: + - `20260417_003355_engine_v3_feasibility` +- repo summary: + - `docs/pitch_root_cause_research_20260417.md` + +Root-cause verdicts: +- rank 1: + - transition ownership and boundary timing drift + - evidence: + - hard-case adaptive boundary timing still peaks around `18.92 ms` + - frozen engine-v2 still collapses at hard-case entry with entry mel `7.017` +- rank 2: + - mixed transient and first-voiced-cycle content is still being handled by one renderer family + - evidence: + - hard-case transient entry/exit max stayed about `1.614 / 6.723` + - engine-v2 still failed with transient bypass enabled +- rank 3: + - current formant preservation is too weak and too local to survive hard transitions + - evidence: + - adaptive formant drift remained around `0.364` on `pitchOrg` and `0.071` on `pitchTest` + - engine-v2 kept envelope correction on but still lost the hard case + +ML benchmark verdict: +- result: + - `blocked_no_stronger_restorer` +- environment: + - runtime ready `true` + - backend `cuda` +- candidate check: + - `voicefixer`: not installed + - `demucs`: not installed + - `audio_separator`: available but not suitable for note-local restoration + - `proxy_ml_restore_v1`: explicitly excluded and already rejected +- decision: + - do not reopen local ML restoration on another proxy + - only reopen once a materially stronger restorer is available + +Engine-v3 feasibility verdict: +- decomposition probe results: + - `pitchOrg_plus4`: score `0.511`, verdict `stop` + - `pitchTest_plus4`: score `0.505`, verdict `stop` +- decision: + - do not open a long `engine-v3` implementation branch from this probe + - only revisit `engine-v3` after defining a materially stronger decomposition and transition-pair ownership model + +Meaning for the program: +- current best renderer remains `pitch_only_adaptive_selector` +- engine-v2 remains frozen as comparison evidence only +- the unsolved product issues are now understood as architectural rather than just tuning leftovers +- next bounded action should be: + - stronger external/licensed or materially stronger ML benchmark, or + - a better pre-defined decomposition/transition design before any future `engine-v3` + +### 2026-04-16: Richer Fixture Close-Out Progress, `H3/H4` Closed For Current Canonical Corpus + +Purpose: +- stop leaving the remaining iteration budget tied to smoke-only fixture coverage +- close out richer transient and formant suites on the current canonical clip families +- tighten the scrub-suite truth so the only remaining mandatory gap is the real multi-note selection-change case + +What changed: +- added richer manifests: + - `tests/fixtures/pitch-regression/suites/transient_richer_suite.json` + - `tests/fixtures/pitch-regression/suites/formant_richer_suite.json` +- extended the scrub-suite plumbing to request a selection-change scenario when available +- ran richer scrub suites on both canonical clip families: + - `20260416_231821_pitchOrg_scrub_suite_richer_r1` + - `20260416_234516_pitchTest_scrub_suite_richer_r1` +- ran richer regression suites: + - transient: `20260416_231844_transient_suite_richer_r1` + - formant: `20260416_233426_formant_suite_richer_r1` + +Scrub-suite truth: +- first drag, repeated drag, and after-transport-cycle are all audible on both clip families +- representative scrub figures: + - `pitchOrg` + - start / stop latency about `26-28 ms` + - repeat stability `0.4699` + - last peak `0.1123` + - `pitchTestOrg` + - start / stop latency about `26-28 ms` + - repeat stability `0.7770` + - last peak `0.1313` +- verdict: + - `H1` is still only partial + - the suite now covers the major stopped-transport scrub scenarios + - but true selection-change is still unproven because the current scrub fixtures resolve to one-note jobs, so the added scenario does not yet exercise a real second-note handoff + +Transient richer-suite highlights: +- `pitchOrg +4` richer entry/exit runs: + - note mel about `7.095` + - onset artifact about `1.212` + - formant drift about `0.367` +- `pitchTestOrg +4` richer entry/exit runs: + - note mel about `4.915` + - entry artifact about `1.58-1.61` + - exit artifact about `6.36-6.72` + - boundary timing error about `11.60-18.92 ms` +- `pitchTestOrg -4` richer entry/exit runs: + - note mel about `5.743` + - exit artifact about `7.90-8.41` + - boundary timing error about `16.54-35.40 ms` +- verdict: + - `H3` is now complete for the current canonical local fixture corpus + - the suite confirms the remaining weakness is still boundary behavior, especially hard-case exits and downward guards + +Formant richer-suite highlights: +- `pitchOrg +4` body / transition formant runs: + - formant drift about `0.360-0.367` + - entry / exit artifact still about `6.59-7.05` +- `pitchTestOrg +4` body / transition formant runs: + - formant drift about `0.063-0.080` + - entry artifact about `1.58-1.77` + - exit artifact about `4.22-6.72` +- `pitchTestOrg -4` body formant run: + - formant drift `0.0569` + - note/body cents still off on the downward guard +- verdict: + - `H4` is now complete for the current canonical local fixture corpus + - the suite confirms formant drift is still materially worse on the easier `pitchOrg` family than on `pitchTestOrg`, while boundary artifacts still dominate the listening problem overall + +Current plan truth: +- remaining mandatory iterations: `1` +- remaining conditional iterations: `+2` +- the one mandatory item still open is: + - true multi-note `H1` scrub selection-change coverage + +### 2026-04-16: `H1` Scrub Selection-Change Close-Out + +Purpose: +- close the last remaining mandatory harness gap instead of leaving scrub selection-change as a half-wired scenario +- prove the app-path scrub job can exercise a real second-note handoff, not just repeat `first_drag` + +What changed: +- fixed scrub note-array flattening in `tools/run-ui-pitch-scrub-regression.ps1` +- added multi-note scrub fixtures: + - `tests/fixtures/pitch-regression/example_pitchOrg_scrub_multinote.json` + - `tests/fixtures/pitch-regression/example_pitchTest_scrub_multinote.json` +- added scrub debug/result fields in the app-path regression flow so the selection-change scenario could be verified directly + +Verification runs: +- direct debug run: + - `20260416_235624_pitchTest_scrub_selection_debug_r1` + - result: + - `scrubPreviewSelectionChangeAudible=true` + - scenario count `2` + - scenario names: + - `first_drag` + - `selection_change` +- full multi-note scrub suite: + - `20260416_235640_pitchTest_scrub_suite_multinote_r3` + - result: + - `first_drag`: audible `true` + - `repeated_drag`: audible `true` + - `after_transport_cycle`: audible `true` + - `selection_change`: audible `true` + - selection-change case reports: + - `scrubPreviewSelectionChangeAudible=true` + - start latency `27.2 ms` + - stop latency `27.6 ms` + - repeat stability `0.4174` + - last peak `0.0991` + +Verdict: +- `H1` is now complete for the current canonical local fixture corpus +- there are no mandatory harness-closeout iterations left +- the remaining work from here is no longer “unfinished implementation”; it is product-quality improvement work, plus the optional conditional phase-locking branch if we decide to open it + +Current plan truth: +- remaining mandatory iterations: `0` +- remaining conditional iterations: `+2` + +### 2026-04-16: Scrub Preview Natural-Segment Fix Landed, Boundary/Formant Work Still Pending + +Purpose: +- fix the user-facing scrub-preview failure instead of treating the older infrastructure-only pass as complete +- make drag preview audible on first drag and stop the tiny-loop tearing behavior +- keep the boundary/formant work explicitly marked as still incomplete + +What changed: +- scrub preview now serializes note/frame payloads across the native bridge for a more reliable native parse +- scrub extraction now prefers the clip's active audio source instead of the preserved original-file path +- scrub extraction now falls back to deriving note bounds from analyzed frames if note metadata arrives incomplete +- scrub preview now uses: + - a longer natural note-local voiced segment + - gain normalization + - repeat-stability telemetry + - preview armed / first-callback / first-drag-audible status flags + +Fresh scrub regression: +- run: `20260416_200227_pitchOrg_scrub_preview_r8` +- result: + - `scrubPreviewAudible=true` + - `scrubPreviewFirstDragAudible=true` + - start latency `26.9 ms` + - stop latency `26.8 ms` + - base pitch `365.17 Hz` + - loop duration `240.0 ms` + - last peak `0.1123` + - repeat stability `0.4699` + +Verdict: +- the scrub path is now materially closer to the intended product behavior and no longer falls back to the near-silent `40 ms / 55 Hz` path in the harness +- this closes the core `S1/S2` scrub implementation work and most of `S3` +- boundary tuning, formant tuning, the full boundary/transient/formant suites, and the full adaptive/engine-v2 tuning program are still not complete + +### 2026-04-16: Dedicated RAM Scrub Voice Implemented, First Full Engine-V2 Audio Pass Active + +Purpose: +- finish the remaining implementation work that the 2-tier pitch-editor program still needed +- replace the old “interactive preview piggyback” assumption with a true RAM scrub monitor path +- move `engine-v2` from diagnostics/scaffold territory into a real benchmarkable audio renderer + +What was implemented: +- dedicated scrub-preview pipeline + - native bridge entrypoints: + - `startPitchScrubPreview` + - `updatePitchScrubPreview` + - `stopPitchScrubPreview` + - scrub loops are extracted from the stable interior of the selected note, not the onset + - loops live entirely in RAM and are mixed directly by `PlaybackEngine` + - stereo clips are supported by deriving loop bounds from mono analysis and extracting matching multichannel audio +- first real full `engine-v2` audio pass: + - Signalsmith-based voiced-core render in the transition window + - cepstral envelope restoration on voiced-support frames + - spectral-flatness transient/unvoiced bypass mask + - residual carry reinjection from shared own-engine analysis + - transition compositor layered on top of the frozen adaptive selector output + +Primary truth-case results: +- `pitchOrg +4` + - adaptive benchmark: + - run: `20260416_115127_pitchOrg_plus4_note_hq_adaptive_r2` + - note mel `7.085` + - env `1.376` + - entry `7.078` + - exit `7.027` + - onset artifact `1.80` + - engine-v2 full audio: + - run: `20260416_124252_pitchOrg_plus4_note_hq_engine_v2_program_impl_r2` + - note mel `9.653` + - env `1.782` + - entry `10.430` + - exit `11.219` + - onset artifact `3.23` + - note/body/core cents `-18.72 / -18.72 / -37.23` + - `spectralEnvelopeCorrectionUsed=true` + - `engineV2Used=true` + - verdict: + - the new renderer is real, but it is still not keepable; it is pulling the easy truth case too far away from the adaptive benchmark +- `pitchTestOrg +4` + - adaptive benchmark: + - run: `20260416_115302_pitchTestOrg_plus4_note_hq_adaptive_r2` + - note mel `2.810` + - env `0.470` + - entry `1.530` + - exit `1.632` + - onset artifact `0.70` + - engine-v2 full audio: + - run: `20260416_124622_pitchTestOrg_plus4_note_hq_engine_v2_program_impl_r3` + - note mel `11.303` + - env `1.425` + - entry `8.389` + - exit `10.274` + - onset artifact `0.85` + - note/body/core cents `-17.94 / -17.94 / -36.45` + - `spectralEnvelopeCorrectionUsed=true` + - `engineV2Used=true` + - verdict: + - the hard truth case is still much worse than the frozen adaptive benchmark + +Decision: +- keep all code in place +- mark the family `Active Test / User Pending` +- do not remove the new scrub voice or engine-v2 code +- next tuning step must narrow engine-v2 engagement around the transition nucleus instead of letting it own such a wide region + +### 2026-04-16: Engine-V2 Transition-Nucleus Tightening + +Purpose: +- reduce the catastrophic full-note drift from the first engine-v2 audio pass +- keep the adaptive selector as the dominant carrier +- let engine-v2 touch only the onset-transition nucleus for upward note changes + +What changed: +- transition ownership moved from the wider island/body window to a much tighter note-entry window +- engine-v2 now adds deltas on top of the adaptive baseline instead of composing a broader replacement mix +- voiced-core wet level and residual carry were reduced +- transient preservation was made more dominant at the entry edge + +Primary truth-case results: +- `pitchOrg +4` + - run: `20260416_125615_pitchOrg_plus4_note_hq_engine_v2_program_impl_r4` + - result: + - note mel `7.323` + - env `1.419` + - entry mel `7.918` + - exit mel `7.027` + - onset artifact `1.66` + - note/body/core cents `0.00 / 0.00 / 0.00` + - verdict: + - much safer than the first full engine-v2 pass and exact on whole-note/body pitch, but still worse than adaptive overall because entry quality drifted and note/env did not improve enough +- `pitchTestOrg +4` + - run: `20260416_125615_pitchTestOrg_plus4_note_hq_engine_v2_program_impl_r4` + - result: + - note mel `3.250` + - env `0.536` + - entry mel `4.277` + - exit mel `1.632` + - onset artifact `1.95` + - note/body/core cents `0.00 / 0.00 / -18.32` + - verdict: + - far better than the original catastrophic engine-v2 attempt, but still clearly worse than the adaptive selector on the hard truth case + +Follow-up tuning: +- `r5` tightened wet ownership even further: + - `20260416_125930_pitchOrg_plus4_note_hq_engine_v2_program_impl_r5` + - `20260416_125930_pitchTestOrg_plus4_note_hq_engine_v2_program_impl_r5` +- result: + - metrics stayed effectively the same as `r4` + - this indicates the current engine-v2 topology is now failing in a stable way rather than exploding numerically + +Additional tightening: +- `r6` protected the first voiced cycles longer and pushed engine-v2 takeover deeper into the entry: + - `20260416_131617_pitchOrg_plus4_note_hq_engine_v2_program_impl_r6` + - `20260416_131617_pitchTestOrg_plus4_note_hq_engine_v2_program_impl_r6` +- result: + - `pitchOrg +4` + - note mel `7.313` + - env `1.365` + - entry mel `8.121` + - onset artifact `1.48` + - exact note/body/core cents `0 / 0 / 0` + - `pitchTestOrg +4` + - note mel `3.429` + - env `0.508` + - entry mel `5.278` + - onset artifact `3.81` + - note/body cents exact but core still `-18.32` +- verdict: + - the easy truth case became somewhat safer and preserved pitch/body exactly + - the hard truth case still loses badly on note-entry quality, so this compositor family is now plateauing as a waveform-correction overlay + +Current interpretation: +- implementation is complete enough to audition: + - dedicated RAM scrub preview exists + - engine-v2 voiced-core + cepstral envelope + transient bypass + residual carry exists +- the main audible issues are still not solved +- the remaining problem is now narrower: + - engine-v2 is no longer catastrophically unstable + - but its current transition compositor still makes the hard note entry worse than the adaptive selector + - the next meaningful move is likely a different engine-v2 role, such as using engine-v2 primarily as a formant/timbre correction layer on top of the adaptive carrier instead of as a waveform-correction overlay + +### 2026-04-16: `FAM-ENGINE-V2` `V2-1` Scaffold Started + +Purpose: +- begin the engine-v2 fallback program without risking the current best editor +- make the new branch fail-closed to `pitch_only_adaptive_selector` +- prove we can extract transition-native support signals before changing audible output + +What was implemented: +- new safe benchmark branch: + - `pitch_only_engine_v2_program` +- new engine-v2 diagnostics flowing through the native/app regression path: + - `engineV2Used` + - `engineV2FallbackUsed` + - `engineV2TransitionCount` + - `engineV2TransitionStartSec` + - `engineV2TransitionEndSec` + - `engineV2HarmonicSupportPeak` + - `engineV2ResidualSupportPeak` + - `engineV2EnvelopeSupportPeak` +- current `V2-1` behavior: + - render exactly the frozen adaptive-selector output + - compute transition/harmonic/residual/envelope scaffold diagnostics from `OwnPitchEngine` shared analysis + - report those diagnostics through the regression summaries + +Primary smoke runs: +- `pitchOrg +4` + - run: `20260416_104358_v2_scaffold_pitchOrg_plus4_smoke_r4` + - result: + - requested/actual branch: `pitch_only_engine_v2_program / pitch_only_engine_v2_program` + - output SHA: `F4BFBE23347CA853EEDE27E48E70F24DC5983AABE37498D02CD8F6D518B3EC72` + - `engineV2Used=true` + - transition count: `1` + - transition window: `0.090–0.830s` + - support peaks: harmonic `0.982`, residual `0.075`, envelope `2.379` + - verdict: + - safe and bit-identical to the frozen adaptive benchmark, with real scaffold engagement +- `pitchTestOrg +4` + - run: `20260416_104610_v2_scaffold_pitchTestOrg_plus4_smoke_r1` + - result: + - requested/actual branch: `pitch_only_engine_v2_program / pitch_only_engine_v2_program` + - output SHA: `1164BB74D7FC7EC87B163790DAF763E94B1B79812B7BE36F3FCD97966CFA740A` + - `engineV2Used=true` + - transition count: `1` + - transition window: `0.090–1.260s` + - support peaks: harmonic `0.982`, residual `0.075`, envelope `2.192` + - verdict: + - safe and bit-identical to the frozen adaptive benchmark on the harder truth case too + +Program state after `V2-1`: +- engine-v2 is now an active family rather than a placeholder +- the scaffold is fail-closed, measurable, and safe on both main `+4` truth clips +- this does not solve the stutter/formant problem yet; it just gives us a trustworthy base for `V2-2` + +### 2026-04-16: `FAM-ENGINE-V2` First `V2-2` Core-Blend Attempt Rejected + +Purpose: +- take one conservative audible step beyond the scaffold +- keep the engine-v2 branch on the same diagnostics/transition windows +- test whether a light own-engine voiced-core bleed-in could help without destabilizing the frozen adaptive output + +What was tried: +- a conservative stable-core blend on top of `pitch_only_engine_v2_program` +- the blend only allowed own-engine output into the analyzed voiced support region +- the rest of the branch still followed the adaptive-selector output + +Primary truth-case result: +- `pitchOrg +4` + - run: `20260416_105142_v2_2_pitchOrg_plus4_g1` + - result: + - note mel `7.085 -> 7.175` + - env `1.376 -> 1.421` + - entry mel `7.078 -> 7.321` + - exit mel `7.027 -> 7.418` + - onset artifact `1.80 -> 1.75` + - verdict: + - not keepable; the note, entry, and exit all drifted the wrong way for only a tiny onset gain +- `pitchTestOrg +4` + - run: `20260416_105142_v2_2_pitchTestOrg_plus4_g1` + - result: + - note mel `6.292 -> 8.192` + - env `1.130 -> 1.236` + - entry mel `1.530 -> 9.507` + - exit mel `1.632 -> 2.027` + - note cents `-125.96 -> -141.08` + - onset artifact `0.70 -> 1.13` + - verdict: + - clearly worse on the harder truth case + +Decision: +- rejected immediately under the stop-fast rule +- restored `pitch_only_engine_v2_program` to the safe `V2-1` scaffold after the trial + +Parity restore check after rollback: +- `pitchOrg +4` + - run: `20260416_110003_v2_scaffold_pitchOrg_plus4_parity_check_r6` + - result: + - SHA back to `F4BFBE23347CA853EEDE27E48E70F24DC5983AABE37498D02CD8F6D518B3EC72` + - still `engineV2Used=true` with transition/support diagnostics intact +- `pitchTestOrg +4` + - run: `20260416_110152_v2_scaffold_pitchTestOrg_plus4_parity_check_r2` + - result: + - SHA back to `1164BB74D7FC7EC87B163790DAF763E94B1B79812B7BE36F3FCD97966CFA740A` + - still `engineV2Used=true` with transition/support diagnostics intact + +Current engine-v2 state: +- scaffold remains active and safe +- first audible `V2-2` attempt is rejected +- the next `V2-2` candidate must be structurally different from a simple stable-core blend + +### 2026-04-15: Remaining Research Close-Out + +Purpose: +- close the remaining credible families cleanly instead of leaving half-open work items in the queue +- freeze the current best editor on measured evidence +- decide whether the next move is another local family or the engine-v2 fallback + +What was implemented: +- added an analysis-only regression harness: + - [run-ui-pitch-analysis-regression.ps1](c:/Users/srvds/Documents/Codes/Studio13-v3/tools/run-ui-pitch-analysis-regression.ps1) + - shared job-driver support in [pitchRegressionDriver.ts](c:/Users/srvds/Documents/Codes/Studio13-v3/frontend/src/utils/pitchRegressionDriver.ts) +- used that harness to close out `FAM-ANALYZER-PYIN` +- reran the four canonical truth cases on `pitch_only_adaptive_selector` to freeze it as the benchmark branch +- checked the repo for a stronger ML restorer than `ml_restore_proxy_v1` +- checked the remaining PV status against the actual local implementation inventory + +Analyzer close-out: +- `A1` direct-YIN + decoder on `pitchOrg` + - run: `20260415_210932_A1_pitchOrg_direct` + - result: + - notes detected / expected: `7 / 1` + - voiced frame ratio: `0.802` + - median voiced confidence: `0.974` + - matched expected note window: yes + - overlap ratio: `0.964` +- `A1` direct-YIN + decoder on `pitchTestOrg` + - run: `20260415_210932_A1_pitchTestOrg_direct` + - result: + - notes detected / expected: `6 / 1` + - voiced frame ratio: `0.676` + - median voiced confidence: `0.976` + - matched expected note window: yes + - overlap ratio: `0.796` +- `A2` FFT-YIN parity decision + - runs: + - `20260415_210932_A2_pitchOrg_fft` + - `20260415_210932_A2_pitchTestOrg_fft` + - result: + - `0` detected notes on both fixtures + - voiced frame ratio `0.000` on both fixtures + - verdict: + - direct-YIN + decoder is frozen as the kept analyzer path + - FFT-YIN is formally rejected-for-now and remains gated off + +Broader adaptive-selector validation: +- runs: + - `20260415_211004_closeout_pitchOrg_plus4_adaptive` + - `20260415_211004_closeout_pitchOrg_minus4_adaptive` + - `20260415_211004_closeout_pitchTestOrg_plus4_adaptive` + - `20260415_211004_closeout_pitchTestOrg_minus4_adaptive` +- result: + - `pitchOrg +4` + - note mel `7.085` + - env `1.376` + - entry `7.078` + - exit `7.027` + - onset artifact `1.80` + - `pitchOrg -4` + - note mel `6.623` + - env `1.102` + - entry `6.585` + - exit `7.475` + - onset artifact `2.80` + - `pitchTestOrg +4` + - note mel `2.810` + - env `0.470` + - entry `1.530` + - exit `1.632` + - onset artifact `0.70` + - `pitchTestOrg -4` + - note mel `3.505` + - env `1.024` + - entry `3.090` + - exit `1.835` + - onset artifact `3.64` +- verdict: + - `pitch_only_adaptive_selector` is now frozen as the benchmark branch for any future engine-v2 work + +Remaining-family close-out decision: +- `FAM-ML-RESTORATION` + - no materially stronger trained restorer exists locally beyond `ml_restore_proxy_v1` + - family is deferred behind engine-v2 +- `FAM-PVDR` + - no materially different stable implementation is ready locally beyond the rejected resample-plus-phase-lock attempt + - family stays closed unless a genuinely different design is prepared + +Program conclusion: +- remaining close-out work is complete +- the current product issues are still unresolved: + - stutter on note changes + - formant/timbre change on note changes +- the next serious step is no longer another local family patch +- the next serious step is the `FAM-ENGINE-V2` fallback program + +### 2026-04-15: `FAM-TRANSITION-HQ` Stop-Fast Verdict + +Purpose: +- try a proper note-change-specific HQ renderer instead of another full-note shell swap +- keep `pitch_only_adaptive_selector` frozen as the live working editor +- benchmark a transition-only shell/core/residual overlay on top of the adaptive output before even considering an ML finish + +What was implemented for the trial: +- temporary branch key: + - `pitch_only_transition_hq` +- temporary HQ-only transition overlay on top of `pitch_only_adaptive_selector`: + - preserve original shell content near the note-change edge + - blend in own-engine voiced core support only through the transition window + - reinject a small residual carry inside the transition core + - apply simple local envelope correction between adaptive and own-engine outputs +- reusable diagnostics were added to the regression/native result flow: + - `transitionHqUsed` + - `transitionHqFallbackUsed` + - `transitionStartSec` + - `transitionEndSec` + - `transitionTransientPeak` + - `transitionVoicedCorePeak` + - `transitionResidualPeak` + - `transitionEnvelopeCorrectionUsed` + +Truth-case results versus the current adaptive selector: +- `pitchOrg +4` + - adaptive control run: `20260415_185731_pitchOrg_plus4_adaptive_transition_cmp` + - transition HQ run: `20260415_185258_pitchOrg_plus4_transition_hq_m1` + - result: + - note mel `7.085 -> 7.395` + - env `1.376 -> 1.434` + - entry mel `7.078 -> 7.723` + - exit mel `7.027 -> 7.027` + - onset artifact `1.80 -> 1.59` + - note/body/core cents stayed exact at `0 / 0 / 0` + - `transitionHqUsed=true` + - transition peaks `0.829 / 0.840 / 0.073` + - verdict on this case: + - onset improved a little, but note, envelope, and entry all got worse +- `pitchTestOrg +4` + - adaptive control run: `20260415_185942_pitchTestOrg_plus4_adaptive_transition_cmp` + - transition HQ run: `20260415_185446_pitchTestOrg_plus4_transition_hq_m1` + - result: + - note mel `2.810 -> 4.084` + - env `0.470 -> 0.772` + - entry mel `1.530 -> 15.735` + - exit mel `1.632 -> 1.632` + - onset artifact `0.70 -> 3.73` + - core cents stayed `-18.32`, matching the underlying hard-case pitch issue + - `transitionHqUsed=true` + - transition peaks `0.574 / 0.840 / 0.073` + - verdict on this case: + - the overlay made the hard truth case dramatically worse + +Verdict: +- rejected after the first DSP iteration +- it failed the keep gate decisively: + - one case only improved the onset score while harming note/body entry quality + - the other primary case regressed heavily on note mel, envelope, entry, and onset artifact + - there was no justification to open the optional ML-finish stage + +Cleanup: +- removed the temporary `pitch_only_transition_hq` branch support from the renderer and harness after the verdict +- kept the transition diagnostics plumbing in place for future benchmark reporting if a materially different engine-v2 path needs similar measurements +- the live editor remains `pitch_only_adaptive_selector` + +### 2026-04-15: Research Synthesis Before The Next Renderer Family + +Purpose: +- stop guessing between "modern sounding" ideas and actually research the remaining families that are still credible +- separate what the repo already tried from what the literature still supports +- update the queue so the next work is evidence-backed instead of another near-duplicate shell variation + +Primary-source takeaways: +- median-filter HPSS is a real and useful separation technique, but it is a decomposition step, not a complete vocal pitch-editor renderer + - source: FitzGerald 2010 DAFx paper +- WSOLA is strong for local continuity and time-scale seams, but not a complete answer to vocal pitch/body/formant artifacts + - source: Verhelst and Roelands 1993 +- research-grade phase-vocoder work is still genuinely untried here + - the repo only tried a lightweight boundary-alignment variant + - sources: Laroche and Dolson 1999, and "Phase Vocoder Done Right" 2022 +- DDSP is the strongest engine-v2 style research direction if we accept a larger redesign + - source: DDSP 2020 +- the shallow-diffusion singing restoration paper is the closest direct match to the repo’s real product problem + - start from a usable pitch-shifted render + - then restore natural singing quality while preserving melody and timing + - source: Liu and Akama 2026 +- WORLD remains useful as a support decomposition for envelope/aperiodicity features, not as the final renderer target + - source: WORLD 2016 + +Decision from the literature pass: +- stop treating more seam-only or shell-only DSP tweaks as the highest-value next step +- keep the current best working editor (`pitch_only_adaptive_selector`) frozen as the live benchmark +- move the next queue to: + - `FAM-PVDR` + - `FAM-ML-RESTORATION` + - broader validation on `FAM-ADAPTIVE-SELECTOR` + +Repo note: +- the full research note and source list is now in [pitch_renderer_research_notes.md](c:/Users/srvds/Documents/Codes/Studio13-v3/docs/pitch_renderer_research_notes.md) + +### 2026-04-15: `FAM-ML-RESTORATION` Proxy Benchmark Verdict + +Purpose: +- execute the next research-backed direction without touching the live editor path +- keep `pitch_only_adaptive_selector` frozen as the base renderer +- test restoration-after-render as an offline benchmark, not a shipping path + +What was implemented for the trial: +- new offline benchmark script: [ml_restore_benchmark.py](c:/Users/srvds/Documents/Codes/Studio13-v3/tools/ml_restore_benchmark.py) +- regression harness integration in [run-ui-pitch-regression.ps1](c:/Users/srvds/Documents/Codes/Studio13-v3/tools/run-ui-pitch-regression.ps1) +- benchmark-only diagnostics recorded through the existing summary flow: + - `mlRestoreUsed` + - `mlRestoreModelId` + - `mlRestoreWindowSec` + - `mlRestoreBaseBranch` +- first proxy model id: + - `ml_restore_proxy_v1` +- `M1` proxy behavior: + - render with `pitch_only_adaptive_selector` + - restore only the note window offline + - use pYIN-derived F0 conditioning from the original window + - use original-window energy conditioning + - apply transient-emphasis blending near note entry + - apply harmonic spectral-envelope correction on the rendered candidate + +Truth-case results versus the current adaptive selector: +- `pitchOrg +4` + - adaptive control run: `20260415_142514_pitchOrg_plus4_adaptive_m1_cmp` + - ML restore run: `20260415_141959_pitchOrg_plus4_ml_restore_m1` + - result: + - note mel `7.085 -> 6.736` + - env `1.376 -> 1.291` + - entry mel `7.078 -> 6.716` + - exit mel `7.027 -> 6.629` + - onset artifact `1.80 -> 1.77` + - note/body/core cents stayed exact at `0 / 0 / 0` + - verdict on this case: + - promising on the easier upward truth case +- `pitchTestOrg +4` + - adaptive control run: `20260415_142514_pitchTestOrg_plus4_adaptive_m1_cmp` + - ML restore run: `20260415_142207_pitchTestOrg_plus4_ml_restore_m1` + - result: + - note mel `2.810 -> 3.004` + - env `0.470 -> 0.463` + - entry mel `1.530 -> 2.048` + - exit mel `1.632 -> 1.957` + - onset artifact `0.70 -> 1.06` + - body/core cents stayed `0.00 / -18.32`, matching the base renderer's pitch behavior + - verdict on this case: + - materially worse on the harder truth case even though envelope RMSE moved slightly in the right direction + +Verdict: +- rejected after `M1` +- it failed the equal-weight keep gate: + - the proxy restorer improved `pitchOrg +4` + - but it materially harmed `pitchTestOrg +4` on note mel, entry mel, exit mel, and onset artifact + - there is no justification for `M2` on this proxy family + +What we are keeping: +- the offline benchmark infrastructure stays in the repo +- the live editor remains frozen on `pitch_only_adaptive_selector` +- `FAM-ML-RESTORATION` should only reopen with: + - a trained restorer + - or a materially different restoration method than `ml_restore_proxy_v1` + +### 2026-04-15: `FAM-PVDR` Stop-Fast Verdict + +Purpose: +- execute the next DSP-first family after the ML benchmark stop +- keep `pitch_only_adaptive_selector` as the base editor behavior +- test a genuinely different long-note phase-vocoder overlay instead of the older lightweight boundary-alignment pass + +What was implemented for the trial: +- temporary branch key: + - `pitch_only_pvdr` +- temporary long-upward overlay on top of `pitch_only_adaptive_selector`: + - resample the long upward note region + - apply a custom phase-vocoder stretch back to original duration + - use identity-style peak phase locking around detected spectral peaks + - blend only into the stable long-upward core region +- reused existing phase-lock diagnostics: + - `phaseLockUsed` + - `phaseLockFallbackUsed` + - `phaseAlignedEntry` + - `phaseAlignedExit` + - `phasePeakCount` + +Truth-case results: +- `pitchOrg +4` + - run: `20260415_181645_pitchOrg_plus4_pvdr_p1` + - result: + - byte-identical to the adaptive selector + - SHA stayed `F4BFBE23347CA853EEDE27E48E70F24DC5983AABE37498D02CD8F6D518B3EC72` + - `phaseLockUsed=false` +- `pitchTestOrg +4` + - run: `20260415_181645_pitchTestOrg_plus4_pvdr_p1` + - result: + - note/body/core cents `-628.27 / -628.27 / -664.72` + - onset artifact exploded to `+200512709616936000` + - entry mel `337.641` + - exit mel `324.358` + - `phaseLockUsed=true` + - `phaseAlignedEntry=true` + - `phaseAlignedExit=true` + - `phasePeakCount=14710` + - verdict on this case: + - catastrophic failure on the harder long-upward truth case + +Verdict: +- rejected after `P1` +- stop-fast rule triggered immediately: + - the easy `+4` case did not improve + - the hard `+4` case failed catastrophically + - there was no reason to run the `-4` guards + +Cleanup: +- removed the temporary `pitch_only_pvdr` branch support from the renderer and harness +- kept the existing phase-lock diagnostics only +- the live editor remains `pitch_only_adaptive_selector` + +### 2026-04-15: `FAM-WSOLA-SEAM` Stop-Fast Verdict + +Purpose: +- try the first genuinely untried DSP-first family from the new queue +- keep the existing `pitch_only_adaptive_selector` routing intact and only replace seam handling at short-upward shoulders with a WSOLA-style similarity search +- compare directly against the current adaptive branch on all four canonical truth cases + +What was implemented for the trial: +- temporary branch key: `pitch_only_wsola_seam` +- temporary WSOLA-style shoulder pass layered on top of the adaptive selector: + - mono-sum similarity search + - short-upward notes only + - entry and exit shoulder realignment with a raised-cosine overlap +- reusable seam-search diagnostics added to the native/regression result flow: + - `wsolaUsed` + - `wsolaFallbackUsed` + - `wsolaEntryLagSamples` + - `wsolaExitLagSamples` + - `wsolaCorrelationScore` + +Truth-case results versus the current adaptive selector: +- `pitchOrg +4` + - adaptive control run: `20260415_122753_pitchOrg_plus4_adaptive_cmp` + - WSOLA run: `20260415_121916_pitchOrg_plus4_wsola_w1` + - WSOLA engaged: + - `wsolaUsed=true` + - entry/exit lag `326 / -349` samples + - correlation `0.667` + - result: + - note mel `7.085 -> 7.102` + - env `1.376 -> 1.377` + - entry mel `7.078 -> 7.314` + - exit mel `7.027 -> 7.151` + - onset artifact `1.80 -> 2.29` + - verdict on this case: + - the only case that changed got worse across the seam-sensitive metrics we care about +- `pitchOrg -4` + - adaptive control run: `20260415_122930_pitchOrg_minus4_adaptive_cmp` + - WSOLA run: `20260415_122117_pitchOrg_minus4_wsola_w1` + - result: + - byte-identical output to the adaptive selector + - SHA stayed `6BDCD513FAE4F2267564769D722FCF1FC48E69D981E4493F4E5D2BBBFFD027A0` +- `pitchTestOrg +4` + - adaptive control run: `20260415_123106_pitchTestOrg_plus4_adaptive_cmp` + - WSOLA run: `20260415_122251_pitchTestOrg_plus4_wsola_w1` + - result: + - byte-identical output to the adaptive selector + - SHA stayed `1164BB74D7FC7EC87B163790DAF763E94B1B79812B7BE36F3FCD97966CFA740A` +- `pitchTestOrg -4` + - adaptive control run: `20260415_123320_pitchTestOrg_minus4_adaptive_cmp` + - WSOLA run: `20260415_122513_pitchTestOrg_minus4_wsola_w1` + - result: + - byte-identical output to the adaptive selector + - SHA stayed `9B3E951F0D60DB8828A1CA82AC162A7987EE3C7BA4CCAACCF187B5E75BBD7F37` + +Verdict: +- rejected after `W1` +- it failed the stop-fast gate: + - no primary truth case improved + - the only engaged case (`pitchOrg +4`) regressed on note mel, entry, exit, and onset artifact + - the other three canonical cases stayed exactly on the adaptive baseline + +Cleanup: +- removed the temporary `pitch_only_wsola_seam` renderer branch support after the verdict +- kept the WSOLA/seam diagnostics in the regression/native result flow because they are reusable if a later seam-search family is tried in a meaningfully different form + +### 2026-04-15: `FAM-PHASE-LOCK-PV` Stop-Fast Verdict + +Purpose: +- try the next genuinely untried DSP-first family after the WSOLA stop-fast reject +- keep the existing `pitch_only_adaptive_selector` routing intact and only swap in a boundary-phase-aligned long-upward variant +- compare directly against the current adaptive branch on all four canonical truth cases + +What was implemented for the trial: +- temporary branch key: `pitch_only_phase_lock_pv` +- temporary adaptive-selector variant for long upward notes: + - same adaptive routing as the kept branch + - mono-sum boundary lag search against the simple `ce33` output + - boundary-aligned long-note replacement inside the long upward body span +- reusable phase-lock diagnostics added to the native/regression result flow: + - `phaseLockUsed` + - `phaseLockFallbackUsed` + - `phaseAlignedEntry` + - `phaseAlignedExit` + - `phasePeakCount` + +Truth-case results versus the current adaptive selector: +- `pitchOrg +4` + - adaptive control run: `20260415_122753_pitchOrg_plus4_adaptive_cmp` + - phase-lock run: `20260415_130513_pitchOrg_plus4_phase_lock_p1` + - result: + - byte-identical output to the adaptive selector + - SHA stayed `F4BFBE23347CA853EEDE27E48E70F24DC5983AABE37498D02CD8F6D518B3EC72` + - `phaseLockUsed=false` +- `pitchOrg -4` + - adaptive control run: `20260415_122930_pitchOrg_minus4_adaptive_cmp` + - phase-lock run: `20260415_130647_pitchOrg_minus4_phase_lock_p1` + - result: + - byte-identical output to the adaptive selector + - SHA stayed `6BDCD513FAE4F2267564769D722FCF1FC48E69D981E4493F4E5D2BBBFFD027A0` + - `phaseLockUsed=false` +- `pitchTestOrg +4` + - adaptive control run: `20260415_123106_pitchTestOrg_plus4_adaptive_cmp` + - phase-lock run: `20260415_130820_pitchTestOrg_plus4_phase_lock_p1` + - phase-lock engaged: + - `phaseLockUsed=true` + - `phaseAlignedEntry=false` + - `phaseAlignedExit=true` + - `phasePeakCount=114` + - result: + - note mel `2.81 -> 3.003` + - env `0.47 -> 0.479` + - entry mel `1.53 -> 2.278` + - exit mel `1.632 -> 1.940` + - onset artifact `0.705 -> 0.66` + - verdict on this case: + - selective boundary alignment was real, but it still made the hard long-upward truth case worse where the keep gate matters +- `pitchTestOrg -4` + - adaptive control run: `20260415_123320_pitchTestOrg_minus4_adaptive_cmp` + - phase-lock run: `20260415_131038_pitchTestOrg_minus4_phase_lock_p1` + - result: + - byte-identical output to the adaptive selector + - SHA stayed `9B3E951F0D60DB8828A1CA82AC162A7987EE3C7BA4CCAACCF187B5E75BBD7F37` + - `phaseLockUsed=false` + +Verdict: +- rejected after `P1` +- it failed the stop-fast gate: + - three canonical cases stayed exactly on the adaptive baseline + - the only engaged case (`pitchTestOrg +4`) materially regressed on note mel and entry mel + - the slight onset-artifact gain was not enough to offset the note/entry loss + +Cleanup: +- removed the temporary `pitch_only_phase_lock_pv` renderer branch support after the verdict +- kept the phase-lock diagnostics in the regression/native result flow because they are reusable if a later, materially different phase-coherent family is tried + +### 2026-04-15: `FAM-HPSS-MEDIAN` Stop-Fast Verdict + +Purpose: +- retry HPSS in a genuinely different form from the rejected heuristic shell +- use a real STFT median-filter separation idea instead of the old voiced-mask proxy +- keep the current adaptive selector as the base render and only replace short-upward note regions with a median-shell recombine + +What was implemented for the trial: +- temporary branch key: `pitch_only_hpss_median` +- temporary median-shell pass layered on top of the adaptive selector: + - mono-sum STFT analysis + - horizontal median across time and vertical median across frequency + - scalar harmonic/transient weighting projected back to the time domain + - original signal as the transient side, adaptive output as the harmonic side + - short upward notes only +- reused HPSS diagnostics in the regression flow: + - `hpssUsed` + - `hpssFallbackUsed` + - `harmonicMaskPeak` + - `aperiodicMaskPeak` + - `spectralEnvelopeCorrectionUsed` + +Truth-case results versus the current adaptive selector: +- `pitchOrg +4` + - adaptive control run: `20260415_122753_pitchOrg_plus4_adaptive_cmp` + - median HPSS run: `20260415_132239_pitchOrg_plus4_hpss_median_hm1` + - median HPSS engaged: + - `hpssUsed=true` + - harmonic/aperiodic peaks `0.868 / 0.554` + - result: + - note mel `7.085 -> 7.432` + - env `1.376 -> 1.279` + - entry mel `7.078 -> 7.445` + - exit mel `7.027 -> 6.742` + - onset artifact `1.80 -> 1.48` + - note/body/core cents `0 / 0 / 0 -> -55.55 / -55.55 / -55.55` + - verdict on this case: + - the onset side improved again, but the body pitch and note quality collapsed, so this is not a keepable trade +- `pitchOrg -4` + - adaptive control run: `20260415_122930_pitchOrg_minus4_adaptive_cmp` + - median HPSS run: `20260415_132408_pitchOrg_minus4_hpss_median_hm1` + - result: + - byte-identical output to the adaptive selector + - SHA stayed `6BDCD513FAE4F2267564769D722FCF1FC48E69D981E4493F4E5D2BBBFFD027A0` +- `pitchTestOrg +4` + - adaptive control run: `20260415_123106_pitchTestOrg_plus4_adaptive_cmp` + - median HPSS run: `20260415_132536_pitchTestOrg_plus4_hpss_median_hm1` + - result: + - byte-identical output to the adaptive selector + - SHA stayed `1164BB74D7FC7EC87B163790DAF763E94B1B79812B7BE36F3FCD97966CFA740A` +- `pitchTestOrg -4` + - adaptive control run: `20260415_123320_pitchTestOrg_minus4_adaptive_cmp` + - median HPSS run: `20260415_132746_pitchTestOrg_minus4_hpss_median_hm1` + - result: + - byte-identical output to the adaptive selector + - SHA stayed `9B3E951F0D60DB8828A1CA82AC162A7987EE3C7BA4CCAACCF187B5E75BBD7F37` + +Verdict: +- rejected after `Hm1` +- it failed the stop-fast gate: + - the only engaged case (`pitchOrg +4`) regressed on note mel, entry mel, and body pitch + - the harder `pitchTestOrg +4` case still did not engage + - there is no basis for opening `FAM-HPSS-MEDIAN-SF` on top of this shell + +Cleanup: +- removed the temporary `pitch_only_hpss_median` renderer branch support after the verdict +- kept the HPSS diagnostics in the regression/native result flow because they remain reusable for any future materially different HPSS family + +### 2026-04-15: `FAM-HPSS-SHELL` Stop-Fast Verdict + +Purpose: +- test a genuinely new vertical split family instead of another onset/body handoff +- keep transient/noise content original and send only the harmonic body through a pitched layer +- compare directly against the kept adaptive selector on all four canonical truth cases + +What was implemented for the trial: +- temporary branch key: `pitch_only_hpss_shell` +- temporary HPSS-style island composition on top of the current adaptive selector base: + - original signal for transient/noise-dominant regions + - pitched harmonic layer only in stable voiced regions + - outer-island-only splices +- temporary HPSS diagnostics in the regression flow: + - `hpssUsed` + - `hpssFallbackUsed` + - `harmonicMaskPeak` + - `aperiodicMaskPeak` + - `spectralEnvelopeCorrectionUsed` + +Truth-case results: +- `pitchOrg +4` + - run: `20260415_110453_pitchOrg_plus4_hpss_shell_h1` + - HPSS engaged: `hpssUsed=true` + - onset artifact improved `1.810 -> 1.390` + - but note/body quality collapsed: + - note mel `6.209 -> 10.066` + - env `0.961 -> 2.028` + - entry mel `6.928 -> 10.996` + - exit mel `6.076 -> 15.137` + - verdict on this case: + - same old pattern again: onset-side gain, unacceptable body loss +- `pitchTestOrg +4` + - run: `20260415_110628_pitchTestOrg_plus4_hpss_shell_h1` + - HPSS never engaged: + - `hpssUsed=false` + - output stayed on the existing adaptive-selector result + - SHA `1164BB74D7FC7EC87B163790DAF763E94B1B79812B7BE36F3FCD97966CFA740A` +- `pitchOrg -4` + - run: `20260415_110628_pitchOrg_minus4_hpss_shell_h1` + - note mel regressed to `6.623` + - env `1.102` + - core cents drifted to `+11.90` +- `pitchTestOrg -4` + - run: `20260415_110628_pitchTestOrg_minus4_hpss_shell_h1` + - note mel `3.505` + - onset artifact `3.64` + - not safe enough to justify continuing the family + +Verdict: +- rejected after `H1` +- it failed the stop-fast gate: + - the easy upward case improved only the onset metric while destroying note/body quality + - the harder upward truth case did not engage at all + - the `-4` guards did not stay clean enough + +Cleanup: +- removed the temporary `pitch_only_hpss_shell` renderer branch support after the verdict +- kept the HPSS diagnostic fields in the regression/native result flow because they are reusable if a later, structurally different vertical-split family is tried + +### 2026-04-15: `FAM-ANALYZER-PYIN` Implementation Start + +Purpose: +- make the editor-side monophonic pitch analysis the top active family +- stop pretending we already have pYIN when the repo only had basic YIN-style trackers + +What was already true before this change: +- `PitchAnalyzer` already existed as an offline monophonic editor analyzer +- `PitchDetector` already existed as a live YIN-style tracker +- neither path had: + - true pYIN candidate generation + - temporal decoding + - FFT-accelerated YIN + +What landed in this pass: +- `PitchAnalyzer` now uses: + - Hann windowing + - FFT-derived YIN difference calculation + - CMNDF candidate extraction + - per-frame voiced/unvoiced probability + - lightweight Viterbi-style decoding across frames +- external editor-facing result shape stayed the same: + - frame `frequency` + - frame `midiNote` + - frame `confidence` + - frame `rmsDB` + - frame `voiced` + - existing note segmentation output +- `PitchDetector` comments were corrected so the live tracker is described honestly as YIN-style, not pYIN + +Current status: +- implemented and compiling +- now validated in staged form on the real fixture clips: + - safe default is the direct Hann-windowed YIN difference path plus the new multi-candidate / voiced-probability / Viterbi-style decoder + - FFT-derived difference is still implemented, but remains behind `OPENSTUDIO_ANALYZER_USE_FFT_YIN=1` until parity is proven +- this family is now the top active queue item in the master map + +Real fixture validation: +- `pitchOrg +4` using local analyzer fallback and the correct short-note fixture: + - run: `20260415_100627_pitchOrg_plus4_adaptive_selector_analyzer_pyin_safe_g3` + - note/body/core cents `0.00 / 0.00 / 0.00` + - note mel `7.085` + - entry mel `7.078` + - exit mel `7.027` + - output SHA `F4BFBE23347CA853EEDE27E48E70F24DC5983AABE37498D02CD8F6D518B3EC72` +- first `pitchTest +4` validation looked broken, but that run used the wrong note fixture (`example_plus4_notes.json`, the older `pitchOrg` timing), so it is not a valid analyzer verdict for the later `pitchTest` note family +- `pitchTest +4` using local analyzer fallback and the correct clip-specific fixture: + - run: `20260415_101036_pitchTestOrg_plus4_adaptive_selector_analyzer_pyin_safe_g4` + - note/body/core cents `-17.94 / 0.00 / -18.32` + - note mel `5.158` + - entry mel `1.530` + - exit mel `1.632` + - output SHA `1164BB74D7FC7EC87B163790DAF763E94B1B79812B7BE36F3FCD97966CFA740A` + +FFT probe status: +- explicit FFT-YIN probe on `pitchOrg +4` is still not safe + - run: `20260415_101249_pitchOrg_plus4_adaptive_selector_analyzer_fft_probe_g4` + - note/body/core cents `-386.31 / -386.31 / -401.30` + - note mel `8.072` + - output SHA `019D3F5341440BE010F2B3E5E620AE5878CEAFE5C03364FCBED7DBCC5A4BDDC3` +- verdict: + - keep FFT-derived YIN staged behind env until parity work is done + - keep the direct YIN difference path as the active safe default for the new decoder stack + +Pitch-editor scope change: +- the pitch editor is now intentionally mono-only +- stereo vocal clips remain supported: + - editor analysis mixes the clip to mono for contour extraction + - correction still renders against the full multichannel clip in the backend +- the old polyphonic mode UI path was removed from the pitch editor surface so the product now matches the monophonic editor target directly + +Immediate next action: +- compare note segmentation quality and contour stability against the previous analyzer behavior on `pitchOrg` and `pitchTestOrg` +- only then decide whether the staged analyzer upgrade can become the new baseline + +### 2026-04-14: Post-PSOLA Continuation Checks + +#### Path E1, Iteration 1: Voiced-Tail Continuation Body +- Change: + - tried a voiced-tail continuation source for the short-upward body, seeded from a small epoch window around the entry anchor instead of the full PSOLA body path +- Result: + - `pitchOrg +4` + - failed closed to the trusted `r6` baseline + - output SHA stayed `055B3300041A8B5DE93C462C7E18F180BF7740D0134B0C05B6D255A6A36B46BD` + - body replacement used/fallback `false / true` + - `pitchTest +4` + - also stayed at the same shipping-control output + - output SHA stayed `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Verdict: + - stopped early after the first iteration + - no improvement on the active target and the guard stayed flat, so the path did not earn its second slot + +#### Path E2, Iteration 1: Early Continuation Handoff Into Existing Core +- Change: + - narrowed the continuation idea to the fragile early body only + - replacement region would cover only the first continuation span after voiced entry, then hand into the existing hybrid core + - replacement blending was changed to sit on top of the legacy/own base mix so the handoff could blend into the live core instead of only back into dry legacy +- Result: + - `pitchOrg +4` + - again failed closed to the trusted `r6` baseline + - output SHA stayed `055B3300041A8B5DE93C462C7E18F180BF7740D0134B0C05B6D255A6A36B46BD` + - body replacement used/fallback `false / true` + - `pitchTest +4` + - also remained unchanged on the shipping-control SHA + - output SHA stayed `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Verdict: + - stopped early after the first iteration + - the architecture never engaged on the active target, so there was no justification to spend the second slot + +#### Continuation-Family Conclusion +- Both post-PSOLA continuation families were exhausted under the stop-fast rule. +- Neither one engaged successfully on `pitchOrg +4`. +- The active renderer was restored to the trusted `HS-4 / r6` baseline by disabling body replacement again. +- Current live truth remains: + - `pitchOrg +4` + - note mel `6.209` + - env `0.961` + - entry mel `6.928` + - exit mel `6.076` + - onset artifact `1.810` + - `pitchTest +4` + - note mel `3.481` + - env `0.623` + - entry mel `2.484` + - exit mel `7.662` + - onset artifact `3.819` + +### 2026-04-15: Dormant Branch Truth Sweep + +Purpose: +- finish the option-map analysis for every callable renderer branch already present in the app-path runner +- stop guessing which archived branches might still matter and measure them directly on the same `+4` truth cases + +#### `branch_simple_ce33` +- Result: + - `pitchOrg +4` + - note mel `7.810` + - env `0.883` + - body/core cents `0.00 / 0.00` + - entry mel `7.996` + - exit mel `9.295` + - onset artifact `2.50` + - SHA `29313710B708E393623C18128948110C62375473DC95D46B5F18E1DBF7FB939A` + - `pitchTest +4` + - note mel `3.044` + - env `0.486` + - body cents `0.00` + - entry mel `1.660` + - exit mel `6.378` + - onset artifact `0.72` + - SHA `70A4507DC97B4CC3C630B3DC418798B58E0F2539BF8C6D4EE7F2CEAB012189EC` +- Verdict: + - not the best single overall control because it loses clearly to `CTRL-R6` on `pitchOrg +4` + - but it is the strongest measured standalone result so far on `pitchTest +4` + - keep as a secondary benchmark and harvest candidate + +#### `branch_current_advanced` +- Result: + - `pitchOrg +4` + - note mel `7.329` + - env `0.827` + - body/core cents `-37.23 / -37.23` + - entry mel `7.752` + - exit mel `9.438` + - onset artifact `1.94` + - SHA `90D2D3638B7F1DF5655AD1A9871A3DCEDAC611E5198B285B5039B8583CD7DF45` + - `pitchTest +4` + - note mel `8.784` + - env `1.190` + - body/core cents `-35.70 / -36.45` + - entry mel `5.908` + - exit mel `6.373` + - onset artifact `2.38` + - SHA `E562BC9377B0BFB51B0D139CC532FEDED240B2D636D154166BAC0B1652B1E703` +- Verdict: + - clearly below both controls on truth-case pitch accuracy and note/body quality + - treat as rejected standalone archived branch + +#### `pitch_only_psola_core` +- Result: + - `pitchOrg +4` + - note mel `11.053` + - env `1.640` + - body/core cents `-55.55 / -55.55` + - entry mel `7.448` + - exit mel `8.976` + - onset artifact `1.94` + - SHA `C6E61F58A3A99B41FAE27557AEE69F61EEAFFF925AE72332A2ACF21D54DEBAD8` + - `pitchTest +4` + - note mel `17.536` + - env `2.547` + - body/core cents `-104.96 / -124.35` + - entry mel `6.305` + - exit mel `6.329` + - onset artifact `2.38` + - SHA `A59B8C6305E31D95584C15214B1712F98BFF90B2E764FACA5A69F80A9DF61E6F` +- Verdict: + - catastrophic truth-case miss as a standalone branch + - reject as standalone archived core + +#### `pitch_only_model_core` +- Result: + - `pitchOrg +4` + - note mel `10.955` + - env `1.626` + - body/core cents `-37.23 / -37.23` + - entry mel `7.421` + - exit mel `8.971` + - onset artifact `1.94` + - SHA `E2D3A690881BB489EEC0B7409FD2D60EA87F1AE2FE96527F01DFC89908F57DE8` + - `pitchTest +4` + - note mel `17.397` + - env `2.530` + - body/core cents `-104.96 / -124.35` + - entry mel `6.295` + - exit mel `6.330` + - onset artifact `2.38` + - SHA `FF455AB0B313826E75D249BBF69CEB61F740D741B85BF0DB06771CEAC73289BC` +- Verdict: + - also a strong standalone rejection on the truth cases + - no reason to keep it as an active standalone contender + +#### `pitch_only_own_engine` +- Result: + - `pitchOrg +4` + - note mel `6.316` + - env `1.044` + - body/core cents `0.00 / 0.00` + - entry mel `7.783` + - exit mel `6.407` + - onset artifact `1.59` + - SHA `0A72E7027F69E85741E24AC1CC7CBD3958E263F0ED7FA9F4CEC49EA3D5202EE6` + - `pitchTest +4` + - note mel `12.164` + - env `1.667` + - body/core cents `0.00 / -18.32` + - entry mel `17.634` + - exit mel `13.403` + - onset artifact `3.99` + - SHA `96397D63A5F3AD6F835A80605E9FAF0615947D80E515F5A5F02F844A21AC4146` +- Verdict: + - interesting on the easier `pitchOrg +4` clip because it keeps exact body pitch and improves onset artifact versus `CTRL-R6` + - not viable as a standalone editor because `pitchTest +4` collapses badly + - harvest only if a later v2 needs own-engine core behavior on easier upward notes + +#### `formant_only_own_engine` and `pitch_plus_formant_own_engine` +- Result: + - both matched `branch_current_advanced` bit-for-bit on both `+4` truth cases + - `pitchOrg +4` SHA `90D2D3638B7F1DF5655AD1A9871A3DCEDAC611E5198B285B5039B8583CD7DF45` + - `pitchTest +4` SHA `E562BC9377B0BFB51B0D139CC532FEDED240B2D636D154166BAC0B1652B1E703` +- Verdict: + - treat them as archived advanced/formant variants, not distinct truth-case winners + - reject them as standalone candidates + +#### Dormant-Branch Sweep Conclusion +- The callable archived branch set is now mapped much more clearly: + - strongest `pitchTest +4` branch: `branch_simple_ce33` + - strongest kept overall experimental control: `CTRL-R6` + - strongest easier-clip own-engine result: `pitch_only_own_engine` on `pitchOrg +4` +- The branches that are still worth keeping around as references are: + - `CTRL-SHIP` + - `CTRL-R6` + - `branch_simple_ce33` +- The rest of the dormant callable branches are no longer credible standalone pitch-editor candidates on the truth cases. +- Next queue consequence: + - the callable archived branch analysis is now complete enough to justify moving the top queue item to `FAM-V2-SYNTH-CORE` + - that family is now tracked in the master map as the next upward v2 attempt instead of continuing any exhausted handoff or archived-core loop + +### 2026-04-15: `FAM-V2-SYNTH-CORE` Stop-Fast Check + +Purpose: +- test one new upward `v2` family that was still structurally different from the exhausted handoff and island-core loops +- specifically: + - island shell + - directly synthesized voiced core + - explicit residual layer + - outer-only shell behavior + +Implementation note: +- branch name used for the single iteration was `pitch_only_synth_core` +- the branch code was removed immediately after the verdict, per the prune-on-reject workflow + +#### `G1` +- Result: + - `pitchOrg +4` + - note mel `8.135` + - env `1.517` + - whole/body/core cents `-408.27 / 0.00 / 0.00` + - entry mel `10.468` + - exit mel `13.781` + - onset artifact `1.39` + - island native used/fallback `true / false` + - SHA `247BC954F14147C9ABE0B12F04C20E907BD94C9FD400A5D99905CA995C44E572` + - `pitchTest +4` + - failed closed back to the control output + - note mel `3.481` + - env `0.623` + - entry mel `2.484` + - exit mel `7.662` + - onset artifact `3.82` + - island native used/fallback `false / false` + - SHA `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Verdict: + - decisive reject after the first iteration + - onset improved again on `pitchOrg +4`, but body/entry/exit and whole-note pitch behavior regressed too much + - no improvement and no engagement on `pitchTest +4` + - no `G2` + - branch code removed immediately + +### 2026-04-15: `FAM-ADAPTIVE-SELECTOR` + +Purpose: +- stop forcing one renderer family to solve both truth clips by itself +- harvest the two strongest measured behaviors we now have: + - `CTRL-R6` for short upward notes like `pitchOrg +4` + - `branch_simple_ce33` for long upward notes like `pitchTest +4` + +Implementation: +- new branch: `pitch_only_adaptive_selector` +- rule for this first family: + - short upward notes -> use `CTRL-R6` behavior + - long upward notes -> use `branch_simple_ce33` + - downward notes remain unchanged on the current baseline behavior + +#### `G1` +- Result: + - `pitchOrg +4` + - note mel `6.210` + - env `0.961` + - entry mel `6.978` + - exit mel `6.029` + - onset artifact `1.80` + - SHA `E039C0AF68908D14ECC2BC17A6D335DDDA7C713B6DEFE10BF4D1C59799D3B0E0` + - `pitchTest +4` + - note mel `7.300` + - env `0.840` + - body/core cents `-35.70 / -54.39` + - entry mel `5.911` + - exit mel `6.375` + - onset artifact `2.38` + - SHA `4E0F29A3472F1B0A51A9B5D7D3FB8050DF374FB32034DD20C3049E143192FCAB` +- Diagnosis: + - the “simple” sub-render inside the selector was not actually using the real `branch_simple_ce33` path + - so `G1` was not a fair test of the harvested selector idea + +#### `G2` +- Fix: + - replaced the selector’s long-up simple source with the true whole-file `SignalsmithShifter::processPitchOnlyCe33Base` path used by standalone `branch_simple_ce33` +- Result: + - `pitchOrg +4` + - note mel `6.210` + - env `0.961` + - entry mel `6.978` + - exit mel `6.029` + - onset artifact `1.80` + - SHA `E039C0AF68908D14ECC2BC17A6D335DDDA7C713B6DEFE10BF4D1C59799D3B0E0` + - `pitchTest +4` + - note mel `3.095` + - env `0.493` + - body/core cents `0.00 / -18.32` + - entry mel `1.530` + - exit mel `7.350` + - onset artifact `0.70` + - SHA `F52F4B97CACE7E048767F3243C7DAFD949FD08F2491DDC1CB649B6B8E9B8F24E` +- Guard runs: + - `pitchOrg -4` + - exact same trusted control SHA `6E73A2F0FA1E053488C65DD801D97F1F30564F0684DE076924CF7CCF2C4394AE` + - `pitchTest -4` + - exact same trusted control SHA `1B6A789C4D36AB0D330CD043E41A72EB27B858120DF34B271293764A197F909C` +- Verdict: + - keep + - this is the first family that improved `pitchTest +4` materially without materially harming `pitchOrg +4` + - downward guards remained unchanged, so the branch is safe enough to freeze as the new experimental base for the next queue item + +### 2026-04-15: Initial `-4` Support Scan For The Next Queue + +Purpose: +- start the downward-family work on top of the newly kept adaptive branch +- check whether the archived support branches already contain an obvious `-4` winner we should harvest before designing a new downward-specific hybrid + +#### `branch_simple_ce33` +- Result: + - `pitchOrg -4` + - note mel `8.397` + - env `3.882` + - onset artifact `2.51` + - SHA `C4BCE0BE13F3ED88EEC2B716ECD1C4328244AA57BD884D9927A6976F5815C325` + - `pitchTest -4` + - note mel `3.931` + - env `3.401` + - note/body cents `-5.74 / -5.74` + - entry mel `3.098` + - exit mel `6.106` + - onset artifact `1.07` + - SHA `2732128716D791837D24CD6328B4FD8FA0E4476FC2940EB12FFB4A9FD2A1A600` +- Verdict: + - not a standalone downward winner + - interesting for onset/exit behavior on `pitchTest -4`, but the envelope error is too large to promote directly + +#### `pitch_only_own_engine` +- Result: + - `pitchOrg -4` + - note mel `5.560` + - env `1.544` + - note/body cents `+11.90 / +11.90` + - exit mel `7.135` + - onset artifact `2.79` + - SHA `451786635BD29F262125F7A93355D9267B08CDD8FDA2E8567029FA300C3111AD` + - `pitchTest -4` + - note mel `7.727` + - env `1.049` + - entry mel `10.313` + - exit mel `13.998` + - onset artifact `4.01` + - SHA `88EA8E2FE77F77E4E316B49D362FEC2599C35E48EBA096C40F100F11794978E7` +- Verdict: + - interesting only as a possible easy-clip downward body trait + - not a viable standalone `-4` path because it collapses badly on `pitchTest -4` + +#### Downward-Scan Conclusion +- There is no immediate archived branch we can promote as the downward answer. +- The adaptive selector remains frozen as the active experimental base. +- The next downward family should be a new hybrid that: + - keeps the adaptive branch unchanged for `+4` + - borrows only narrowly proven `-4` traits + - does not replace the whole renderer with either `ce33` or standalone own-engine on downward notes + +### 2026-04-15: `FAM-ADAPTIVE-SELECTOR` Downward Harvest Keep + +Purpose: +- take the best narrow `-4` trait from the support scan +- land it inside the already-kept `pitch_only_adaptive_selector` branch instead of inventing another standalone renderer family + +#### Cleanup Before The Real Test +- I first tried adding a separate branch key for the downward selector experiment. +- That path was not trustworthy: + - the harness label was correct + - but the app still reported `requested / actual = branch_hybrid_reset / branch_hybrid_reset` +- Instead of burning more time on branch-key plumbing, I removed that dead `_down` key and folded the downward trial directly into the working `pitch_only_adaptive_selector` branch. + +#### Iteration G3: Light Short-Downward Own-Engine Support +- Change: + - kept the adaptive selector's upward policy unchanged: + - `CTRL-R6` on short upward notes + - `branch_simple_ce33` on long upward notes + - added one bounded downward trait: + - light own-engine contribution on shorter edited downward notes + - protected entry and exit shoulders + - max own weight `0.24` +- Result: + - `pitchOrg -4` + - note mel `7.405 -> 6.570` + - env `1.094 -> 1.100` + - entry mel `6.902 -> 6.571` + - exit mel `8.778 -> 7.565` + - onset artifact `2.743 -> 2.802` + - SHA `1D75C81C2FC23779F7B07114A30B72B8564E43A638A28276244C84DFEE91F46D` + - `pitchTest -4` + - remained byte-identical to the prior adaptive baseline: + - SHA `1B6A789C4D36AB0D330CD043E41A72EB27B858120DF34B271293764A197F909C` + - note mel `3.775` + - env `1.062` + - entry mel `3.090` + - exit mel `7.134` + - onset artifact `3.642` + - `+4` guards + - both adaptive-selector winners stayed unchanged: + - `pitchOrg +4` SHA `E039C0AF68908D14ECC2BC17A6D335DDDA7C713B6DEFE10BF4D1C59799D3B0E0` + - `pitchTest +4` SHA `F52F4B97CACE7E048767F3243C7DAFD949FD08F2491DDC1CB649B6B8E9B8F24E` +- Verdict: + - keep + - this is the first downward-side improvement that: + - improves a real `-4` truth case + - keeps the harder `pitchTest -4` case safe + - leaves the newly won `+4` adaptive outputs untouched + +#### Current Best Experimental Truth +- `pitchOrg +4` + - note mel `6.210` + - env `0.961` + - entry mel `6.978` + - exit mel `6.029` + - onset artifact `1.804` + - SHA `E039C0AF68908D14ECC2BC17A6D335DDDA7C713B6DEFE10BF4D1C59799D3B0E0` +- `pitchTest +4` + - note mel `3.095` + - env `0.493` + - entry mel `1.530` + - exit mel `7.350` + - onset artifact `0.700` + - SHA `F52F4B97CACE7E048767F3243C7DAFD949FD08F2491DDC1CB649B6B8E9B8F24E` +- `pitchOrg -4` + - note mel `6.570` + - env `1.100` + - entry mel `6.571` + - exit mel `7.565` + - onset artifact `2.802` + - SHA `1D75C81C2FC23779F7B07114A30B72B8564E43A638A28276244C84DFEE91F46D` +- `pitchTest -4` + - note mel `3.775` + - env `1.062` + - entry mel `3.090` + - exit mel `7.134` + - onset artifact `3.642` + - SHA `1B6A789C4D36AB0D330CD043E41A72EB27B858120DF34B271293764A197F909C` + +#### Conclusion +- `pitch_only_adaptive_selector` is now the best overall experimental editor path in the repo: + - short upward notes: + - `CTRL-R6` + - long upward notes: + - `branch_simple_ce33` + - shorter downward notes: + - bounded own-engine support blended into the adaptive output +- The harder `pitchTest -4` problem is not solved yet, but basic downward support is no longer “missing entirely.” + +## Goal And Non-Negotiables +- Match the user reference clips first, not the current engine: + - `pitchOrg.wav -> pitchOrg+4s.wav` + - `pitchOrg.wav -> pitchOrg-4s.wav` + - `pitchTestOrg.wav -> pitchTestOrg+4s.wav` + - `pitchTestOrg.wav -> pitchTestOrg-4s.wav` +- Hard requirements: + - exact local pitch change + - no word/consonant cutoff + - no clipping-like grit + - no previous-note drag + - preview under about `1s` +- Historical safety reference: + - commit `ce33d14f8a11f3315797876eff2711ef71a7cdf0` +- Sonic target: + - user-supplied reference WAVs, even when another render sounds nicer by ear + +## Current Baseline +- Current pitch-only baseline: + - corrected note-local `single` render routing + - safer `ce33`-family carrier + - minimal note-core Stage B +- Why this is the current baseline: + - fixed the major `single` vs `preview_segment` correctness bug + - exact target pitch is now healthy again on the corrected `pitchTest` family + - onset behavior is safer than the old advanced branch + - still fast enough for note-local preview +- Current main pain point: + - `pitchOrg -> +4` still sounds darker / more synthetic than the reference + - neighbor-note metrics are still too high + +### Current Measured Baseline +| Case | Note mel | Note env RMSE | Cents error | Onset jump | Current note-local latency | +| --- | ---: | ---: | ---: | ---: | ---: | +| `pitchOrg -> +4` | `8.299` | `1.018` | `0.00` | `+0.65 dB` | `226 ms` | +| `pitchOrg -> -4` | `7.408` | `1.114` | `0.00` | `+1.64 dB` | `191 ms` | +| `pitchTestOrg -> +4` | `3.449` | `0.620` | `0.00` | `0.00 dB` | `283 ms` | +| `pitchTestOrg -> -4` | `3.769` | `1.071` | `0.00` | `0.00 dB` | `268 ms` | + +- Preview note: + - the bounded note-local preview path has stayed in the same `~200-300 ms` class during harness checks, comfortably under the `<1s` requirement +- Interpretation: + - `pitchTest` correctness is now healthy + - `pitchOrg +4` remains the main sonic gap + +## Reality-Check Reset (2026-04-13) +- Status: + - do **not** treat the current app as research-level or sample-level + - do **not** treat the current own-engine branch as promotion-ready +- Why this reset happened: + - manual listening on the first upward `+4` note move still revealed: + - formant drift + - neighboring-word cutoff / crackle + - robotic shifted-note color + - that means older "close enough" interpretations were too generous +- Fresh frozen report: + - matrix root: `D:\test projects\os tests\reports\pitch-reality-check_20260413_172356` + - matrix markdown: `D:\test projects\os tests\reports\pitch-reality-check_20260413_172356\pitch_reality_check_matrix.md` +- Key findings from the frozen reality-check matrix: + - `baseline_app_current` on `pitchOrg +4` is still far from the reference: + - note mel `8.299` + - entry/exit mel `7.690 / 8.511` + - formant body harmonic drift `0.523` + - low/mid/high band delta `-2.826 / +0.807 / -4.847 dB` + - F1/F2 proxy drift `-23.4 / -46.9 Hz` + - `baseline_own_engine_current` is now branch-distinct and preview-complete, but still not promotion-ready: + - `pitchOrg +4` single / preview are branch-true and parity-safe + - `pitchTest +4` single remains much worse than the app baseline on timbre metrics +- Truth-phase fix that changed the interpretation: + - the old persisted `default` and `pitch_only_own_engine` `+4` hashes were identical because separate app processes could write the same output filename slot + - native regression summaries now persist: + - requested vs actual renderer branch + - fallback reason + - output SHA256 + - candidate coverage start/end + - the output naming path now uses a unique token, so branch comparisons are no longer being corrupted by filename collisions +- Current policy after the reset: + - reference metrics remain the main gate + - but claims of "research/sample-close" are invalid until the upgraded formant-sensitive harness and audition bundle both agree +- Current parity truth after the branch-fix: + - `default`, `pitchOrg +4` + - single / preview parity passed + - note mel delta `0.509 dB` + - note env delta `0.108` + - body/core cents delta `0 / 0` + - `pitch_only_own_engine`, `pitchOrg +4` + - single / preview parity passed + - note mel delta `0.063 dB` + - note env delta `0.194` + - body/core cents delta `0 / 0` + - this removes preview/single parity as the immediate blocker on `pitchOrg +4` + +### Track A: Current App Cleanup After Reset +#### Iteration TA-1: Upward shoulder / entry protection softening +- Hypothesis: + - the first audible failure might be mostly an onset/shoulder problem in the current hybrid-reset app path +- Change: + - made upward-note shoulders longer and drier at note entry/exit +- Result: + - `pitchOrg +4` improved only slightly at the note edges: + - onset peak `+0.65 -> +0.61 dB` + - exit mel `8.511 -> 8.275` + - but `pitchTest +4` regressed clearly: + - note mel `3.449 -> 3.489` + - entry mel `1.941 -> 2.779` + - centroid drift `-39.4 -> -53.9 Hz` +- Verdict: + - rejected +- Learning: + - shoulder tuning alone is not the main blocker on the current app path + +#### Iteration TA-2: Short-upward note-core spectral rebalance +- Hypothesis: + - the current app path might be over-boosting mid bands and over-trimming low/high bands for short upward notes +- Change: + - added a short-upward-only note-core spectral rebalance inside the Stage B envelope-anchor path +- Result: + - effectively neutral on both checked cases +- Verdict: + - rejected +- Learning: + - the current app-path miss is not being driven by a small short-upward Stage B weighting error + +#### Iteration TA-3: Upward carrier swap inside the hybrid app path +- Hypothesis: + - the old `ce33` upward carrier itself is the source of the formant issue, so replacing it with the newer note-local Stage A carrier might immediately improve `+4` +- Change: + - swapped only upward islands in the hybrid branch to the newer note-local carrier while keeping the hybrid blending and Stage B +- Result: + - pitch correctness broke badly: + - `pitchOrg +4` cents `0.00 -> -37.23` + - `pitchTest +4` cents `0.00 -> -35.70` + - timbre metrics did move, but not in a usable product direction +- Verdict: + - rejected and reverted +- Learning: + - the current hybrid branch depends on the `ce33` carrier assumptions more strongly than expected + - fixing the app-path formant issue will need a cleaner carrier replacement plan, not an inline swap + +#### Iteration TA-4: `ce33` carrier with preserve-style formant guidance +- Hypothesis: + - the `ce33` hybrid carrier might be darkening `pitchOrg +4` because it hardcodes `1 / pitchRatio` timbre fallback, so feeding it voiced pitch guidance and preserve-style formant handling could reduce the audible formant drift +- Change: + - routed the `ce33` pitch-only carrier through preserve-mode-style formant handling instead of the old ratio-driven fallback +- Result: + - `pitchOrg +4` regressed badly: + - note mel `8.299 -> 8.833` + - note env `1.018 -> 1.298` + - centroid drift `-189.7 -> +84.8 Hz` + - entry/exit mel `7.690 / 8.511 -> 11.240 / 10.962` + - onset peak `+0.65 -> +2.52 dB` + - `pitchTest +4` also regressed badly: + - note mel `3.449 -> 8.516` + - note env `0.620 -> 1.703` + - centroid drift `-39.4 -> +355.6 Hz` +- Verdict: + - rejected and reverted +- Learning: + - the current hybrid app path is not simply "too dark because `1 / ratio` is wrong" + - moving the `ce33` carrier toward preserve-mode behavior blows up the note body and boundaries instead of fixing the reference mismatch + +#### Iteration TA-5: Short-upward Stage B de-darkening rebalance +- Hypothesis: + - the primary `pitchOrg +4` miss might be recoverable without touching the fragile `ce33` carrier by de-darkening only the short upward Stage B weighting: + - slightly less mid emphasis + - less low trim + - less high-band air protection +- Change: + - added a stronger short-upward-only spectral rebalance inside the hybrid Stage B path +- Result: + - `pitchOrg +4` was mixed: + - note mel `8.299 -> 8.170` + - note env `1.018 -> 1.102` + - centroid drift `-189.7 -> -26.5 Hz` + - entry/exit mel worsened to `9.109 / 9.461` + - harmonic drift worsened `0.523 -> 0.771` + - `pitchTest +4` regressed clearly: + - note mel `3.449 -> 6.366` + - note env `0.620 -> 1.246` + - centroid drift `-39.4 -> +144.5 Hz` +- Verdict: + - rejected and reverted +- Learning: + - stronger short-upward de-darkening can move centroid drift in the right direction on the hard case, but it is too destructive to the healthier clip family + - the next app-path move should not be a broader Stage B spectral push; it needs a cleaner upward carrier or decomposition idea + +#### Iteration TA-6: Upward shoulder protection tightening on the shipping branch +- Hypothesis: + - the stitched / cutoff feeling at the start of upward notes is partly coming from too much wet shoulder exposure in the hybrid-reset shipping branch +- Change: + - for upward notes only, tightened the audible entry/exit shoulders and lowered the wet floor inside the protected entry/exit zones + - kept the voiced body / Stage B timbre logic unchanged +- Result: + - `pitchOrg +4` improved slightly and stayed exact: + - note mel `8.299 -> 8.289` + - note env `1.018 -> 1.017` + - onset peak `+0.65 -> +0.61 dB` + - exit mel `8.511 -> 8.203` + - harmonic drift `0.523 -> 0.522` + - `pitchTest +4` stayed within the guard budget, but did not improve: + - note mel `3.449 -> 3.481` + - note env `0.620 -> 0.623` + - neighbor mel stayed effectively unchanged +- Verdict: + - kept +- Learning: + - the onset / stitching feel can be improved a little without destabilizing the guard case + - but the main miss is still note-core timbre / formant behavior, not shoulders alone + +#### Iteration TA-7: Upward frame-local envelope smoothing +- Hypothesis: + - the upward hybrid-reset app path might still be too broad in its envelope estimate, so tightening the envelope smoothing window could make the correction more local-frame and more reference-like +- Change: + - reduced the upward hybrid-reset envelope smoothing window inside the Stage B anchor pass +- Result: + - `pitchOrg +4` was effectively unchanged at the saved-output level before the guard rerun + - `pitchTest +4` shifted slightly but without any useful improvement +- Verdict: + - rejected and reverted +- Learning: + - the current app-path miss is not being caused by envelope smoothing width alone + +#### Iteration TA-8: Upward core-only high-band detail reinjection +- Hypothesis: + - the app path might still sound dark / robotic on `pitchOrg +4` because the upward hybrid-reset branch is over-smoothing high-band detail inside the voiced core +- Change: + - increased capped upward high-band detail reinjection only inside the hybrid-reset core path +- Result: + - `pitchOrg +4` regressed overall: + - note mel `8.289 -> 8.304` + - note env `1.017 -> 1.021` + - entry/exit mel `7.774 / 8.203 -> 7.829 / 8.238` + - harmonic drift improved only trivially `0.5221 -> 0.5210` + - `pitchTest +4` stayed at the same weaker guard result as the previous rejected branch: + - note mel `3.504` + - note env `0.614` +- Verdict: + - rejected +- Learning: + - extra high-band detail alone is not enough to make the current app path more reference-like + - the remaining app-path gap is more structural than just "too little air" + +## Research Timeline +### Milestone 1: Early DSP tuning before the harness was good enough +- Prior belief: + - broad note mel and general listening were enough to guide tuning +- New evidence: + - onset cracking, note-start weakness, and neighbor damage were still audible even when broad metrics looked acceptable +- Decision: + - stop trusting note-wide metrics alone + +### Milestone 2: Harness upgrade for onset / release / neighbor damage +- Prior belief: + - if note mel improved, pitch quality was probably improving in the right way +- New evidence: + - onset and neighbor windows could fail badly while overall note metrics stayed merely "okay" +- Decision: + - add entry/core/exit windows, pre/post-neighbor windows, onset peak jump, onset high-band burst, and audition bundles + +### Milestone 3: Wrong `pitchTest` fixture discovery +- Prior belief: + - the `pitchTest` comparisons were valid enough to compare branches +- New evidence: + - the old `pitchTest` runs were using the wrong note payload / wrong window family +- Decision: + - add clip-family-specific fixtures for `pitchTestOrg` and stop reusing the older `pitchOrg` note fixture + +### Milestone 4: Preview vs `single` divergence discovery +- Prior belief: + - preview and normal apply were basically the same engine with different quality settings +- New evidence: + - corrected `pitchTestOrg -> +4` preview was around `-36 cents`, while `single` was once around `-282 cents` +- Decision: + - treat the old `single` path as a core product bug, not a minor tuning issue + +### Milestone 5: Note-local `single` render fix +- Prior belief: + - pitch correctness problems were mainly inside Stage A / Stage B DSP +- New evidence: + - most of the giant miss came from how `single` rendered and spliced the edited region +- Decision: + - make `single` use note-local windowing logic consistent with the preview family + +### Milestone 6: Architecture bakeoff +- Prior belief: + - the current advanced branch might simply be the global winner +- New evidence: + - branch results depended strongly on clip family and on whether the corrected fixtures were used +- Decision: + - stop assuming the most complex branch is best; compare branches only through the corrected harness + +### Milestone 7: Safer hybrid becomes the current baseline +- Prior belief: + - a mixed carrier branch might deliver the best of both worlds +- New evidence: + - carrier mixing was unstable after the `single` routing fix +- Decision: + - use the safer `ce33`-family carrier plus a minimal note-core Stage B as the current baseline + +## Approaches Tried +### 1. Simple `ce33`-style Signalsmith baseline +- Intent: + - recover the historically safer articulation path +- Why it seemed promising: + - earlier stable behavior, cleaner entry/exit, fewer synthetic artifacts +- Result: + - exact pitch and cleaner onset behavior in many cases + - still too plain / not close enough to the reference timbre on `pitchOrg` +- Failure / plateau reason: + - safer than richer branches, but often too far from the sonic target +- Status: + - `plateaued but informative` + +### 2. Current advanced note-island Signalsmith branch +- Intent: + - improve the carrier and recover timbre with a serial note-core Stage B +- Why it seemed promising: + - better broad quality than simple `ce33` on `pitchOrg` + - fast preview +- Result: + - good broad note metrics on some cases + - onset and neighbor damage remained + - looked better than it really was before the harness upgrade +- Failure / plateau reason: + - too artifact-prone at note boundaries and not robust enough across clip families +- Status: + - `plateaued but informative` + +### 3. Carrier-mix hybrid (`ce33` shoulders + current core carrier) +- Intent: + - keep safer shoulders while using the stronger current carrier in the core +- Why it seemed promising: + - matched the intuition that shoulders and core want different behavior +- Result: + - unstable after the corrected `single` path + - could blow up centroid drift and quality badly +- Failure / plateau reason: + - mixing two carriers inside one note island is too fragile in the current design +- Status: + - `dead end` + +### 4. Safer hybrid (`ce33` carrier + minimal Stage B) +- Intent: + - keep the safer carrier and add only a small note-core recovery layer +- Why it seemed promising: + - lowest-risk path after the `single` routing fix +- Result: + - exact pitch on corrected `pitchTest` + - safer onset behavior + - good enough to become the current baseline +- Failure / plateau reason: + - still below the research/sample target, especially on `pitchOrg +4` +- Status: + - `current baseline` + +### 5. PSOLA branch +- Intent: + - use a voiced-core pitch-synchronous recovery path +- Why it seemed promising: + - should preserve articulation well in voiced regions +- Result: + - no meaningful app-path win +- Failure / plateau reason: + - did not beat the safer baseline enough to justify the extra complexity +- Status: + - `dead end` + +### 6. Model-style experimental branch +- Intent: + - recover more natural timbre through a heavier local refinement branch +- Why it seemed promising: + - could, in theory, close the realism gap more quickly +- Result: + - no app-path win strong enough to justify it +- Failure / plateau reason: + - not a practical product path in the current form +- Status: + - `dead end for now` + +### 7. Bungee benchmark +- Intent: + - benchmark an interesting outside shifter against Signalsmith +- Why it seemed promising: + - partial wins on some upward note-island cases +- Result: + - interesting benchmark, mixed across directions +- Failure / plateau reason: + - not strong enough or clean enough to replace the current shipping path +- Status: + - `plateaued but informative` + +## Failures And Root Causes +### Failure: word cutoff / broken onset +- Root cause: + - processing too wide a region + - shoulders treated like core + - old `single` render routing bug +- Learning: + - strict note-local rendering is mandatory + +### Failure: clipping-like crackle without true digital clipping +- Root cause: + - interference-style artifacts from overly aggressive blending and unstable branch combinations +- Learning: + - safer carrier + serial correction only + +### Failure: pitch-up sounding brighter, pitch-down darker +- Root cause: + - pitch-directional timbre drift and naive fallback behavior +- Learning: + - up/down probably need distinct note-core handling + +### Failure: branch looked good numerically but sounded wrong +- Root cause: + - harness originally missed onset/release/neighbor artifacts +- Learning: + - decision-making must use artifact-sensitive metrics and short audition files + +### Failure: misleading `pitchTest` conclusions +- Root cause: + - wrong note fixture and wrong analysis window +- Learning: + - every clip family needs its own verified fixture when note placement differs + +## What Actually Improved +- note-island rendering +- corrected `single` render routing +- artifact-aware harness +- simpler `ce33`-family carrier behavior +- smaller, safer Stage B + +## Own-Engine Architecture Branch +- Goal: + - stop building around `generic shifter -> corrective overlays` + - create a shared decomposition engine that can later support: + - `pitch_only_own_engine` + - `formant_only_own_engine` + - `pitch_plus_formant_own_engine` +- What is implemented now: + - new shared-analysis module in: + - `Source/OwnPitchEngine.h` + - `Source/OwnPitchEngine.cpp` + - current analysis output includes: + - note islands + - voiced masks + - F0 track + - epochs + - harmonic model + - residual model + - spectral envelope model + - new experimental branch routing in: + - `Source/PitchResynthesizer.cpp` + - `tools/run-ui-pitch-regression.ps1` + - current mode support: + - `pitch_only_own_engine`: real experimental renderer + - `formant_only_own_engine`: shared-analysis scaffold, legacy render fallback + - `pitch_plus_formant_own_engine`: own pitch base scaffold, legacy formant overlay fallback +- Why this branch exists: + - the old Signalsmith-centered family plateaued below the target + - future global formant work needs a clean place in the engine graph, not another retrofit +- Current status: + - `still viable as architecture` + - `not viable yet as a shipping renderer` + +### First Own-Engine Benchmark Snapshot +| Case | Branch | Note mel | Note env RMSE | Cents error | Centroid drift | Onset jump | Latency | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `pitchOrg -> +4` | `default` | `7.288` | `0.789` | `-18.32` | `-115.9 Hz` | `+0.65 dB` | `184 ms` | +| `pitchOrg -> +4` | `pitch_only_own_engine` v1 | `13.917` | `1.840` | `-762.28` | `+349.9 Hz` | `+15.14 dB` | `49 ms` | +| `pitchOrg -> +4` | `pitch_only_own_engine` v2 | `14.169` | `1.893` | `-762.28` | `+355.2 Hz` | `+17.63 dB` | `46 ms` | +| `pitchTestOrg -> +4` | `default` | `3.449` | `0.620` | `0.00` | `-39.4 Hz` | `0.00 dB` | `266 ms` | +| `pitchTestOrg -> +4` | `pitch_only_own_engine` v1 | `19.693` | `2.782` | `0.00` | `+1776.0 Hz` | `+2.29 dB` | `63 ms` | +| `pitchTestOrg -> +4` | `pitch_only_own_engine` v2 | `19.822` | `2.805` | `0.00` | `+1781.8 Hz` | `+4.83 dB` | `61 ms` | + +- Interpretation: + - the own-engine branch is very fast already + - the current synthesis prototype is badly wrong on timbre and onset behavior + - `pitchOrg +4` also has a major pitch-takeover failure in the current prototype + - the small wet-mask fix did not solve the core problem +- Current decision: + - keep the architecture branch + - do **not** treat the current own-engine renderer as competitive yet + - next own-engine work must focus on: + - correct voiced-core takeover + - target-pitch integrity on `pitchOrg +4` + - less synthetic harmonic rendering before any global-formant module is turned on + +### Own-Engine Iteration Log +#### Iteration OE-1: Use the app correction curve as pitch intent +- Change: + - stop deriving target pitch directly from note MIDI fields + - use the same per-sample correction curve the legacy renderer uses +- Why: + - keeps pitch intent consistent across engine families + - avoids fixture-specific note-payload interpretation bugs +- Result: + - `pitchOrg +4` pitch correctness recovered from `-762.28 cents` to `-18.32 cents` + - timbre was still far too bright and synthetic +- Learning: + - the biggest remaining own-engine problem after OE-1 was synthesis color, not pitch-target interpretation + +#### Iteration OE-2: Replace additive carrier with epoch-grain carrier where possible +- Change: + - use a pitch-synchronous grain carrier inside the voiced core when epoch density is good +- Why: + - keeps more of the original waveform character than pure additive synthesis +- Result: + - `pitchTest +4` improved materially: + - note mel `19.478 -> 9.447` + - note envelope `2.819 -> 1.234` + - centroid drift `+2321.8 Hz -> -18.2 Hz` + - `pitchOrg +4` barely changed in that pass +- Learning: + - the new carrier family can work + - clip families do not fail in the same way, so direction and clip behavior matter + +#### Iteration OE-3: Tame core wetness and brightness +- Change: + - lower core wet cap + - make shoulders drier + - add a conservative low-pass to the prototype carrier +- Why: + - the epoch carrier was still too bright and too intrusive in the note body +- Result: + - `pitchOrg +4` improved strongly and became the first genuinely competitive own-engine result: + - note mel `12.012 -> 5.945` + - note envelope `1.989 -> 1.099` + - cents `-18.32 -> 0.00` + - centroid drift `+1323.2 Hz -> +67.1 Hz` + - onset jump `+17.64 dB -> +0.81 dB` + - `pitchOrg -4` was mixed: + - note mel `6.396` + - note envelope `1.154` + - cents `+59.09` + - `pitchTest +4` stayed improved versus the broken prototype, but remained worse than baseline: + - note mel `9.603` + - note envelope `1.144` + - cents `-8.95` + - `pitchTest -4` also remained worse than baseline: + - note mel `8.779` + - note envelope `1.125` + - cents `0.00` +- Learning: + - the own engine is now promising on the hardest `pitchOrg +4` case + - the current prototype is not robust yet across directions and clip families + - next own-engine work should be direction-specific and carrier-shaping-specific, not another broad architecture rewrite + +#### Iteration OE-4: Direction- and duration-aware carrier shaping +- Change: + - separate carrier settings for: + - short upward notes + - longer upward notes + - downward notes +- Why: + - `pitchOrg +4` and the `pitchTest` family were clearly not asking for the same wetness / brightness balance +- Result: + - improved `pitchTest +4` and both `-4` cases + - but softened the best `pitchOrg +4` result too much +- Decision: + - treat the first direction-aware pass as informative, not the final keeper +- Learning: + - direction-aware tuning is necessary + - but the main objective must still protect the `pitchOrg +4` win + +#### Iteration OE-5: Stronger voiced-core takeover with short-downward pitch bias +- Change: + - keep the direction-aware carrier shaping + - add stronger voiced-core takeover + - apply a small target-ratio bias only for short downward notes +- Why: + - the remaining own-engine errors were dominated by partial dry-pitch leakage on `-4` +- Result: + - `pitchOrg -4` cents fixed from `+59.09` to `0.00` + - `pitchTest +4` and `pitchTest -4` stayed exact at `0.00` + - `pitchOrg +4` stayed clearly better than the legacy baseline on note mel, but not as strong as OE-3 +- Learning: + - the own engine now has a credible cross-case baseline + - the next gap is mostly timbre realism / spectral balance, not gross pitch correctness + +#### Iteration OE-6: Longer-upward shoulder easing +- Change: + - keep OE-5 as the base + - reduce dry shoulder protection only for longer upward notes +- Why: + - `pitchTest +4` was still failing more at entry/exit shape than at pitch correctness +- Result: + - `pitchTest +4` improved slightly without harming the other three tracked cases: + - note mel `9.207 -> 9.143` + - note envelope `1.116 -> 1.120` (flat) + - cents stayed `0.00` +- Learning: + - long upward notes do want slightly less dry shoulder protection + - the remaining gap is now a smaller timbre realism problem, not a routing or gross takeover failure + +#### Iteration OE-7: WORLD-style decomposition pass 1 +- Change: + - replace the heuristic own-engine decomposition with a real source-filter pass: + - pitch-adaptive spectral envelope analysis + - harmonic model driven from the source envelope + - band aperiodicity / residual model + - source-filter correction applied on top of the epoch carrier +- Why: + - the own engine was now routing and pitching correctly often enough that the remaining gap looked like a decomposition realism problem, not a carrier-routing problem +- Result: + - this became the first kept decomposition upgrade: + - `pitchOrg +4` improved: + - note mel `6.528 -> 6.293` + - note envelope `1.076 -> 1.049` + - centroid drift `-223.7 Hz -> -139.1 Hz` + - cents stayed `-18.32` + - `pitchTest +4` stayed exact at `0.00 cents`, but timbre remained weak: + - note mel `9.143 -> 9.260` + - note envelope `1.120 -> 1.109` + - centroid drift improved `-435.9 Hz -> -230.7 Hz` + - `pitchTest -4` stayed exact and slightly steadier in centroid: + - note mel `7.915 -> 7.907` + - centroid drift `-247.3 Hz -> -25.4 Hz` +- Learning: + - the source-filter direction is real and worth continuing + - the main remaining miss is now concentrated in long upward note entry/exit and spectral balance, not gross pitch intent + +#### Iteration OE-8: Aperiodicity-aware envelope correction +- Change: + - damp spectral-envelope correction in high-aperiodicity bands + - reduce residual reinjection in noisier regions +- Why: + - the first decomposition pass still looked too blunt on `pitchTest`, especially in regions that behave more like noisy / mixed content than clean harmonics +- Result: + - essentially neutral: + - `pitchOrg +4` note mel `6.293 -> 6.277` + - `pitchOrg -4` note mel `7.033 -> 7.040` + - `pitchTest +4` note mel `9.260 -> 9.265` + - `pitchTest -4` note mel `7.907 -> 7.901` +- Decision: + - rejected as a useful next lever +- Learning: + - the remaining `pitchTest` gap is not mainly caused by over-correcting noisy bins + +#### Iteration OE-9: Fade source-filter correction at core boundaries +- Change: + - fade the source-filter correction and residual reinjection in and out inside the core +- Why: + - the own engine was still trailing the baseline most clearly at note entry and exit on long upward notes +- Result: + - also effectively neutral: + - `pitchOrg +4` note mel `6.277 -> 6.275` + - `pitchTest +4` note mel `9.265 -> 9.259` + - `pitchOrg -4` and `pitchTest -4` moved only trivially +- Decision: + - rejected; keep OE-7 as the current own-engine baseline +- Learning: + - `pitchTest +4` is not primarily failing because the source-filter correction reaches too close to the core edges + - the shared `pitchTest` pre/post-neighbor metrics are also high on the default baseline, so that metric is not the differentiator there + +#### Iteration OE-10: Local-frame spectral-envelope correction +- Change: + - keep the source-filter decomposition from OE-7 + - switch the correction from a single broad note-average envelope to a locally interpolated frame envelope with a small global blend for stability +- Why: + - long upward notes still looked too flattened in timbre, which suggested the correction was too coarse over time rather than fundamentally wrong +- Result: + - this is the new kept decomposition baseline: + - `pitchTest +4` improved: + - note mel `9.260 -> 9.069` + - note envelope `1.109 -> 1.071` + - `pitchTest -4` improved: + - note mel `7.907 -> 7.727` + - note envelope `1.077 -> 1.049` + - `pitchOrg +4` regressed slightly: + - note mel `6.293 -> 6.342` + - note envelope `1.049 -> 1.053` + - `pitchOrg -4` also regressed slightly: + - note mel `7.033 -> 7.060` +- Learning: + - temporal envelope locality matters + - the long-note `pitchTest` gap was not just a carrier problem + - this change is worth keeping even with the small `pitchOrg` regressions because it improves the weaker family materially + +#### Iteration OE-11: Short-upward pitch-ratio bias tuning +- Change: + - add a tiny upward-only pitch-ratio bias for shorter notes, reusing the same structural hook that already fixed the short-downward miss +- Why: + - after OE-10, `pitchOrg +4` still carried the old `-18.32` cents miss even though `pitchTest +4` was exact +- Result: + - the first bias attempt overshot and was rejected + - the reduced bias was kept: + - `pitchOrg +4` cents `-18.32 -> 0.00` + - `pitchOrg +4` note envelope `1.053 -> 1.017` + - `pitchOrg +4` note mel `6.342 -> 6.318` + - `pitchTest +4` stayed unchanged at `9.069` note mel / `0.00` cents + - both `-4` cases stayed unchanged from OE-10 +- Learning: + - the short-upward pitch miss was a small ratio calibration issue, not a deeper decomposition failure + - the own engine now has a meaningfully better `+4` story than it did at OE-7 + +### Latest Own-Engine Snapshot +| Case | Branch | Note mel | Note env RMSE | Cents error | Centroid drift | Onset jump | Latency | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `pitchOrg -> +4` | `default` | `7.288` | `0.789` | `-18.32` | `-115.9 Hz` | `+0.65 dB` | `184 ms` | +| `pitchOrg -> +4` | `pitch_only_own_engine (OE-11)` | `6.318` | `1.017` | `0.00` | `-170.6 Hz` | `+0.73 dB` | `57 ms` | +| `pitchOrg -> -4` | `default baseline` | `7.408` | `1.114` | `0.00` | n/a | `+1.64 dB` | `191 ms` | +| `pitchOrg -> -4` | `pitch_only_own_engine (OE-11)` | `7.060` | `1.279` | `0.00` | `+217.3 Hz` | `+0.95 dB` | `54 ms` | +| `pitchTestOrg -> +4` | `default baseline` | `3.449` | `0.620` | `0.00` | `-39.4 Hz` | `0.00 dB` | `266 ms` | +| `pitchTestOrg -> +4` | `pitch_only_own_engine (OE-11)` | `9.069` | `1.071` | `0.00` | `-225.9 Hz` | `0.00 dB` | `74 ms` | +| `pitchTestOrg -> -4` | `default baseline` | `3.769` | `1.071` | `0.00` | n/a | `0.00 dB` | `268 ms` | +| `pitchTestOrg -> -4` | `pitch_only_own_engine (OE-11)` | `7.727` | `1.049` | `0.00` | `-37.9 Hz` | `0.00 dB` | `76 ms` | + +- Current decision after OE-11: + - keep the own-engine branch active + - keep OE-11 as the current own-engine baseline + - OE-8 and OE-9 were neutral and should not be re-tried first + - the own engine is now exact on all four tracked pitch cases + - it is still not ready to replace the current baseline across all clip families + - next work should prioritize: + - improve long upward-note timbre realism on `pitchTest +4` + - bring the `pitchOrg` and `pitchTest` envelope / spectral realism closer together without giving back exact pitch + - compare entry/exit behavior against the default baseline, not just against the reference + - then global-formant field work on top of the stabilized pitch carrier + +### Own-Engine Preview Reality Check +- Fresh current-code preview truth: + - `pitch_only_own_engine` preview no longer times out or drops the result file + - it is still not usable, because the preview render is sonically far away from both the reference and the own-engine single render +- Current `pitchOrg -> +4`: + - `single`: note mel `6.318`, note env `1.017`, note/body/core cents `0.00 / +18.92 / +18.92` + - `preview_segment`: note mel `12.758`, note env `1.434`, note/body/core cents `-458.43 / -880.59 / 0.00` +- Current `pitchTestOrg -> +4`: + - `single`: note mel `9.069`, note env `1.071`, note/body/core cents `0.00 / 0.00 / -18.32` + - `preview_segment`: note mel `16.584`, note env `2.051`, note/body/core cents `-978.55 / -978.55 / -984.54` +- Learning: + - the own-engine preview problem is no longer "job failed" + - it is now a **preview/single parity** problem + - the core of `pitchOrg +4` preview can still hit the right pitch while the broader note body is badly wrong, which points to body/coverage/takeover behavior rather than pure target-pitch math + +#### Iteration OE-12: Short-upward body-takeover increase +- Hypothesis: + - the own-engine upward short-note miss might be caused by too much dry original pitch leaking outside the strict voiced core, especially in preview +- Change: + - increased short-upward body entry/exit wetness and outside-core wet scaling +- Result: + - no measurable change on the checked runs: + - `pitchOrg +4 single`: stayed `6.318 / 1.017 / 0.00 cents` + - `pitchOrg +4 preview`: stayed `12.758 / 1.434 / -458.43 cents` + - `pitchTest +4 single`: stayed `9.069 / 1.071 / 0.00 cents` +- Verdict: + - rejected and reverted +- Learning: + - the current own-engine preview failure is not fixed by a simple increase in short-upward body wetness + - the next own-engine move should target preview/single parity more explicitly, not just "more corrected body" + +## Remaining Approaches +### Phase 1: Timbre Recovery On Safer Hybrid Baseline +- **Iteration budget:** `2` +- Why this could be better: + - pitch correctness is now acceptable + - the remaining gap is mainly note-core timbre realism, especially `pitchOrg +4` + - this is the lowest-risk path because it keeps the corrected note-local routing +- What would count as failure: + - after 2 bounded iterations, `pitchOrg +4` still sounds clearly robotic/dark and metrics do not materially improve +- What happens if it is worse: + - revert the failed timbre change immediately + - keep the safer hybrid baseline unchanged +- **Iterations left after Phase 1 completes:** `6` + +### Phase 2: Direction-Specific Note-Core Correction +- **Iteration budget:** `2` +- Why this could be better: + - `+pitch` and `-pitch` have already shown different needs + - one shared timbre rule is likely holding quality back +- What would count as failure: + - separate up/down tuning gives only tiny wins or causes regressions across clip families +- What happens if it is worse: + - keep only any clearly winning directional constant + - otherwise revert to the safer hybrid baseline +- **Iterations left after Phase 2 completes:** `4` + +### Phase 3: Shoulder / Onset Protection Refinement +- **Iteration budget:** `2` +- Why this could be better: + - the user still reports slight break/cutoff/robotic onset behavior + - the upgraded harness now measures those failures directly +- What would count as failure: + - onset gets cleaner but note-core quality worsens too much + - or metrics improve while the audible issue remains unchanged +- What happens if it is worse: + - lock the best shoulder behavior already found + - do not keep stretching the shoulder region just because one artifact metric improves +- **Iterations left after Phase 3 completes:** `2` + +### Phase 4: Harmonic-Relative Stage B +- **Iteration budget:** `2` +- Why this could be better: + - this is the last serious DSP-family change still likely to move closer to the research renders without another Stage A rewrite + - it targets harmonic structure more directly than the current envelope-style correction +- What would count as failure: + - no meaningful gain after 2 bounded iterations + - or latency / instability / artifacts regress +- What happens if it is worse: + - stop incremental DSP tuning on this family + - do not open another broad branch without a new explicit decision +- **Iterations left after Phase 4 completes:** `0` + +## Iteration Budget And Phase Gates +- Total remaining serious research budget: **`8` bounded iterations** +- Phase breakdown: + - Phase 1: `2` + - Phase 2: `2` + - Phase 3: `2` + - Phase 4: `2` +- At the end of each phase, record: + - what changed + - what improved + - what regressed + - whether the next phase is still justified +- Phase gates: + - start: `8` iterations left + - after Phase 1: `6` iterations left + - after Phase 2: `4` iterations left + - after Phase 3: `2` iterations left + - after Phase 4: `0` iterations left +- Rule: + - no phase gets more than its 2-iteration budget without an explicit reset of the plan + +## Iteration Results +### Phase 1, Iteration 1 of 2: Upward hybrid envelope-anchor relaxation +- Hypothesis: + - `pitchOrg +4` was still too dark, so a lighter upward-only hybrid envelope anchor might recover some timbre without disturbing onset safety +- Change: + - slightly increased upward hybrid mid-band lift + - slightly reduced upward low-band trim + - slightly relaxed upward air protection + - slightly increased upward hybrid detail retention +- Result: + - `pitchOrg -> +4`: note mel `8.414 -> 8.411`, note env `1.019 -> 1.019`, centroid `-194.0 -> -193.3 Hz` + - `pitchTestOrg -> +4`: note mel `3.441 -> 3.439`, note env `0.620 -> 0.621`, centroid `-34.2 -> -33.6 Hz` + - onset metrics stayed effectively unchanged +- Verdict: + - no material improvement + - kept in place because it was neutral-to-slightly-better, but it does not count as a meaningful Phase 1 win + +### Phase 1, Iteration 2 of 2: Stronger upward hybrid Stage B +- Hypothesis: + - a slightly stronger upward Stage B might improve the darker `pitchOrg +4` note core more directly +- Change: + - benchmarked stronger upward Stage B wetness only + - tested `OPENSTUDIO_PITCH_STAGEB_SCALE_UP=0.25` + - tested `OPENSTUDIO_PITCH_STAGEB_SCALE_UP=0.30` +- Result: + - `0.25`: + - `pitchOrg -> +4`: note mel `8.207`, note env `1.018`, centroid `-182.5 Hz` + - `pitchTestOrg -> +4`: note mel `3.549`, note env `0.664`, centroid `-28.9 Hz` + - `0.30`: + - `pitchOrg -> +4`: note mel `8.207`, note env `1.018`, centroid `-182.5 Hz` + - `pitchTestOrg -> +4`: note mel `3.658`, note env `0.699`, centroid `-24.2 Hz` +- Verdict: + - rejected + - the `pitchOrg +4` gain was too small to justify the regression on the already healthier `pitchTestOrg +4` case + +### Phase 1 Status +- Outcome: + - Phase 1 completed without a meaningful win + - current baseline remains the safer hybrid with the neutral Iteration 1 refinement and without the stronger upward Stage B override +- Iterations remaining: + - total remaining after Phase 1 completion: `6` + - next phase to start: `Phase 2: Direction-Specific Note-Core Correction` + +### Phase 2, Iteration 1 of 2: Upward short-note bias inside envelope-anchor strength +- Hypothesis: + - the shorter `pitchOrg +4` note might need stronger upward note-core correction than the longer `pitchTest +4` note +- Change: + - added an upward-only short-note boost inside the envelope-anchor strength calculation +- Result: + - `pitchOrg -> +4`: note mel `8.411 -> 8.410`, note env `1.019 -> 1.019`, centroid `-193.3 -> -193.0 Hz` + - `pitchTestOrg -> +4`: note mel `3.439 -> 3.439`, note env `0.621 -> 0.621`, centroid `-33.6 -> -33.6 Hz` +- Verdict: + - effectively neutral + - kept because it did not hurt anything, but it was too small to matter by itself + +### Phase 2, Iteration 2 of 2: Upward short-note wet-cap bonus +- Hypothesis: + - the stronger upward Stage B setting from Phase 1 helped the short `pitchOrg +4` note but hurt the longer `pitchTest +4` note, so the wet-cap bonus should apply only to short upward notes +- Change: + - added an upward-only wet-cap bonus for shorter edited notes in the safer hybrid branch +- Result: + - `pitchOrg -> +4`: note mel `8.411 -> 8.302`, note env `1.019 -> 1.019`, centroid `-193.3 -> -187.4 Hz` + - `pitchTestOrg -> +4`: note mel `3.439 -> 3.439`, note env `0.621 -> 0.621`, centroid `-33.6 -> -33.6 Hz` + - downward sanity check: + - `pitchOrg -> -4` stayed `7.408 / 1.114 / 0.00 cents` +- Verdict: + - kept + - this is the first meaningful gain that improved the hard upward case without damaging the healthier clip family + +### Phase 2 Status +- Outcome: + - Phase 2 completed with one kept directional improvement + - current baseline now includes the upward short-note wet-cap bonus +- Iterations remaining: + - total remaining after Phase 2 completion: `4` + - next phase to start: `Phase 3: Shoulder / Onset Protection Refinement` + +### Phase 3, Iteration 1 of 2: Drier hybrid shoulders and stronger edge protection +- Hypothesis: + - the safer hybrid still needed slightly more shoulder protection, especially at note boundaries, to reduce onset stress without undoing the Phase 2 timbre gain +- Change: + - shortened audible shoulders for the hybrid branch + - increased entry/exit protection lengths + - lowered hybrid edge wet floors at note entry and exit +- Result: + - `pitchOrg -> +4`: note mel `8.302 -> 8.297`, note env `1.019 -> 1.019`, centroid `-187.4 -> -189.3 Hz`, onset jump `+0.68 -> +0.65 dB` + - `pitchTestOrg -> +4`: note mel `3.439 -> 3.454`, note env `0.621 -> 0.622`, centroid `-33.6 -> -39.1 Hz` +- Verdict: + - kept only because the hard case improved slightly and onset moved in the right direction + - not a strong win + +### Phase 3, Iteration 2 of 2: Longer hybrid entry-side splice fade +- Hypothesis: + - a longer entry-side splice fade might reduce the remaining note-start stress without changing the note core +- Change: + - increased hybrid entry-side splice fade length only +- Result: + - no measurable change relative to Phase 3 iteration 1 on either `pitchOrg +4` or `pitchTestOrg +4` +- Verdict: + - no effect + - the earlier Phase 3 iteration 1 state is the only part worth keeping + +### Phase 3 Status +- Outcome: + - Phase 3 completed without a meaningful onset-specific win + - current baseline keeps the tiny improvement from Phase 3 iteration 1 only +- Iterations remaining: + - total remaining after Phase 3 completion: `2` + - next phase to start: `Phase 4: Harmonic-Relative Stage B` + +### Phase 4, Iteration 1 of 2: More harmonic-focused upward hybrid weighting +- Hypothesis: + - the hybrid upward path was still too broadband, so a more harmonic-focused envelope weight might move the result closer to the reference timbre +- Change: + - reduced the broadband floor of the harmonic weighting for upward hybrid blocks + - increased upward hybrid detail retention slightly +- Result: + - `pitchOrg -> +4`: note mel `8.297 -> 8.303`, note env `1.019 -> 1.018`, centroid `-189.3 -> -190.6 Hz` + - `pitchTestOrg -> +4`: note mel `3.454 -> 3.458`, note env `0.622 -> 0.616`, centroid `-39.1 -> -40.4 Hz` +- Verdict: + - rejected as a meaningful direction + - the change broadened in the wrong way and did not produce a real win + +### Phase 4, Iteration 2 of 2: Narrower harmonic neighborhoods for upward hybrid blocks +- Hypothesis: + - if the broadening was the problem, a narrower harmonic neighborhood might keep correction closer to true harmonics without washing across the note +- Change: + - reverted the broader harmonic weighting from iteration 1 + - narrowed the harmonic-mask sigma for upward hybrid blocks +- Result: + - `pitchOrg -> +4`: note mel `8.297 -> 8.299`, note env `1.019 -> 1.018`, centroid `-189.3 -> -189.7 Hz` + - `pitchTestOrg -> +4`: note mel `3.454 -> 3.449`, note env `0.622 -> 0.620`, centroid `-39.1 -> -39.4 Hz` +- Verdict: + - neutral + - too small to count as a real Phase 4 win + +### Phase 4 Status +- Outcome: + - Phase 4 completed without a meaningful DSP-family breakthrough + - the current frozen fallback baseline is still the safer hybrid family with: + - corrected note-local `single` routing + - short-note upward wet-cap bonus from Phase 2 + - tiny shoulder refinement from Phase 3 +- Iterations remaining: + - total remaining after Phase 4 completion: `0` + - next state: `freeze fallback baseline and hand off unless a new plan is created` + +## Decision Gates +- Continue on the current path only if: + - pitch remains within `+/-5 cents` + - no word cutoff or crackle returns + - `pitchOrg +4` timbre improves materially by ear and metrics + - preview remains under `1s` +- Freeze the current safer hybrid as fallback baseline if: + - later phases fail to improve timbre materially + - but the baseline still preserves pitch correctness, locality, and onset safety +- Escalate / hand off to another agent if: + - all `8` remaining iterations are used + - and the output is still clearly below the user reference by ear + - especially if `pitchOrg +4` remains robotic/dark + +## Failure / Handoff Plan +- When iterations reach `0`: + - stop doing more parameter loops + - freeze the best fallback baseline + - hand off with: + - current best baseline branch + - harness commands + - winning and losing metrics + - dead ends not worth retrying first + - open hypotheses for the next agent +- Handoff framing: + - the product path is now functionally corrected + - the remaining gap is mainly sonic realism, especially on upward note-local shifts + - the next agent should start from the corrected note-local baseline, not the old broken path + +## Bottom Line +- The biggest hidden bug was not just DSP quality; it was the mismatch between preview-style note-local rendering and the old `single` apply path. +- After fixing that, the project moved to a much safer baseline. +- We are no longer mainly fighting gross pitch correctness on the later `pitchTest` note. +- The remaining work is bounded: + - `8` serious iterations left + - `4` remaining approach families + - after that, freeze the best baseline and hand off cleanly instead of looping indefinitely. + +## Persistent Comparison Harness +- Canonical persistent artifact root: + - `D:\test projects\os tests` +- Canonical manifest: + - `D:\test projects\os tests\manifests\pitch_artifacts.json` +- Best saved app/own-engine artifacts are now registered in the manifest instead of being left in temp folders. +- New comparison tooling: + - `tools/pitch_harness_common.ps1` + - `tools/register-pitch-artifact.ps1` + - `tools/pitch_spectrogram_compare.py` + - `tools/run-pitch-output-comparison.ps1` +- Verified persistent comparison reports: + - `pitchOrg +4` + - matrix: `D:\test projects\os tests\reports\pitch_output_comparison_20260413_160935.md` + - report dir: `D:\test projects\os tests\reports\pitchOrg__4\20260413_160935` + - `pitchTestOrg +4` + - matrix: `D:\test projects\os tests\reports\pitch_output_comparison_20260413_161412.md` + - report dir: `D:\test projects\os tests\reports\pitchTestOrg__4\20260413_161412` +- Research-reference comparison is now handled explicitly by the harness: + - if a research render is not registered in the manifest, the report records that as missing instead of silently skipping it +- Current comparison finding to keep in mind: + - for the saved `+4` reports, the current `default` and `pitch_only_own_engine` candidates came out numerically identical in the stored comparison runs, so branch routing needs to be re-checked before treating those as meaningfully distinct outputs + +## April 13 Reality-Check Follow-Through + +### Truth Refresh +- Branch-truth and output-truth were re-verified through the real app path. +- Current shipping fallback remains: + - `default` -> actual branch `branch_hybrid_reset` +- Current hard-case app baseline remains: + - `pitchOrg +4`: note mel `8.299`, env `1.018`, cents `0.00`, pre/post-neighbor mel `8.017 / 4.235`, onset peak `+0.649 dB`, harmonic drift `0.523` +- Current downward app baseline remains: + - `pitchOrg -4`: note mel `7.405`, env `1.094`, cents `0.00`, onset peak `+1.416 dB`, harmonic drift `0.469` +- Current guard-case app baseline remains: + - `pitchTest +4`: note mel `3.449`, env `0.620`, cents `0.00`, pre/post-neighbor mel `15.956 / 19.285` + +### Harness Upgrade: Stutter-Sensitive Metrics +- Added new onset artifact diagnostics to `tools/reference_audio_match.py`: + - `onsetDerivativeJumpDb` + - `onsetRepeatSimilarityExcess` + - `onsetSpectralFluxDelta` + - `onsetArtifactScore` +- These are now carried through: + - `tools/run-ui-pitch-regression.ps1` + - `tools/run-pitch-reality-check.ps1` +- Purpose: + - make the stitched / repeated-attack / stutter-like onset failure visible in saved reports instead of relying only on broad entry-window metrics + +### App Iteration A: Onset-Safe Splice Relocation +- Hypothesis: + - anchoring the wet handoff later for upward notes would reduce the stitched/stutter feel at note start +- Change: + - first tried voiced-onset-anchored handoff + - then tried a small mandatory minimum delay for the upward wet ramp +- Result: + - both variants produced no measurable output change on the real app path + - `pitchOrg +4` stayed: + - note mel `8.289` + - env `1.017` + - onset peak `+0.61 dB` + - onset derivative / repeat / flux / artifact `+1.84 / 0.000 / +1.50 / +1.84` + - `pitchTest +4` stayed: + - note mel `3.481` + - env `0.623` + - onset derivative / repeat / flux / artifact `0.00 / 0.636 / +0.59 / +3.82` +- Verdict: + - rejected + - the current app-path onset problem is real, but this splice-only tweak was not strong enough to move the rendered output + +### App Iteration B: Dedicated Downward App Law +- Hypothesis: + - downward pitch shifts need their own low/mid/high envelope law and should not share the same timbre correction as upward shifts +- Change: + - tested a downward-only hybrid Stage B weighting change: + - stronger low-harmonic relocation + - different air protection + - lower downward detail preservation +- Result: + - no meaningful win worth keeping + - `pitchOrg -4` stayed effectively at baseline: + - note mel `7.405` + - env `1.094` + - onset derivative / repeat / flux / artifact `+2.74 / 0.000 / +0.74 / +2.74` + - `pitchTest -4` remained around the old level and still not acceptable by ear: + - note mel `3.775` + - env `1.062` + - onset derivative / repeat / flux / artifact `0.00 / 0.607 / +1.00 / +3.64` +- Verdict: + - rejected + - the current app branch appears to be at its ceiling for these small downward-law pushes + +### App-Path Status After Final Two Micro-Iterations +- Outcome: + - the harness is now better at agreeing with the audible onset complaint + - the two final app-path micro-iterations did not produce a keepable improvement + - shipping app baseline remains the last kept `TA-6` shoulder-protection state +- App-path iterations left: + - `0` +- Decision: + - stop further micro-tuning on the current app branch + - pivot active recovery to the own-engine / source-filter path for the next structural work + +## Own-Engine Recovery Restart + +### Own-Engine Baseline Before Restart +- Fresh own-engine truth runs showed: + - `pitchOrg +4` + - note mel `6.484` + - env `1.302` + - cents `+18.92` + - harmonic drift `0.423` + - onset artifact `1.76` + - `pitchOrg -4` + - note mel `7.307` + - env `1.634` + - cents `-46.79` + - harmonic drift `0.320` + - onset artifact `2.79` +- Working conclusion: + - the own engine was already more promising than the shipping app on some timbre metrics + - but it was still not trustworthy because pitch correctness had drifted again + +### Own-Engine Iteration OE-R1: Upward Onset-Safe Wet Handoff +- Hypothesis: + - the own-engine wet mask was handing too much synthesized content into the perceptual note onset, causing entry roughness and obscuring the core renderer quality +- Change: + - for upward notes only, made the own-engine body entry much drier at the start of the note + - added a short dry-hold inside the note body before the wet ramp rises + - left downward onset handling unchanged +- Result: + - `pitchOrg +4` + - note mel `6.484 -> 6.530` + - env `1.302 -> 1.356` + - cents `+18.92 -> 0.00` + - harmonic drift `0.423 -> 0.390` + - centroid `-310.7 -> -279.2 Hz` + - `pitchOrg -4` + - note mel `7.307 -> 5.560` + - env `1.634 -> 1.544` + - cents `-46.79 -> +11.90` + - harmonic drift `0.320 -> 0.176` + - `pitchTest +4` + - remained effectively unchanged: + - note mel `9.072` + - env `1.075` + - core cents `-18.32` + - `pitchTest -4` + - stayed exact on pitch: + - note mel `7.727` + - env `1.049` + - cents `0.00` +- Verdict: + - kept as the new experimental own-engine baseline + - this is the first structural own-engine iteration after the app-path stop point that materially improved both `pitchOrg +4` and `pitchOrg -4` + - it still does not beat the shipping app on the `pitchTest` family, so it remains experimental only + +### Own-Engine Iteration OE-R2: Stronger Long-Upward Source-Filter Correction +- Hypothesis: + - `pitchTest +4` is a long upward note, so the own-engine source-filter correction for long bodies might simply be too weak and too broad +- Change: + - increased long-upward local envelope correction strength + - tightened the gain limits and high-band cap for long upward notes only +- Result: + - `pitchTest +4` + - stayed essentially unchanged: + - note mel `9.070` + - env `1.073` + - core cents `-18.32` + - onset artifact `4.01` + - `pitchOrg +4` + - stayed effectively unchanged and healthy for the current own-engine baseline: + - note mel `6.520` + - env `1.352` + - cents `0.00` +- Verdict: + - rejected + - `pitchTest +4` is not mainly blocked by a simple long-upward correction-strength issue in the current source-filter step + +### Own-Engine Iteration OE-R3: Long-Upward Drier Shoulders +- Hypothesis: + - the long upward note in `pitchTest +4` might still be failing because too much synthesized signal is entering the body shoulders, not because the core timbre model is weak +- Change: + - made long upward notes use a drier entry shoulder, longer exit protection, and lower outside-core wetness +- Result: + - `pitchOrg +4` + - improved onset-facing metrics a bit: + - onset peak `+0.73 -> +0.49 dB` + - onset artifact `1.76 -> 1.59` + - pitch stayed exact + - `pitchTest +4` + - essentially unchanged: + - note mel `9.070` + - env `1.072` + - onset artifact `3.99` +- Verdict: + - rejected as the next baseline + - useful for diagnosis because it showed `pitchTest +4` is not mainly a shoulder wetness problem + +### Own-Engine Iteration OE-R4: Long-Upward Tighter Core Window +- Hypothesis: + - if `pitchTest +4` is not a wet-mask problem, its synthesized core may still be too wide and invading phonetic shoulders before epoch rendering begins +- Change: + - increased long-upward core entry/exit protection inside the shared analysis stage +- Result: + - `pitchTest +4` + - did not improve: + - note mel `9.084` + - env `1.079` + - onset artifact `4.01` + - `pitchOrg +4` + - stayed unchanged and healthy relative to the current own-engine baseline +- Verdict: + - rejected + - `pitchTest +4` is not mainly blocked by a simple long-note core-window size issue + +### Own-Engine Budget Status +- Used in the current structural own-engine cycle: + - `4` bounded iterations + - `OE-R1` kept + - `OE-R2` rejected + - `OE-R3` rejected + - `OE-R4` rejected +- Remaining serious own-engine iterations in the current budget: + - `2` +- Current best experimental own-engine baseline remains: + - `OE-R1`: upward onset-safe wet handoff + +### Own-Engine Iteration OE-R5: Long-Upward Epoch-Carrier Remap +- Hypothesis: + - long upward notes may sound repetitive because extra target epochs are being matched to the nearest discrete source epoch instead of interpolating between source epochs +- Change: + - for long upward notes only, interpolated the epoch carrier between adjacent source epochs instead of hard-rounding to one source epoch +- Result: + - `pitchTest +4` + - mixed: + - note mel `9.085` + - env `1.077` + - harmonic drift improved to `0.221` + - but onset/body quality did not improve enough + - `pitchOrg +4` + - unchanged and still exact +- Verdict: + - rejected + - useful because it confirmed the long-note upward miss is partly carrier-related, but not solved by epoch interpolation alone + +### Own-Engine Iteration OE-R6: Long-Upward Harmonic-Carrier Split +- Hypothesis: + - the long-note upward miss might need a different carrier family altogether; keep short-note upward and downward notes on the stronger path, but force long upward notes onto the harmonic core renderer +- Change: + - long upward notes bypass the epoch carrier and use the harmonic carrier path instead +- Result: + - `pitchTest +4` + - first meaningful improvement on this clip family in this bounded cycle: + - note mel `9.084 -> 8.785` + - env `1.079 -> 1.045` + - harmonic drift `0.290 -> 0.232` + - `pitchOrg +4` + - unchanged and still strong for the current own-engine baseline: + - note mel `6.520` + - env `1.352` + - cents `0.00` + - `pitchOrg -4` + - unchanged +- Verdict: + - kept + - this becomes the new experimental own-engine baseline + +### Own-Engine Final Budget Status +- Used in the current bounded structural own-engine cycle: + - `6` serious iterations + - kept: + - `OE-R1` + - `OE-R6` + - rejected: + - `OE-R2` + - `OE-R3` + - `OE-R4` + - `OE-R5` +- Remaining serious own-engine iterations in the current budget: + - `0` +- Current best experimental own-engine baseline is now: + - `OE-R6`: long-upward harmonic-carrier split on top of the earlier upward onset-safe handoff + +## April 13, 2026: Hybrid Structural Branch For `pitchTestOrg` + +### Goal +- Start the next recovery cycle on the user-confirmed truth case: + - original: `D:\test projects\pitchTestOrg.wav` + - references: + - `D:\test projects\pitchTestOrg+4s.wav` + - `D:\test projects\pitchTestOrg-4s.wav` +- Freeze the current shipping app path as control and route new work into an explicit experimental branch: + - `pitch_only_hybrid_structural` + +### Harness / Branch Infrastructure +- Added explicit experimental branch routing: + - requested / actual branch name: + - `pitch_only_hybrid_structural` +- Updated the UI regression runner so this branch can be exercised through the same persistent app-path harness and reports. + +### Hybrid Structural Iteration HS-1: Voiced-Onset / Voiced-Exit Anchored Wet Mask +- Hypothesis: + - the `pitchTest` truth case is failing because the pitch renderer is entering and exiting too close to the perceptual attack and handoff, creating stitched boundaries +- Change: + - added a branch-specific experimental route + - added voiced-onset / voiced-exit detection support in the own-engine wet-mask path + - first experimental mask delayed wet entry and advanced wet exit around detected voiced activity +- Result: + - `pitchTest +4` + - got much worse: + - note mel `11.889` + - env `1.620` + - entry mel `15.982` + - exit mel `12.982` + - `pitchTest -4` + - also underperformed the shipping control badly: + - note mel `7.718` + - env `1.044` + - entry mel `10.380` + - exit mel `14.244` +- Verdict: + - rejected + - diagnosis: + - simply keeping the shoulder drier is not enough + - on this truth case, the shoulder still needs pitch-rendered content rather than a long dry hold + +### Hybrid Structural Iteration HS-2: Downward-Specific Source-Filter Law +- Hypothesis: + - `pitchTest -4` is a true downward timbre problem and needs a separate branch-specific low/mid/high correction law +- Change: + - added branch-specific downward source-filter parameters for: + - correction strength + - gain limits + - local envelope blend + - residual reinjection +- Result: + - `pitchTest -4` + - only tiny movement, still far from control: + - note mel `7.727` + - env `1.045` + - exit mel `14.072` + - `pitchTest +4` + - regressed further: + - note mel `12.181` + - env `1.671` + - entry mel `18.285` +- Verdict: + - rejected + - diagnosis: + - a downward-specific law is necessary eventually + - but not on top of the current failed shoulder/core handoff variant + +### Isolation Cleanup +- The hybrid-structural behavior is now explicitly isolated to the `pitch_only_hybrid_structural` branch so the older own-engine route is not unintentionally affected by these failed experiments. + +### Current State After This Cycle +- Shipping control remains the current app baseline. +- `pitch_only_hybrid_structural` exists as a distinct experimental branch in the harness. +- No keepable quality win was found yet on the user-confirmed `pitchTestOrg +/-4` truth case. +- Most important learning: + - this clip family is not improved by a simple "drier shoulders" move + - the next structural attempt must preserve more pitch-rendered content at the note entry while still making the splice behavior smoother + +### Hybrid Structural Iteration HS-3: Short-Upward Blend Cap And Later Ramp +- Hypothesis: + - on the hybrid branch, the short-upward own-engine blend was still too aggressive across the full core, so we were leaving performance on the table at the note attack and handoff +- Change: + - kept the existing short-upward-only hybrid blend policy + - increased entry/exit protection inside the hybrid blend mask: + - entry `40 ms -> 50 ms` + - exit `50 ms -> 60 ms` + - capped own-engine contribution inside the short-upward core at `0.82` instead of fully replacing the legacy carrier +- Result: + - `pitchOrg +4` + - meaningful improvement over the previous kept hybrid baseline: + - note mel `6.252 -> 6.214` + - env `1.019 -> 0.966` + - entry mel `7.429 -> 7.069` + - exit mel `6.393 -> 6.076` + - onset artifact rose only slightly: + - `1.753 -> 1.796` + - `pitchTest +4` + - preserved exact shipping-control behavior by SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` + - `pitchTest -4` + - preserved exact shipping-control behavior by SHA: + - `1B6A789C4D36AB0D330CD043E41A72EB27B858120DF34B271293764A197F909C` +- Verdict: + - kept + - new experimental hybrid baseline +- Learning: + - the hybrid branch is now doing the right thing structurally: + - it stays pinned to the shipping control on the user-confirmed `pitchTest` truth case + - while still allowing bounded short-upward improvements on `pitchOrg +4` + - next short-upward work should improve onset smoothness without giving back the note-body gain + +### Hybrid Structural Iteration HS-4: Softer Early-Core Short-Upward Blend +- Hypothesis: + - the stitched feel is no longer coming from the fully dry attack itself, but from the first few milliseconds after the hybrid branch enters the rendered core +- Change: + - left the hybrid branch pinned to shipping-control behavior for long and downward notes + - within the short-upward path only: + - kept the later `50 ms` entry and `60 ms` exit protection from `HS-3` + - reduced the first `20 ms` of the rendered core to a lower own-engine blend cap (`0.62`) before ramping to the full short-upward cap (`0.82`) +- Result: + - `pitchOrg +4` + - another small but real gain over `HS-3`: + - note mel `6.214 -> 6.202` + - env `0.966 -> 0.964` + - entry mel `7.069 -> 6.788` + - harmonic drift `0.4001 -> 0.3997` + - onset artifact did not improve yet: + - `1.796 -> 1.840` + - `pitchTest +4` + - still preserved exact shipping-control behavior by SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Verdict: + - kept as the new experimental hybrid baseline + - but with a caution flag: + - this improved the short-upward note match again without touching the truth-case fallback + - it did not solve the onset-artifact score, so the next step still needs to be onset-focused rather than more broad note-body tuning + +### Hybrid Structural Iteration HS-5: Post-Core Legacy Hold +- Hypothesis: + - the stitched onset might be caused by the hybrid branch entering the early rendered core too soon, so holding the very start of the core on the legacy carrier for a few extra milliseconds could reduce the splice artifact +- Change: + - inserted an additional `8 ms` post-core legacy hold before the short-upward hybrid branch began ramping into the early-core own-engine blend +- Result: + - `pitchOrg +4` + - mixed and too small to keep: + - note mel `6.202 -> 6.209` (worse) + - env `0.964 -> 0.961` (better) + - entry mel `6.788 -> 6.928` (worse) + - onset artifact `1.841 -> 1.809` (slightly better) + - `pitchTest +4` + - still preserved exact shipping-control behavior by SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Verdict: + - rejected + - reverted to `HS-4` baseline +- Learning: + - delaying the rendered-core handoff later by itself is not enough + - the remaining onset problem is probably about the transition shape between legacy and hybrid content, not simply "start later" + +### Hybrid Structural Iteration HS-6: Onset-Side Blend Smoothing +- Hypothesis: + - the remaining short-upward onset artifact might come from a derivative spike at the legacy/hybrid boundary, so a tiny post-blend smoothing pass over the first `12 ms` after hybrid entry could soften the splice without changing note-body behavior +- Change: + - added an onset-only smoothing pass on the already-blended short-upward output, applied only over the first `12 ms` after the hybrid branch enters the short-upward rendered core +- Result: + - `pitchOrg +4` + - mixed and not good enough to keep: + - note mel `6.202 -> 6.226` (worse) + - env `0.964 -> 0.960` (better) + - entry mel `6.788 -> 6.804` (worse) + - onset artifact `1.841 -> 1.822` (slightly better) + - harmonic drift `0.3997 -> 0.3948` (better) + - `pitchTest +4` + - still preserved exact shipping-control behavior by SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Verdict: + - rejected + - reverted to `HS-4` baseline +- Learning: + - onset-only waveform smoothing can shave a little off the artifact score, but it is too expensive in note match if applied this bluntly + - the next onset fix likely needs a smarter transition law between legacy and hybrid content, not a generic local smoothing pass + +### Hybrid Structural Iteration HS-7: Coherence-Aware Onset Weighting +- Hypothesis: + - the onset artifact might come from the hybrid branch using too much own-engine signal in moments where the legacy and own waveforms disagree sharply, so early onset-side blend should be reduced only when the two signals are locally incoherent +- Change: + - added a short-upward-only onset-focus mask over the first `18 ms` after hybrid core entry + - inside that onset window only, modulated the hybrid blend by a local coherence score derived from: + - slope agreement between legacy and own signals + - instantaneous amplitude agreement +- Result: + - `pitchOrg +4` + - collapsed to the same mixed outcome as the prior rejected onset-law attempt: + - note mel `6.202 -> 6.209` (worse) + - env `0.964 -> 0.961` (better) + - entry mel `6.788 -> 6.928` (worse) + - onset artifact `1.841 -> 1.809` (slightly better) + - `pitchTest +4` + - still preserved exact shipping-control behavior by SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Verdict: + - rejected + - reverted to `HS-4` baseline +- Learning: + - a simple local coherence heuristic is still not changing the transition in the right way + - the next real onset attempt likely needs to operate on the transition curve or phase relationship more fundamentally, not just attenuate the blend where mismatch is detected + +### Hybrid Structural Iteration HS-8: Phase-Aligned Onset Bridge Scaffold +- Hypothesis: + - the remaining stitched onset might come from a phase-misaligned handoff between legacy and hybrid content, so replacing the short-upward scalar onset law with a local aligned bridge could reduce the stutter without disturbing the proven guard-case fallback +- Change: + - added hybrid onset-bridge scaffolding in `PitchResynthesizer.cpp`: + - local alignment search + - bridge-safe RMS matching + - bridge diagnostics plumbed through native regression results + - also updated the regression result merge so bridge diagnostics now survive into saved JSON/Markdown reports +- Result: + - first active bridge attempt on `pitchOrg +4` was clearly not keepable: + - note mel `6.202 -> 6.392` (worse) + - env `0.964 -> 1.006` (worse) + - entry mel `6.788 -> 9.951` (much worse) + - onset artifact `1.841 -> 1.810` (slightly better only) + - bridge diagnostics finally reported truthfully: + - `bridge used / fallback: true / false` + - `bridge lag / score / gain: 23 / 0.759 / -1.5 dB` + - `pitchTest +4` + - stayed pinned to shipping-control SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Verdict: + - rejected as an active renderer change + - bridge activation is now disabled again, while the diagnostics plumbing stays in the repo + - active experimental baseline restored to the last known-good `HS-4 / r6` behavior +- Reconfirmed active baseline after disabling bridge activation: + - `pitchOrg +4` + - note mel `6.209` + - env `0.961` + - entry mel `6.928` + - onset artifact `1.810` + - `bridge used / fallback: false / true` + - `pitchTest +4` + - unchanged shipping-control SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Learning: + - explicit onset-bridge diagnostics were worth adding + - the first bridge formulation confirmed the real problem is not “whether to align,” but “how to enter the aligned content without blowing up entry match” + - the next viable onset attempt should not replace the whole onset blend law at once; it should treat alignment as a constrained adjustment inside the already-good `r6` blend family + +### Hybrid Structural Iteration HS-9: Constrained Onset Alignment Assist +- Hypothesis: + - a full onset bridge was too aggressive, but a tiny alignment-only assist inside the existing `r6` early-core ramp might improve the stitched feel without disturbing the proven body/guard-case behavior +- Change: + - tried using local onset alignment only as a helper inside the existing short-upward `r6` blend law, rather than replacing that law +- Result: + - after rebuilding cleanly, the constrained assist was a true no-op: + - `pitchOrg +4` + - note mel stayed `6.209` + - env stayed `0.961` + - entry mel stayed `6.928` + - onset artifact stayed `1.810` + - `pitchTest +4` + - still preserved exact shipping-control SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Verdict: + - rejected + - cleaned back out so the branch stays on the real `HS-4 / r6` baseline +- Learning: + - the current onset issue is not being solved by small alignment assists layered onto the existing short-upward ramp + - the next meaningful improvement will likely need a different short-upward transition model entirely, not another tiny variant of the same ramp + +### Hybrid Structural Iteration HS-10: Four-Zone Short-Upward Transition Scaffold +- Hypothesis: + - replacing the old short-upward scalar ramp with an explicit four-zone onset/exit model might improve the stitched feel without touching long notes or downward notes +- Change: + - replaced the short-upward `r6` ramp with a four-zone model: + - dry shoulder + - transition pre-core + - stabilized early-core + - separate exit taper + - kept bridge rendering disabled and kept `pitchTest +4` on the same long-note fallback route +- Result: + - `pitchOrg +4` + - not keepable: + - note mel stayed flat at `6.209` + - env worsened `0.961 -> 0.963` + - entry mel worsened `6.928 -> 7.019` + - exit mel worsened `6.076 -> 6.310` + - onset artifact worsened `1.810 -> 1.840` + - `pitchTest +4` + - remained unchanged on the shipping-control SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Verdict: + - rejected +- Learning: + - the first explicit transition-model scaffold was too dry at the start and too weak at the handoff back out + - simply breaking the ramp into named zones did not improve the actual onset problem + +### Hybrid Structural Iteration HS-11: Onset-Confidence Preset Tables +- Hypothesis: + - the new short-upward transition model might need different preset tables based on onset strength, with shorter/softer notes entering the hybrid content earlier than the default table +- Change: + - added internal `high / medium / soft` onset classification + - then tried a soft-biased preset for short, non-spiky upward notes +- Result: + - `pitchOrg +4` + - the first classifier pass stayed on the same losing table as HS-10 + - after biasing short notes toward the soft table, the result still got worse: + - note mel `6.209 -> 6.215` + - env `0.961 -> 0.974` + - entry mel `6.928 -> 7.140` + - exit mel `6.076 -> 6.306` + - onset artifact `1.810 -> 1.840` + - `pitchTest +4` + - still remained pinned to the same shipping-control SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Verdict: + - rejected + - reverted fully back to the trusted `HS-4 / r6` short-upward blend +- Reconfirmed active baseline after the revert: + - `pitchOrg +4` + - note mel `6.209` + - env `0.961` + - entry mel `6.928` + - onset artifact `1.810` + - output SHA `055B3300041A8B5DE93C462C7E18F180BF7740D0134B0C05B6D255A6A36B46BD` +- Learning: + - short-upward onset strength tables still do not rescue the current scalar-ramp family + - the next meaningful transition-model attempt probably needs a different render structure, not another preset table layered onto the same blend shape + +### Hybrid Structural Iteration HS-12: Short-Upward Exit Handoff +- Hypothesis: + - the remaining miss on the short-upward hybrid path might be more about the note exit than the note entry, so handing back to the legacy renderer earlier and drier near note end could reduce exit damage without touching the frozen `pitchTest +4` fallback +- Change: + - changed the short-upward exit portion of the hybrid blend only: + - start the handoff back to legacy earlier + - cap the exit region to a lower own-engine weight + - leave the onset side and long/downward behavior unchanged +- Result: + - `pitchOrg +4` + - clearly not keepable: + - note mel `6.209 -> 6.214` + - env `0.961 -> 0.973` + - exit mel `6.076 -> 6.350` + - exit env `1.198 -> 1.306` + - onset artifact stayed effectively flat at `1.81` + - `pitchTest +4` + - remained pinned to the same shipping-control SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Verdict: + - rejected + - reverted back to the trusted `HS-4 / r6` short-upward baseline +- Learning: + - the current short-upward family is not mainly being held back by an exit-only taper problem + - onset, exit, and body behavior are still coupled tightly enough that taper-only changes keep degrading the note before they help the splice + +### Hybrid Structural Iteration HS-13: Plateau-Style Core Takeover +- Hypothesis: + - the current `r6` family may be plateaued because it still behaves like a long scalar ramp, so a structurally different short-upward model with a dry shoulder, fast equal-power entry, steady plateau, and explicit fade-out might break through that ceiling +- Change: + - replaced the short-upward scalar-style blend with a plateau takeover: + - dry shoulder + - fixed equal-power fade into full own-engine plateau + - fixed equal-power fade back out near note end + - left long-note and downward behavior untouched +- Result: + - `pitchOrg +4` + - clearly worse than the trusted `r6` baseline: + - note mel `6.209 -> 6.228` + - env `0.961 -> 0.977` + - entry mel `6.928 -> 7.498` + - exit mel `6.076 -> 6.312` + - onset artifact `1.807 -> 1.840` + - `pitchTest +4` + - stayed pinned to the same shipping-control SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Verdict: + - rejected + - reverted back to the trusted `HS-4 / r6` baseline +- Learning: + - even a structurally different plateau takeover still made the short-upward note worse + - the next step is no longer another transition law; it needs a different short-upward render structure than legacy/own scalar blending entirely + +### Structural Recovery Cycle: Attack-Preserve Body Replacement +- Goal: + - stop iterating on whole-note legacy/own blend masks and test a true short-upward structural replacement: + - dry attack + - stable voiced entry lock + - rendered body + - dry exit + - keep `pitchOrg +4` and `pitchTest +4` as equal-weight primary targets + - hard-stop any structural path after `2` bounded iterations if it still does not produce a keepable win +- Infrastructure kept: + - added persistent body-replacement diagnostics to the native regression result and saved summaries: + - `bodyReplacementUsed` + - `bodyReplacementFallbackUsed` + - `entryLockStartSec` + - `entryLockLengthMs` + - `exitLockStartSec` + - `renderedBodyStartSec` + - `renderedBodyEndSec` + +#### Path A, Iteration 1: Dry Attack + Existing Own-Engine Body +- Change: + - replaced the short-upward note body with the current own-engine body only between a deterministic voiced entry lock and exit lock + - preserved dry attack and dry exit outside that span +- Result: + - `pitchOrg +4` + - onset artifact improved: + - `1.810 -> 1.603` + - but the note became materially worse overall: + - note mel `6.209 -> 6.282` + - env `0.961 -> 1.036` + - entry mel `6.928 -> 7.759` + - exit mel `6.076 -> 6.517` + - whole-note cents `0.00 -> -18.32` + - `pitchTest +4` + - stayed flat on the same shipping-control SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Verdict: + - rejected after `1` iteration + - no equal-weight win, and the non-changing `pitchTest +4` meant Path A did not justify a second try +- Learning: + - removing attack-region blending alone is not enough if the rendered body entering after the lock is still structurally mismatched + +#### Path B, Iteration 1: Dry Attack + Epoch-Anchored Body +- Change: + - kept the same entry/exit locks + - replaced the inserted body with an epoch-anchored overlap-add body synthesized from the stable voiced region +- Result: + - `pitchOrg +4` + - large body improvements: + - note mel `6.209 -> 5.418` + - env `0.961 -> 0.616` + - entry mel `6.928 -> 5.250` + - exit mel `6.076 -> 5.636` + - formant drift `0.400 -> 0.126` + - but onset collapsed badly: + - onset artifact `1.810 -> 4.360` + - whole-note cents `0.00 -> +37.23` + - `pitchTest +4` + - still stayed flat on the same shipping-control SHA +- Verdict: + - promising but not keepable + - advanced to a second and final bounded iteration + +#### Path B, Iteration 2: Later, Longer Entry Lock +- Change: + - started the epoch body later and lengthened the entry crossfade to keep more of the original attack dry +- Result: + - `pitchOrg +4` + - onset improved slightly versus Path B iteration 1: + - onset artifact `4.360 -> 4.040` + - but it was still far worse than the kept `r6` baseline + - entry also regressed again: + - entry mel `5.250 -> 7.219` + - `pitchTest +4` + - still stayed flat on the same shipping-control SHA +- Verdict: + - rejected + - Path B exhausted its `2`-iteration budget and did not produce a keepable equal-weight win +- Learning: + - an epoch-anchored body can improve note-core realism, but on this path the onset splice became much worse than the baseline + - the body renderer and the onset handoff are still too tightly coupled here + +#### Path C, Iteration 1: Dry Attack + Harmonic Body +- Change: + - attempted a coherent harmonic-envelope body replacement instead of grain copying +- Result: + - the harmonic body never engaged on `pitchOrg +4` + - render failed closed back to the `r6` baseline: + - note mel `6.209` + - env `0.961` + - onset artifact `1.810` + - output SHA `055B3300041A8B5DE93C462C7E18F180BF7740D0134B0C05B6D255A6A36B46BD` +- Verdict: + - inconclusive first try + - moved to a second and final iteration so the harmonic path could engage deterministically + +#### Path C, Iteration 2: Static Spectral-Envelope Harmonic Body +- Change: + - added a static spectral-envelope fallback so the harmonic body would still render when frame-level harmonic tracks were too sparse +- Result: + - `pitchOrg +4` + - catastrophic failure: + - note mel `6.209 -> 19.738` + - env `0.961 -> 4.495` + - whole-note cents `0.00 -> -762.28` + - entry mel `6.928 -> 25.128` + - exit mel `6.076 -> 28.915` + - harmonic drift `0.400 -> 1.818` + - `pitchTest +4` + - still stayed flat on the same shipping-control SHA +- Verdict: + - rejected immediately + - Path C exhausted its `2`-iteration budget +- Learning: + - the current harmonic-only fallback is not viable as a body renderer in this product path + +#### Baseline Restore After Structural Cycle +- Action: + - restored the experimental hybrid branch to the last trusted `HS-4 / r6` behavior while keeping the new body-replacement diagnostics infrastructure +- Reconfirmed baseline: + - `pitchOrg +4` + - note mel `6.209` + - env `0.961` + - entry mel `6.928` + - exit mel `6.076` + - onset artifact `1.810` + - output SHA `055B3300041A8B5DE93C462C7E18F180BF7740D0134B0C05B6D255A6A36B46BD` + - `pitchTest +4` + - output SHA `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +- Structural-cycle conclusion: + - Path A: rejected after `1/2` + - Path B: rejected after `2/2` + - Path C: rejected after `2/2` + - no structural path beat the trusted `r6` baseline on both equal-weight targets + - the renderer is back on the safe baseline, and the repo now contains full diagnostics for why each body-replacement path failed + - the next step is no longer “another transition law”; it needs a different short-upward render structure than legacy/own scalar blending entirely +### 2026-04-14: Path D, Voiced-Entry-Locked PSOLA Body + +#### Path D, Iteration 1: Fixed-Threshold PSOLA Body +- Change: + - added a short-upward-only voiced-entry-locked TD-PSOLA body path inside the hybrid renderer + - kept long upward and all downward notes on the existing fallback path + - required stable voiced entry, stable voiced exit, at least `5` usable source epochs, and equal-power dry-attack/body/dry-exit splices +- Result: + - `pitchOrg +4` + - strong body improvement versus the `r6` baseline: + - note mel `6.209 -> 5.147` + - env `0.961 -> 0.488` + - entry mel `6.928 -> 5.607` + - exit mel `6.076 -> 5.482` + - harmonic drift `0.400 -> 0.146` + - but onset collapsed badly: + - onset artifact `1.810 -> 4.400` + - onset derivative `+1.81 -> +2.89` + - onset flux `+1.48 -> +4.40` + - `pitchTest +4` + - stayed pinned to the shipping-control SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` + - body replacement did not engage on this clip +- Verdict: + - promising but not keepable + - advanced to a second and final bounded iteration because the note-body metrics improved materially while the guard case stayed flat + +#### Path D, Iteration 2: Earlier Lock, Longer Entry Fade, Drier Tail +- Change: + - tuned only the allowed PSOLA parameters: + - voiced threshold `0.65 -> 0.60` + - entry sustain `12 ms -> 10 ms` + - entry fade `10 ms -> 12 ms` + - dry exit tail `20 ms -> 24 ms` +- Result: + - `pitchOrg +4` + - body stayed clearly improved versus baseline: + - note mel `6.209 -> 5.105` + - env `0.961 -> 0.474` + - exit mel `6.076 -> 5.383` + - harmonic drift `0.400 -> 0.127` + - onset improved slightly versus Path D iteration 1, but was still much worse than `r6`: + - onset artifact `4.400 -> 3.620` + - entry mel `5.607 -> 6.093` + - still far above baseline onset artifact `1.810` + - `pitchTest +4` + - still stayed pinned to the same shipping-control SHA + - body replacement still did not engage +- Verdict: + - rejected + - Path D exhausted its `2`-iteration budget without beating the equal-weight gate +- Learning: + - voiced-entry-locked PSOLA can produce the best note-body match seen so far on `pitchOrg +4` + - but in this architecture the onset handoff is still too destructive, and the path does not generalize to `pitchTest +4` + +#### Baseline Restore After Path D +- Action: + - restored the active renderer to the trusted `HS-4 / r6` baseline by disabling the PSOLA body path while keeping the diagnostics scaffolding in place +- Reconfirmed baseline: + - `pitchOrg +4` + - note mel `6.209` + - env `0.961` + - entry mel `6.928` + - exit mel `6.076` + - onset artifact `1.810` + - output SHA `055B3300041A8B5DE93C462C7E18F180BF7740D0134B0C05B6D255A6A36B46BD` + - body replacement used/fallback `false / true` + - `pitchTest +4` + - note mel `3.481` + - env `0.623` + - entry mel `2.484` + - exit mel `7.662` + - onset artifact `3.819` + - output SHA `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` + - body replacement used/fallback `false / false` +- Path D conclusion: + - exhausted after `2/2` + - improved note-body fidelity on `pitchOrg +4` more than the prior structural paths + - still failed the gate because onset quality remained substantially worse than the trusted baseline +### 2026-04-14: Path F, Island-Native Renderer With Transient-Preserve Core + +#### Path F, Iteration 1: Fixed-Mask Island-Native Render +- Change: + - added a new experimental branch `pitch_only_island_native` + - rendered short upward note islands as one internal object instead of doing an internal body handoff + - used transient-preserve onset and exit bands from the original signal plus an own-engine voiced-core render, with only outer-island splices + - added island-native diagnostics to the native regression result and saved summaries +- Result: + - `pitchOrg +4` + - the path engaged successfully: + - island native used/fallback `true / false` + - island render span `0.08 -> 0.8300 s` + - transient/core mask peak `1.000 / 0.996` + - onset improved materially versus the `r6` baseline: + - onset artifact `1.810 -> 1.388` + - but note and boundary quality regressed too much: + - note mel `6.209 -> 7.746` + - env `0.961 -> 1.479` + - entry mel `6.928 -> 10.191` + - exit mel `6.076 -> 8.275` + - `pitchTest +4` + - stayed pinned to the shipping-control SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` + - island-native path did not engage on this clip: + - island native used/fallback `false / false` +- Verdict: + - promising enough for one final bounded tuning pass + - advanced to Iteration 2 because onset improved substantially on the active target while the equal-weight guard stayed flat + +#### Path F, Iteration 2: Lower Voiced-Core Threshold +- Change: + - applied the plan's only allowed F2 adjustment for the observed failure mode: + - lowered voiced-core threshold `0.65 -> 0.60` + - left onset hold, exit hold, and outer splice lengths unchanged +- Result: + - `pitchOrg +4` + - island-native stayed engaged: + - island native used/fallback `true / false` + - island render span `0.08 -> 0.8300 s` + - transient/core mask peak `1.000 / 0.996` + - improved a little versus Path F iteration 1, but still clearly lost to baseline: + - note mel `7.746 -> 7.511` + - env `1.479 -> 1.420` + - entry mel `10.191 -> 9.818` + - exit mel `8.275 -> 7.915` + - onset artifact stayed improved versus baseline: + - `1.810 -> 1.388` + - still failed the equal-weight gate because body and boundary regressions remained far beyond tolerance + - `pitchTest +4` + - still pinned to the same shipping-control SHA + - island-native path still did not engage +- Verdict: + - rejected + - Path F exhausted its `2`-iteration budget without beating the trusted `r6` baseline +- Learning: + - eliminating the internal renderer handoff can improve the onset metric on `pitchOrg +4` + - but this fixed-mask island-native model still damages note body and entry/exit too much + - the path also does not generalize to `pitchTest +4`, because it never activates there under the current gating + +#### Path F Conclusion +- Status: + - exhausted after `2/2` + - not promoted + - shipping fallback and active trusted experimental baseline remain unchanged +- Current trusted live baseline remains: + - `pitchOrg +4` + - note mel `6.209` + - env `0.961` + - entry mel `6.928` + - exit mel `6.076` + - onset artifact `1.810` + - output SHA `055B3300041A8B5DE93C462C7E18F180BF7740D0134B0C05B6D255A6A36B46BD` + - `pitchTest +4` + - note mel `3.481` + - env `0.623` + - entry mel `2.484` + - exit mel `7.662` + - onset artifact `3.819` + - output SHA `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` +### 2026-04-14: Path G, Adaptive Island-Native PSOLA-Core + +#### Path G, Iteration 1: Fixed-Threshold Island Shell + PSOLA Core +- Change: + - added a new experimental branch `pitch_only_island_native_psola` + - kept the Path F island-native shell and outer-only splice policy + - replaced the island voiced-core source with a PSOLA core synthesized only from stable voiced epochs inside the island + - added one relaxed engagement retry for `pitchTest`-type clips: + - voiced threshold `0.55` + - sustain `8 ms` + - left long upward and all downward notes on fallback behavior +- Result: + - `pitchOrg +4` + - branch engaged successfully: + - island native used/fallback `true / false` + - island render span `0.08 -> 0.8300 s` + - transient/core mask peak `1.000 / 0.996` + - onset stayed improved versus the trusted `r6` baseline: + - onset artifact `1.810 -> 1.388` + - but note and boundary quality regressed even harder than the fixed-mask own-engine-core island path: + - note mel `6.209 -> 8.053` + - env `0.961 -> 1.608` + - entry mel `6.928 -> 11.972` + - exit mel `6.076 -> 16.651` + - note cents `0.00 -> +18.52` + - `pitchTest +4` + - stayed pinned to the same shipping-control SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` + - the relaxed engagement retry still did not activate the path: + - island native used/fallback `false / false` +- Verdict: + - rejected after `1/2` + - the path failed the equal-weight gate immediately, so no G2 tuning pass was allowed +- Learning: + - combining the Path F shell with a PSOLA core preserved the onset-side gain on `pitchOrg +4` + - but the PSOLA core destabilized pitch/body/exit inside the island + - the path still does not generalize to `pitchTest +4`, even with the relaxed engagement retry + +#### Path G Conclusion +- Status: + - stopped after `1/2` under the stop-fast rule + - not promoted + - shipping fallback and trusted `HS-4 / r6` experimental baseline remain unchanged +### 2026-04-15: Family `FAM-V2-HSR`, Engine V2 Harmonic/Source-Filter + Residual Shell + +#### Iteration 1: `pitch_only_engine_v2` +- Change: + - added a temporary `pitch_only_engine_v2` branch + - reused the island-native shell for short upward notes only + - replaced the fixed own-engine core with an explicit v2 composition: + - original transient-preserve shell + - own-engine harmonic/source-filter core + - explicit residual layer from the island residual model + - kept long upward and all downward behavior on fallback +- Result: + - `pitchOrg +4` + - onset stayed improved versus `r6`: + - onset artifact `1.810 -> 1.388` + - but the note and boundaries regressed clearly: + - note mel `6.209 -> 7.975` + - env `0.961 -> 1.454` + - entry mel `6.928 -> 9.737` + - exit mel `6.076 -> 12.703` + - note cents `0.00 -> -18.32` + - `pitchTest +4` + - stayed pinned to the same shipping-control SHA: + - `1A9B12D8EA57EFEAC7B3505735BC9D4DA5F666D98703DA1333549314719CC043` + - the branch still did not engage on this clip +- Verdict: + - rejected after `1/2` + - it failed the equal-weight gate decisively, so it did not earn a second iteration +- Cleanup: + - removed the temporary `pitch_only_engine_v2` branch code immediately after the reject +- Learning: + - adding an explicit residual layer to the island-native shell did not solve the onset/body tradeoff + - the family preserved the onset-side gain seen in prior island-native attempts + - but it still could not preserve note body or generalize to `pitchTest +4` + +### 2026-04-16: Stopped-Transport Scrub Preview Fix + `engine-v2` Tuning R7 + +#### Scrub Preview Audibility +- Change: + - moved the RAM scrub loop out of the transport-gated clip playback path + - added a dedicated preview voice rendered directly from the main audio callback + - added start/stop ramps and native status counters for scrub regression + - added a dedicated scrub regression runner: + - `tools/run-ui-pitch-scrub-regression.ps1` +- Result: + - run: `20260416_153028_pitchOrg_scrub_preview_r2` + - stopped-transport scrub preview is now measurably active: + - `scrubPreviewAudible=true` + - start latency `27.9 ms` + - stop latency `27.5 ms` + - mixed callback count `3` + - mixed sample count `3072` + - last peak `0.0005417` +- Verdict: + - structural scrub-audibility blocker fixed + - still needs user audition in the real editor path, but the old “silent while stopped” wiring gap is no longer hidden + +#### Render Tuning R7 +- Change: + - kept `pitch_only_engine_v2_program` active + - made the cepstral lifter F0-adaptive + - raised overlap inside the envelope-restore stage + - tightened transient ownership and shifted entry focus earlier + - widened the transition lead/tail slightly while keeping the adaptive carrier underneath + - added boundary timing metrics to the comparison script: + - `entryLagMs` + - `exitLagMs` + - `onsetDelayMs` + - `boundaryTimingErrorMs` +- Result: + - `pitchOrg +4` + - run: `20260416_152736_pitchOrg_plus4_note_hq_engine_v2_tune_r7` + - note mel `7.314` + - env `1.347` + - entry mel `7.396` + - exit mel `7.027` + - onset artifact `1.388` + - formant body harmonic drift `0.415` + - `pitchTestOrg +4` + - run: `20260416_152736_pitchTestOrg_plus4_note_hq_engine_v2_tune_r7` + - note mel `3.540` + - env `0.521` + - entry mel `7.513` + - exit mel `1.632` + - onset artifact `4.013` + - onset delay `1.479 ms` + - formant body harmonic drift `0.069` + - fresh adaptive controls: + - `pitchOrg +4`: `20260416_153028_pitchOrg_plus4_note_hq_adaptive_cmp_r3` + - note mel `7.085` + - entry mel `7.078` + - onset artifact `1.803` + - harmonic drift `0.369` + - `pitchTestOrg +4`: `20260416_153028_pitchTestOrg_plus4_note_hq_adaptive_cmp_r3` + - note mel `2.810` + - entry mel `1.530` + - onset artifact `0.705` + - harmonic drift `0.066` +- Verdict: + - the tuned `engine-v2` path is now safer on the easy clip than the earlier catastrophic versions + - but it still loses clearly to `pitch_only_adaptive_selector` on the hard note-entry case + - the current repo truth remains: + - adaptive is still the audible winner + - scrub preview audibility is structurally fixed + - boundary/formant tuning still needs more work + +### 2026-04-16: Harness Close-Out Progress, Scrub Suite + Transient/Formant Smoke Suites + +#### Multi-scenario scrub suite +- Change: + - extended the scrub regression job to accept: + - repeated drag cycles + - optional transport play/stop cycle before scrub + - added `tools/run-ui-pitch-scrub-suite.ps1` + - isolated suite outputs into their own `case_runs` directory to avoid cross-run collisions +- Result: + - run: `20260416_205833_pitchOrg_scrub_suite_smoke_r3` + - case summary: + - `first_drag` + - audible `true` + - first-drag audible `true` + - start / stop latency `26.8 / 26.7 ms` + - `repeated_drag` + - audible `true` + - first-drag audible `true` + - start / stop latency `27.0 / 26.3 ms` + - scenario count recorded as `3` + - `after_transport_cycle` + - audible `true` + - first-drag audible `true` + - start / stop latency `27.5 / 26.4 ms` + - scenario count recorded as `2` +- Verdict: + - scrub harness coverage is now broader than the old single happy-path run + - but `H1` is still only partial because: + - selection-change scrub is not yet covered + - per-scenario pass breakdown is not fully propagated into the result JSON + - the audible “breaking loop” complaint is still a listening issue, not a solved benchmark issue + +#### Transient and formant suite harnesses +- Change: + - added a manifest-driven suite runner: + - `tools/run-ui-pitch-regression-suite.ps1` + - added wrappers: + - `tools/run-ui-pitch-transient-suite.ps1` + - `tools/run-ui-pitch-formant-suite.ps1` + - added smoke manifests: + - `tests/fixtures/pitch-regression/suites/transient_smoke_suite.json` + - `tests/fixtures/pitch-regression/suites/formant_smoke_suite.json` + - fixed suite output collisions by isolating per-suite `case_runs` +- Result: + - transient smoke run: `20260416_205335_transient_suite_smoke_r3` + - `pitchOrg +4` + - note mel `6.671` + - onset artifact `1.825` + - entry / exit artifact `6.918 / 7.138` + - onset delay `-0.104 ms` + - boundary timing error `8.417 ms` + - formant drift `0.374` + - `pitchTestOrg +4` + - note mel `2.802` + - onset artifact `2.051` + - entry / exit artifact `1.489 / 1.627` + - onset delay `-0.625 ms` + - boundary timing error `0.958 ms` + - formant drift `0.050` + - formant smoke run: `20260416_205335_formant_suite_smoke_r3` + - same canonical smoke cases now flow through a dedicated formant-focused suite wrapper and summary +- Verdict: + - `H3` and `H4` harness plumbing now exist and run cleanly + - but they are still only partial because the repo does not yet contain the full transient/formant fixture set from the plan + +#### Remaining plan truth +- The full recovery plan is still not complete. +- Remaining mandatory iterations: `14` +- Remaining conditional iterations: `+2` + +### 2026-04-16: Adaptive Carrier Boundary/Formant Correction Pass R2 + +#### Adaptive selector correction-layer pass +- Change: + - kept `pitch_only_adaptive_selector` as the carrier + - added a narrow adaptive boundary-correction layer in `PitchResynthesizer.cpp` + - applied transient/unvoiced bypass, cepstral envelope restore, and residual carry only inside note entry/exit windows + - exposed correction diagnostics through the existing render summary so the run proves whether the path actually engaged +- Result: + - suite run: `20260416_210838_adaptive_boundary_tune_r2` + - `pitchOrg +4` + - note mel `6.548` + - env `1.044` + - entry mel `7.383` + - exit mel `7.007` + - onset artifact `3.259` + - boundary timing error `8.417 ms` + - formant drift `0.372` + - `spectralEnvelopeCorrectionUsed=true` + - `pitchTestOrg +4` + - note mel `3.029` + - env `0.511` + - entry mel `1.993` + - exit mel `5.269` + - onset artifact `0.824` + - boundary timing error `5.458 ms` + - formant drift `0.038` + - `spectralEnvelopeCorrectionUsed=true` + - comparison against the prior adaptive smoke baseline: + - `pitchOrg +4` + - slightly better note mel/env + - much worse onset artifact + - entry slightly worse + - exit slightly better + - `pitchTestOrg +4` + - onset artifact improved sharply + - formant proxy improved slightly + - note mel/env regressed + - exit and boundary timing regressed badly +- Verdict: + - this pass is real and engaged; it is not another stale-binary false read + - it is still not keepable in its current shape + - the failure pattern says the next adaptive-carrier work should focus on: + - `A3` transient handoff crossfade sweep + - `A4` entry timing compensation + - `A5` F0-adaptive cepstral lifter retune + +#### Remaining plan truth +- The full recovery plan is still not complete. +- Remaining mandatory iterations: `13` +- Remaining conditional iterations: `+2` + +### 2026-04-16: Adaptive Carrier Boundary/Formant Correction Pass R3 + +#### Adaptive selector correction-layer retune +- Change: + - kept the adaptive carrier correction path active + - split entry vs exit defaults so note exit ownership is much lighter than note entry + - widened the outer correction crossfade, pushed entry focus slightly earlier, and reduced correction wetness near the raw boundary +- Result: + - suite run: `20260416_211448_adaptive_boundary_tune_r3` + - `pitchOrg +4` + - note mel `6.537` + - env `1.040` + - entry mel `7.435` + - exit mel `7.028` + - onset artifact `2.383` + - boundary timing error `8.396 ms` + - formant drift `0.373` + - `spectralEnvelopeCorrectionUsed=true` + - `pitchTestOrg +4` + - note mel `2.960` + - env `0.493` + - entry mel `1.876` + - exit mel `4.067` + - onset artifact `1.540` + - boundary timing error `2.250 ms` + - formant drift `0.039` + - `spectralEnvelopeCorrectionUsed=true` + - comparison against `r2`: + - `pitchOrg +4` + - note mel/env improved slightly + - onset artifact improved sharply + - entry stayed a little worse than the pre-correction baseline + - `pitchTestOrg +4` + - note mel/env/entry/exit all improved versus `r2` + - boundary timing also improved strongly + - but exit and boundary timing still remain materially worse than the original adaptive baseline +- Verdict: + - `r3` is a real improvement over `r2` + - it is still not the fix: + - the easy clip still has too much onset damage + - the hard clip still has too much exit damage + - the next honest priorities remain: + - `A3` transient handoff crossfade sweep + - `A4` entry timing compensation + - `A5` cepstral lifter retune + +#### Remaining plan truth +- The full recovery plan is still not complete. +- Remaining mandatory iterations: `12` +- Remaining conditional iterations: `+2` + +### 2026-04-16: Adaptive Carrier Boundary/Formant Correction Pass R5 + +#### Adaptive selector compromise profile +- Change: + - kept the adaptive carrier correction path active + - tuned toward a compromise profile after the `r4` onset-safe and exit-safe sweeps: + - lower flatness center and slightly higher RMS gate + - shorter entry crossfade + - earlier entry bias + - lighter entry and exit wetness + - lighter residual carry + - promoted this profile into the default code path because it is the best adaptive correction run so far +- Result: + - suite run: `20260416_212441_adaptive_boundary_tune_r5_compromise` + - `pitchOrg +4` + - note mel `6.533` + - env `1.039` + - entry mel `7.462` + - exit mel `7.044` + - onset artifact `2.261` + - boundary timing error `8.396 ms` + - formant drift `0.373` + - `spectralEnvelopeCorrectionUsed=true` + - `pitchTestOrg +4` + - note mel `2.935` + - env `0.487` + - entry mel `1.833` + - exit mel `3.665` + - onset artifact `1.748` + - boundary timing error `2.250 ms` + - formant drift `0.040` + - `spectralEnvelopeCorrectionUsed=true` + - comparison against earlier adaptive correction passes: + - this is the best mixed result so far + - it improves `pitchTestOrg +4` note/entry/exit/formant metrics versus `r2` and `r3` + - it also improves `pitchOrg +4` onset artifact versus `r2` and `r3` + - but it still does not beat the plain adaptive baseline on the note-start/note-end problems that matter most +- Verdict: + - the adaptive correction layer is now a real tunable path, not just a failed idea + - but it is still not the proper fix yet + - remaining priority order should be: + - `A4` entry timing compensation + - `A5` cepstral lifter retune + - then either: + - one more `A3` cleanup pass if start roughness still dominates listening, or + - freeze the adaptive correction layer as a non-default experiment if it keeps failing against plain adaptive + +#### Remaining plan truth +- The full recovery plan is still not complete. +- Remaining mandatory iterations: `11` +- Remaining conditional iterations: `+2` + +### 2026-04-16: Adaptive Carrier Timing/Formant Retune R6 + Onset Cleanup R7 + +#### Timing/formant retune +- Change: + - added explicit entry/exit pre/post window tuning to the adaptive correction path + - made cepstral lifter scale and correction strength tunable instead of fixed + - softened the default cepstral profile and reduced correction authority near the raw boundary +- Result: + - run: `20260416_213431_adaptive_boundary_tune_r6_timing_formant` + - `pitchOrg +4` + - note mel `6.541` + - env `1.038` + - entry mel `7.298` + - exit mel `7.112` + - onset artifact `2.261` + - formant drift `0.373` + - `pitchTestOrg +4` + - note mel `2.830` + - env `0.470` + - entry mel `1.755` + - exit mel `1.887` + - onset artifact `1.748` + - formant drift `0.050` +- Verdict: + - this was a real improvement on the hard clip, especially at exit + - formant drift did not materially improve + - the easy clip still kept too much start-side damage + +#### Onset cleanup follow-up +- Change: + - raised the onset RMS gate slightly + - reduced entry wetness again + - shortened the entry pre/post ownership window + - promoted the resulting profile into the default code path because it outperformed the earlier adaptive correction runs overall +- Result: + - run: `20260416_213810_adaptive_boundary_tune_r7_onsetcleanup` + - `pitchOrg +4` + - note mel `6.526` + - env `1.023` + - entry mel `6.890` + - exit mel `7.112` + - onset artifact `2.261` + - formant drift `0.374` + - `pitchTestOrg +4` + - note mel `2.828` + - env `0.470` + - entry mel `1.707` + - exit mel `1.887` + - onset artifact `1.714` + - formant drift `0.050` +- Verdict: + - `r7` is the strongest adaptive correction profile so far + - it narrows the hard-case gap substantially + - but it still does not solve the two main user-facing issues: + - `pitchOrg +4` note start is still too artifacted versus plain adaptive + - `pitchTestOrg +4` note exit and onset are still not as clean as the plain adaptive baseline + +#### Remaining plan truth +- The full recovery plan is still not complete. +- Remaining mandatory iterations: `10` +- Remaining conditional iterations: `+2` + +### 2026-04-16: Adaptive Carrier Formant Sweep R8 + Residual Sweep R9 + +#### Cepstral/formant sweep +- Change: + - ran two formant-focused profiles on the adaptive correction path: + - `r8 strong`: lower lifter scale and stronger cepstral correction + - `r8 soft`: higher lifter scale and softer cepstral correction +- Result: + - runs: + - `20260416_223105_adaptive_formant_tune_r8_strong` + - `20260416_223105_adaptive_formant_tune_r8_soft` + - both profiles were effectively flat on the smoke suite + - differences were tiny enough that they do not justify a default change yet +- Verdict: + - `A5` is not the dominant lever on the current smoke cases + - richer sustained-vowel fixtures are still needed before declaring cepstral tuning closed in a product sense + +#### Residual carry sweep +- Change: + - compared a dry residual profile against a wetter residual profile on the same formant smoke suite +- Result: + - runs: + - `20260416_223535_adaptive_residual_tune_r9_dry` + - `20260416_223535_adaptive_residual_tune_r9_wet` + - `r9 dry` won slightly but consistently: + - `pitchOrg +4` + - note mel `6.525` + - env `1.022` + - entry mel `6.885` + - onset artifact `2.257` + - `pitchTestOrg +4` + - note mel `2.828` + - env `0.469` + - entry mel `1.703` + - onset artifact `1.688` + - the wetter residual profile was slightly worse on both smoke cases +- Verdict: + - `A6` currently favors keeping residual reinjection off in the adaptive correction layer + - the default code path now uses the dry residual profile + +#### Remaining plan truth +- The full recovery plan is still not complete. +- Remaining mandatory iterations: `8` +- Remaining conditional iterations: `+2` + +### 2026-04-16: Adaptive Carrier STFT Sweep R10 + +#### STFT correction-layer sweep +- Change: + - exposed the cepstral correction stage FFT order and hop divisor as explicit tuning controls + - compared: + - `1024 / 8` + - `2048 / 8` + - `2048 / 4` +- Result: + - runs: + - `20260416_224905_adaptive_stft_tune_r10_1024o8` + - `20260416_224905_adaptive_stft_tune_r10_2048o8` + - `20260416_224905_adaptive_stft_tune_r10_2048o4` + - all three profiles were effectively identical on both smoke cases + - representative values: + - `pitchOrg +4` + - note mel `6.525` + - env `1.022` + - entry `6.885` + - exit `7.112` + - onset artifact `2.257` + - formant drift `0.374` + - `pitchTestOrg +4` + - note mel `2.828` + - env `0.469` + - entry `1.703` + - exit `1.887` + - onset artifact `1.688` + - formant drift `0.050` +- Verdict: + - `A7` is effectively flat on the current adaptive correction path + - FFT size / hop is not the dominant remaining lever here + - remaining budget should move to: + - richer fixture completion for `H1/H3/H4` + - challenger close-out `B1-B4` + +#### Remaining plan truth +- The full recovery plan is still not complete. +- Remaining mandatory iterations: `7` +- Remaining conditional iterations: `+2` + +### 2026-04-16: Engine-v2 Challenger Narrow Close-Out R8 + +#### Narrowed/drier engine-v2 pass +- Change: + - narrowed engine-v2 ownership further around the edited transition nucleus + - made its transient flatness gates, entry bias, core wetness, residual wetness, and cepstral tuning explicit + - dried the path down substantially and shortened its transition window so it behaved more like a challenger overlay than a broad note takeover +- Result: + - runs: + - `20260416_230357_enginev2_narrow_pitchOrg_plus4_r8` + - `20260416_230555_enginev2_narrow_pitchTest_plus4_r8` + - `pitchOrg +4` + - note mel `6.608` + - env `1.009` + - entry mel `7.620` + - exit mel `7.112` + - onset artifact `1.390` + - formant drift `0.395` + - `pitchTestOrg +4` + - note mel `3.143` + - env `0.498` + - entry mel `7.017` + - exit mel `1.887` + - onset artifact `4.000` + - formant drift `0.062` +- Verdict: + - this is enough to close the challenger honestly + - the narrowed engine-v2 pass still loses clearly on the hard clip + - it is worth keeping in the repo for reference/audition, but not for more main-budget tuning + - the adaptive carrier remains the only live path worth carrying forward + +#### Remaining plan truth +- The full recovery plan is still not complete. +- Remaining mandatory iterations: `3` +- Remaining conditional iterations: `+2` + +### 2026-04-27: Direction-Specific Note-HQ Timbre + Exit Polish + +#### Renderer decision +- Change: + - moved production `note_hq` pitch-only rendering to native direction-specific HQ. + - upward edits keep the current native adaptive branch. + - downward edits now use a guarded formant law instead of sharing the upward compensation path. + - Rubber Band remains installed/diagnosed and can be forced for benchmarks with `OPENSTUDIO_PITCH_USE_RUBBERBAND_HQ=1`, but it is not quality-promoted for production note-HQ. +- Reason: + - the previous native bug was signal-chain correctness, not a renderer-family opening: detected F0 guidance had been passed as `formantRatios` in pitch-only calls. + - the remaining audible downshift issue was the native compensation law, not a reason to require Rubber Band. + - the measured Rubber Band benchmark still failed the `+4` mid-band formant gate and had worse boundary timing than native. + +#### Downshift formant guard +- Change: + - pitch-only downward edits use `pow(1 / ratio, alpha)` with default alpha `0.58`. + - downshift envelope anchoring was strengthened around the voiced `200-3500 Hz` body. + - native result diagnostics now report direction, selected branch, downshift guard usage, guard alpha, effective note-HQ range, and Rubber Band quality-promotion status. +- Result: + - primary `pitchOrg -4` run: `tmp_pitch_runs/20260427_160222_direction_guard_v3_pitchOrg_minus4`. + - harmonic-envelope drift improved from the prior native `0.469` to `0.353`. + - body low/mid/high deltas were `-0.420 / -1.280 / +0.208 dB`. + - body/core cents were `0.00 / +11.90`. + - onset artifact was `1.40`; exit-next artifact was `1.927`. +- Verdict: + - the downshift timbre problem is materially improved and now passes the practical gate. + - the aspirational harmonic-drift target remains `<= 0.35`; this pass landed just above it at `0.353`, so future work should treat that as polish, not an emergency renderer swap. + +#### Exit-to-next-note polish +- Change: + - note-HQ transition ownership is asymmetric by default: shorter pre-note shoulder, longer post-note shoulder, and a small next-note head when adjacent notes touch. + - final native compositing uses a wider dry-protected crossfade at the effective commit range. + - the harness now reports exit-to-next-note discontinuity metrics and writes `cand_exit_next.wav`. +- Result: + - `pitchOrg +4`: `tmp_pitch_runs/20260427_160222_direction_guard_v3_pitchOrg_plus4`, exit-next artifact `1.110`. + - `pitchOrg -4`: `tmp_pitch_runs/20260427_160222_direction_guard_v3_pitchOrg_minus4`, exit-next artifact `1.927`. + - boundary suite passed: `tmp_pitch_runs/20260427_162913_direction_guard_v4_boundary_plus4`. +- Verdict: + - the remaining word-break issue is now handled as an edited-note exit handoff metric rather than being hidden inside broad note/window scores. + +#### Validation +- Passed: + - primary `pitchOrg +4` and `pitchOrg -4` note-HQ acceptance runs. + - export parity for both directions: + - `tmp_pitch_runs/20260427_162615_direction_guard_v4_export_parity_plus4` + - `tmp_pitch_runs/20260427_162615_direction_guard_v4_export_parity_minus4` + - richer formant suite: + - `tmp_pitch_runs/20260427_161118_direction_guard_v4_formant_richer` + - richer transient suite: + - `tmp_pitch_runs/20260427_161118_direction_guard_v4_transient_richer` + - boundary suite: + - `tmp_pitch_runs/20260427_162913_direction_guard_v4_boundary_plus4` +- Harness correction: + - phrase/full-clip note-HQ candidates are scored as full-clip audio. + - window-local semantics remain only for actual segment renders such as preview segments. + - downshift F1/F2 hard gates use the more stable note-body proxy on the canonical window; core F2 remains reported but is too unstable to gate alone on this fixture. + +### 2026-04-27: Two-Sided Note-HQ Boundary Follow-Up + +#### Problem +- User feedback: + - the edited-note exit/next-note handoff was improved. + - a stutter/word-break became audible on the note or word before the edited note. +- Root cause: + - note-HQ rendering had enough left context, but the dry-protected compositor used a generic fixed fade. + - with a left shoulder, that let wet audio become fully committed before the edited note body actually began. + +#### Change +- Commit ranges now retain: + - effective shoulder start/end. + - true note body start/end. +- Entry behavior: + - dry-to-wet fade spans the whole left shoulder. + - full wet is reached at the edited note body start, not before it. +- Exit behavior: + - wet-to-dry release starts with a small `12 ms` lead-in before the note body end, then fades through the right shoulder. + - this lowers the derivative discontinuity at the edited-note exit without returning to the previous-word artifact. +- Harness: + - added `preCommitArtifactScore`. + - added `cand_pre_commit.wav` audition excerpt. + - dry-neighbor residual checks now evaluate only the unowned dry neighbor region outside the effective note-HQ commit range. + +#### Result +- `pitchOrg +4`: + - run: `tmp_pitch_runs/20260427_202041_direction_entry_guard_v6_plus4` + - pre-commit/onset artifact: `0.055 / 1.791` + - exit-next artifact: `2.307` + - body/core cents: `0.00 / 0.00` + - harmonic drift: `0.378` +- `pitchOrg -4`: + - run: `tmp_pitch_runs/20260427_202204_direction_entry_guard_v6_minus4` + - pre-commit/onset artifact: `0.093 / 1.398` + - exit-next artifact: `0.091` + - body/core cents: `0.00 / +11.90` + - harmonic drift: `0.352` +- Boundary suite: + - run: `tmp_pitch_runs/20260427_boundary_entry_guard_v6b/20260427_202849_direction_entry_guard_v6_boundary` + - summary: `pitch_boundary_suite_summary.md` + - start-edge cases stayed clean on the new pre-commit metric. + +#### Verdict +- The correct ownership model is not a larger shoulder by itself. +- The renderer should have phrase/shoulder context, but the final commit blend must be note-body-aware so the previous word remains dry until the transition actually belongs to the edited note. diff --git a/docs/pitch_recovery_master_map.md b/docs/pitch_recovery_master_map.md new file mode 100644 index 0000000..e3ddb12 --- /dev/null +++ b/docs/pitch_recovery_master_map.md @@ -0,0 +1,324 @@ +# Pitch Recovery Master Map + +## Current Controls +- 2026-05-02 Vienna VSF audition pivot: + - user audition rejected the adaptive family for the start/pre-start artifact on the Vienna clip; VSF HQ hybrids-disabled was the only tested family that did not have that start artifact. + - `vienna_vsf_residual_less_055_plus4_full` and `vienna_vsf_residual_less_055_minus4_full` are the current best audition baselines, with residual scale `0.55`; the remaining target is +4 edited-note body clashing/distortion, while -4 nasal tone is regression context. + - body/dry blend variants are rejected because they sounded like two voices singing the edited pitch; do not continue with reduced core wet or dry/body blend candidates for this issue. + - current repair direction is VSF epoch-carrier only: optional upward source-epoch interpolation plus upward grain-radius scaling, with diagnostics reporting residual scale, epoch interpolation use, and effective grain scale. + - iterative fine-tuning round 1 is staged in `tmp_pitch_runs\vienna_vsf_iter_20260502_041147`: exact relative `+4.00`, VSF HQ, hybrids disabled, residual scale `0.55`, core wet `1.0`, and candidate WAVs for baseline `grain065`, interpolation strengths `0.75/0.50`, grain radius `0.60/0.70` at interpolation `0.75`, and `-1.5 dB` upward body presence trim variants. All deterministic harness gates passed, but clashing/distortion, doubled voice, formant/timbre, naturalness, and start artifact remain `not_asserted` until user audition. + - user audition currently ranks `vienna_vsf_iter_grain070_interp075_plus4_full` as the best +4 candidate so far; this is best-by-audition, not a fixed/completed claim. + - local diagnostic-only helper `local_tools\pitch\pitch_residual_hotspot_report.py` reported residual hotspots by time slice and broad band against the batch-local `grain065` baseline; these residual/hotspot metrics are not quality claims and the helper is not tracked. + - downshift nasal iteration is staged in `tmp_pitch_runs\vienna_vsf_minus4_iter_20260502_044712`: exact relative `-4.00`, VSF HQ, hybrids disabled, residual scale `0.55`, core wet `1.0`, no grain/epoch tuning, and candidate WAVs for baseline, `-1.5 dB` nasal trims at `900/1100/1300 Hz`, plus `1100 Hz` nasal trim with `+1.0 dB` body compensation at `430 Hz`. All deterministic harness gates passed, and EQ variants are not byte-identical to baseline. User audition currently ranks `vienna_vsf_iter_minus4_nasal1100_full` as the best -4 candidate so far; nasal tone, distortion, doubled voice, formant/timbre, naturalness, and start artifact remain audition-gated and not a fixed/completed claim. + - default VSF tuning now follows the current user-picked paths: upward shifts enable source-epoch interpolation by default with interpolation strength `0.75` and upward grain radius scale `0.70`; downward shifts apply `-1.5 dB` body nasal trim at `1100 Hz` by default. Env overrides remain available for diagnostics. +- 2026-04-29 doubled-core recovery: + - app audition still reported an artificial doubled vocal even after mono spectrogram gates passed, so the current default is treated as audition-not-done until the stable edited-note core clears a dedicated double-voice QA pass. + - historical state at that point: product/default note-HQ pitch-only selection was restored to `pitch_only_adaptive_selector` with `legacy_natural` recovery; this is superseded by the 2026-05-02 Vienna VSF audition pivot above. + - likely cause addressed first: the vocal source/filter output was being blended with the adaptive-selector core on long notes by default. That core hybrid is now opt-in via `OPENSTUDIO_VSF_CORE_HYBRID_ENABLE=1` or the legacy diagnostic override `OPENSTUDIO_VSF_CORE_HYBRID_DISABLE=0`. + - `tools\pitch_double_voice_analyze.py` is now part of reference-backed pitch-only `note_hq` runs and gates original-F0 leakage, secondary pitch excess, stereo correlation drift, mid/side drift, comb/notch excess, and dry-correlation excess. + - mid/side drift uses an audible-side floor, so nearly mono side residue is reported as raw drift without falsely failing the doubled-core gate. + - layer dumps for diagnosis are available via `-DumpPitchLayers` / `OPENSTUDIO_VSF_LAYER_DUMP_ENABLE=1`; they write dry input, source/filter core, residual/noise, wet envelope, adaptive hybrid output when engaged, and final output. + - lesson: doubled-core artifacts are phase/stereo/layering failures, so mono mel and formant proxies cannot be the final naturalness proof. +- 2026-04-29 spectrogram-first QA correction: + - completion claims for reference-backed pitch-only `note_hq` renders now require a final spectrogram/mel/waveform-envelope report for both upshift and downshift; if the spectrogram done gate fails, the work is explicitly "not done" and the next failing region is named. + - the failure baseline is `D:\test projects\os tests\runs\20260429_074056_vocal_source_filter_pitchTest_plus4_transient_smoke`: `cand_phrase.wav` vs `ref_phrase.wav` measured about `12.15 dB` whole-phrase mel MAE, phrase short-RMS envelope correlation `0.177`, phrase high-band delta about `+5.25 dB`, core high-band delta about `+6.23 dB`, entry RMS about `+7.6 dB`, exit RMS about `-5.45 dB`, onset peak jump about `+28.8 dB`, onset high-band burst about `+12.9 dB`, and candidate/reference lag about `54 ms`. + - lesson: pitch cents, branch diagnostics, and proxy formant gates can pass while the rendered vocal is still unusable. Spectrogram/mel/waveform-envelope checks and human audition now veto completion. + - next repair order is entry/gating burst first, residual/noise smear second, voiced-core naturalness third, and timing/ownership audit fourth. Do not widen dry-protected neighbor commits just to match references that changed wider phrase audio. +- 2026-04-29 source/filter artifact repair status: + - current code adds PSOLA overlap-weight normalization, longer dry entry ownership, delayed entry-only gain/EQ shaping, duration-specific long-note RMS trim, and bounded long-note adaptive hybrids for entry/core/exit in the `pitch_only_vocal_source_filter_hq` renderer. + - latest canonical `pitchOrg` reports remain close inside the edited note after the long-note-only hybrid change: `20260429_122136_spectrogram_gate_pitchOrg_plus4_after_exit_hybrid_diag` measured core/entry/exit mel `4.68/7.41/5.71 dB`, and `20260429_122322_spectrogram_gate_pitchOrg_minus4_after_exit_hybrid_diag` measured `4.35/6.14/5.61 dB`. + - latest harder `pitchTestOrg` reports have the note-owned spectrogram regions inside the current gates: `20260429_123450_spectrogram_gate_pitchTest_plus4_core_hybrid_diag` measured core/entry/exit mel `6.84/4.29/6.17 dB`, and `20260429_123742_spectrogram_gate_pitchTest_minus4_core_hybrid_diag` measured `6.47/4.27/7.74 dB`. + - these runs are still not a final completion claim until app audition is clean. The remaining phrase-wide envelope failure is non-actionable on these references because original-vs-reference pre/post-neighbor mismatch is about `15/19 dB`; the harness now reports that mismatch and skips the phrase-envelope done gate in that case. Positive onset bursts remain hard failures, but quieter-than-reference onset deltas are reported as target mismatch rather than burst artifacts. + - next concrete fix if audition still sounds wrong: inspect the long-note `cand_core.wav` and `cand_exit.wav` spectrograms manually against `ref_core.wav`/`ref_exit.wav`, then tune only the core hybrid amount or replace the adaptive core support. Do not widen dry-protected neighbor commits just to improve phrase correlation. +- 2026-04-29 default vocal source/filter hard-gate follow-up: + - short upward entries now use a shorter `24 ms` dry shell and a near-neutral short-up entry mid cut (`OPENSTUDIO_VSF_SHORT_UP_ENTRY_MID_CUT_DB`, default `-3 dB`) to remove the residual onset-flux failure without affecting downshift or long-note policies. + - the four default-branch hard checks now pass pitch-quality and spectrogram gates with `actualRendererBranch=pitch_only_vocal_source_filter_hq`: + - `20260429_132647_spectrogram_gate_pitchOrg_plus4_short_entry_eq3_hard_check`: core/entry/exit mel `4.09/5.37/5.70 dB`. + - `20260429_132935_spectrogram_gate_pitchOrg_minus4_after_short_entry_eq3_hard_check`: `4.35/6.14/5.61 dB`. + - `20260429_132935_spectrogram_gate_pitchTest_plus4_after_short_entry_eq3_hard_check`: `6.84/4.29/6.17 dB`. + - `20260429_132935_spectrogram_gate_pitchTest_minus4_after_short_entry_eq3_hard_check`: `6.47/4.27/7.74 dB`. + - final status remains audition-gated: these metrics say the renderer is no longer the obvious ghost/robot failure from the spectrogram baseline, but app listening still wins over the proxy gates. +- 2026-04-28 Signalsmith pitch-only formant-contract correction: + - pitch-only vocal note edits now call `setFormantFactor(1.0f, true)` in the Signalsmith carrier, matching live preview and the real-time corrector. + - active adaptive-selector pitch-only carriers pass detected F0 through `setFormantBase(...)` when available, and the offline transpose map uses the same stage-A tonality-limit controls as live preview. + - explicit formant edits pass their requested ratio directly with `compensatePitch=true`; pitch ratio is not folded into the formant factor. + - downshift-specific timbre support remains a bounded adaptive-selector blend/envelope-transfer concern (`OPENSTUDIO_PITCH_DOWNSHIFT_OWN_BLEND` defaults to `0.42`), not inverse-ratio carrier compensation. +- 2026-04-28 entry contour-handoff correction: + - the start artifact was reclassified as a pitch-trajectory/ownership mismatch: the renderer needs pre-note ratio context, but final audible ownership must still be controlled by the entry bridge. + - measured tuning showed that hard/unknown `pitchOrg` entries must keep render pre-roll and reach the target by `note.startTime`; adding an audible body ramp there regressed entry lag and onset gates. + - entry pitch handoff is therefore enabled only for explicit continuous/internal transitions (`soft_legato`, `internal_bend`, `internal_vibrato`, or adjacent edited notes inside the same island). Hard/unknown entries keep dry-protected audio ownership and render-context pre-roll without delaying the shifted body. + - diagnostics now include `noteHqEntryPitchHandoffUsed`, handoff start/end, pre/body milliseconds, slope-jump, and acceleration-limit status; the frontend bridge and regression summaries preserve those fields. + - the harness now reports entry F0 slope/acceleration metrics. The hard gate applies only when a real pitch handoff is used; canonical hard/unknown step edits still report the metric as diagnostic because the reference itself behaves like a step edit. + - verification: + - `D:\test projects\os tests\runs\20260428_115125_entry_contour_handoff_plus4_final3`: `pitchOrg +4` passed; onset artifact `1.501`, onset derivative `1.05`, exit-next `2.156`, body/core pitch `0.00/0.00 cents`, harmonic drift `0.382`. + - `D:\test projects\os tests\runs\20260428_115314_entry_contour_handoff_minus4_final`: `pitchOrg -4` passed; onset artifact `1.187`, onset derivative `0.92`, exit-next `0.207`, downshift harmonic drift `0.339`, low/mid/high `-0.911/-2.011/-0.202 dB`. + - `D:\test projects\os tests\runs\20260428_115719_entry_contour_handoff_two_adjacent_plus4_r2`: adjacent selected notes still render as one edit island (`noteHqEditIslandCount=1`, `noteHqEditedNoteCount=2`) and report the internal pitch handoff instead of a second dry/wet bridge. +- 2026-04-28 emergency word-grouping repair: + - product default is restored to single-note ownership: clicking or dragging one note selects and edits only that note. + - `wordGroupId` remains metadata for diagnostics and controlled render-island decisions, but it no longer automatically expands visible selection or pitch-drag edits. + - the large word-group hull overlay and multi-note "Word" inspector display were removed because broad/incorrect grouping made unrelated notes appear and move together. + - analyzer merging is conservative again: automatic merges use a short `40 ms` gap plus an about `1 st` pitch-distance guard, rather than swallowing nearby material solely because it is within the dropout bridge. + - strong acoustic `hard_word_like` boundary candidates may split default analyzer regions; pure pitch hysteresis, pitch corner, and `internal_vibrato` candidates remain non-destructive diagnostics. + - harness diagnostics now treat hard acoustic splits separately from destructive pitch-corner/pitch-jump failures and add expected-region overhang checks to catch collapsed words. + - verification: + - analyzer run `D:\test projects\os tests\runs\20260428_110455_emergency_word_group_repair_analysis_pitchOrg`: `noteCount=5`, `wordGroupCount=5`, `destructiveCornerSplitCount=0`, `destructivePitchJumpSplitCount=0`, hard acoustic splits `2`, max fragments `1`, max overhang `0.459`. + - UI ownership test `pitchEditorSingleNoteOwnership.test.ts`: click/select, drag update, and selected pitch move all affect only explicit note IDs even when notes share `wordGroupId`. + - primary note-HQ runs `D:\test projects\os tests\runs\20260428_110516_emergency_repair_pitchOrg_plus4` and `D:\test projects\os tests\runs\20260428_110641_emergency_repair_pitchOrg_minus4` passed the current gates; measured onset artifacts were `1.48` and `1.598`, exit-next artifacts `2.31` and `0.091`, downshift harmonic drift `0.343`. + - adjacent selected-note run `D:\test projects\os tests\runs\20260428_110807_emergency_repair_two_adjacent_plus4` reported `noteHqEditIslandCount=1` and `noteHqEditedNoteCount=2`; it is kept as the double-voice safety check, with exit-next artifact still a polish risk on that stress fixture. +- 2026-04-28 phrase-first vibrato-safe word detection: + - the older running-average pitch-jump split is no longer allowed to destructively cut editable notes; sustained pitch movement now becomes `boundaryCandidates` with `pitch_hysteresis_*` reasons. + - analyzer segmentation is phrase-first: short voiced detector dropouts up to `80 ms` are bridged, while automatic note cuts are reserved for hard acoustic evidence such as long unvoiced gaps or sustained energy breaks. + - vibrato-like periodic reversals are reported as `internal_vibrato` diagnostics and remain non-destructive by default. + - close fragments are merged across short non-hard gaps without blocking on pitch distance alone, so melisma/bend/vibrato movement does not create separate editable words. + - superseded by the emergency repair above: `wordGroupId` is assistive metadata, not default UI ownership. + - harness diagnostics now also report `pitchDeviationCandidateCount`, `destructivePitchJumpSplitCount`, and `vibratoSuppressedCandidateCount`; destructive pitch-jump splits must be `0` by default. + - verification: + - analyzer run `D:\test projects\os tests\runs\20260428_040921_phrase_first_word_detection_pitchOrg`: `noteCount=3`, `wordGroupCount=3`, `destructivePitchJumpSplitCount=0`, `destructiveCornerSplitCount=0`, edited-word overlap `1.000`, max fragments `1`. + - primary note-HQ runs `D:\test projects\os tests\runs\20260428_040951_phrase_first_pitchOrg_plus4` and `D:\test projects\os tests\runs\20260428_041117_phrase_first_pitchOrg_minus4` kept the prior pitch, onset, exit, and downshift formant gates. + - adjacent-fragment diagnostic `D:\test projects\os tests\runs\20260428_041244_phrase_first_two_adjacent_plus4` reported `noteHqEditIslandCount=1` and `noteHqEditedNoteCount=2`, confirming one ownership island for two supplied fragments. + - lesson: demoting pitch-corner splits was not enough because the older pitch-deviation splitter could still fragment continuous sung words before word grouping ran. +- 2026-04-28 word-group + edit-island correction: + - pitch-curve corners are now exported as boundary candidates instead of destructive note splits by default; `OPENSTUDIO_ANALYZER_APPLY_CORNER_SPLITS=1` is research-only. + - analyzer output now includes `wordGroupId`, so close voiced fragments without a hard acoustic break remain one editable word/phrase group. + - superseded by the emergency repair above: normal pitch moves affect only explicitly selected notes. + - final note-HQ builds commit ownership per edit island, not per note, so adjacent moved notes get one outer entry bridge and one outer exit bridge instead of internal dry/wet handoffs. + - merged commit ranges no longer average pitch ratios with `sqrt(previous * current)`; variable-ratio islands keep ownership separate from the actual per-sample pitch curve. + - harness diagnostics now report boundary candidates, destructive corner split count, word-group overlap, edit-island count, and edited-note count. + - verification: + - analyzer run `D:\test projects\os tests\runs\20260428_022002_word_group_analysis_pitchOrg_r2`: `noteCount=10`, `wordGroupCount=4`, `cornerCandidateCount=2`, `destructiveCornerSplitCount=0`, min expected word-group overlap `0.928`. + - primary note-HQ runs `D:\test projects\os tests\runs\20260428_022028_word_group_island_plus4` and `D:\test projects\os tests\runs\20260428_022028_word_group_island_minus4` kept the prior +4/-4 pitch, onset, exit, and downshift formant gates. + - adjacent-fragment diagnostic `D:\test projects\os tests\runs\20260428_023520_word_group_two_adjacent_plus4_r4` reported `noteHqEditIslandCount=1` and `noteHqEditedNoteCount=2` in the raw result, confirming one ownership island for two moved fragments. + - lesson: more segmentation can improve seam metrics while making vocal editing worse; editable note boundaries, vocal word groups, and render islands must stay separate. +- 2026-04-27 segmentation + timbre-stability update: + - note segmentation now has conservative pitch-corner boundary detection: sharp smoothed-F0 direction reversals can split a note only when supported by energy, confidence, nearby unvoiced/noise, or strong pitch-prominence evidence. + - detected note boundaries now carry `entryBoundaryKind` / `exitBoundaryKind` diagnostics (`hard_word_like`, `soft_legato`, `internal_bend`, or `unknown`) plus reason and score. + - note-HQ commit policy is boundary-kind aware: hard word-like entries get a tighter audible bridge, while soft legato/sustain entries may keep the wider phrase bridge needed for continuous vocal gestures. + - downshift pitch-only renders now apply voiced-core spectral envelope transfer on edited note bodies after the native directional render; this makes formant stability depend on the original vowel envelope rather than only on ratio compensation. + - the failed lesson is recorded: smoothing the wrong detected note boundary can move the stitch artifact around without fixing it, so analysis boundaries, vocal boundaries, and render ownership are now treated separately. + - primary verification: + - `D:\test projects\os tests\runs\20260428_013135_seg_corner_timbre_plus4`: body/core `0.00 / 0.00 cents`, onset artifact `1.479`, onset derivative `1.087`, exit-next `2.307`, harmonic drift `0.379`. + - `D:\test projects\os tests\runs\20260428_013650_seg_corner_timbre_minus4_mix005`: body/core `0.00 / +11.90 cents`, `spectralEnvelopeCorrectionUsed=true`, onset artifact `1.598`, onset derivative `1.598`, exit-next `0.091`, harmonic drift `0.343`, low/mid/high `-1.083 / -2.067 / -0.053 dB`. + - rejected aggressive envelope-transfer run `D:\test projects\os tests\runs\20260428_013344_seg_corner_timbre_minus4` regressed harmonic drift to `0.615`, so the kept pass uses a small voiced-support-weighted transfer mix. + - analyzer diagnostic run `D:\test projects\os tests\runs\20260428_013900_seg_corner_boundary_analysis` serialized boundary diagnostics and reported `2` corner-boundary notes; use manual vocal-boundary references before increasing split aggressiveness. +- 2026-04-27 signal-chain correctness update: + - native pitch-only renders no longer pass detected F0 curves through the `formantRatios` argument by accident; pitch-only entrypoints now route `ratios + detectedPitchHz` separately and keep `formantCurveUsed=false`. + - note-HQ compare semantics are corrected: preview segments are window-local, but phrase/full-clip note-HQ candidates are scored as full-clip audio. + - note-HQ apply now renders with phrase/transition context but dry-protected final compositing keeps audio before the edited note start dry; `pitchOrg` renders with context around `0.604s-1.683s`, effective ownership `0.860s-1.610s`, and audible commit `0.900s-1.610s` for a `0.900s-1.550s` note body. + - follow-up direction-specific decision: production note-HQ pitch-only defaults to native directional HQ; Rubber Band/offline HQ remains benchmark-only unless explicitly requested. + - follow-up runtime fix: `tools/rubberband/sndfile.dll`, `vcruntime140.dll`, and `vcruntime140_1.dll` are now bundled so Rubber Band starts and reports version `4.0.0`; `20260427_145800_rubberband_runtime_fixed_pitchOrg_plus4` confirms `phraseHqExternalUsed=true`. + - external Rubber Band HQ is still benchmark-gated, not quality-promoted: `20260427_145847_rubberband_runtime_fixed_pitchOrg_plus4` failed strict formant/boundary gates (`midBandDeltaDb=+7.866`, boundary `38.54 ms`). + - final diagnostic native-fallback runs: + - `20260427_135220_after_fix_pitchOrg_plus4_native_override_final`: body/core pitch `0.00/0.00 cents`, formant body drift `0.378`, low/mid/high `-2.46/+0.53/-3.59 dB`, core F1/F2 drift `-46.9/+23.4 Hz`, boundary `8.42 ms`. + - `20260427_135413_after_fix_pitchOrg_minus4_native_override_final`: body/core pitch `0.00/+11.90 cents`, formant body drift `0.469`, low/mid/high `-0.41/+0.83/+1.12 dB`, core F1/F2 drift `+46.9/-70.3 Hz`, boundary `25.38 ms`. + - richer formant suite passed under explicit native fallback: `20260427_135901_after_fix_formant_richer_native_override`. + - richer transient suite passed under explicit native fallback: `20260427_141431_after_fix_transient_richer_native_override`. + - boundary-variant suite is not fully green under the new strict gate: `20260427_142907_after_fix_boundary_pitchOrg_plus4_native_override` passed `start_earlier` and `start_later`, then failed synthetic `end_earlier` at boundary timing `38.938 ms > 32 ms`. +- Shipping fallback: `branch_hybrid_reset` +- Trusted experimental control: `pitch_only_hybrid_structural` (`HS-4 / r6`) +- Active best experimental branch: `pitch_only_adaptive_selector` with harvested long-upward and short-downward support +- Analyzer close-out state: direct-YIN + decoder is the frozen kept path; FFT-YIN stays rejected-for-now behind `OPENSTUDIO_ANALYZER_USE_FFT_YIN=1` +- Scrub preview state: natural-segment scrub preview is now active and benchmarked via `20260416_200227_pitchOrg_scrub_preview_r8` (`scrubPreviewAudible=true`, `scrubPreviewFirstDragAudible=true`, loop duration `240 ms`, base pitch `365 Hz`, last peak `0.112`); repeat-stability tuning is still open +- Root-cause research state: + - completed in `20260417_012542_pitch_root_cause_research` + - ranked causes: + - transition ownership and boundary timing drift + - mixed transient and first-voiced-cycle handling inside one renderer family + - formant preservation that is too weak and too local for hard transitions + - repo summary: + - [pitch_root_cause_research_20260417.md](c:/Users/srvds/Documents/Codes/Studio13-v3/docs/pitch_root_cause_research_20260417.md) +- ML benchmark state: + - completed in `20260417_003355_pitch_ml_benchmark` + - verdict: `blocked_no_stronger_restorer` + - local environment has no materially stronger note-local restorer ready now +- Engine-v3 feasibility state: + - completed in `20260417_003355_engine_v3_feasibility` + - `V3-1` decomposition probe verdict: `stop` + - do not open a longer engine-v3 implementation branch from the current decomposition probe +- Harness close-out state: + - `H2` boundary suite is implemented + - `H1` scrub suite is still partial, but it now has richer multi-scenario runs on both canonical clip families: + - `20260416_231821_pitchOrg_scrub_suite_richer_r1` + - `20260416_234516_pitchTest_scrub_suite_richer_r1` + - true multi-note scrub fixtures now exist: + - `tests/fixtures/pitch-regression/example_pitchOrg_scrub_multinote.json` + - `tests/fixtures/pitch-regression/example_pitchTest_scrub_multinote.json` + - multi-note scrub suite run: + - `20260416_235640_pitchTest_scrub_suite_multinote_r3` + - first drag, repeated drag, after-transport-cycle, and selection-change are now all exercised end-to-end + - `H1` scrub suite is now complete for the current canonical local fixture corpus + - `H3` transient suite is now complete for the current canonical fixture corpus with richer up/down boundary-focused coverage: + - `20260416_231844_transient_suite_richer_r1` + - `H4` formant suite is now complete for the current canonical fixture corpus with richer body/transition coverage: + - `20260416_233426_formant_suite_richer_r1` + - first adaptive-carrier boundary/formant correction pass is now benchmarked: + - latest run: `20260416_224905_adaptive_stft_tune_r10_*` + - `spectralEnvelopeCorrectionUsed=true` on both `+4` truth clips + - result: best adaptive correction profile so far, still mixed and not keepable yet + - `A7` STFT sweep is now closed as effectively flat (`1024/8`, `2048/8`, `2048/4` all matched within noise) + - engine-v2 challenger close-out: + - run: `20260416_230357_enginev2_narrow_pitchOrg_plus4_r8` + - run: `20260416_230555_enginev2_narrow_pitchTest_plus4_r8` + - verdict: freeze challenger; still loses clearly on the hard clip + - remaining mandatory iterations from the previous close-out program: `0` + - remaining conditional iterations from the previous close-out program: `+2` +- Current renderer research reference: [pitch_renderer_research_notes.md](c:/Users/srvds/Documents/Codes/Studio13-v3/docs/pitch_renderer_research_notes.md) +- Pitch editor scope: monophonic only, with stereo vocal clips supported by analyzing a mono sum while preserving multichannel render output +- Canonical truth cases: + - `pitchOrg.wav -> pitchOrg+4s.wav` + - `pitchOrg.wav -> pitchOrg-4s.wav` + - `pitchTestOrg.wav -> pitchTestOrg+4s.wav` + - `pitchTestOrg.wav -> pitchTestOrg-4s.wav` +- Workflow rules: + - Track every family here first. + - Test one active family at a time. + - Compare every run against `CTRL-SHIP` and `CTRL-R6`. + - Maximum `2` serious iterations per family. + - If a family is rejected, remove renderer-specific code quickly and record the rejection in the chronological log. + +## 2026-04-27 Production Note-HQ Directional Update +- Classification: signal-chain correctness correction, not a reopened renderer-family experiment. +- Product decision: + - `note_hq` pitch-only production rendering now defaults to native direction-specific HQ. + - Upward edits keep the existing native adaptive path. + - Superseded carrier detail: downward edits used a gentler formant guard for a period, but pitch-only Signalsmith now uses neutral formant preservation and leaves downshift timbre support to bounded post-render correction. + - Rubber Band remains available for diagnostics and benchmark runs, but it is not quality-promoted for production `note_hq` unless `OPENSTUDIO_PITCH_USE_RUBBERBAND_HQ=1`. +- Downshift timbre fix: + - native pitch-only downshifts now keep the Signalsmith carrier formant-neutral with `setFormantFactor(1.0f, true)`. + - active pitch-only carriers keep detected F0 as `setFormantBase(...)` guidance and use the live-preview stage-A tonality limit before applying neutral formant compensation. + - the adaptive selector's bounded own-engine downshift blend now defaults to `0.42`; envelope matching remains the secondary bounded body-color support focused on the voiced `200-3500 Hz` body while protecting transient and air bands. +- Boundary polish: + - note-HQ ownership is now asymmetric around edited notes, with a wider post-note shoulder and a small next-note head when adjacent notes touch. + - final compositing uses a wider dry-protected crossfade at the effective commit range instead of cutting at the note body boundary. + - the regression harness now reports an explicit exit-to-next-note artifact score. +- 2026-04-27 two-sided boundary correction: + - the first exit-focused fix exposed an audible pre-note/previous-word handoff because the dry-protected compositor could become fully wet before the edited note body started. + - commit ranges now carry both the effective shoulder and the true note body start/end. + - final audible compositing now starts at the edited note body start, never at the left effective shoulder by default. + - `[effectiveStartTime, note.startTime)` is copied from the original/dry audio exactly; the dry-to-wet entry fade happens inside the first `12 ms` of the edited note body. + - release compositing starts a small `12 ms` lead-in before the note body end, then fades through the right shoulder, reducing the derivative jump at the edited-note exit. + - the failed lesson is recorded: widening left shoulder ownership can move the stutter backward into the previous word, so render context and audible commit ownership must stay separate. + - the harness now reports `preBodyTailOriginalResidualDb`, `candidateActiveDifferenceStartSec`, `preBodyTailArtifactScore`, and writes `orig_pre_body_tail.wav`, `cand_pre_body_tail.wav`, and `diff_pre_body_tail.wav`. +- Primary measured results: + - `pitchOrg +4`: run `tmp_pitch_runs/20260427_160222_direction_guard_v3_pitchOrg_plus4`; body/core cents `0.00 / 0.00`, harmonic drift `0.378`, band deltas `-2.464 / +0.531 / -3.590 dB`, F1/F2 proxy drift `-46.875 / +23.438 Hz`, onset artifact `1.79`, exit-next artifact `1.110`. + - `pitchOrg -4`: run `tmp_pitch_runs/20260427_160222_direction_guard_v3_pitchOrg_minus4`; body/core cents `0.00 / +11.90`, harmonic drift improved from the prior native `0.469` to `0.353`, band deltas `-0.420 / -1.280 / +0.208 dB`, onset artifact `1.40`, exit-next artifact `1.927`. + - two-sided boundary follow-up `pitchOrg +4`: run `tmp_pitch_runs/20260427_202041_direction_entry_guard_v6_plus4`; pre-commit/onset artifacts `0.055 / 1.791`, exit-next artifact `2.307`, body/core cents `0.00 / 0.00`. + - two-sided boundary follow-up `pitchOrg -4`: run `tmp_pitch_runs/20260427_202204_direction_entry_guard_v6_minus4`; pre-commit/onset artifacts `0.093 / 1.398`, exit-next artifact `0.091`, harmonic drift `0.352`, body/core cents `0.00 / +11.90`. + - final pre-body dry ownership `pitchOrg +4`: run `tmp_pitch_runs/20260427_213815_pre_body_dry_v3_plus4`; body/core cents `0.00 / 0.00`, pre-body residual `-170.878 dB`, active difference start `0.900063s`, onset artifact `1.706`, exit-next artifact `2.307`, harmonic drift `0.379`. + - final pre-body dry ownership `pitchOrg -4`: run `tmp_pitch_runs/20260427_213940_pre_body_dry_v3_minus4`; body/core cents `0.00 / +11.90`, pre-body residual `-169.515 dB`, active difference start `0.900063s`, onset artifact `2.721`, exit-next artifact `0.091`, harmonic drift `0.352`. + - entry-bridge follow-up: + - product rule is now bridge-aware: render context may extend before the edited note, and final apply may use a bounded entry bridge, but audio before `noteHqEntryBridgeStartSec` must remain original/dry. + - upward edits use a tight in-body bridge (`0.900s -> 0.916s` on `pitchOrg +4`) because the pre-note bridge regressed the upward onset audit. + - downward edits use a bounded pre-note bridge (`0.876s -> 0.980s` on `pitchOrg -4`) with a `-22.0 ms` wet-read offset, `+1.7 dB` entry envelope correction, and `10 ms` dry transient preservation. + - the corrected lesson is recorded: fully dry pre-note ownership prevented previous-word mutation, but could leave a phase/envelope discontinuity exactly at the edited-note entry. + - final `pitchOrg +4` run `tmp_pitch_runs/20260428_004839_entry_bridge_v15_plus4`: body/core cents `0.00 / 0.00`, entry lag `-0.125 ms`, onset artifact `1.479`, onset derivative `1.087`, protected pre-bridge residual `-172.823 dB`, exit-next artifact `2.307`, harmonic drift `0.379`. + - final `pitchOrg -4` run `tmp_pitch_runs/20260428_004708_entry_bridge_v15_minus4`: body/core cents `0.00 / +11.90`, entry lag `+0.771 ms`, onset artifact `1.598`, onset derivative `1.598`, protected pre-bridge residual `-240.000 dB`, exit-next artifact `0.091`, harmonic drift `0.344`. + - export parity passed in `tmp_pitch_runs/20260428_005155_entry_bridge_v15_plus4_export` and `tmp_pitch_runs/20260428_005458_entry_bridge_v15_minus4_export`; the source-vs-export dry residual remains informational because the mixer/export path changes the full file from time zero. + - Export parity passed for both directions in `tmp_pitch_runs/20260427_214538_pre_body_dry_v4_plus4_export` and `tmp_pitch_runs/20260427_214814_pre_body_dry_v4_minus4_export`; the source-vs-export dry residual readout is informational only because the mixer/export path changes the full file from time zero. + - Richer formant suite passed in `tmp_pitch_runs/pre_body_dry_v2_formant_richer/20260427_212751_pre_body_dry_v2_formant_richer`. + - Richer transient suite passed in `tmp_pitch_runs/pre_body_dry_v4_transient_richer/20260427_215055_pre_body_dry_v4_transient_richer`. + - Boundary suite completed in `tmp_pitch_runs/pre_body_dry_v2_boundary/20260427_212152_pre_body_dry_v2_boundary`; the primary real `pitchOrg` cases stay under the exit-next gate, while the synthetic shortened-end stress case still shows a high exit-next artifact and remains a stress warning. +- Rubber Band benchmark status: + - runtime availability diagnostics stay in place. + - pitch maps now use `effectiveStart/effectiveEnd` transition shoulders for benchmark ramps. + - current benchmark evidence is not production-promotable because the earlier `+4` run failed the mid-band formant gate and had worse boundary timing than native. + +## Approach Status Table +| Family ID | Approach | Class | Status | Last Result | Best Observed Gain | Main Failure | Can Harvest | Code State | Next Action | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `CTRL-SHIP` | Shipping fallback `branch_hybrid_reset` | Control | `Kept Control` | Shipping-safe app path; still current fallback baseline | Safest all-around fallback, exact pitch on truth cases | Still far from samples on onset, exit, neighbors | Yes | `Active branch` | Keep frozen as shipping control | +| `CTRL-R6` | `pitch_only_hybrid_structural` / `HS-4 / r6` | Control | `Kept Control` | Trusted experimental baseline; best kept short-upward branch | `pitchOrg +4` note mel `6.209`, env `0.961`, entry `6.928`, exit `6.076`, onset artifact `1.810` | Does not improve `pitchTest +4`; stutter still present | Yes | `Active branch` | Keep frozen as experimental control | +| `FAM-ANALYZER-PYIN` | Native editor-first analyzer upgrade: staged direct-YIN baseline plus FFT-YIN probe, multi-candidate extraction, voiced/unvoiced probabilities, and Viterbi-style decoding in `PitchAnalyzer` | Analysis upgrade | `Promising` | 2026-04-15 close-out: direct Hann-windowed YIN + decoder matched the target note window on both analysis fixtures (`pitchOrg`: `7` detected / `1` expected, overlap ratio `0.964`; `pitchTestOrg`: `6` detected / `1` expected, overlap ratio `0.796`) with high median voiced confidence (`0.974` / `0.976`), while FFT-YIN produced `0` detected notes and `0.000` voiced-frame ratio on both clips when forced on | First native editor-side path that adds pYIN-like candidate generation and temporal decoding without new dependencies, while keeping the editor contract stable and the direct path usable on real clips | Direct path still oversplits full-clip note segmentation on the current fixtures, and FFT-derived difference computation is still not parity-safe enough to promote | Yes | `Active branch` | Freeze the direct-YIN + decoder path as the kept analyzer implementation, keep FFT-YIN gated off, and treat any future analyzer work as segmentation polish rather than an open parity question | +| `FAM-SCALAR` | Scalar blend/ramp/timing/smoothing/coherence/bridge family | Exhausted tweak family | `Rejected` | Multiple rejected HS iterations; no keep beyond `HS-4 / r6` | Safer short-upward early-core blend in `HS-4` | Could not fix onset/body tradeoff; all later variants plateaued | Yes | `Disabled` | Do not revisit without a new structural reason | +| `FAM-BODY-A` | Dry attack + existing own-engine body | Body replacement | `Rejected` | Path A iteration 1 | Onset artifact improved on `pitchOrg +4` | Note mel/env and entry worsened; no equal-weight win | No | `Disabled` | Keep rejected | +| `FAM-BODY-B` | Dry attack + epoch-copy body | Body replacement | `Rejected` | Path B exhausted after 2 iterations | Strong note-body gain on `pitchOrg +4` | Onset collapsed badly; not keepable | Yes | `Disabled` | Harvest only if a future v2 needs epoch-copy body evidence | +| `FAM-BODY-C` | Dry attack + harmonic-only body | Body replacement | `Rejected` | Path C exhausted after 2 iterations | None; C1 failed closed | C2 collapsed pitch/body badly | No | `Disabled` | Keep rejected | +| `FAM-BODY-D` | Dry attack + PSOLA body | Body replacement | `Rejected` | Path D exhausted after 2 iterations | Best body realism on `pitchOrg +4`: note mel `5.105`, env `0.474`, harmonic drift `0.127` | Onset artifact stayed far worse than `r6`; did not generalize to `pitchTest +4` | Yes | `Disabled` | Harvest PSOLA body realism only, not the handoff architecture | +| `FAM-CONT-E1` | Voiced-tail continuation body | Continuation | `Rejected` | Failed closed on first iteration | None | Never engaged on active target | No | `Disabled` | Keep rejected | +| `FAM-CONT-E2` | Early continuation handoff into existing core | Continuation | `Rejected` | Failed closed on first iteration | None | Never engaged on active target | No | `Disabled` | Keep rejected | +| `FAM-ISLAND-F` | Fixed-mask island-native with own-engine core | Island-native | `Rejected` | Path F exhausted after 2 iterations | Onset artifact improved to `1.388` on `pitchOrg +4` | Body, entry, and exit regressed too much; never engaged on `pitchTest +4` | Yes | `Disabled` | Harvest onset-side gain only | +| `FAM-ISLAND-G` | Island-native + PSOLA core | Island-native | `Rejected` | Path G stopped after 1 iteration | Preserved Path F onset-side gain | Pitch/body/exit destabilized badly; still no `pitchTest +4` engagement | No | `Disabled` | Keep rejected | +| `FAM-CE33-SIMPLE` | Legacy simple `ce33` path (`branch_simple_ce33`) | Archived baseline | `Promising` | Truth sweep 2026-04-15; `pitchOrg +4` note mel `7.810`, env `0.883`, entry `7.996`, exit `9.295`, onset artifact `2.50`; `pitchTest +4` note mel `3.044`, env `0.486`, entry `1.660`, exit `6.378`, onset artifact `0.72` | Best measured `pitchTest +4` single-case result so far | Still behind `CTRL-R6` on `pitchOrg +4`; not a single best overall control | Yes | `Active branch` | Keep as a secondary benchmark and harvest candidate, not as the main control | +| `FAM-ADAPTIVE-SELECTOR` | Harvested hybrid selector: `CTRL-R6` for short upward notes, `branch_simple_ce33` for long upward notes, light own-engine support on short downward notes | Hybrid kept path | `Kept Control` | 2026-04-15 broader validation sweep: `pitchOrg +4` note mel `7.085`, env `1.376`, entry `7.078`, exit `7.027`, onset artifact `1.80`; `pitchOrg -4` note mel `6.623`, env `1.102`, entry `6.585`, exit `7.475`, onset artifact `2.80`; `pitchTestOrg +4` note mel `2.810`, env `0.470`, entry `1.530`, exit `1.632`, onset artifact `0.70`; `pitchTestOrg -4` note mel `3.505`, env `1.024`, entry `3.090`, exit `1.835`, onset artifact `3.64` | First branch to beat `CTRL-R6` on `pitchTest +4`, keep the easier `+4` case competitive, and carry a useful harvested `pitchOrg -4` improvement without destabilizing the truth set | Still does not fully solve stutter/formant note-change quality, and `pitchTestOrg -4` remains only acceptable rather than a clear win | Yes | `Active branch` | Freeze as the benchmark branch for the rest of the research close-out and any engine-v2 comparisons | +| `FAM-ADVANCED` | Advanced legacy branch (`branch_current_advanced`) | Archived baseline | `Rejected` | Truth sweep 2026-04-15; `pitchOrg +4` note mel `7.329`, env `0.827`, cents `-36.45`; `pitchTest +4` note mel `8.784`, env `1.190`, cents `-35.70` | None beyond historical reference value | Misses pitch on both truth cases and loses clearly to both controls | No | `Active branch` | Do not use standalone; prune after tracker-based cleanup reaches archived branches | +| `FAM-CORE-PSOLA` | Standalone PSOLA core (`pitch_only_psola_core`) | Archived core | `Rejected` | Truth sweep 2026-04-15; `pitchOrg +4` note mel `11.053`, env `1.640`, body cents `-55.55`; `pitchTest +4` note mel `17.536`, env `2.547`, body cents `-104.96` | None as a standalone truth-case path | Severe pitch/body drift on both truth cases | No | `Active branch` | Do not use standalone; prune after tracker-based cleanup reaches archived cores | +| `FAM-CORE-MODEL` | Standalone model core (`pitch_only_model_core`) | Archived core | `Rejected` | Truth sweep 2026-04-15; `pitchOrg +4` note mel `10.955`, env `1.626`, body cents `-37.23`; `pitchTest +4` note mel `17.397`, env `2.530`, body cents `-104.96` | None as a standalone truth-case path | Severe truth-case loss with large body drift and poor envelope match | No | `Active branch` | Do not use standalone; prune after tracker-based cleanup reaches archived cores | +| `FAM-OWN-PITCH` | Standalone own-engine pitch path (`pitch_only_own_engine`) | Archived core | `Rejected` | Truth sweep 2026-04-15; `pitchOrg +4` note mel `6.316`, env `1.044`, entry `7.783`, exit `6.407`, onset artifact `1.59`; `pitchTest +4` note mel `12.164`, env `1.667`, entry `17.634`, exit `13.403`, onset artifact `3.99` | Decent exact-pitch `pitchOrg +4` body with a better onset artifact than `CTRL-R6` | Collapses badly on `pitchTest +4`; not a viable standalone editor | Yes | `Active branch` | Harvest only if a future v2 needs own-engine core behavior on easy upward clips | +| `FAM-FORMANT-ONLY` | Own-engine formant-only path (`formant_only_own_engine`) | Archived formant path | `Rejected` | Truth sweep 2026-04-15 matched `FAM-ADVANCED` bit-for-bit on both `+4` truth cases | None distinct from `FAM-ADVANCED` | Not a competitive truth-case path and appears equivalent to the advanced branch on current tests | No | `Active branch` | Treat as an archived alias-like formant path; prune with `FAM-ADVANCED` | +| `FAM-PITCH-PLUS-FORMANT` | Own-engine pitch-plus-formant path (`pitch_plus_formant_own_engine`) | Archived formant path | `Rejected` | Truth sweep 2026-04-15 matched `FAM-ADVANCED` bit-for-bit on both `+4` truth cases | None distinct from `FAM-ADVANCED` | Not a competitive truth-case path and appears equivalent to the advanced branch on current tests | No | `Active branch` | Treat as an archived alias-like formant path; prune with `FAM-ADVANCED` | +| `FAM-BASELINE-SAFE` | `baseline_safe` runner option | Alias / cleanup item | `Superseded` | Code inspection 2026-04-15: runner validate-set contains it, renderer parser does not map it to a distinct branch | None; not a distinct renderer family | Appears to be an alias/config leftover rather than a real option | No | `Not started` | Treat as non-distinct; clean up the option when touching runner dispatch next | +| `FAM-V2-SYNTH-CORE` | Island shell + directly synthesized voiced core + explicit residual layer | Major revamp | `Rejected` | `G1` on 2026-04-15; `pitchOrg +4` onset artifact `1.810 -> 1.39` but note mel `6.209 -> 8.135`, env `0.961 -> 1.517`, entry `6.928 -> 10.468`, exit `6.076 -> 13.781`, whole-note cents `-408.27`; `pitchTest +4` failed closed to control SHA | Preserved the familiar onset-side gain pattern on `pitchOrg +4` | Body, entry, exit, and pitch stability regressed too much; no reason to spend `G2` | No | `Removed` | Do not continue this family; escalate to a deeper redesign definition | +| `FAM-V2-HSR` | Transient layer + harmonic/source-filter core + residual layer | Major revamp | `Rejected` | `pitch_only_engine_v2` G1 on 2026-04-15; `pitchOrg +4` onset artifact `1.810 -> 1.388` but note mel `6.209 -> 7.975`, env `0.961 -> 1.454`, entry `6.928 -> 9.737`, exit `6.076 -> 12.703`; `pitchTest +4` unchanged and still did not engage | Onset-side gain on `pitchOrg +4` matched island-native family | Body/exit regressed too much and the family still did not generalize to `pitchTest +4` | No | `Removed` | Do not continue this family; move to a deeper redesign definition | +| `FAM-HPSS-SHELL` | HPSS-style vertical split: original transient/noise shell plus pitched harmonic layer | Major revamp | `Rejected` | `H1` on 2026-04-15; `pitchOrg +4` engaged and improved onset artifact `1.810 -> 1.390`, but note mel worsened `6.209 -> 10.066`, env `0.961 -> 2.028`, entry `6.928 -> 10.996`, exit `6.076 -> 15.137`; `pitchTest +4` failed closed with `hpssUsed=false`; `-4` guards also lost versus the adaptive control | Another confirmation that vertical transient preservation can reduce the onset score on the easy upward clip | Harmonic body collapsed badly on `pitchOrg +4`, did not engage on `pitchTest +4`, and did not stay safe on the `-4` guards | No | `Removed` | Stop the family at `H1`; do not open `H2/H3` on this shell | +| `FAM-HPSS-SF` | HPSS shell plus source-filter harmonic core | Major revamp | `Researched` | Not started because `FAM-HPSS-SHELL` failed the stop-fast gate immediately | Plausible path for vertical split plus timbre preservation if a stronger shell ever wins | Blocked by the failed shell; current planned version should not be opened on top of a losing `H1` | Yes | `Not started` | Only revisit if a structurally different HPSS shell is defined later | +| `FAM-HPSS-SF-APER` | HPSS shell plus source-filter core plus explicit aperiodic layer | Major revamp | `Researched` | Not started because `FAM-HPSS-SHELL` failed the stop-fast gate immediately | Keeps the full transient/harmonic/aperiodic decomposition idea on the map | Blocked by the failed shell; should not be layered onto the rejected `H1` foundation | Yes | `Not started` | Only revisit if a later HPSS shell earns continuation | +| `FAM-WSOLA-SEAM` | WSOLA-style shoulder similarity search on top of the adaptive selector | Major revamp | `Rejected` | `W1` on 2026-04-15; only `pitchOrg +4` changed, and it got worse versus the current adaptive branch: note mel `7.085 -> 7.102`, env `1.376 -> 1.377`, entry `7.078 -> 7.314`, exit `7.027 -> 7.151`, onset artifact `1.80 -> 2.29`; `pitchOrg -4`, `pitchTestOrg +4`, and `pitchTestOrg -4` stayed byte-identical to the adaptive branch | Proved the repo can run seam-search diagnostics through the existing truth harness | No truth-case win, and the only engaged case regressed on onset, entry, exit, and note mel | No | `Removed` | Do not revisit this shoulder-only WSOLA implementation; move to phase-coherent DSP families | +| `FAM-PHASE-LOCK-PV` | Phase-vocoder body path with harmonic peak locking and boundary phase alignment | Major revamp | `Rejected` | `P1` on 2026-04-15; `pitchOrg +4`, `pitchOrg -4`, and `pitchTestOrg -4` stayed byte-identical to `FAM-ADAPTIVE-SELECTOR`; `pitchTestOrg +4` was the only engaged case with `phaseLockUsed=true`, `phaseAlignedExit=true`, `phasePeakCount=114`, but it regressed on note mel `2.81 -> 3.003`, entry mel `1.53 -> 2.278`, and exit mel `1.632 -> 1.940` while only slightly improving onset artifact `0.705 -> 0.66` | Proved this lighter boundary-phase-alignment variant can engage selectively on the harder long-upward case without destabilizing the other truth cases | No equal-weight win: the only engaged case materially worsened note and entry metrics, so it did not earn a tuned `P2` | No | `Removed` | Do not revisit this boundary-alignment-only phase-lock pass; move to `FAM-HPSS-MEDIAN` | +| `FAM-HPSS-MEDIAN` | True median-filter HPSS shell with harmonic/transient recombine | Major revamp | `Rejected` | `Hm1` on 2026-04-15; only `pitchOrg +4` changed, with `hpssUsed=true`, harmonic/aperiodic peaks `0.868 / 0.554`, but note mel worsened `7.085 -> 7.432`, entry mel `7.078 -> 7.445`, and note/body/core cents collapsed to `-55.55`; `pitchOrg -4`, `pitchTestOrg +4`, and `pitchTestOrg -4` stayed byte-identical to `FAM-ADAPTIVE-SELECTOR` | Proved the repo can run a real median-filter HPSS shell through the truth harness instead of just the earlier heuristic mask shell | Still did not generalize to `pitchTest +4`, and the only engaged case regressed on body pitch and note/entry quality | No | `Removed` | Do not revisit this scalar median-shell implementation; keep only the diagnostics and stop list entry | +| `FAM-HPSS-MEDIAN-SF` | Median HPSS shell plus harmonic spectral-envelope preservation | Major revamp | `Researched` | Not started because `FAM-HPSS-MEDIAN` failed the stop-fast gate at `Hm1` | Puts true HPSS and explicit harmonic timbre preservation together in one family | Gated behind a promising `FAM-HPSS-MEDIAN` shell result; should not open on a losing shell | Yes | `Not started` | Leave blocked unless a materially stronger median HPSS shell is defined later | +| `FAM-PVDR` | Research-grade phase-coherent phase-vocoder family with proper phase locking or phase-gradient integration | Major revamp | `Rejected` | `P1` on 2026-04-15 benchmarked a long-upward resample-plus-phase-lock PVDR overlay on top of `pitch_only_adaptive_selector`; `pitchOrg +4` stayed byte-identical, but `pitchTestOrg +4` catastrophically failed with note/body/core cents `-628.27 / -628.27 / -664.72`, onset artifact exploding to `+200512709616936000`, and `phaseLockUsed=true`, `phasePeakCount=14710` | Proved the repo can route a genuinely different PV-style long-note benchmark through the truth harness without touching the live branch | The first real PVDR attempt was numerically unstable and failed the stop-fast gate decisively on the harder truth case, and there is no materially different stable PV implementation ready locally right now | No | `Removed` | Keep the broader PV family closed unless a genuinely different phase-gradient or otherwise more stable implementation is ready to benchmark | +| `FAM-TRANSITION-HQ` | HQ-only transition-native note-change overlay on top of `pitch_only_adaptive_selector` | Major revamp | `Rejected` | `M1` on 2026-04-15 benchmarked `pitch_only_transition_hq`; it engaged on both `+4` truth cases with transition diagnostics, but `pitchOrg +4` regressed from note mel `7.085 -> 7.395`, env `1.376 -> 1.434`, entry `7.078 -> 7.723`, onset artifact `1.80 -> 1.59`, while `pitchTestOrg +4` regressed much harder from note mel `2.810 -> 4.084`, env `0.470 -> 0.772`, entry `1.530 -> 15.735`, onset artifact `0.70 -> 3.73`, and core cents stayed off at `-18.32` | Proved the repo can benchmark a transition-focused HQ overlay with explicit shell/core/residual diagnostics on stereo-safe mono analysis | The overlay harmed both primary truth cases, especially `pitchTestOrg +4`, so it did not earn a second DSP iteration or the optional ML finish | No | `Removed` | Stop this family at `M1`; the next escalation is not another local overlay but a materially larger engine redesign if we revisit note-change rendering | +| `FAM-ML-RESTORATION` | Offline-only post-render restoration/refinement benchmark | Research only | `Blocked` | `M1` on 2026-04-15 benchmarked `ml_restore_proxy_v1` on top of `pitch_only_adaptive_selector`; it helped the easier clip but materially harmed `pitchTestOrg +4`. Follow-up environment benchmark on 2026-04-17 returned `blocked_no_stronger_restorer`: `voicefixer` and `demucs` are not installed, `audio_separator` is not suitable for note-local restoration, and the proxy path is explicitly excluded. | First offline benchmark infrastructure that can score a restoration pass on top of the kept renderer without touching the live editor path | No materially stronger local note-local restorer is available right now, so reopening the family locally would just repeat the rejected proxy path | Yes | `Disabled` | Keep the benchmark harness and diagnostics, but only reopen this family once a genuinely stronger local or external/licensed restorer is available | +| `FAM-ENGINE-V2` | Full note-transition-aware engine redesign with explicit harmonic core, residual/noise path, formant envelope, and transition compositor | Full redesign fallback | `Frozen Reference` | `V2-1` scaffold remains parity-safe on both primary `+4` truth clips with branch `pitch_only_engine_v2_program`, and the branch contains the full transition-native audio pass: dedicated RAM scrub preview, Signalsmith-based voiced-core render, cepstral envelope restoration, spectral-flatness transient bypass, residual carry, and transition composition. Latest narrowed runs on 2026-04-16 still lose to the adaptive selector, and the 2026-04-17 root-cause pass confirmed the branch is evidence only, not the recommended active path. | The branch now has the full implementation scaffolding we actually need: note-local HQ infrastructure, dedicated scrub monitoring, transition-native diagnostics, and a benchmarkable engine-v2 sound path that no longer fails numerically | The current waveform-correction overlay shape plateaued: it can be made safer on the easy clip, but the hard truth-case entry still loses badly versus the adaptive selector | Yes | `Active branch` | Keep the code for comparison and audition, but do not continue this branch as the main recovery path without a materially stronger decomposition/transition model | +| `FAM-ENGINE-V3` | Clean-sheet transition-pair renderer with explicit transient shell, voiced core, residual path, and built-in formant handling | Long-term redesign | `Blocked` | 2026-04-17 feasibility probe `20260417_003355_engine_v3_feasibility` scored only `0.511` on `pitchOrg_plus4` and `0.505` on `pitchTest_plus4`, both with verdict `stop` at `V3-1` decomposition stage | Keeps a true clean-sheet DSP option on the map instead of another local overlay family | Current decomposition probe is not strong enough to justify immediate build-out and risks repeating engine-v2 if opened blindly | Yes | `Not started` | Only reopen after defining a materially stronger decomposition and transition-pair ownership design; do not continue from the current probe | +| `FAM-V2-HSR-DOWN` | Separate downward law on top of the active experimental branch | Major revamp | `Harvested` | Support scan plus adaptive integration on 2026-04-15: `branch_simple_ce33` was not a clean `-4` winner; `pitch_only_own_engine` improved easy downward body metrics; harvested light own-engine support into `FAM-ADAPTIVE-SELECTOR`, improving `pitchOrg -4` note mel `7.405 -> 6.570`, entry `6.902 -> 6.571`, exit `8.778 -> 7.565` while `pitchTest -4` stayed on the same SHA | Demonstrated that a small own-engine contribution helps easier short downward notes without harming the hard guard case | Harvested result still does not materially improve the harder `pitchTest -4` truth case | Yes | `Active branch` | Treat the first downward trait as harvested; only start a deeper standalone downward family if later validation shows `pitchTest -4` still needs a dedicated win | +| `FAM-V2-LONG` | Separate long-note policy on top of v2 | Major revamp | `Researched` | Not implemented yet | Lets short-note and long-note solutions diverge cleanly | Blocked until a new short-upward v2 family actually wins | Yes | `Not started` | Blocked pending a new upward v2 control | +| `FAM-NEURAL-FINAL` | Final-only restoration benchmark | Research only | `Researched` | Not implemented yet | Possible late-stage restoration after DSP render | Too early; should not be next coding task | Yes | `Not started` | Queue only if DSP v2 plateaus | +| `FAM-SIGNALSMITH-CARRIER` | Signalsmith as carrier/fallback inside a larger v2 engine | Support architecture | `Researched` | Current repo history already proves Signalsmith-family strength | Strong stability and fallback behavior | Current Signalsmith-centered handoff family is plateaued | Yes | `Active branch` | Reuse as a support role inside v2, not as more local patching | +| `FAM-WORLD-FEATURES-V2` | WORLD-style decomposition features only for envelope / aperiodicity support | Support architecture | `Researched` | Not implemented as a distinct branch; kept as an internal-analysis option only | Useful support direction for future envelope and residual estimation | Pure WORLD rendering is still out of scope and non-competitive here | Yes | `Not started` | Use only as support features inside a future renderer, never as a standalone path | +| `FAM-DECOMP-FEATURES` | WORLD-style decomposition ideas as internal features only | Support architecture | `Researched` | Research direction only; no pure WORLD retry | Useful for F0/envelope/residual decomposition signals | Pure WORLD render path is not competitive for this product | Yes | `Not started` | Use only as internal analysis support if helpful in v2 | + +## Harvestable Traits +| Trait ID | Trait | Source Family | Evidence | Adopted? | Target Architecture | +| --- | --- | --- | --- | --- | --- | +| `TR-HS4-EARLYCORE` | Softer early-core short-upward blend | `CTRL-R6` | Best kept short-upward compromise in current branch | Yes | `CTRL-R6` control | +| `TR-D-BODY` | PSOLA body realism in stable voiced core | `FAM-BODY-D` | `pitchOrg +4` note mel `6.209 -> 5.105`, env `0.961 -> 0.474`, harmonic drift `0.400 -> 0.127` | No | `FAM-V2-HSR` | +| `TR-F-ONSET` | Island-native onset improvement with outer-only splices | `FAM-ISLAND-F` | `pitchOrg +4` onset artifact `1.810 -> 1.388` | No | `FAM-V2-HSR` | +| `TR-SIGNALSMITH-STABILITY` | Stable fallback/carrier behavior | `CTRL-SHIP`, `FAM-SIGNALSMITH-CARRIER` | Shipping path remains safest exact-pitch fallback across truth cases | Yes | Shipping fallback and future v2 fallback role | +| `TR-DECOMP-SUPPORT` | Internal decomposition as analysis features, not renderer | `FAM-DECOMP-FEATURES` | Useful structural separation for F0 / envelope / residual planning | No | `FAM-V2-HSR` | +| `TR-CE33-PITCHTEST` | Strong `pitchTest +4` truth-case behavior from simple `ce33` | `FAM-CE33-SIMPLE` | `pitchTest +4` note mel `3.044`, env `0.486`, entry `1.660`, exit `6.378`, onset artifact `0.72` | No | Future hybrid / v2 benchmark role | +| `TR-OWN-PITCH-EASY-UP` | Own-engine exact-pitch short-upward behavior on easier clips | `FAM-OWN-PITCH` | `pitchOrg +4` note mel `6.316`, onset artifact `1.59`, exact body/core pitch | No | Future v2 core selection / fallback study | +| `TR-ADAPTIVE-LONGUP` | Long-upward selector rule: keep `CTRL-R6` on short upward notes and use `ce33` on long upward notes | `FAM-ADAPTIVE-SELECTOR` | `pitchTest +4` improved from `3.481 / 0.623 / 2.484 / 7.662 / 3.819` to `3.095 / 0.493 / 1.530 / 7.350 / 0.70` while `pitchOrg +4` stayed effectively flat | Yes | `FAM-ADAPTIVE-SELECTOR` | +| `TR-ADAPTIVE-DOWN-SHORT` | Light own-engine support on shorter downward notes inside the adaptive selector | `FAM-V2-HSR-DOWN` | `pitchOrg -4` improved from note mel `7.405` to `6.570`, entry `6.902` to `6.571`, exit `8.778` to `7.565`, while `pitchTest -4` stayed byte-identical and both `+4` truth cases stayed on their kept adaptive outputs | Yes | `FAM-ADAPTIVE-SELECTOR` | +| `TR-OWN-DOWN-EASY` | Own-engine note-body gain on easier downward clips | `FAM-OWN-PITCH` | `pitchOrg -4` note mel `7.405 -> 5.560` with exact core pitch, but no generalization to `pitchTest -4` | No | Future downward-specific hybrid study | +| `TR-PHASE-LOCK-COHERENCE` | Peak-locked phase coherence around harmonic bodies | `FAM-PHASE-LOCK-PV` | `P1` only showed a selective `pitchTestOrg +4` engagement, but the resulting note/entry regression was not worth harvesting as a kept trait | No | Future phase-coherent DSP family only if a materially different implementation is defined | +| `TR-HPSS-MEDIAN-TRANSIENT` | Median-filter transient preservation without body invasion | `FAM-HPSS-MEDIAN` | `Hm1` engaged on `pitchOrg +4`, but the shell still dragged the note body off pitch (`-55.55` cents) and worsened note/entry mel, so there is no keepable transient-preserve trait yet | No | Future HPSS median family only if a materially different shell is defined | +| `TR-SF-HARMONIC-ENVELOPE` | Harmonic-only spectral-envelope preservation on top of a stronger shell | `FAM-HPSS-MEDIAN-SF` | Pending family implementation | No | Future HPSS median + source-filter family | +| `TR-ML-RESTORE` | Offline restoration of residual pitch-shift artifacts | `FAM-ML-RESTORATION` | `M1` proxy benchmark proved the harness can improve the easier `pitchOrg +4` case, but it materially harmed `pitchTestOrg +4`, so there is no keepable restore trait yet | No | Future offline restoration benchmark | +| `TR-TRANSITION-HQ` | Transition-focused shell/core/residual overlay | `FAM-TRANSITION-HQ` | `M1` engaged cleanly and exposed useful diagnostics, but it worsened both `+4` truth cases and produced no keepable transition trait | No | Future engine-v2 work only if a materially different transition model is defined | + +## Active Queue +1. Stronger ML / external benchmark only + - keep the offline benchmark harness and diagnostics + - do not reopen local ML restoration on another proxy or tuning-only pass + - only continue once a materially stronger restorer or outside/licensed benchmark path is available +2. `FAM-ENGINE-V2` + - keep the branch frozen as comparison evidence and for user audition only + - do not spend main tuning budget here unless a materially stronger decomposition/transition model is defined first +3. `FAM-ENGINE-V3` only after a stronger decomposition design exists + - do not continue from the current `V3-1` probe + - require a materially stronger transient/core/residual decomposition and transition-pair ownership design first +4. `FAM-PVDR` reopen only if a materially different stable implementation is ready + - do not reuse the rejected resample-plus-phase-lock overlay + - only reopen with a more stable phase-gradient or otherwise fundamentally different phase-coherent implementation +5. External benchmark decision + - trained ML restoration program + - or outside/licensed renderer benchmark + +## Stop List +- Do not revisit `FAM-SCALAR` without a new structural reason. +- Do not revisit `FAM-BODY-A`, `FAM-BODY-B`, `FAM-BODY-C`, or `FAM-BODY-D` as standalone handoff architectures. +- Do not revisit `FAM-CONT-E1` or `FAM-CONT-E2`. +- Do not revisit `FAM-ISLAND-F` or `FAM-ISLAND-G` as-is. +- Do not revisit `FAM-V2-HSR` as implemented on 2026-04-15; that residual-layer shell has been tested and removed. +- Do not revisit `FAM-V2-SYNTH-CORE` as implemented on 2026-04-15; that directly synthesized island-core pass has been tested and removed. +- Do not revisit `FAM-HPSS-SHELL` as implemented on 2026-04-15; it improved the easy-note onset score but collapsed note/body quality and never generalized to `pitchTest +4`. +- Do not revisit `FAM-WSOLA-SEAM` as implemented on 2026-04-15; the only engaged case (`pitchOrg +4`) got worse, and the other canonical truth cases stayed identical to `FAM-ADAPTIVE-SELECTOR`. +- Do not revisit `FAM-PHASE-LOCK-PV` as implemented on 2026-04-15; the only engaged case (`pitchTestOrg +4`) still lost on note mel and entry despite selective boundary alignment. +- Do not revisit `FAM-HPSS-MEDIAN` as implemented on 2026-04-15; it finally used a real median-filter shell, but the only engaged case (`pitchOrg +4`) still regressed on body pitch, note mel, and entry. +- Do not revisit `FAM-PVDR` as implemented on 2026-04-15; the first resample-plus-phase-lock overlay failed catastrophically on `pitchTestOrg +4`. +- Do not revisit `FAM-TRANSITION-HQ` as implemented on 2026-04-15; the transition overlay engaged on both `+4` truth cases but worsened both, especially `pitchTestOrg +4`. +- Do not revisit `FAM-ADVANCED`, `FAM-FORMANT-ONLY`, or `FAM-PITCH-PLUS-FORMANT` as standalone truth-case candidates. +- Do not revisit `FAM-CORE-PSOLA`, `FAM-CORE-MODEL`, or `FAM-OWN-PITCH` as standalone editors; harvest only explicitly proven traits. +- Do not retry pure WORLD end-to-end rendering. +- Do not spend more tuning cycles on local Stage B / blend-shape patching in the current Signalsmith-centered handoff family. +- Do not keep rejected renderer families alive just because one trait was useful; harvest the trait and remove the dead path. diff --git a/docs/pitch_renderer_research_notes.md b/docs/pitch_renderer_research_notes.md new file mode 100644 index 0000000..1d83073 --- /dev/null +++ b/docs/pitch_renderer_research_notes.md @@ -0,0 +1,202 @@ +# Pitch Renderer Research Notes + +Date: 2026-04-15 + +Purpose: +- capture primary-source research before more renderer implementation +- separate "we tried this in code" from "the literature actually supports this family" +- guide the next queue using evidence instead of repeating near-duplicate shell experiments + +## Current repo truth +- 2026-04-28 Signalsmith pitch-only formant-contract correction: this is a DSP contract fix, not a renderer-family reopening. + - Native pitch-only renders now use `setFormantFactor(1.0f, true)` for every pitch-only block, matching live preview and `S13PitchCorrector`. + - The active adaptive selector no longer throws away detected F0 on its Signalsmith pitch-only carriers: detected F0 guidance goes through `setFormantBase(...)` when available and is never passed as a formant-ratio curve. + - Offline pitch-only transpose maps use the same stage-A tonality-limit controls as live preview (`OPENSTUDIO_PITCH_STAGEA_TONALITY_LIMIT_HZ_*`) while keeping the formant factor neutral. + - Explicit formant edits pass the requested ratio directly with `compensatePitch=true`; the pitch ratio is not divided into the formant factor. + - `OPENSTUDIO_PITCH_DOWNSHIFT_FORMANT_ALPHA` no longer drives Signalsmith pitch-only formant scaling. Downshift body-color support stays in the bounded adaptive selector own-engine blend (`OPENSTUDIO_PITCH_DOWNSHIFT_OWN_BLEND`, default `0.42`) plus envelope-transfer/post-correction stages, not inverse-ratio carrier compensation. +- 2026-04-28 entry contour-handoff correction: this is a signal-chain ownership fix, not a renderer-family reopening. + - Hard/unknown note entries need pitch-ratio render pre-roll for shifter history, but the audible commit must still be governed by the entry bridge. + - A delayed body ramp on canonical hard/unknown `pitchOrg` edits measured worse, so production keeps the target ratio reached by `note.startTime` for those cases. + - Explicit continuous/internal transitions can use a minimum-jerk pitch handoff, and adjacent edited notes inside one island report that internal handoff instead of creating another dry/wet bridge. + - New diagnostics and harness metrics track entry pitch handoff ranges plus F0 slope/acceleration; slope gates are meaningful only when a handoff is actually used. +- 2026-04-28 emergency word-grouping repair: this is a product ownership correction, not a renderer-family reopening. + - `wordGroupId` is now assistive metadata only; normal click/drag editing no longer expands to the whole group. + - Explicitly selected adjacent notes can still render as one note-HQ island to avoid doubled ownership. + - Analyzer hard acoustic candidates can split broad phrase regions, but pitch hysteresis/vibrato remains diagnostic. + - The corrected lesson is that analyzer grouping must not silently decide the user's edit target. + - Measured after repair: analyzer destructive corner/pitch-jump splits are `0/0`, `pitchOrg +4/-4` note-HQ primary runs pass, downshift harmonic drift is `0.343`, and adjacent selected notes report one edit island for two edited notes. +- 2026-04-28 phrase-first word detection update: this is an analyzer/product-model correction, not a renderer-family reopening. + - The previous pitch-corner demotion was incomplete because an older running-average pitch-jump rule could still destructively split continuous vibrato/bend passages. + - Pitch jumps, hysteresis-style sustained deviations, and vibrato reversals now surface as diagnostics/boundary candidates instead of default editable cuts. + - Short voiced detector dropouts are bridged inside a phrase, while hard automatic splits require stronger acoustic evidence. + - Superseded by the emergency repair: the editor does not treat `wordGroupId` as the primary pitch-edit object by default. +- 2026-04-28 word-group/edit-island update: this is a product-model and signal-chain correction, not a renderer-family reopening. + - Pitch-corner detection is now diagnostic by default; it surfaces `boundaryCandidates` without splitting editable notes unless a research env flag is set. + - The analyzer exports `wordGroupId`, but pitch editing no longer defaults to moving the whole word group. + - Note-HQ final apply now commits per edit island, so adjacent selected notes do not create internal dry/wet bridges or doubled ownership. + - Commit range merging no longer averages pitch ratios; pitch ownership and the per-sample pitch curve are separate concepts. + - The corrected lesson is that segmentation can become too "successful": cutting every curve corner can hide a stitch problem while creating broken-word editing. +- 2026-04-27 segmentation/timbre update: this is still a signal-chain correction, not a renderer-family reopening. + - The analyzer now distinguishes pitch contour tracking from editable note/vocal boundaries by adding conservative pitch-corner splits with boundary-kind diagnostics. + - Note-HQ bridge ownership consumes those boundary kinds: hard word-like boundaries stay tightly dry-protected, while soft legato/sustain boundaries can use wider smoothing. + - Downshift timbre preservation now adds voiced-core spectral envelope transfer after the native directional render so the shifted vowel body is pulled back toward the original envelope. + - This targets the failure mode where a good compositor is smoothing the wrong acoustic boundary. + - An aggressive envelope-transfer probe regressed `pitchOrg -4` harmonic drift to `0.615`, so the kept implementation uses a small voiced-support-weighted mix and passed the primary downshift gate at `0.343`. +- 2026-04-27 update: the latest pitch editor work was a signal-chain correctness pass, not a new renderer-family experiment. + - The main timbre bug was an argument-order bug: detected F0 guidance was being supplied where `formantRatios` belong in pitch-only `SignalsmithShifter::process(...)` calls. + - Pitch-only paths now call explicit pitch-only entrypoints and keep explicit formant rendering separate. + - The note-HQ harness now scores phrase/full-clip results as full-clip audio instead of always slicing as if the candidate were window-local. + - Transition shoulders are now owned by the note-HQ commit range, which reduces hard body-edge switching without reopening the rejected seam-tweak families. +- 2026-04-27 direction-specific update: the follow-up fix is also a signal-chain correction, not a new renderer-family experiment. + - Production `note_hq` pitch-only now defaults to native direction-specific HQ. + - Upward edits keep the existing native adaptive path. + - Superseded carrier detail: downward edits used a gentler `pow(1 / ratio, alpha)` formant guard for a period, but pitch-only Signalsmith now preserves formants with neutral `setFormantFactor(1.0f, true)`. + - Rubber Band stays available as a diagnostic/benchmark backend, and its benchmark pitch map now uses effective transition shoulders, but it is not quality-promoted for production until it passes the same formant and boundary gates as native. + - New diagnostics include pitch direction, renderer branch, downshift guard usage, guard alpha, effective transition range, and Rubber Band quality-promotion status. + - The harness now includes an exit-to-next-note artifact score so edited-note release and next-note head issues are measured directly. + - Primary `pitchOrg -4` harmonic drift improved from the prior native `0.469` to `0.353`; primary `pitchOrg +4` stayed stable at `0.378`. +- 2026-04-27 two-sided boundary follow-up: + - This is still a signal-chain/compositor correction, not a new renderer family. + - The renderer needs shoulder context, but final audible commit ownership must not include the previous word. + - Commit ranges now carry both context/effective range and note-body start/end. + - The compositor copies `[effectiveStartTime, note.startTime)` dry, fades dry-to-wet inside the first `12 ms` of the edited note body, and keeps the exit release fade through the right shoulder. + - The failed lesson is explicit: widening left shoulder ownership can move the stutter backward into the previous word. + - The harness now reports `preBodyTailOriginalResidualDb`, `candidateActiveDifferenceStartSec`, and `preBodyTailArtifactScore` in addition to onset and exit-next artifacts. + - Primary final runs passed with pre-body residuals below `-169 dB`, active difference starting at `0.900063s`, onset artifacts `1.706 / 2.721`, and exit-next artifacts `2.307 / 0.091` for `pitchOrg +4 / -4`. +- 2026-04-27 entry-bridge follow-up: + - This remains a signal-chain/compositor correction, not a renderer-family experiment. + - The fully dry pre-note rule protected the previous word, but left a phase/envelope discontinuity at the edited-note entry. + - Final apply may now use a bounded bridge before `note.startTime`; everything before `noteHqEntryBridgeStartSec` must remain dry. + - Upward edits use a tight in-body bridge; downward edits use a bounded pre-note bridge with a temporary wet-read delay and capped envelope correction. + - Primary final runs: `pitchOrg +4` entry lag `-0.125 ms`, onset artifact `1.479`, pre-bridge residual `-172.823 dB`; `pitchOrg -4` entry lag `+0.771 ms`, onset artifact `1.598`, harmonic drift `0.344`, pre-bridge residual `-240.000 dB`. +- Best working editor path is still `pitch_only_adaptive_selector`. +- We have already exhausted: + - blend/ramp handoff families + - dry-attack/body-replacement families + - continuation families + - heuristic HPSS shell + - WSOLA shoulder-only seam pass + - lightweight phase-lock boundary-alignment pass + - median HPSS shell without envelope correction +- The persistent audible problems are still: + - onset stutter / seam artifact + - formant/timbre drift on note changes + - weak generalization from `pitchOrg` to `pitchTestOrg` + +## Primary-source findings + +### 1. Median-filter HPSS is real, but it is a separation tool, not a complete vocal pitch-editor solution +- FitzGerald 2010 presents a fast harmonic/percussive separation method using median filtering on the spectrogram. +- The paper supports HPSS as a useful structural decomposition step. +- It does not claim that median HPSS alone solves vocal pitch-edit rendering, onset/body continuity, or formant preservation. +- This matches the repo result: + - onset-side improvement is plausible + - body/pitch/timbre can still collapse if the harmonic renderer is weak + +Source: +- Derry FitzGerald, "Harmonic/Percussive Separation Using Median Filtering," DAFx-10, 2010 +- https://dafx.de/paper-archive/2010/DAFx10/DerryFitzGerald_DAFx10_P15.pdf + +### 2. WSOLA is strong for local continuity and time-scale seams, but it is not a full answer to vocal pitch-edit artifacts +- Verhelst and Roelands 1993 introduced WSOLA for high-quality time-scale modification of speech. +- It is good at local alignment and reducing overlap-add discontinuities. +- It does not by itself solve harmonic-body retuning, formant preservation, or residual/noise modeling. +- This matches the repo result: + - shoulder-only WSOLA did not beat the current adaptive editor + +Source: +- Werner Verhelst and Marc Roelands, "An Overlap-Add Technique Based on Waveform Similarity (WSOLA) for High Quality Time-Scale Modification of Speech," ICASSP 1993 +- publication record: https://www.etrovub.be/research/publications/publication-details/overview/3449/ + +### 3. Stronger phase-vocoder work is still genuinely untried here +- Laroche and Dolson 1999 directly target classical phase-vocoder weaknesses with peak-centered frequency shifting and better phase handling. +- "Phase Vocoder Done Right" (2022) goes further and explicitly argues that phase-gradient-based integration can avoid typical phase-vocoder artifacts without separate transient handling. +- The repo only tried a lighter boundary-alignment variant, not a research-grade phase-coherent PV path. +- Therefore: + - the phase family is not exhausted in the literature sense + - only the lightweight local branch we tried is exhausted + +Sources: +- Jean Laroche and Mark Dolson, "New Phase-Vocoder Techniques for Pitch-Shifting, Harmonizing and Other Exotic Effects," 1999 +- https://www.ee.columbia.edu/~dpwe/papers/LaroD99-pvoc.pdf +- "Phase Vocoder Done Right," arXiv 2202.07382 +- https://arxiv.org/abs/2202.07382 + +### 4. DDSP is the strongest research-backed engine-redesign direction +- DDSP combines harmonic/noise DSP structure with learned control. +- The paper specifically highlights interpretable, modular control over pitch, loudness, and timbre-relevant components. +- This is a strong fit for our real problem: + - harmonic body + - noise/residual + - timbre preservation + - controllable pitch change +- It is not a small patch; it is a true engine-v2 direction. + +Source: +- Jesse Engel et al., "DDSP: Differentiable Digital Signal Processing," 2020 +- https://arxiv.org/abs/2001.04643 + +### 5. The strongest paper match to our current product problem is restoration-after-shift +- The 2026 shallow-diffusion singing restoration paper reframes pitch shifting as restoration. +- This is highly relevant to the repo because our best current path already produces a usable pitch-correct render, but still leaves: + - stutter + - robotic coloration + - formant drift +- The paper explicitly targets those tradeoffs while preserving melody and timing. +- This makes offline restoration a stronger next benchmark than more small DSP shell variations. + +Source: +- Yunyi Liu and Taketo Akama, "Self-supervised restoration of singing voice degraded by pitch shifting using shallow diffusion," 2026 +- https://arxiv.org/abs/2601.10345 + +### 6. WORLD remains useful as a support decomposition, not necessarily as the final renderer +- WORLD gives explicit decomposition into: + - F0 + - spectral envelope + - aperiodicity +- That is valuable for support features and diagnostics. +- The repo history still does not support pure WORLD-style rendering as the final pitch-editor path. + +Source: +- Masanori Morise et al., "WORLD: A Vocoder-Based High-Quality Speech Synthesis System for Real-Time Applications," 2016 +- https://www.jstage.jst.go.jp/article/transinf/E99.D/7/E99.D_2015EDP7457/_pdf/-char/en + +## Research-backed conclusions + +### Exhausted enough for now +- WSOLA as a shoulder-only seam fix +- heuristic HPSS shell +- scalar median HPSS shell +- lightweight boundary-aligned phase-lock variant + +### Still genuinely viable +- a true research-grade phase-vocoder family: + - identity / peak phase locking + - or phase-gradient / RTPGHI-style integration +- an offline restoration benchmark on top of the best current renderer +- DDSP-style engine-v2 work if we accept a larger revamp + +### Lower priority or support only +- WORLD as decomposition support only +- further HPSS shell variants without a stronger harmonic/timbre model +- more seam-only local overlap tweaks + +## Recommended next queue +1. `FAM-PVDR` + - proper phase-coherent phase-vocoder family + - not another boundary-alignment patch + - long/stable voiced regions first +2. `FAM-ML-RESTORATION` + - offline benchmark on top of `pitch_only_adaptive_selector` + - treat artifacts as restoration rather than trying to make the base shifter perfect +3. Broader validation sweep on `FAM-ADAPTIVE-SELECTOR` + - keep the current best editor stable while new work stays benchmark-only + +## Why this matters +- The current repo pattern is clear: + - shell/onset ideas can improve onset metrics on the easy case + - but they do not reliably preserve body, pitch exactness, or hard-case generalization +- The literature supports moving either: + - deeper into true phase-coherent DSP + - or into restoration-after-render +- It does not support more time on small seam-only tweaks as the highest-value next move diff --git a/docs/pitch_root_cause_research_20260417.md b/docs/pitch_root_cause_research_20260417.md new file mode 100644 index 0000000..23ca2fa --- /dev/null +++ b/docs/pitch_root_cause_research_20260417.md @@ -0,0 +1,109 @@ +# Pitch Root-Cause Research 2026-04-17 + +## Summary +This pass was meant to answer three bounded questions: + +1. Why are the two remaining product issues still happening? +2. Is there a fast local ML/external-style restoration path available now? +3. Is a clean-sheet `engine-v3` DSP reset credible enough to continue immediately? + +The answers are: + +- The failures are primarily architectural, not just tuning-related. +- No materially stronger local ML restorer is available in this environment right now. +- The first `engine-v3` feasibility probe does not justify immediate build-out. + +Baseline remains: + +- kept renderer: `pitch_only_adaptive_selector` +- frozen comparison-only challenger: `pitch_only_engine_v2_program` + +## Top Causes +| Rank | Cause | Confidence | Evidence | Likely fixes | +| --- | --- | --- | --- | --- | +| 1 | Transition ownership and boundary timing drift | high | Hard-case adaptive boundary timing stays high even after correction (`18.92 ms` boundary max; `18.92 ms` transient max), and frozen engine-v2 still collapses on hard-case entry (`entry mel 7.017`). | Own entry and exit as a transition pair instead of as per-note local patches; move shell/core alignment earlier at note entry instead of relying on late correction; only benchmark architectures that separate transient shell from voiced core before pitch rendering | +| 2 | Mixed transient and first-voiced-cycle content is still being handled by one renderer family | medium | Adaptive correction improves some easy windows but hard-case transient metrics stay elevated (`pitchTest` transient entry/exit max `1.614 / 6.723`), and engine-v2 still fails with transient bypass enabled. | Use a true shell/core/residual decomposition before note-change rendering; treat the first voiced cycles as their own ownership zone; prefer learned restoration or a clean-sheet transition renderer over more local handoff tuning | +| 3 | Current formant preservation is too weak and too local to survive hard transitions | high | Formant drift remains in the adaptive path even after correction (`pitchOrg` formant mean `0.364`; `pitchTest` formant mean `0.071`), while engine-v2 keeps spectral envelope correction on but still loses the hard case (`formant drift 0.062`, `entry mel 7.017`). | Use an explicitly conditioned source-filter or learned restoration stage instead of a light cepstral patch; keep pitch exactness in the carrier and repair timbre only inside the affected transition region; only continue DSP work if the new architecture bakes formant handling into the core renderer | + +## Key Evidence +- Original vs reference hard case: + - boundary timing `39.833 ms` + - formant drift `0.528` +- Adaptive best hard-case transient means: + - entry `1.598` + - exit `6.543` + - timing max `18.917 ms` +- Adaptive best hard-case formant mean drift: + - `0.071` +- Frozen engine-v2 hard-case run: + - entry mel `7.017` + - onset artifact `4.000` + - formant drift `0.062` + +Primary artifact files: + +- root-cause run: + - `D:\test projects\os tests\runs\20260417_012542_pitch_root_cause_research\pitch_root_cause_research.md` + - `D:\test projects\os tests\runs\20260417_012542_pitch_root_cause_research\pitch_root_cause_research.json` +- ML benchmark: + - `D:\test projects\os tests\runs\20260417_003355_pitch_ml_benchmark\pitch_ml_benchmark_summary.md` +- engine-v3 feasibility: + - `D:\test projects\os tests\runs\20260417_003355_engine_v3_feasibility\pitch_engine_v3_feasibility_summary.md` + +## ML Benchmark Verdict +Local ML restoration is blocked for now. + +Environment result: + +- verdict: `blocked_no_stronger_restorer` +- runtime ready: `true` +- selected backend: `cuda` + +Candidate check: + +- `voicefixer`: not installed +- `demucs`: not installed +- `audio_separator`: available, but not note-local restoration +- `proxy_ml_restore_v1`: explicitly excluded and already rejected + +Meaning: + +- do not reopen `FAM-ML-RESTORATION` locally on another proxy +- the next ML step is to source a genuinely stronger restorer or benchmark an outside/licensed path + +## Engine-v3 Feasibility Verdict +The first `V3-1` decomposition probe says `stop`, not `continue`. + +Results: + +| Case | Decomposition score | Verdict | +| --- | ---: | --- | +| `pitchOrg_plus4` | `0.511` | `stop` | +| `pitchTest_plus4` | `0.505` | `stop` | + +Interpretation: + +- the first shell/core/residual split heuristic is not yet stronger than the current adaptive carrier +- there is no evidence yet that an immediate `engine-v3` build-out would avoid repeating the `engine-v2` failure pattern + +Meaning: + +- do not start a long `engine-v3` branch from this decomposition probe alone +- only reopen `engine-v3` if we define a materially stronger decomposition and transition-pair ownership model first + +## Decision +Current best action order is: + +1. Keep `pitch_only_adaptive_selector` as the working editor. +2. Stop local ML retries until a stronger restorer is actually available. +3. Do not expand `engine-v3` from the current feasibility probe. +4. If we want the fastest chance of a real audible improvement, benchmark an external/licensed or materially stronger ML restorer next. +5. If we want a new in-house DSP engine later, begin with a better decomposition/transition design doc first, not another immediate renderer branch. + +## What This Research Changed +This pass closes the main uncertainty from the earlier tuning program: + +- the problem is not mainly unresolved because we missed one more STFT or crossfade tweak +- the problem remains because the current family does not own the transition pair strongly enough, still mixes transient and first voiced cycles too early, and only preserves formants with a weak local correction + +That means the next step should be bounded and evidence-led, not another open-ended renderer tuning loop. diff --git a/docs/pr_footprint_cleanup_20260501.md b/docs/pr_footprint_cleanup_20260501.md new file mode 100644 index 0000000..96c4481 --- /dev/null +++ b/docs/pr_footprint_cleanup_20260501.md @@ -0,0 +1,24 @@ +# PR Footprint Cleanup - 2026-05-01 + +## Keep + +- Source, frontend, docs, tools, and test fixture files that are part of active feature work or regression coverage. +- AI runtime installer/configuration files such as `tools/install_ai_tools.py`, `tools/prepare-ai-runtime.ps1`, runtime install-plan JSON files, `tools/openstudio_ace_runner.py`, and the lightweight `tools/openstudio_ace_backend` bridge sources. +- Pitch regression scripts and fixtures needed to reproduce the pitch-renderer work. + +## Ignore Or Remove + +- `tmp_pitch_runs/` generated pitch regression runs. Pre-cleanup status showed 2951 untracked files in this directory. +- `tools/openstudio_ace_backend/vendor_runtime/` downloaded/generated ACE/Comfy runtime payloads. Pre-cleanup status showed 429 untracked files in this directory. The installer/probe code reconstructs this runtime from the pinned runtime plan and cache; it should not be committed. +- Generated pitch regression WAV/PNG/JSONL reports under `tests/fixtures/pitch-regression`. +- Local debug capture output under `pitch_debug_captures/`. + +## Needs Review + +- Untracked source/docs/frontend/test files outside the generated buckets are not deleted here. They appear to be intentional feature or regression work and should be reviewed by feature owner before commit. +- Tracked modified files remain untouched by this cleanup pass. + +## Actions + +- Added `.gitignore` coverage for generated pitch runs, pitch debug captures, generated report media, and the ACE vendor runtime payload. +- Removed generated local runtime/run folders from the working tree after verifying they were inside the repository root. diff --git a/docs/qwen-daw-assistant-turboquant-plan.md b/docs/qwen-daw-assistant-turboquant-plan.md new file mode 100644 index 0000000..f5999d1 --- /dev/null +++ b/docs/qwen-daw-assistant-turboquant-plan.md @@ -0,0 +1,584 @@ +# Qwen DAW Assistant, TurboQuant, GGUF, and ACE Step Plan + +Date: 2026-04-27 + +Status: planning document only. Do not treat this file as implementation. + +## Goal + +Build a local-only Qwen assistant for Studio13 that can understand plain English, inspect project/audio context, plan DAW/plugin actions, explain the impact of those actions, and execute only after user confirmation. + +The assistant must also control the existing ACE Step 1.5 XL Turbo music-generation integration. + +The installer must check hardware first, download only one supported model/runtime profile, verify that it actually runs, and fail clearly if unsupported. No runtime fallback and no downloading a larger model that cannot run on the machine. + +## Key Decision + +Use Qwen-only audio/music-capable models. + +Do not use Gemma. + +Do not use text-only Qwen models as the main assistant, even if they are easier to run, because the requirement is audio/music understanding. + +ACE Step's internal `qwen_4b_ace15.safetensors` model is not the Studio13 assistant brain. It is part of the ACE generation stack and should stay there. + +## What TurboQuant Actually Gives Us + +TurboQuant is KV-cache compression for vLLM inference. It is not a model-weight quantizer. + +That means: + +- TurboQuant helps with longer context after the base model has loaded. +- TurboQuant can reduce KV memory pressure for large project/audio summaries. +- TurboQuant does not make an oversized model's weights fit into VRAM by itself. +- TurboQuant applies to the vLLM path, not the GGUF/llama.cpp path. +- For quality-sensitive audio/music planning, prefer 4-bit values where possible. TurboQuant's own README identifies low-bit value quantization as the main quality bottleneck. + +TurboQuant's published hardware tests are not audio-model tests: + +| TurboQuant-tested model | Hardware | Relevance to Studio13 | +| --- | --- | --- | +| `Qwen3.5-27B-AWQ` | 1x RTX 5090 32GB | Useful proof of vLLM KV compression, but not an audio/music Omni assistant target. | +| `Qwen3.5-35B-A3B` | 8x RTX 3090 24GB | Useful MoE/KV reference, but not a practical single-workstation Studio13 target. | + +Studio13 should use TurboQuant as an optimization layer only after a Qwen audio model has passed a real load test. + +## Runtime Families + +The installer should probe both runtime families before downloading a model, then select exactly one verified profile. + +### Primary: vLLM/vLLM-Omni + AWQ + TurboQuant + +Use when WSL2 CUDA validates cleanly. + +Includes: + +- WSL2/Linux CUDA runtime. +- vLLM or vLLM-Omni. +- AWQ or AWQ-Marlin where supported. +- TurboQuant KV-cache compression. +- Triton kernels from vLLM/TurboQuant. +- FlashAttention or vLLM's optimized attention backend. +- PagedAttention. +- Fixed context, batch, and audio-window limits based on VRAM. + +Preferred 16GB VRAM candidate: + +- `Qwen/Qwen2.5-Omni-7B-AWQ` + +Why: + +- It supports text, image, audio, and video input. +- Its model card explicitly targets lower-VRAM GPUs. +- Its official table lists substantially lower memory than BF16: 11.77GB for 15s video, 17.84GB for 30s video, 30.31GB for 60s video. +- For Studio13, we should use audio-only input and `return_audio=false`, so the practical audio-only load can be lower than the video table, but it still must be verified locally. + +### Secondary: llama.cpp + GGUF + mtmd/mmproj + +Use only if the exact GGUF model and audio path verify successfully. + +Includes: + +- llama.cpp CUDA build. +- GGUF language/model file. +- matching multimodal projector or mtmd-compatible assets when required. +- GPU layer offload. +- llama.cpp flash attention where supported. +- audio-only verification prompt. + +Important: + +- TurboQuant does not apply to GGUF. +- GGUF model weight quantization and TurboQuant KV compression are separate technologies. +- llama.cpp multimodal/audio support is actively evolving, so GGUF profiles must be marked candidate-only until Studio13's installer proves that the exact model can accept audio and return reliable text/tool-plan output. + +## Qwen Model Map: Small to Large + +Only models with audio/music understanding are allowed in the assistant model map. + +The installer should maintain a strict profile manifest. A model is "loadable" only if both the hardware prefilter and the real startup test pass. + +| Tier | Model | Role | Audio/music support | Runtime candidates | Minimum install rule | +| --- | --- | --- | --- | --- | --- | +| Small | `Qwen/Qwen2.5-Omni-3B` | Lowest-footprint direct audio assistant | Yes. Official card reports music/audio reasoning benchmarks. | vLLM-Omni, Transformers, GGUF candidate | Install only after a real 10s audio load test. Prefer this when 7B-AWQ fails on 16GB VRAM. | +| Recommended | `Qwen/Qwen2.5-Omni-7B-AWQ` | Main consumer-GPU assistant | Yes. Better than 3B on most audio/text tasks and has low-VRAM AWQ profile. | vLLM-Omni/Transformers AWQ | Primary target for 32GB RAM / 16GB VRAM. Use 10-15s audio batches by default. | +| Medium | `Qwen/Qwen2.5-Omni-7B` BF16 | Higher-quality 7B baseline | Yes | vLLM-Omni/Transformers | Reject on 16GB VRAM. Require large-VRAM verification, roughly 48GB+ practical headroom. | +| Large analysis sidecar | `Qwen/Qwen3-Omni-30B-A3B-Captioner` | Detailed audio caption/context extractor | Yes, audio-focused text output | vLLM/Transformers, GGUF candidate | Not the main DAW control brain. Require high-VRAM verification or remote lab machine. | +| Large reasoning | `Qwen/Qwen3-Omni-30B-A3B-Thinking` | Best multimodal reasoning candidate | Yes | vLLM/Transformers, GGUF candidate | Reject on 16GB VRAM. Official BF16 table is 68.74GB minimum for 15s video. | +| Largest | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | Full thinker/talker model | Yes | vLLM/Transformers, GGUF candidate | Reject on 16GB VRAM. Official BF16 table is 78.85GB minimum for 15s video. Disable talker when text output only. | + +Legacy candidate: + +- `Qwen/Qwen2-Audio-7B-Instruct` can understand audio, but Qwen2.5-Omni should be preferred because it has stronger documented multimodal/audio/music performance and better current runtime paths. + +## Hardware Selection Rules + +These are installer prefilter rules. The final authority is a real local model-load and inference test. + +| Hardware | Selected profile | +| --- | --- | +| 16GB VRAM, 32GB RAM, WSL2 CUDA passes | `Qwen/Qwen2.5-Omni-7B-AWQ` through vLLM/vLLM-Omni, TurboQuant enabled after load. | +| 16GB VRAM, vLLM path fails but llama.cpp audio GGUF path verifies | `Qwen2.5-Omni-3B-GGUF` first; `Qwen2.5-Omni-7B-GGUF` only if audio load test passes with headroom. | +| 24GB VRAM | `Qwen/Qwen2.5-Omni-7B-AWQ` with longer audio windows, or verified `Qwen2.5-Omni-7B-GGUF`. | +| 48GB VRAM | `Qwen/Qwen2.5-Omni-7B` BF16 or AWQ with longer batches. | +| 80GB+ VRAM | Consider `Qwen3-Omni-30B-A3B-Thinking`. | +| 96GB+ VRAM or multi-GPU | Consider `Qwen3-Omni-30B-A3B-Instruct`. | +| No verified profile | Do not download a model. Mark assistant unsupported. | + +For the current target class of 32GB RAM / 16GB VRAM, the realistic plan is: + +1. Try `Qwen/Qwen2.5-Omni-7B-AWQ` with audio-only inference, `return_audio=false`, 10s default audio windows. +2. If it cannot be loaded during pre-download verification planning, try the 3B Omni profile. +3. If neither profile can be proven loadable, do not install the assistant model. + +## Installer Requirements + +The existing AI runtime installer should gain a separate "assistant runtime" section while keeping ACE Step installation intact. + +Probe before download: + +- Windows version. +- WSL2 availability. +- NVIDIA driver and CUDA visibility inside WSL2. +- GPU model, VRAM, compute capability, and free VRAM. +- System RAM and free disk. +- Existing ACE Step runtime status. +- Whether vLLM/vLLM-Omni can start. +- Whether TurboQuant can import and attach its vLLM integration. +- Whether llama.cpp CUDA build can run an mtmd/audio smoke test. + +Profile manifest fields: + +- `profileId` +- `modelRepo` +- `modelRevision` +- `runtimeFamily` +- `quantization` +- `requiresAudioInput` +- `requiresMmproj` +- `minRamGb` +- `minVramGb` +- `minFreeDiskGb` +- `defaultAudioWindowSeconds` +- `maxVerifiedAudioWindowSeconds` +- `contextTokens` +- `gpuOffloadPolicy` +- `turboQuantEnabled` +- `flashAttentionEnabled` +- `tritonRequired` +- `startupTestPrompt` +- `audioSmokeTestFile` +- `sha256` or pinned file checksums where possible + +Verification after download: + +1. Load the model. +2. Load multimodal/audio projector if required. +3. Run a 10s audio/music understanding test. +4. Run a strict JSON action-plan test. +5. Validate against Studio13's assistant action schema. +6. For vLLM path, verify TurboQuant integration and the attention backend. +7. For GGUF path, verify llama.cpp audio input through its supported local API/CLI. +8. Record the verified profile in a runtime status file. + +No fallback policy: + +- The installer may probe multiple profiles before download. +- The installer must download only the selected profile. +- After download, if verification fails, mark assistant unsupported. +- Do not silently install a smaller model after a failed selected install. +- User can explicitly rerun setup to select a different profile. + +## Audio Context Strategy + +Do not feed all multitrack audio directly into the LLM. + +Use deterministic project/audio analysis first, then send compact structured context plus selected raw audio windows only when needed. + +Per clip context: + +- file path/id +- duration +- sample rate +- channels +- clip gain +- fades +- offset/trim +- loudness +- peak/true peak +- silence regions +- transient density +- pitch/key estimate where relevant +- tempo hints +- spectral tilt +- stereo width + +Per track context: + +- track name/type guess +- clips +- volume/pan/mute/solo/arm +- routing/sends +- FX chain and plugin params +- automation summary +- loudness and peak range +- frequency balance +- role guess such as vocal, drums, bass, guitar, keys, FX, bus, master + +Master context: + +- integrated LUFS +- short-term LUFS +- true peak +- crest factor +- clipping risk +- spectral balance +- stereo width +- phase/correlation +- dominant problem regions + +Batching rules: + +| Situation | Audio window policy | +| --- | --- | +| Selected clip edit | Send selected clip summary plus 5-10s raw audio around selection. | +| Track-level mix command | Send full track summary plus 10s representative windows. | +| Master mix command on small project | Send all track summaries plus representative 10s windows from loud/active sections. | +| Master mix command on large project | Batch by buses/stems first, then request track-level details only for problematic groups. | +| Many tracks or long project | Use section summaries first. Raw audio only for selected sections. | +| Micro timing/glitch tasks | Use 1s windows around transient/problem locations. | +| Arrangement/structure tasks | Use 30-100s symbolic/summary windows, not raw audio by default. | + +Default for 16GB VRAM: + +- 10s audio windows. +- No video input. +- Text output only. +- `return_audio=false`. +- TurboQuant enabled only on the vLLM path after model load. + +## Assistant Action Surface + +All actions must be typed, validated, reversible where possible, and confirmation-gated. + +The assistant should not directly mutate state. It should produce an action plan that Studio13 executes through existing store/bridge paths. + +Every clip/track/project data mutation must use undo-aware command paths. + +### Transport and Timeline + +- Play, stop, pause, record. +- Seek to time/bar/marker. +- Set loop region. +- Toggle metronome/count-in. +- Set tempo/time signature. +- Add, remove, rename, and navigate markers/regions. +- Set snap/grid/zoom/tool mode. + +### Tracks and Mixer + +- Add, duplicate, remove, rename, color, reorder tracks. +- Create folder tracks, buses, VCAs, groups. +- Set volume, pan, width, phase, mute, solo, arm, monitoring. +- Set input/output routing. +- Add/remove/adjust sends. +- Create mix snapshots. +- Compare or restore mixer snapshots. + +### Clips and Editing + +- Import audio/MIDI. +- Add generated audio clips. +- Move, split, trim, resize, duplicate, delete clips. +- Set clip gain, mute, color, lock, group, reverse. +- Apply fades/crossfades. +- Quantize, nudge, align, slip edit. +- Dynamic split by transients. +- Normalize or adjust gain. +- Render/bounce selected clips. + +### Plugins and FX Chains + +- Insert built-in FX or scanned plugins. +- Remove, bypass, enable, reorder plugins. +- Read plugin parameters. +- Set plugin parameters. +- Load presets where available. +- Create common chains, such as vocal cleanup, bass control, drum bus glue, mastering limiter. +- Explain expected sonic impact before applying. + +### Automation + +- Create/show/hide automation lanes. +- Add, move, delete automation points. +- Set automation mode. +- Draw fades, ramps, ducking, pan moves, filter sweeps. +- Summarize automation conflicts. + +### Pitch, Stem, and Audio Analysis + +- Analyze pitch contour. +- Apply pitch correction. +- Extract MIDI where supported. +- Separate stems. +- Detect tempo/key. +- Detect transients/silence. +- Analyze loudness/spectrum/stereo. +- Suggest corrective chains based on analysis. + +### Render and Export + +- Configure render format. +- Render master. +- Render selected tracks/stems only when backend support exists. +- Normalize/tail options where supported. +- Explain unsupported render options rather than pretending they work. + +### ACE Step 1.5 XL Turbo + +The assistant must be able to control ACE Step through Studio13's existing AI generation pipeline. + +Actions: + +- `ai.getRuntimeStatus` +- `ai.openSetup` +- `ai.createAITrack` +- `ai.setWorkflow` +- `ai.setGenerationParams` +- `ai.generateMusic` +- `ai.cancelGeneration` +- `ai.pollGeneration` +- `ai.insertGeneratedClip` + +ACE workflow params to expose: + +- `workflowId` +- `prompt` +- `lyrics` +- `seed` +- `bpm` +- `duration` +- `timesignature` +- `language` +- `keyscale` +- `generate_audio_codes` +- `inferenceSteps` +- `cfg_scale` +- `guidance_scale` +- `shift` +- `temperature` +- `top_p` +- `top_k` +- `min_p` + +The assistant should not call `tools/generate_music.py` directly. It should use the app's existing NativeBridge/store flow so progress, cancellation, UI status, and generated clip insertion stay consistent. + +If the user requests continuation from an existing clip, the assistant should only use ACE continuation if that workflow is implemented and marked available. Otherwise it should say continuation is unavailable and offer a prompt/style-based generation using extracted clip context. + +## Example Action Chains + +### "Make the vocal clearer" + +Plan: + +1. Analyze selected vocal track loudness, spectral balance, silence, and FX chain. +2. Add or adjust high-pass EQ. +3. Add gentle compression. +4. Add de-esser if sibilance is detected. +5. Raise presence band only if masking is detected. +6. Lower competing instruments or add sidechain/dynamic EQ if needed. +7. Preview and keep changes undoable. + +Impact: + +- Improves intelligibility. +- May make vocal brighter or more forward. +- Can increase harshness if overdone, so cap boosts conservatively. + +### "Make the master louder without clipping" + +Plan: + +1. Analyze master LUFS, true peak, crest factor, clipping risk. +2. Identify tracks/buses causing peak spikes. +3. Lower or compress those tracks first. +4. Add/adjust master bus compression only if needed. +5. Add limiter ceiling, e.g. -1.0 dBTP. +6. Increase gain to target loudness. +7. Re-analyze and show before/after. + +Impact: + +- Raises perceived loudness. +- May reduce dynamics. +- Safer than only pushing a limiter because it fixes source peaks first. + +### "Generate a dark synthwave intro" + +Plan: + +1. Read project tempo/key if present. +2. Create or select an AI track. +3. Set ACE Step workflow to `text-to-music`. +4. Translate request into ACE params: prompt, BPM, key, duration, seed. +5. Confirm with the user. +6. Start ACE generation. +7. Poll progress. +8. Insert generated clip. +9. Name/color/route track. + +Impact: + +- Adds a new generated audio clip. +- Does not modify existing audio. +- Can be regenerated with a seed change. + +### "Use this clip as context and create drums that fit" + +Plan: + +1. Analyze selected clip tempo/key/energy/transients. +2. Summarize style and rhythmic feel. +3. Build an ACE prompt from the summary. +4. Generate a drum-focused clip on a new AI track. +5. Align clip start to selection. +6. Set gain conservatively and route to drum bus if present. + +Impact: + +- Creates a new generated drum layer. +- Existing clip remains untouched. +- Exact continuation depends on whether ACE continuation workflow is available. + +## UI Plan + +Add a collapsible chat bar on the right side of the DAW. + +States: + +- collapsed icon button +- expanded chat +- context selected +- thinking/planning +- awaiting confirmation +- executing +- success +- failed/unsupported + +The assistant response should show: + +- interpreted intent +- affected tracks/clips/plugins +- proposed action chain +- expected sonic impact +- risk/destructive status +- undo availability +- confirmation controls + +Execution policy: + +- Never auto-execute data-changing actions from plain text. +- Always ask for confirmation. +- Allow safe read-only analysis without confirmation. +- Let the user inspect the exact action chain before execution. + +## Feasibility Report + +Feasible, with the right constraints. + +Most realistic for 32GB RAM / 16GB VRAM: + +- `Qwen/Qwen2.5-Omni-7B-AWQ` +- vLLM/vLLM-Omni in WSL2 CUDA +- audio-only input +- `return_audio=false` +- 10s default raw audio windows +- project summaries for larger context +- TurboQuant for KV/context pressure after load + +Likely not feasible on 16GB VRAM: + +- Qwen3-Omni 30B BF16 variants. +- Long raw multitrack audio context in one prompt. +- Full raw master mix analysis for many tracks without batching. + +GGUF feasibility: + +- Useful as a secondary path. +- Must be verified per exact model because llama.cpp audio/multimodal support is still moving quickly. +- Do not assume a GGUF file is enough; audio input support and projector compatibility must pass a real smoke test. + +ACE Step feasibility: + +- Already integrated as a music-generation engine. +- Very useful as an assistant-controlled tool. +- Not suitable as the general DAW assistant model. +- The assistant should use ACE for generation, not for reasoning/planning. + +## Implementation Phases + +### Phase 1: Documentation and manifests + +- Create assistant model profile manifest. +- Add action schema draft. +- Define installer probe outputs. +- Define verification prompts and audio smoke-test assets. + +### Phase 2: Runtime probe + +- Extend AI runtime probe for assistant runtime. +- Add WSL2 CUDA checks. +- Add vLLM/vLLM-Omni import checks. +- Add TurboQuant import/hook checks. +- Add llama.cpp CUDA/mtmd checks. + +### Phase 3: Installer selection + +- Select exactly one model profile before download. +- Download only selected model. +- Run verification. +- Save verified runtime status. +- Fail clearly if unsupported. + +### Phase 4: Assistant service + +- Local HTTP service wrapper around selected runtime. +- Strict JSON output mode. +- Tool/action schema validation. +- Project/audio context assembler. +- Plan generation and confirmation flow. + +### Phase 5: DAW integration + +- Collapsible right chat bar. +- Action preview UI. +- Confirmation and execution state. +- Undo-aware action execution. +- ACE Step action control. + +### Phase 6: Audio intelligence + +- Clip/track/master feature summaries. +- Representative audio window selector. +- Batch planner for large projects. +- Before/after analysis for mix actions. + +## Acceptance Criteria + +- On unsupported hardware, no assistant model is downloaded. +- On supported hardware, exactly one assistant model profile is downloaded. +- Startup verification proves audio input, JSON planning, and schema validation. +- The assistant can explain and confirm a DAW action chain before running. +- The assistant can control ACE Step generation through Studio13's existing AI generation flow. +- Master mix requests use track/bus analysis and batched context instead of dumping all audio into the LLM. +- All data-changing actions remain undoable where Studio13 supports undo. + +## Sources + +- TurboQuant: https://github.com/0xSero/turboquant +- vLLM-Omni supported models: https://docs.vllm.ai/projects/vllm-omni/en/latest/models/supported_models/ +- Qwen2.5-Omni-3B: https://huggingface.co/Qwen/Qwen2.5-Omni-3B +- Qwen2.5-Omni-7B-AWQ: https://huggingface.co/Qwen/Qwen2.5-Omni-7B-AWQ +- Qwen3-Omni-30B-A3B-Instruct: https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct +- llama.cpp multimodal/mtmd notes: https://github.com/ggml-org/llama.cpp/blob/master/tools/mtmd/README.md + diff --git a/docs/release-runbook.md b/docs/release-runbook.md index f266dde..2475b46 100644 --- a/docs/release-runbook.md +++ b/docs/release-runbook.md @@ -39,8 +39,10 @@ Use this flow instead: 8. Verify the published direct-download URLs: - `https://github.com/<org>/<repo>/releases/latest/download/OpenStudio-Setup-x64.exe` - `https://github.com/<org>/<repo>/releases/latest/download/OpenStudio-macOS.dmg` + - `https://github.com/<org>/<repo>/releases/latest/download/OpenStudio-Linux.AppImage` - `https://github.com/<org>/<repo>/releases/download/<ai-runtime-tag>/OpenStudio-AI-Runtime-windows-base-x64.zip` - `https://github.com/<org>/<repo>/releases/latest/download/OpenStudio-AI-Runtime-macos-arm64.zip` + - `https://github.com/<org>/<repo>/releases/download/<ai-runtime-tag>/OpenStudio-AI-Runtime-linux-x64.tar.gz` 9. Verify the website repo finishes its deploy and the public metadata/redirect URLs on `openstudio.org.in` return JSON/XML/302 responses instead of the SPA HTML shell. The stable installer/runtime filenames are part of the public download contract. The website repo is now the only publisher of public metadata and redirects. @@ -147,6 +149,7 @@ That publish-asset set contains: - `OpenStudio-ai-runtime-stable-latest.json` - `OpenStudio-appcast-windows-stable.xml` - `OpenStudio-appcast-macos-stable.xml` +- `OpenStudio-appcast-linux-stable.xml` - `OpenStudio-checksums.txt` The website repo should fetch those assets after the desktop release publishes, place them into its deploy-input area, and then deploy `openstudio.org.in`. diff --git a/docs/release-smoke-checklist.md b/docs/release-smoke-checklist.md index 2aa16d5..af4e180 100644 --- a/docs/release-smoke-checklist.md +++ b/docs/release-smoke-checklist.md @@ -73,6 +73,7 @@ Use this checklist for every release candidate before publishing installers, man - `OpenStudio-ai-runtime-stable-latest.json` - `OpenStudio-appcast-windows-stable.xml` - `OpenStudio-appcast-macos-stable.xml` + - `OpenStudio-appcast-linux-stable.xml` - Confirm `OpenStudio-checksums.txt` matches the published binaries. - Confirm `appcast/windows-stable.xml` points to the published Windows installer. - Confirm `appcast/macos-stable.xml` points to the published macOS DMG. @@ -80,7 +81,7 @@ Use this checklist for every release candidate before publishing installers, man - Confirm the website repo deploy has completed before validating `openstudio.org.in`. - Confirm the public metadata and appcast URLs on `openstudio.org.in` no longer return the SPA HTML shell. - Confirm `https://openstudio.org.in/releases/ai-runtime/latest.json` and `https://openstudio.org.in/releases/ai-runtime/stable/latest.json` are live and uncached. -- Confirm `https://openstudio.org.in/download/ai-runtime/windows/latest` and `https://openstudio.org.in/download/ai-runtime/macos/latest` redirect cleanly if those convenience endpoints are enabled. +- Confirm `https://openstudio.org.in/download/ai-runtime/windows/latest`, `https://openstudio.org.in/download/ai-runtime/macos/latest`, and `https://openstudio.org.in/download/ai-runtime/linux/latest` redirect cleanly if those convenience endpoints are enabled. - Run `./tools/validate-published-release.ps1 -MetadataDir dist/release-metadata -Channel stable -ReleaseSiteUrl https://openstudio.org.in -ValidateRedirects` after deploy and confirm it passes. ## Launch Sign-Off diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 0dedb4a..b49b8e8 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -15,6 +15,7 @@ import { publishCurrentMixerUISnapshot, startMixerUISync, } from "./utils/mixerWindowSync"; +import { maybeRunPitchRegressionDriver } from "./utils/pitchRegressionDriver"; import { Button } from "./components/ui"; import { Timeline } from "./components/Timeline"; import { TimelineRuler } from "./components/TimelineRuler"; @@ -156,6 +157,7 @@ function App() { showMasterAutomation, showStemSeparation, showAiToolsSetup, + closeAiToolsSetup, } = useDAWStore( useShallow((state) => ({ tracks: state.tracks, @@ -224,6 +226,7 @@ function App() { showMasterAutomation: state.showMasterAutomation, showStemSeparation: state.showStemSeparation, showAiToolsSetup: state.showAiToolsSetup, + closeAiToolsSetup: state.closeAiToolsSetup, })) ); @@ -259,6 +262,10 @@ function App() { const isProjectLoading = useDAWStore((state) => state.isProjectLoading); const projectLoadingMessage = useDAWStore((state) => state.projectLoadingMessage); + useEffect(() => { + void maybeRunPitchRegressionDriver(); + }, []); + // Toast notification state const toastVisible = useDAWStore((state) => state.toastVisible); const toastMessage = useDAWStore((state) => state.toastMessage); @@ -275,7 +282,21 @@ function App() { const showToast = useDAWStore((state) => state.showToast); const previousAiToolsStateRef = useRef(aiToolsStatus.state); const previousAiToolsInstallInProgressRef = useRef(aiToolsStatus.installInProgress); - const [showAiToolsInstallBlocker, setShowAiToolsInstallBlocker] = useState(false); + const hasNativeMusicProfile = + (aiToolsStatus.musicGenerationAvailableProfiles ?? []).length === 0 + || (aiToolsStatus.musicGenerationAvailableProfiles ?? []).includes("native-xl-turbo"); + const previousAiToolsFullReadyRef = useRef(Boolean( + aiToolsStatus.musicGenerationReady + && aiToolsStatus.musicGenerationLayoutValid + && (aiToolsStatus.musicGenerationPerformanceReady ?? true) + && hasNativeMusicProfile, + )); + const [aiToolsInstallStatusStale, setAiToolsInstallStatusStale] = useState(false); + const aiToolsHasByteProgress = (aiToolsStatus.bytesTotal ?? 0) > 0 && (aiToolsStatus.bytesDownloaded ?? 0) >= 0; + const aiToolsVisualProgressRatio = aiToolsHasByteProgress + ? Math.max(0, Math.min((aiToolsStatus.bytesDownloaded ?? 0) / Math.max(aiToolsStatus.bytesTotal ?? 1, 1), 1)) + : Math.max(0, Math.min(aiToolsStatus.progress ?? 0, 1)); + const aiToolsVisualProgressPercent = Math.round(aiToolsVisualProgressRatio * 100); useEffect(() => startMixerUISync(), []); @@ -292,7 +313,7 @@ function App() { }, [applyAiToolsStatusUpdate]); useEffect(() => { - const shouldPollAiToolsStatus = aiToolsStatus.state === "checking"; + const shouldPollAiToolsStatus = aiToolsStatus.state === "checking" || aiToolsStatus.installInProgress; if (!shouldPollAiToolsStatus) { return; @@ -300,29 +321,50 @@ function App() { const timer = window.setInterval(() => { void refreshAiToolsStatus(); - }, 1500); + }, aiToolsStatus.installInProgress ? 2000 : 1500); return () => window.clearInterval(timer); }, [aiToolsStatus.installInProgress, aiToolsStatus.state, refreshAiToolsStatus]); useEffect(() => { if (!aiToolsStatus.installInProgress) { - setShowAiToolsInstallBlocker(false); + setAiToolsInstallStatusStale(false); return; } const timer = window.setInterval(() => { const staleMs = Date.now() - aiToolsStatusLastUpdatedAt; - setShowAiToolsInstallBlocker(staleMs >= 8000); + const isStale = staleMs >= 8000; + setAiToolsInstallStatusStale(isStale); + if (isStale) { + void refreshAiToolsStatus(); + } }, 1000); return () => window.clearInterval(timer); - }, [aiToolsStatus.installInProgress, aiToolsStatusLastUpdatedAt]); + }, [aiToolsStatus.installInProgress, aiToolsStatusLastUpdatedAt, refreshAiToolsStatus]); useEffect(() => { const previousState = previousAiToolsStateRef.current; const previousInstallInProgress = previousAiToolsInstallInProgressRef.current; + const previousFullReady = previousAiToolsFullReadyRef.current; const currentState = aiToolsStatus.state; + const currentFullReady = Boolean( + currentState === "ready" + && aiToolsStatus.musicGenerationReady + && aiToolsStatus.musicGenerationLayoutValid + && (aiToolsStatus.musicGenerationPerformanceReady ?? true) + && hasNativeMusicProfile, + ); + const currentPartialReady = Boolean( + currentState === "ready" + && ( + !aiToolsStatus.musicGenerationReady + || !aiToolsStatus.musicGenerationLayoutValid + || !(aiToolsStatus.musicGenerationPerformanceReady ?? true) + || !hasNativeMusicProfile + ), + ); const installAttemptStates = new Set([ "checking", "fetching_runtime_manifest", @@ -336,10 +378,28 @@ function App() { "downloading_model", ]); const wasInstallAttempt = previousInstallInProgress || installAttemptStates.has(previousState); + const readyStateChanged = previousState !== currentState; + const fullReadyChanged = previousFullReady !== currentFullReady; - if (previousState !== currentState) { - if (currentState === "ready") { + if (readyStateChanged || fullReadyChanged) { + if (currentFullReady && (readyStateChanged || !previousFullReady)) { showToast("AI tools are ready", "success"); + if (wasInstallAttempt) { + closeAiToolsSetup(); + } + } else if (currentPartialReady && readyStateChanged) { + showToast( + (!hasNativeMusicProfile + ? "Stem separation is ready, but the OpenStudio ACE split profile is still missing required music-generation assets." + : "") + || + aiToolsStatus.musicGenerationPerformanceStatusMessage + || "Stem separation is ready, but music generation is not fully ready yet.", + "info", + ); + if (wasInstallAttempt) { + openAiToolsSetup(); + } } else if (currentState === "error" && aiToolsStatus.error) { showToast(aiToolsStatus.error, "error"); if (wasInstallAttempt) { @@ -357,7 +417,21 @@ function App() { previousAiToolsStateRef.current = currentState; previousAiToolsInstallInProgressRef.current = aiToolsStatus.installInProgress; - }, [aiToolsStatus.error, aiToolsStatus.installInProgress, aiToolsStatus.state, openAiToolsSetup, showToast]); + previousAiToolsFullReadyRef.current = currentFullReady; + }, [ + aiToolsStatus.error, + aiToolsStatus.installInProgress, + aiToolsStatus.musicGenerationAvailableProfiles, + aiToolsStatus.musicGenerationLayoutValid, + aiToolsStatus.musicGenerationPerformanceReady, + aiToolsStatus.musicGenerationPerformanceStatusMessage, + aiToolsStatus.musicGenerationReady, + aiToolsStatus.state, + closeAiToolsSetup, + hasNativeMusicProfile, + openAiToolsSetup, + showToast, + ]); useEffect(() => { const unsubscribe = nativeBridge.subscribe("mixerWindowClosed", (data) => { @@ -913,6 +987,12 @@ function App() { void createTrackOfType("instrument"); }, }, + { + label: "Add AI Track", + onClick: () => { + void createTrackOfType("ai"); + }, + }, { label: "Add MIDI Track", onClick: () => { @@ -1600,49 +1680,6 @@ function App() { )} {/* File Drop Overlay — shown when dragging files from OS file explorer */} - {showAiToolsInstallBlocker && ( - <div className="fixed inset-0 z-[10002] flex items-center justify-center bg-black/75 backdrop-blur-sm"> - <div className="w-[min(52rem,calc(100vw-2rem))] rounded-2xl border border-daw-accent/30 bg-neutral-950 shadow-2xl"> - <div className="border-b border-neutral-800 px-5 py-4"> - <p className="text-base font-semibold text-white">Installing AI Tools</p> - <p className="mt-1 text-sm text-neutral-300"> - OpenStudio is still preparing large AI components. Progress is being monitored in the background. - </p> - </div> - <div className="space-y-4 p-5"> - <div className="space-y-2"> - <div className="flex items-center justify-between text-sm"> - <span className="text-neutral-200">{aiToolsStatus.stepLabel || aiToolsStatus.message || "Installing AI Tools..."}</span> - <span className="text-neutral-400"> - {Math.max(0, Math.round((aiToolsStatus.progress ?? 0) * 100))}% - </span> - </div> - <div className="h-2 overflow-hidden rounded-full bg-neutral-800"> - <div - className="h-full rounded-full bg-daw-accent transition-all duration-300" - style={{ width: `${Math.max(6, Math.min((aiToolsStatus.progress ?? 0) * 100, 100))}%` }} - /> - </div> - </div> - <div className="grid gap-2 text-xs text-neutral-400 sm:grid-cols-3"> - <span>Phase: <span className="text-neutral-200">{aiToolsStatus.lastPhase || aiToolsStatus.state}</span></span> - <span>Elapsed: <span className="text-neutral-200">{Math.max(0, Math.round((aiToolsStatus.elapsedMs ?? 0) / 1000))}s</span></span> - <span>Session: <span className="text-neutral-200">{aiToolsStatus.installSessionId || "n/a"}</span></span> - </div> - <div className="rounded-xl border border-neutral-800 bg-black px-4 py-3 font-mono text-xs text-green-300"> - <div className="max-h-56 space-y-1 overflow-y-auto"> - {(aiToolsStatus.activityLines?.length ? aiToolsStatus.activityLines : [aiToolsStatus.message || "Installing AI Tools..."]).map((line, index) => ( - <div key={`${index}-${line}`} className="break-words"> - {line} - </div> - ))} - </div> - </div> - </div> - </div> - </div> - )} - {isDraggingFiles && ( <div className="fixed inset-0 z-9999 flex items-center justify-center bg-black/60 backdrop-blur-sm pointer-events-none"> <div className="flex flex-col items-center gap-3 px-8 py-6 rounded-xl border-2 border-dashed border-daw-accent bg-neutral-900/90 shadow-2xl"> @@ -1693,15 +1730,19 @@ function App() { <div className="h-2 overflow-hidden rounded-full bg-neutral-800"> <div className="h-full rounded-full bg-daw-accent transition-all duration-300" - style={{ width: `${Math.max(6, Math.min((aiToolsStatus.progress ?? 0) * 100, 100))}%` }} + style={{ width: `${Math.max(6, Math.min(aiToolsVisualProgressPercent, 100))}%` }} /> </div> <div className="mt-2 flex items-center justify-between text-[11px] text-neutral-400"> <span>{aiToolsStatus.stepLabel || aiToolsStatus.message || "Preparing optional AI tools..."}</span> - <span>{Math.round((aiToolsStatus.progress ?? 0) * 100)}%</span> + <span>{aiToolsVisualProgressPercent}%</span> </div> {aiToolsStatus.statusWarning ? ( <p className="mt-2 text-[11px] text-yellow-300">{aiToolsStatus.statusWarning}</p> + ) : aiToolsInstallStatusStale ? ( + <p className="mt-2 text-[11px] text-yellow-300"> + Progress updates are delayed, but OpenStudio is still checking the installer in the background. + </p> ) : null} </div> diff --git a/frontend/src/__tests__/aiToolsSetupModalWindowsLock.test.ts b/frontend/src/__tests__/aiToolsSetupModalWindowsLock.test.ts new file mode 100644 index 0000000..3e9abef --- /dev/null +++ b/frontend/src/__tests__/aiToolsSetupModalWindowsLock.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, it } from "vitest"; +import modalSource from "../components/AiToolsSetupModal.tsx?raw"; + +describe("AI tools setup modal Windows lock guidance", () => { + it("keeps the runtime lock failure reasons and recovery copy in the modal", () => { + expect(modalSource).toContain("runtime_locked_rebuild_failed"); + expect(modalSource).toContain("runtime_rebuild_remove_failed"); + expect(modalSource).toContain("runtime-only rebuild"); + expect(modalSource).toContain("Retry keeps the downloaded stem models and ACE-Step checkpoints in place."); + expect(modalSource).toContain("Reset AI Tools"); + }); +}); diff --git a/frontend/src/__tests__/aiWorkflowParams.test.ts b/frontend/src/__tests__/aiWorkflowParams.test.ts new file mode 100644 index 0000000..3b0e1e1 --- /dev/null +++ b/frontend/src/__tests__/aiWorkflowParams.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "vitest"; +import { + getDefaultWorkflowParams, + normalizeWorkflowParams, +} from "../data/aiWorkflows"; + +describe("AI workflow params", () => { + it("provides the fixed ACE-Step parameter surface for text-to-music", () => { + const defaults = getDefaultWorkflowParams("text-to-music"); + + expect(defaults).toMatchObject({ + prompt: "", + lyrics: "", + seed: -1, + bpm: 120, + duration: 30, + timesignature: "4/4", + language: "en", + keyscale: "C major", + generate_audio_codes: true, + cfg_scale: 2, + guidance_scale: 1, + shift: 3, + temperature: 0.85, + top_p: 0.9, + top_k: 0, + min_p: 0, + }); + }); + + it("normalizes and clamps AI workflow params to the pinned schema", () => { + const normalized = normalizeWorkflowParams("lyrics+style", { + prompt: 123, + lyrics: null, + seed: "42", + bpm: "999", + duration: "-50", + timesignature: "3", + language: "xx", + keyscale: "not-a-key", + generate_audio_codes: "false", + cfg_scale: "4.5", + guidance_scale: "21", + shift: "9.5", + temperature: "1.25", + top_p: "0.95", + top_k: "12", + min_p: "0.123", + ignored: "value", + }); + + expect(normalized).toEqual({ + prompt: "123", + lyrics: "", + seed: 42, + bpm: 240, + duration: 5, + timesignature: "3/4", + language: "en", + keyscale: "C major", + generate_audio_codes: false, + inferenceSteps: 8, + cfg_scale: 4.5, + guidance_scale: 20, + shift: 5, + temperature: 1.25, + top_p: 0.95, + top_k: 12, + min_p: 0.123, + }); + }); + + it("preserves explicit musical metadata in the native parity request surface", () => { + const normalized = normalizeWorkflowParams("text-to-music", { + bpm: 170, + duration: 191, + timesignature: "3/4", + keyscale: "C# minor", + }); + + expect(normalized).toMatchObject({ + bpm: 170, + duration: 191, + timesignature: "3/4", + keyscale: "C# minor", + generate_audio_codes: true, + }); + }); + + it("keeps generate_audio_codes enabled by default", () => { + const normalized = normalizeWorkflowParams("text-to-music", { + generate_audio_codes: true, + }); + + expect(normalized).toMatchObject({ + bpm: 120, + duration: 30, + timesignature: "4/4", + keyscale: "C major", + generate_audio_codes: true, + }); + }); +}); diff --git a/frontend/src/__tests__/pitchEditorSingleNoteOwnership.test.ts b/frontend/src/__tests__/pitchEditorSingleNoteOwnership.test.ts new file mode 100644 index 0000000..973c5a7 --- /dev/null +++ b/frontend/src/__tests__/pitchEditorSingleNoteOwnership.test.ts @@ -0,0 +1,86 @@ +import { beforeEach, afterEach, describe, expect, it, vi } from "vitest"; +import type { PitchNoteData } from "../services/NativeBridge"; +import { usePitchEditorStore } from "../store/pitchEditorStore"; + +function makeNote(id: string, correctedPitch: number, wordGroupId = "word_a"): PitchNoteData { + const index = Number(id.replace(/\D/g, "")) || 0; + return { + id, + startTime: index * 0.5, + endTime: index * 0.5 + 0.4, + effectiveStartTime: index * 0.5, + effectiveEndTime: index * 0.5 + 0.4, + detectedPitch: correctedPitch, + correctedPitch, + driftCorrectionAmount: 0, + vibratoDepth: 1, + vibratoRate: 0, + transitionIn: 40, + transitionOut: 60, + formantShift: 0, + gain: 0, + voiced: true, + wordGroupId, + pitchDrift: [], + }; +} + +describe("pitch editor single-note ownership", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.stubGlobal("window", { + location: { hostname: "test.local" }, + }); + usePitchEditorStore.setState({ + trackId: "track-a", + clipId: "clip-a", + clipDuration: 3, + contour: null, + notes: [ + makeNote("note1", 60, "word_a"), + makeNote("note2", 62, "word_a"), + makeNote("note3", 64, "word_b"), + ], + selectedNoteIds: [], + undoStack: [], + redoStack: [], + renderCoverage: [], + applyState: "idle", + applyMessage: "", + lastApplyRequestId: null, + activeLogicalRequestId: null, + }); + }); + + afterEach(() => { + vi.clearAllTimers(); + vi.unstubAllGlobals(); + vi.useRealTimers(); + }); + + it("selects only the clicked note even when nearby notes share a word group", () => { + usePitchEditorStore.getState().selectNote("note1"); + + expect(usePitchEditorStore.getState().selectedNoteIds).toEqual(["note1"]); + }); + + it("updates only the requested note during a drag", () => { + usePitchEditorStore.getState().updateNote("note1", { correctedPitch: 66 }); + + const notes = usePitchEditorStore.getState().notes; + expect(notes.find((note) => note.id === "note1")?.correctedPitch).toBe(66); + expect(notes.find((note) => note.id === "note2")?.correctedPitch).toBe(62); + expect(notes.find((note) => note.id === "note3")?.correctedPitch).toBe(64); + }); + + it("bulk pitch actions affect only explicitly selected notes", () => { + usePitchEditorStore.setState({ selectedNoteIds: ["note2"] }); + + usePitchEditorStore.getState().moveSelectedPitch(1); + + const notes = usePitchEditorStore.getState().notes; + expect(notes.find((note) => note.id === "note1")?.correctedPitch).toBe(60); + expect(notes.find((note) => note.id === "note2")?.correctedPitch).toBe(63); + expect(notes.find((note) => note.id === "note3")?.correctedPitch).toBe(64); + }); +}); diff --git a/frontend/src/__tests__/rulerClickSnap.test.ts b/frontend/src/__tests__/rulerClickSnap.test.ts new file mode 100644 index 0000000..bb04a08 --- /dev/null +++ b/frontend/src/__tests__/rulerClickSnap.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; +import { getRulerClickSnapTime } from "../utils/rulerClickSnap"; + +const TS_4_4 = { numerator: 4, denominator: 4 }; + +describe("getRulerClickSnapTime", () => { + it("prefers nearest beat when within the ruler snap threshold", () => { + expect( + getRulerClickSnapTime({ + time: 0.52, + pixelsPerSecond: 100, + tempo: 120, + timeSignature: TS_4_4, + gridSize: "quarter_beat", + snapEnabled: false, + }) + ).toBe(0.5); + }); + + it("falls back to grid snap when beat snap is too far and grid snap is close", () => { + expect( + getRulerClickSnapTime({ + time: 0.37, + pixelsPerSecond: 100, + tempo: 120, + timeSignature: TS_4_4, + gridSize: "quarter_beat", + snapEnabled: true, + }) + ).toBe(0.375); + }); + + it("returns the raw time when no snap target is within threshold", () => { + expect( + getRulerClickSnapTime({ + time: 0.28, + pixelsPerSecond: 100, + tempo: 120, + timeSignature: TS_4_4, + gridSize: "minute", + snapEnabled: true, + }) + ).toBe(0.28); + }); + + it("uses beat snap even when ctrl bypass is active", () => { + expect( + getRulerClickSnapTime({ + time: 0.52, + pixelsPerSecond: 100, + tempo: 120, + timeSignature: TS_4_4, + gridSize: "quarter_beat", + snapEnabled: true, + ctrlBypass: true, + }) + ).toBe(0.5); + }); + + it("skips grid fallback when ctrl bypass is active", () => { + expect( + getRulerClickSnapTime({ + time: 0.37, + pixelsPerSecond: 100, + tempo: 120, + timeSignature: TS_4_4, + gridSize: "quarter_beat", + snapEnabled: true, + ctrlBypass: true, + }) + ).toBe(0.37); + }); +}); diff --git a/frontend/src/components/AITrackHeader.tsx b/frontend/src/components/AITrackHeader.tsx new file mode 100644 index 0000000..7e9659d --- /dev/null +++ b/frontend/src/components/AITrackHeader.tsx @@ -0,0 +1,898 @@ +import React, { useEffect, useRef, useState } from "react"; +import { createPortal } from "react-dom"; +import classNames from "classnames"; +import { Sparkles, SlidersHorizontal, Wand2 } from "lucide-react"; +import { useShallow } from "zustand/react/shallow"; +import { AI_WORKFLOWS, getAIWorkflow } from "../data/aiWorkflows"; +import { nativeBridge, type AIGenerationProgress } from "../services/NativeBridge"; +import { + getEffectiveTrackHeight, + type AITrackGenerationState, + type Track, + useDAWStore, +} from "../store/useDAWStore"; +import { ColorPicker } from "./ColorPicker"; +import { AIWorkflowModal } from "./AIWorkflowModal"; +import { Button, Input, Select } from "./ui"; +import { + TCP_HEADER_BUTTON_PAIR_CLASS, + TCP_HEADER_PRIMARY_BUTTON_CLASS, + TCP_HEADER_TOGGLE_BUTTON_CLASS, +} from "./tcpHeaderButtonStyles"; + +interface AITrackHeaderProps { + track: Track; + isSelected?: boolean; +} + +function formatPhaseLabel(phase?: string) { + if (!phase) { + return "Preparing"; + } + + return phase + .split("_") + .filter(Boolean) + .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) + .join(" "); +} + +function formatElapsedLabel(elapsedMs?: number) { + if (!elapsedMs || elapsedMs <= 0) { + return ""; + } + + const totalSeconds = Math.max(0, Math.floor(elapsedMs / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return minutes > 0 ? `${minutes}m ${seconds}s` : `${seconds}s`; +} + +function formatEtaLabel(etaMs?: number) { + if (!etaMs || etaMs <= 0) { + return ""; + } + + const totalSeconds = Math.max(0, Math.floor(etaMs / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return minutes > 0 ? `${minutes}m ${seconds}s left` : `${seconds}s left`; +} + +function formatProgressAgeLabel(ageMs?: number) { + if (!ageMs || ageMs <= 0) { + return ""; + } + + const totalSeconds = Math.max(0, Math.floor(ageMs / 1000)); + if (totalSeconds < 1) { + return "<1s since progress"; + } + + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return minutes > 0 + ? `${minutes}m ${seconds}s since progress` + : `${seconds}s since progress`; +} + +function formatRuntimeProfileLabel(profile?: string) { + switch (profile) { + case "native-xl-turbo": + case "openstudio-ace-split": + return "OpenStudio ACE Split"; + default: + return profile ? formatPhaseLabel(profile) : ""; + } +} + +function formatLmModelLabel(lmModel?: string) { + if (!lmModel) { + return ""; + } + if (lmModel === "auto") { + return "Auto LM"; + } + if (lmModel.endsWith(".safetensors")) { + return lmModel + .replace("qwen_", "Qwen ") + .replace("_ace15.safetensors", "") + .replace("_", " ") + .replace("b", "B"); + } + return lmModel.replace("acestep-5Hz-lm-", "LM "); +} + +function formatSessionModeLabel(sessionMode?: string) { + switch (sessionMode) { + case "persistent": + return "Persistent session"; + case "oneshot-fallback": + return "One-shot fallback"; + case "oneshot": + return "One-shot"; + default: + return sessionMode ? formatPhaseLabel(sessionMode) : ""; + } +} + +function formatAttemptModeLabel(attemptMode?: string) { + switch (attemptMode) { + case "lm_dit": + return "LM First"; + case "dit_only": + return "Direct DiT"; + case "native_split_graph": + return "Native Split Graph"; + case "legacy_ace_wrapper": + return "Legacy Wrapper"; + default: + return attemptMode ? formatPhaseLabel(attemptMode) : ""; + } +} + +function formatFailureKindLabel(failureKind?: string) { + switch (failureKind) { + case "native_asset_missing": + return "Native asset missing"; + case "native_conditioning_failure": + return "Native conditioning failure"; + case "native_sampling_failure": + return "Native sampling failure"; + case "native_decode_failure": + return "Native decode failure"; + default: + return failureKind ? formatPhaseLabel(failureKind) : ""; + } +} + +function formatProgressLabel(progress: number) { + return `${Math.max(0, Math.round(progress * 100))}%`; +} + +function getProgressWidth(progress: number) { + return `${Math.max(4, Math.round(progress * 100))}%`; +} + +function getDisplayState( + progress: AIGenerationProgress, +): AITrackGenerationState { + if (progress.state === "error") { + return "error"; + } + + if (progress.state === "loading") { + return "loading"; + } + + return "generating"; +} + +function getStatusHeadline(track: Track) { + const workflow = getAIWorkflow(track.aiWorkflow); + + if (workflow.available === false) { + return workflow.availabilityNote || "Workflow unavailable"; + } + + if (track.aiGenerationState === "error" && track.aiGenerationError) { + if (track.aiGenerationFailureKind === "worker_protocol") { + return `Worker session failed to start: ${track.aiGenerationError}`; + } + if (track.aiGenerationFailureKind === "decode_stalled") { + return `Decode stalled: ${track.aiGenerationError}`; + } + if (track.aiGenerationFailureKind === "native_conditioning_failure") { + return `ACE-Step conditioning failed: ${track.aiGenerationError}`; + } + return track.aiGenerationError; + } + + if (track.aiGenerationState === "loading" || track.aiGenerationState === "generating") { + return track.aiGenerationMessage + || `${formatPhaseLabel(track.aiGenerationPhase)} ${formatProgressLabel(track.aiGenerationProgress ?? 0)}`; + } + + return "Ready to generate"; +} + +function getStatusMeta(track: Track) { + if (track.aiGenerationState === "error") { + const parts = [ + track.aiGenerationFailureKind === "worker_protocol" + ? "Worker protocol failure" + : track.aiGenerationFailureKind === "decode_stalled" + ? "Decode stalled" + : formatFailureKindLabel(track.aiGenerationFailureKind), + track.aiGenerationPhase + ? `Last phase: ${formatPhaseLabel(track.aiGenerationPhase)}` + : "Generation failed", + track.aiGenerationSessionMode + ? formatSessionModeLabel(track.aiGenerationSessionMode) + : "", + track.aiGenerationAttemptMode + ? formatAttemptModeLabel(track.aiGenerationAttemptMode) + : "", + track.aiGenerationLmStage + ? `LM stage: ${formatPhaseLabel(track.aiGenerationLmStage)}` + : "", + track.aiGenerationAttemptIndex && track.aiGenerationAttemptIndex > 1 + ? `Attempt ${track.aiGenerationAttemptIndex}` + : "", + formatProgressAgeLabel(track.aiGenerationLastProgressAgeMs), + track.aiGenerationWorkerExitCode + ? `Exit ${track.aiGenerationWorkerExitCode}` + : "", + ].filter(Boolean); + + return parts.join(" · "); + } + + if (track.aiGenerationState === "loading" || track.aiGenerationState === "generating") { + const parts = [ + track.aiGenerationRunMode ? track.aiGenerationRunMode.toUpperCase() : "", + track.aiGenerationBackend + ? track.aiGenerationBackend.toUpperCase() + : "", + track.aiGenerationSessionMode + ? formatSessionModeLabel(track.aiGenerationSessionMode) + : "", + track.aiGenerationAttemptMode + ? formatAttemptModeLabel(track.aiGenerationAttemptMode) + : "", + track.aiGenerationLmStage + ? `LM stage: ${formatPhaseLabel(track.aiGenerationLmStage)}` + : "", + track.aiGenerationAttemptIndex && track.aiGenerationAttemptIndex > 1 + ? `Attempt ${track.aiGenerationAttemptIndex}` + : "", + formatElapsedLabel(track.aiGenerationElapsedMs), + track.aiGenerationFailureKind === "decode_stalled" + ? "" + : formatEtaLabel(track.aiGenerationEtaMs), + (track.aiGenerationLastProgressAgeMs ?? 0) >= 10000 + ? formatProgressAgeLabel(track.aiGenerationLastProgressAgeMs) + : "", + formatRuntimeProfileLabel(track.aiGenerationRuntimeProfile), + formatLmModelLabel(track.aiGenerationLmModel), + ].filter(Boolean); + + return parts.length > 0 ? parts.join(" • ") : "Worker active"; + } + + return track.aiGenerationBackend + ? `Last backend: ${track.aiGenerationBackend.toUpperCase()}` + : "Music generation idle"; +} + +export const AITrackHeader = React.memo(function AITrackHeader({ + track, + isSelected, +}: AITrackHeaderProps) { + const { + updateTrack, + toggleTrackMute, + setAITrackWorkflow, + setAITrackParams, + setAITrackGenerationState, + addGeneratedAudioClip, + trackHeight, + aiToolsStatus, + openAiToolsSetup, + } = useDAWStore( + useShallow((state) => ({ + updateTrack: state.updateTrack, + toggleTrackMute: state.toggleTrackMute, + setAITrackWorkflow: state.setAITrackWorkflow, + setAITrackParams: state.setAITrackParams, + setAITrackGenerationState: state.setAITrackGenerationState, + addGeneratedAudioClip: state.addGeneratedAudioClip, + trackHeight: state.trackHeight, + aiToolsStatus: state.aiToolsStatus, + openAiToolsSetup: state.openAiToolsSetup, + })), + ); + + const colorBarRef = useRef<HTMLDivElement>(null); + const pollTimeoutRef = useRef<number | null>(null); + const pollActiveRef = useRef(false); + const generationStartTimeRef = useRef<number | null>(null); + const [showColorPicker, setShowColorPicker] = useState(false); + const [showParams, setShowParams] = useState(false); + const workflow = getAIWorkflow(track.aiWorkflow); + const isBusy = + track.aiGenerationState === "loading" + || track.aiGenerationState === "generating"; + const canStartMusicGeneration = + workflow.available !== false + && aiToolsStatus.musicGenerationReady + && aiToolsStatus.musicGenerationLayoutValid + && (aiToolsStatus.musicGenerationPerformanceReady ?? true); + const musicGenerationBlockedMessage = + aiToolsStatus.musicGenerationPerformanceStatusMessage + || aiToolsStatus.musicGenerationStatusMessage + || (!aiToolsStatus.musicGenerationLayoutValid + && aiToolsStatus.musicGenerationModelId + && aiToolsStatus.musicGenerationCheckpointRoot + ? `Pinned ACE-Step native asset layout is not ready in ${aiToolsStatus.musicGenerationCheckpointRoot}.` + : aiToolsStatus.error + || aiToolsStatus.message + || "AI music generation is not ready yet."); + + const stopPolling = () => { + if (pollTimeoutRef.current !== null) { + window.clearTimeout(pollTimeoutRef.current); + pollTimeoutRef.current = null; + } + pollActiveRef.current = false; + }; + + const scheduleNextPoll = (startTime: number) => { + stopPolling(); + pollTimeoutRef.current = window.setTimeout(() => { + void pollGeneration(startTime); + }, 250); + }; + + const applyProgressUpdate = async ( + progress: AIGenerationProgress, + startTime: number, + ) => { + if (progress.state === "error") { + stopPolling(); + generationStartTimeRef.current = null; + setAITrackGenerationState(track.id, "error", { + progress: progress.progress ?? 0, + error: progress.error || progress.message || "Generation failed.", + phase: progress.phase, + message: progress.message || progress.error || "Generation failed.", + backend: progress.backend || track.aiGenerationBackend || "", + elapsedMs: progress.elapsedMs ?? 0, + heartbeatTs: progress.heartbeatTs ?? 0, + phaseProgress: progress.phaseProgress, + etaMs: progress.etaMs, + runMode: progress.runMode, + runtimeProfile: progress.runtimeProfile, + lmModel: progress.lmModel, + statusNote: progress.statusNote, + failureKind: progress.failureKind, + sessionMode: progress.sessionMode, + workerExitCode: progress.workerExitCode, + lastStdoutLine: progress.lastStdoutLine, + lastStderrLine: progress.lastStderrLine, + attemptMode: progress.attemptMode, + attemptIndex: progress.attemptIndex, + protocolVersion: progress.protocolVersion, + scriptVersion: progress.scriptVersion, + requestId: progress.requestId, + priorFailure: progress.priorFailure, + lastProgressAgeMs: progress.lastProgressAgeMs, + tracePath: progress.tracePath, + failureDetail: progress.failureDetail, + lmBackend: progress.lmBackend, + lmStage: progress.lmStage, + }); + return; + } + + if (progress.state === "cancelled") { + stopPolling(); + generationStartTimeRef.current = null; + setAITrackGenerationState(track.id, "idle"); + return; + } + + if (progress.state === "done") { + stopPolling(); + generationStartTimeRef.current = null; + + if (!progress.outputFile) { + setAITrackGenerationState(track.id, "error", { + progress: progress.progress ?? 1, + error: "Generation finished without producing an audio file.", + phase: progress.phase || "done", + message: progress.message || "Generation finished without producing an audio file.", + backend: progress.backend || track.aiGenerationBackend || "", + elapsedMs: progress.elapsedMs ?? 0, + heartbeatTs: progress.heartbeatTs ?? 0, + phaseProgress: progress.phaseProgress, + etaMs: progress.etaMs, + runMode: progress.runMode, + runtimeProfile: progress.runtimeProfile, + lmModel: progress.lmModel, + statusNote: progress.statusNote, + failureKind: progress.failureKind, + sessionMode: progress.sessionMode, + workerExitCode: progress.workerExitCode, + lastStdoutLine: progress.lastStdoutLine, + lastStderrLine: progress.lastStderrLine, + attemptMode: progress.attemptMode, + attemptIndex: progress.attemptIndex, + protocolVersion: progress.protocolVersion, + scriptVersion: progress.scriptVersion, + requestId: progress.requestId, + priorFailure: progress.priorFailure, + lastProgressAgeMs: progress.lastProgressAgeMs, + tracePath: progress.tracePath, + failureDetail: progress.failureDetail, + lmBackend: progress.lmBackend, + lmStage: progress.lmStage, + }); + return; + } + + try { + await addGeneratedAudioClip(track.id, progress.outputFile, startTime); + setAITrackGenerationState(track.id, "idle"); + setShowParams(false); + } catch (error) { + setAITrackGenerationState(track.id, "error", { + progress: progress.progress ?? 1, + error: + error instanceof Error + ? error.message + : "Audio rendered, but the generated clip could not be imported.", + phase: "import_failed", + message: "Audio rendered, but clip import failed.", + backend: progress.backend || track.aiGenerationBackend || "", + elapsedMs: progress.elapsedMs ?? 0, + heartbeatTs: progress.heartbeatTs ?? 0, + phaseProgress: progress.phaseProgress, + etaMs: progress.etaMs, + runMode: progress.runMode, + runtimeProfile: progress.runtimeProfile, + lmModel: progress.lmModel, + statusNote: progress.statusNote, + failureKind: progress.failureKind, + sessionMode: progress.sessionMode, + workerExitCode: progress.workerExitCode, + lastStdoutLine: progress.lastStdoutLine, + lastStderrLine: progress.lastStderrLine, + attemptMode: progress.attemptMode, + attemptIndex: progress.attemptIndex, + protocolVersion: progress.protocolVersion, + scriptVersion: progress.scriptVersion, + requestId: progress.requestId, + priorFailure: progress.priorFailure, + lastProgressAgeMs: progress.lastProgressAgeMs, + tracePath: progress.tracePath, + failureDetail: progress.failureDetail, + lmBackend: progress.lmBackend, + lmStage: progress.lmStage, + }); + } + return; + } + + setAITrackGenerationState(track.id, getDisplayState(progress), { + progress: progress.progress ?? 0, + error: "", + phase: progress.phase, + message: + progress.message + || `${formatPhaseLabel(progress.phase)} ${formatProgressLabel(progress.progress ?? 0)}`, + backend: progress.backend || track.aiGenerationBackend || "", + elapsedMs: progress.elapsedMs ?? 0, + heartbeatTs: progress.heartbeatTs ?? 0, + phaseProgress: progress.phaseProgress, + etaMs: progress.etaMs, + runMode: progress.runMode, + runtimeProfile: progress.runtimeProfile, + lmModel: progress.lmModel, + statusNote: progress.statusNote, + failureKind: progress.failureKind, + sessionMode: progress.sessionMode, + workerExitCode: progress.workerExitCode, + lastStdoutLine: progress.lastStdoutLine, + lastStderrLine: progress.lastStderrLine, + attemptMode: progress.attemptMode, + attemptIndex: progress.attemptIndex, + protocolVersion: progress.protocolVersion, + scriptVersion: progress.scriptVersion, + requestId: progress.requestId, + priorFailure: progress.priorFailure, + lastProgressAgeMs: progress.lastProgressAgeMs, + tracePath: progress.tracePath, + failureDetail: progress.failureDetail, + lmBackend: progress.lmBackend, + lmStage: progress.lmStage, + }); + + scheduleNextPoll(startTime); + }; + + const pollGeneration = async (startTime: number) => { + if (pollActiveRef.current) { + return; + } + + pollActiveRef.current = true; + + try { + const progress = await nativeBridge.getAIGenerationProgress(); + await applyProgressUpdate(progress, startTime); + } catch (error) { + stopPolling(); + generationStartTimeRef.current = null; + setAITrackGenerationState(track.id, "error", { + progress: 0, + error: error instanceof Error ? error.message : "Generation failed.", + phase: "poll_failed", + message: "The app lost contact with the generation worker.", + tracePath: "", + failureDetail: "", + lmBackend: "", + lmStage: "", + }); + } finally { + pollActiveRef.current = false; + } + }; + + useEffect(() => { + if (!isBusy) { + stopPolling(); + } + + return stopPolling; + }, [isBusy]); + + const handleGenerate = async () => { + if (isBusy) { + await nativeBridge.cancelAIGeneration(); + stopPolling(); + generationStartTimeRef.current = null; + setAITrackGenerationState(track.id, "idle"); + return; + } + + const startTime = useDAWStore.getState().transport.currentTime; + const workflowId = track.aiWorkflow ?? "text-to-music"; + const params = { ...(track.aiWorkflowParams ?? {}) }; + + if (workflow.available === false) { + setAITrackGenerationState(track.id, "error", { + progress: 0, + error: + workflow.availabilityNote + || "This workflow is not currently available in OpenStudio.", + phase: "workflow_unavailable", + message: + workflow.availabilityNote + || "This workflow is not currently available in OpenStudio.", + tracePath: "", + failureDetail: "", + lmBackend: "", + lmStage: "", + }); + return; + } + + if ( + !aiToolsStatus.musicGenerationReady + || !aiToolsStatus.musicGenerationLayoutValid + || !(aiToolsStatus.musicGenerationPerformanceReady ?? true) + ) { + setAITrackGenerationState(track.id, "error", { + progress: 0, + error: musicGenerationBlockedMessage, + phase: "runtime_blocked", + message: musicGenerationBlockedMessage, + tracePath: "", + failureDetail: "", + lmBackend: "", + lmStage: "", + }); + openAiToolsSetup(); + return; + } + + generationStartTimeRef.current = startTime; + setAITrackGenerationState(track.id, "loading", { + progress: 0.01, + error: "", + phase: "starting", + message: "Starting ACE-Step...", + backend: "", + elapsedMs: 0, + heartbeatTs: 0, + phaseProgress: undefined, + etaMs: undefined, + runMode: "cold", + runtimeProfile: "openstudio-ace-split", + lmModel: "", + statusNote: "", + failureKind: "", + sessionMode: "persistent", + workerExitCode: 0, + lastStdoutLine: "", + lastStderrLine: "", + attemptMode: "native_split_graph", + attemptIndex: 1, + protocolVersion: 0, + scriptVersion: "", + requestId: "", + priorFailure: "", + lastProgressAgeMs: 0, + tracePath: "", + failureDetail: "", + lmBackend: "", + lmStage: "", + }); + + try { + const result = await nativeBridge.startAIGeneration( + track.id, + workflowId, + params, + ); + + if (!result.started) { + generationStartTimeRef.current = null; + setAITrackGenerationState(track.id, "error", { + progress: 0, + error: result.error || "Failed to start AI generation.", + phase: "start_failed", + message: result.error || "Failed to start AI generation.", + tracePath: "", + failureDetail: "", + lmBackend: "", + lmStage: "", + }); + return; + } + + void pollGeneration(startTime); + } catch (error) { + stopPolling(); + generationStartTimeRef.current = null; + setAITrackGenerationState(track.id, "error", { + progress: 0, + error: error instanceof Error ? error.message : "Generation failed.", + phase: "start_failed", + message: "Failed to start AI generation.", + tracePath: "", + failureDetail: "", + lmBackend: "", + lmStage: "", + }); + } + }; + + const statusHeadline = getStatusHeadline(track); + const statusMeta = getStatusMeta(track); + + return ( + <> + <div + className={`flex flex-col border-b border-neutral-900 relative overflow-hidden box-border ${isSelected ? "bg-neutral-700" : "bg-neutral-800"}`} + style={{ height: getEffectiveTrackHeight(track, trackHeight) }} + > + <div className="flex shrink-0 overflow-hidden" style={{ height: trackHeight }}> + <div + ref={colorBarRef} + onClick={() => setShowColorPicker(true)} + className="w-2 shrink-0 cursor-pointer hover:brightness-125 transition-all relative group/color" + style={{ background: track.color || "#666" }} + title="Click to change track color" + data-color-bar + data-no-select + data-no-drag + > + <div className="absolute inset-0 bg-white/20 opacity-0 group-hover/color:opacity-100 transition-opacity" /> + </div> + {showColorPicker && + createPortal( + <ColorPicker + currentColor={track.color} + anchorRef={colorBarRef} + onColorChange={(color) => { + updateTrack(track.id, { color }); + setShowColorPicker(false); + }} + onClose={() => setShowColorPicker(false)} + />, + document.body, + )} + + <div className="flex-1 min-w-0 px-2 py-1.5"> + <div className="flex items-center gap-1.5"> + <span className="inline-flex items-center gap-1 rounded-full border border-cyan-500/40 bg-cyan-500/10 px-2 py-0.5 text-[10px] font-semibold uppercase tracking-[0.16em] text-cyan-300"> + <Sparkles size={11} /> + AI + </span> + <div className="min-w-0 flex-1" data-no-drag data-no-select> + <Select + value={track.aiWorkflow ?? "text-to-music"} + onChange={(value) => setAITrackWorkflow(track.id, String(value))} + options={AI_WORKFLOWS.map((entry) => ({ + value: entry.id, + label: entry.label, + disabled: entry.available === false, + }))} + size="sm" + fullWidth + /> + </div> + <Button + variant={isBusy ? "danger" : "primary"} + size="sm" + onClick={() => void handleGenerate()} + disabled={!canStartMusicGeneration && !isBusy} + data-no-drag + data-no-select + className="shrink-0" + > + <Wand2 size={12} /> + {isBusy ? "Cancel" : "Generate"} + </Button> + </div> + + <div className="mt-1.5"> + <Input + type="text" + variant="inline" + size="sm" + value={track.name} + onChange={(event) => updateTrack(track.id, { name: event.target.value })} + placeholder="AI Track Name" + inputClassName="w-full min-w-0" + /> + </div> + + <div className="mt-1.5 flex items-start gap-2"> + <span + className={classNames( + TCP_HEADER_BUTTON_PAIR_CLASS, + "shrink-0", + )} + > + <Button + variant="default" + size="icon-sm" + shape="square" + active={Boolean(track.muted)} + onClick={() => toggleTrackMute(track.id)} + title={track.muted ? "Unmute track" : "Mute track"} + aria-label={track.muted ? "Unmute track" : "Mute track"} + className={TCP_HEADER_PRIMARY_BUTTON_CLASS} + data-no-drag + data-no-select + > + M + </Button> + <Button + variant="default" + size="icon-sm" + shape="square" + active={showParams} + onClick={() => setShowParams(true)} + title="Open AI parameters" + aria-label="Open AI parameters" + className={TCP_HEADER_TOGGLE_BUTTON_CLASS} + data-no-drag + data-no-select + > + <SlidersHorizontal size={12} /> + </Button> + </span> + + <div className="min-w-0 flex-1"> + <div + className={classNames( + "truncate text-[11px] leading-4", + track.aiGenerationState === "error" + ? "text-red-300" + : "text-daw-text", + )} + title={statusHeadline} + > + {statusHeadline} + </div> + <div className="mt-0.5 flex items-center gap-1.5 text-[10px] uppercase tracking-[0.12em] text-daw-text-muted"> + {track.aiGenerationBackend ? ( + <span className="rounded-full border border-neutral-700 bg-neutral-900/80 px-1.5 py-0.5 text-[9px] text-daw-text"> + {track.aiGenerationBackend.toUpperCase()} + </span> + ) : null} + {track.aiGenerationRunMode ? ( + <span className="rounded-full border border-neutral-700 bg-neutral-900/80 px-1.5 py-0.5 text-[9px] text-daw-text"> + {track.aiGenerationRunMode.toUpperCase()} + </span> + ) : null} + <span className="truncate" title={statusMeta}> + {statusMeta} + </span> + </div> + </div> + </div> + + {isBusy ? ( + <div className="mt-1.5 space-y-1"> + <div className="h-1.5 w-full rounded-full bg-neutral-900"> + <div + className="h-1.5 rounded-full bg-cyan-400 transition-all duration-200" + style={{ + width: getProgressWidth(track.aiGenerationProgress ?? 0), + }} + /> + </div> + <div className="flex items-center justify-between text-[10px] uppercase tracking-[0.12em] text-daw-text-muted"> + <span className="truncate">{formatPhaseLabel(track.aiGenerationPhase)}</span> + <span>{formatProgressLabel(track.aiGenerationProgress ?? 0)}</span> + </div> + {track.aiGenerationStatusNote ? ( + <div + className="text-[10px] leading-4 text-daw-text-muted" + title={track.aiGenerationStatusNote} + > + {track.aiGenerationStatusNote} + </div> + ) : null} + {track.aiGenerationPriorFailure ? ( + <div + className="text-[10px] leading-4 text-daw-text-muted" + title={track.aiGenerationPriorFailure} + > + Prior failure: {track.aiGenerationPriorFailure} + </div> + ) : null} + {(track.aiGenerationLastProgressAgeMs ?? 0) >= 10000 ? ( + <div className="text-[10px] leading-4 text-daw-text-muted"> + {formatProgressAgeLabel(track.aiGenerationLastProgressAgeMs)} + </div> + ) : null} + </div> + ) : null} + {track.aiGenerationState === "error" ? ( + <div className="mt-1.5 space-y-1 text-[10px] leading-4 text-daw-text-muted"> + {track.aiGenerationFailureDetail ? ( + <div title={track.aiGenerationFailureDetail}> + Detail: {track.aiGenerationFailureDetail} + </div> + ) : null} + {track.aiGenerationLmBackend ? ( + <div title={track.aiGenerationLmBackend}> + LM backend: {track.aiGenerationLmBackend} + </div> + ) : null} + {track.aiGenerationLmStage ? ( + <div title={track.aiGenerationLmStage}> + LM stage: {formatPhaseLabel(track.aiGenerationLmStage)} + </div> + ) : null} + {track.aiGenerationRequestId ? ( + <div className="truncate" title={track.aiGenerationRequestId}> + Request: {track.aiGenerationRequestId} + </div> + ) : null} + {track.aiGenerationTracePath ? ( + <div className="truncate" title={track.aiGenerationTracePath}> + Trace: {track.aiGenerationTracePath} + </div> + ) : null} + </div> + ) : null} + </div> + </div> + </div> + + <AIWorkflowModal + track={track} + aiToolsStatus={aiToolsStatus} + isOpen={showParams} + onClose={() => setShowParams(false)} + onGenerate={handleGenerate} + onCancel={async () => { + await nativeBridge.cancelAIGeneration(); + stopPolling(); + generationStartTimeRef.current = null; + setAITrackGenerationState(track.id, "idle"); + }} + onOpenAiToolsSetup={openAiToolsSetup} + onWorkflowChange={(workflowId) => setAITrackWorkflow(track.id, workflowId)} + onParamsChange={(params) => setAITrackParams(track.id, params)} + /> + </> + ); +}); diff --git a/frontend/src/components/AIWorkflowModal.tsx b/frontend/src/components/AIWorkflowModal.tsx new file mode 100644 index 0000000..9b2358d --- /dev/null +++ b/frontend/src/components/AIWorkflowModal.tsx @@ -0,0 +1,508 @@ +import { useMemo } from "react"; +import { type AiToolsStatus } from "../services/NativeBridge"; +import { type Track } from "../store/useDAWStore"; +import { + AI_WORKFLOWS, + AI_WORKFLOW_SECTION_LABELS, + type AIWorkflowSection, + getAIWorkflow, + mergeWorkflowParams, +} from "../data/aiWorkflows"; +import { + Button, + Checkbox, + Input, + Modal, + ModalContent, + ModalFooter, + ModalHeader, + Select, + Slider, + Textarea, +} from "./ui"; + +interface AIWorkflowModalProps { + track: Track; + aiToolsStatus: AiToolsStatus; + isOpen: boolean; + onClose: () => void; + onGenerate: () => void | Promise<void>; + onCancel: () => void | Promise<void>; + onOpenAiToolsSetup: () => void; + onWorkflowChange: (workflowId: string) => void; + onParamsChange: (params: Record<string, unknown>) => void; +} + +const SECTION_ORDER: AIWorkflowSection[] = [ + "prompt", + "music", + "sampling", + "generation", +]; + +function formatProgressLabel(progress: number) { + return `${Math.max(0, Math.round(progress * 100))}%`; +} + +function formatPhaseLabel(phase?: string) { + if (!phase) { + return "Preparing"; + } + + return phase + .split("_") + .filter(Boolean) + .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) + .join(" "); +} + +function formatElapsedLabel(elapsedMs?: number) { + if (!elapsedMs || elapsedMs <= 0) { + return ""; + } + + const totalSeconds = Math.max(0, Math.floor(elapsedMs / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return minutes > 0 ? `${minutes}m ${seconds}s elapsed` : `${seconds}s elapsed`; +} + +function formatEtaLabel(etaMs?: number) { + if (!etaMs || etaMs <= 0) { + return ""; + } + + const totalSeconds = Math.max(0, Math.floor(etaMs / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return minutes > 0 ? `${minutes}m ${seconds}s left` : `${seconds}s left`; +} + +function formatRuntimeProfileLabel(profile?: string) { + switch (profile) { + case "native-xl-turbo": + case "openstudio-ace-split": + return "OpenStudio ACE Split"; + default: + return profile ? formatPhaseLabel(profile) : ""; + } +} + +function formatLmModelLabel(lmModel?: string) { + if (!lmModel) { + return ""; + } + if (lmModel === "auto") { + return "Auto LM"; + } + if (lmModel.endsWith(".safetensors")) { + return lmModel + .replace("qwen_", "Qwen ") + .replace("_ace15.safetensors", "") + .replace("_", " ") + .replace("b", "B"); + } + return lmModel.replace("acestep-5Hz-lm-", "LM "); +} + +function formatSessionModeLabel(sessionMode?: string) { + switch (sessionMode) { + case "persistent": + return "Persistent session"; + case "oneshot-fallback": + return "One-shot fallback"; + case "oneshot": + return "One-shot"; + default: + return sessionMode ? formatPhaseLabel(sessionMode) : ""; + } +} + +function formatOptionLabel(paramKey: string, option: string) { + if (paramKey === "runtimeProfile") { + return formatRuntimeProfileLabel(option); + } + return option; +} + +export function AIWorkflowModal({ + track, + aiToolsStatus, + isOpen, + onClose, + onGenerate, + onCancel, + onOpenAiToolsSetup, + onWorkflowChange, + onParamsChange, +}: AIWorkflowModalProps) { + const workflow = getAIWorkflow(track.aiWorkflow); + const params = useMemo( + () => mergeWorkflowParams(workflow.id, track.aiWorkflowParams), + [track.aiWorkflowParams, workflow.id], + ); + const isBusy = + track.aiGenerationState === "loading" + || track.aiGenerationState === "generating"; + const isMusicGenerationReady = Boolean( + aiToolsStatus.musicGenerationReady + && aiToolsStatus.musicGenerationLayoutValid + && (aiToolsStatus.musicGenerationPerformanceReady ?? true), + ); + const musicGenerationBlockedMessage = !isMusicGenerationReady + ? (aiToolsStatus.musicGenerationPerformanceStatusMessage + || aiToolsStatus.musicGenerationStatusMessage + || (!aiToolsStatus.musicGenerationLayoutValid + && aiToolsStatus.musicGenerationModelId + && aiToolsStatus.musicGenerationCheckpointRoot + ? `Pinned ACE-Step native asset layout is not ready in ${aiToolsStatus.musicGenerationCheckpointRoot}.` + : aiToolsStatus.error + || aiToolsStatus.message + || "AI music generation is not ready yet.")) + : ""; + const canSubmitGeneration = + workflow.available !== false && isMusicGenerationReady && !isBusy; + + const paramsBySection = useMemo(() => { + return SECTION_ORDER.map((section) => ({ + section, + params: workflow.params.filter((param) => param.section === section), + })).filter((group) => group.params.length > 0); + }, [workflow.params]); + + const handleParamChange = (key: string, value: unknown) => { + const nextParams = { + ...params, + [key]: value, + }; + onParamsChange(nextParams); + }; + + return ( + <Modal isOpen={isOpen} onClose={onClose} size="xl"> + <ModalHeader title="AI Workflow Parameters" /> + <ModalContent> + <div className="space-y-4"> + <div className="grid gap-4 xl:grid-cols-[minmax(0,1fr)_280px]"> + <div className="rounded border border-neutral-800 bg-neutral-950/70 p-4"> + <div className="flex items-start justify-between gap-3"> + <div className="min-w-0"> + <p className="truncate text-sm font-semibold text-daw-text"> + {track.name} + </p> + <p className="mt-1 text-xs leading-5 text-daw-text-secondary"> + {workflow.description} + </p> + </div> + {track.aiGenerationBackend ? ( + <span className="shrink-0 rounded-full border border-neutral-700 bg-neutral-900 px-2 py-1 text-[10px] font-semibold uppercase tracking-[0.16em] text-daw-text"> + {track.aiGenerationBackend.toUpperCase()} + </span> + ) : null} + </div> + </div> + + <div className="rounded border border-neutral-800 bg-neutral-950/50 p-4"> + <Select + label="Workflow" + value={workflow.id} + onChange={(value) => onWorkflowChange(String(value))} + options={AI_WORKFLOWS.map((entry) => ({ + value: entry.id, + label: entry.label, + disabled: entry.available === false, + }))} + size="sm" + fullWidth + /> + </div> + </div> + + {workflow.available === false ? ( + <div className="rounded border border-yellow-700/40 bg-yellow-950/30 px-4 py-3 text-sm text-yellow-200"> + {workflow.availabilityNote + ?? "This workflow is not available in the current OpenStudio build."} + </div> + ) : null} + + {!isMusicGenerationReady ? ( + <div className="space-y-2 rounded border border-red-700/40 bg-red-950/30 px-4 py-3 text-sm text-red-200"> + <p>{musicGenerationBlockedMessage}</p> + {aiToolsStatus.musicGenerationCheckpointRoot ? ( + <p className="text-xs text-red-100/80 break-all"> + Checkpoint root: {aiToolsStatus.musicGenerationCheckpointRoot} + </p> + ) : null} + </div> + ) : null} + + {track.aiGenerationState === "error" && track.aiGenerationError ? ( + <div className="space-y-1 rounded border border-red-700/40 bg-red-950/30 px-4 py-3 text-sm text-red-200"> + <p>{track.aiGenerationError}</p> + <div className="flex flex-wrap gap-x-3 gap-y-1 text-xs uppercase tracking-[0.12em] text-red-100/80"> + {track.aiGenerationPhase ? ( + <span>Last phase: {formatPhaseLabel(track.aiGenerationPhase)}</span> + ) : null} + {track.aiGenerationSessionMode ? ( + <span>{formatSessionModeLabel(track.aiGenerationSessionMode)}</span> + ) : null} + {track.aiGenerationWorkerExitCode ? ( + <span>Exit code: {track.aiGenerationWorkerExitCode}</span> + ) : null} + </div> + {track.aiGenerationStatusNote ? ( + <p className="text-xs leading-5 text-red-100/80">{track.aiGenerationStatusNote}</p> + ) : null} + {track.aiGenerationLastStderrLine ? ( + <p className="text-xs leading-5 text-red-100/80 break-all"> + Last stderr: {track.aiGenerationLastStderrLine} + </p> + ) : null} + {track.aiGenerationLastStdoutLine ? ( + <p className="text-xs leading-5 text-red-100/80 break-all"> + Last stdout: {track.aiGenerationLastStdoutLine} + </p> + ) : null} + </div> + ) : null} + + {isBusy ? ( + <div className="rounded border border-cyan-800/50 bg-cyan-950/20 p-4"> + <div className="flex flex-col gap-3 md:flex-row md:items-start md:justify-between"> + <div className="min-w-0"> + <p className="text-sm font-semibold text-daw-text"> + {formatPhaseLabel(track.aiGenerationPhase)} + </p> + <p className="mt-1 text-xs leading-5 text-daw-text-secondary"> + {track.aiGenerationMessage || "Music generation is in progress."} + </p> + </div> + + <div className="flex flex-wrap items-center gap-2 text-[10px] font-semibold uppercase tracking-[0.14em] text-daw-text-muted"> + <span className="rounded-full border border-neutral-700 bg-neutral-900/80 px-2 py-1 text-daw-text"> + {formatProgressLabel(track.aiGenerationProgress ?? 0)} + </span> + {track.aiGenerationBackend ? ( + <span className="rounded-full border border-neutral-700 bg-neutral-900/80 px-2 py-1 text-daw-text"> + {track.aiGenerationBackend.toUpperCase()} + </span> + ) : null} + {track.aiGenerationRunMode ? ( + <span className="rounded-full border border-neutral-700 bg-neutral-900/80 px-2 py-1 text-daw-text"> + {track.aiGenerationRunMode.toUpperCase()} + </span> + ) : null} + {track.aiGenerationSessionMode ? ( + <span className="rounded-full border border-neutral-700 bg-neutral-900/80 px-2 py-1 text-daw-text"> + {formatSessionModeLabel(track.aiGenerationSessionMode)} + </span> + ) : null} + {track.aiGenerationElapsedMs ? ( + <span className="rounded-full border border-neutral-700 bg-neutral-900/80 px-2 py-1 text-daw-text"> + {formatElapsedLabel(track.aiGenerationElapsedMs)} + </span> + ) : null} + {track.aiGenerationEtaMs ? ( + <span className="rounded-full border border-neutral-700 bg-neutral-900/80 px-2 py-1 text-daw-text"> + {formatEtaLabel(track.aiGenerationEtaMs)} + </span> + ) : null} + {track.aiGenerationRuntimeProfile ? ( + <span className="rounded-full border border-neutral-700 bg-neutral-900/80 px-2 py-1 text-daw-text"> + {formatRuntimeProfileLabel(track.aiGenerationRuntimeProfile)} + </span> + ) : null} + {track.aiGenerationLmModel ? ( + <span className="rounded-full border border-neutral-700 bg-neutral-900/80 px-2 py-1 text-daw-text"> + {formatLmModelLabel(track.aiGenerationLmModel)} + </span> + ) : null} + </div> + </div> + + <div className="mt-3 h-2.5 w-full rounded-full bg-neutral-900"> + <div + className="h-2.5 rounded-full bg-daw-accent transition-all duration-200" + style={{ + width: `${Math.max(4, Math.round((track.aiGenerationProgress ?? 0) * 100))}%`, + }} + /> + </div> + {track.aiGenerationStatusNote ? ( + <p className="mt-3 text-xs leading-5 text-daw-text-secondary"> + {track.aiGenerationStatusNote} + </p> + ) : null} + </div> + ) : null} + + {workflow.available !== false ? ( + <div className="space-y-4"> + {paramsBySection.map((group) => ( + <section + key={group.section} + className="rounded border border-neutral-800 bg-neutral-950/50 p-4" + > + <div className="mb-3"> + <p className="text-xs font-semibold uppercase tracking-[0.16em] text-daw-text-muted"> + {AI_WORKFLOW_SECTION_LABELS[group.section]} + </p> + </div> + + <div + className={ + group.section === "prompt" + ? "space-y-3" + : "grid grid-cols-1 gap-3 md:grid-cols-2" + } + > + {group.params.map((param) => { + const value = params[param.key]; + const controlDisabled = false; + + if (param.type === "textarea") { + return ( + <Textarea + key={param.key} + label={param.label} + value={String(value ?? "")} + onChange={(event) => + handleParamChange(param.key, event.target.value) + } + placeholder={param.placeholder} + rows={param.key === "lyrics" ? 10 : 6} + fullWidth + /> + ); + } + + if (param.type === "text" || param.type === "number") { + return ( + <Input + key={param.key} + label={param.label} + type={param.type === "number" ? "number" : "text"} + value={String(value ?? "")} + onChange={(event) => + handleParamChange( + param.key, + param.type === "number" + ? Number(event.target.value) + : event.target.value, + ) + } + placeholder={param.placeholder} + size="sm" + disabled={controlDisabled} + fullWidth + /> + ); + } + + if (param.type === "slider") { + const numericValue = + typeof value === "number" + ? value + : Number(value ?? param.default ?? 0); + + return ( + <div + key={param.key} + className="rounded border border-neutral-800 bg-neutral-950/70 p-3" + > + <div className="mb-2 flex items-center justify-between gap-3"> + <span className="text-xs font-medium uppercase tracking-[0.12em] text-daw-text-muted"> + {param.label} + </span> + <span className="rounded border border-neutral-700 bg-neutral-900 px-2 py-0.5 text-[11px] text-daw-text"> + {numericValue} + </span> + </div> + <Slider + value={numericValue} + min={param.min ?? 0} + max={param.max ?? 100} + step={param.step ?? 1} + disabled={controlDisabled} + onChange={(nextValue) => + handleParamChange(param.key, nextValue) + } + /> + </div> + ); + } + + if (param.type === "toggle") { + return ( + <label + key={param.key} + className="flex items-center justify-between gap-3 rounded border border-neutral-800 bg-neutral-950/70 px-3 py-2.5" + > + <span className="text-sm text-daw-text"> + {param.label} + </span> + <Checkbox + checked={Boolean(value)} + disabled={controlDisabled} + onChange={() => + handleParamChange(param.key, !Boolean(value)) + } + /> + </label> + ); + } + + return ( + <Select + key={param.key} + label={param.label} + value={String(value ?? "")} + onChange={(nextValue) => + handleParamChange(param.key, String(nextValue)) + } + options={(param.options ?? []).map((option) => ({ + value: option, + label: formatOptionLabel(param.key, option), + disabled: false, + }))} + size="sm" + disabled={controlDisabled} + fullWidth + /> + ); + })} + </div> + </section> + ))} + </div> + ) : null} + </div> + </ModalContent> + <ModalFooter> + <Button variant="secondary" onClick={onClose} disabled={isBusy}> + Close + </Button> + {isBusy ? ( + <Button variant="danger" onClick={() => void onCancel()}> + Cancel + </Button> + ) : ( + <> + {!isMusicGenerationReady ? ( + <Button variant="secondary" onClick={onOpenAiToolsSetup}> + Open AI Tools Setup + </Button> + ) : null} + <Button + variant="primary" + onClick={() => void onGenerate()} + disabled={!canSubmitGeneration} + > + Generate + </Button> + </> + )} + </ModalFooter> + </Modal> + ); +} diff --git a/frontend/src/components/AddMultipleTracksModal.tsx b/frontend/src/components/AddMultipleTracksModal.tsx index 82086f1..b3fdc21 100644 --- a/frontend/src/components/AddMultipleTracksModal.tsx +++ b/frontend/src/components/AddMultipleTracksModal.tsx @@ -17,12 +17,14 @@ const TRACK_TYPE_OPTIONS = [ { value: "audio", label: "Audio" }, { value: "midi", label: "MIDI" }, { value: "instrument", label: "Instrument" }, + { value: "ai", label: "AI Track" }, ] as const; const DEFAULT_PREFIX: Record<InsertableTrackType, string> = { audio: "Audio", midi: "MIDI", instrument: "Instrument", + ai: "AI", }; export function AddMultipleTracksModal({ diff --git a/frontend/src/components/AiToolsSetupModal.tsx b/frontend/src/components/AiToolsSetupModal.tsx index e5ca7c8..a873327 100644 --- a/frontend/src/components/AiToolsSetupModal.tsx +++ b/frontend/src/components/AiToolsSetupModal.tsx @@ -44,12 +44,80 @@ function formatElapsed(ms?: number): string { return minutes > 0 ? `${minutes}m ${seconds}s` : `${seconds}s`; } +function formatBytes(bytes?: number): string { + const value = Math.max(0, bytes ?? 0); + if (value === 0) return "0 B"; + const units = ["B", "KB", "MB", "GB"]; + let size = value; + let unitIndex = 0; + while (size >= 1024 && unitIndex < units.length - 1) { + size /= 1024; + unitIndex += 1; + } + const precision = unitIndex === 0 ? 0 : size >= 100 ? 0 : size >= 10 ? 1 : 2; + return `${size.toFixed(precision)} ${units[unitIndex]}`; +} + +function formatRuntimeProfileLabel(profile?: string): string { + switch (profile) { + case "native-xl-turbo": + case "openstudio-ace-split": + return "OpenStudio ACE Split"; + default: + return profile ?? ""; + } +} + +function getUnavailableProfileDetails( + runtimeProfiles: Record<string, unknown> | undefined, + unavailableProfiles: Array<Record<string, unknown>>, +): string[] { + const details: string[] = []; + + if (runtimeProfiles && typeof runtimeProfiles === "object") { + for (const [profileId, rawProfile] of Object.entries(runtimeProfiles)) { + if (!rawProfile || typeof rawProfile !== "object") continue; + const profile = rawProfile as Record<string, unknown>; + if (profile.available === true) continue; + const missingAssets = Array.isArray(profile.missingAssets) + ? profile.missingAssets + .map((entry) => String(entry ?? "").trim()) + .filter(Boolean) + : []; + const label = formatRuntimeProfileLabel(String(profile.id ?? profileId)); + details.push( + missingAssets.length > 0 + ? `${label}: ${missingAssets.join(", ")}` + : label, + ); + } + } + + if (details.length > 0) { + return details; + } + + return unavailableProfiles + .map((entry) => { + const label = formatRuntimeProfileLabel(String(entry.id ?? "")); + const missingAssets = Array.isArray(entry.missingAssets) + ? entry.missingAssets + .map((asset) => String(asset ?? "").trim()) + .filter(Boolean) + : []; + if (!label) return ""; + return missingAssets.length > 0 ? `${label}: ${missingAssets.join(", ")}` : label; + }) + .filter(Boolean); +} + export default function AiToolsSetupModal() { - const { showAiToolsSetup, closeAiToolsSetup, installAiTools, aiToolsStatus } = useDAWStore( + const { showAiToolsSetup, closeAiToolsSetup, installAiTools, resetAiTools, aiToolsStatus } = useDAWStore( useShallow((s) => ({ showAiToolsSetup: s.showAiToolsSetup, closeAiToolsSetup: s.closeAiToolsSetup, installAiTools: s.installAiTools, + resetAiTools: s.resetAiTools, aiToolsStatus: s.aiToolsStatus, })), ); @@ -58,13 +126,31 @@ export default function AiToolsSetupModal() { const isPythonMissing = aiToolsStatus.state === "pythonMissing"; const hasInstallError = aiToolsStatus.state === "error" || aiToolsStatus.state === "cancelled"; - const isInstallComplete = aiToolsStatus.available || aiToolsStatus.state === "ready"; + const isStemSeparationReady = aiToolsStatus.available || aiToolsStatus.state === "ready"; + const isMusicGenerationInstalled = Boolean(aiToolsStatus.musicGenerationReady && aiToolsStatus.musicGenerationLayoutValid); + const isMusicGenerationPerformanceReady = aiToolsStatus.musicGenerationPerformanceReady ?? true; + const isMusicGenerationFullyReady = + isMusicGenerationInstalled + && isMusicGenerationPerformanceReady; + const isPartiallyReady = isStemSeparationReady && !isMusicGenerationFullyReady; + const isInstallComplete = isStemSeparationReady && isMusicGenerationFullyReady; + const showSetupModeCard = !isInstallComplete; + const showSetupSteps = !isInstallComplete; + const musicGenerationBlockedMessage = + aiToolsStatus.musicGenerationPerformanceStatusMessage + || (isMusicGenerationInstalled + ? "Music generation is installed, but acceleration is incomplete in this managed runtime." + : aiToolsStatus.musicGenerationStatusMessage + || (!aiToolsStatus.musicGenerationLayoutValid + ? "Pinned ACE-Step native split-model files are still missing." + : "Music generation still needs the OpenStudio ACE split backend.")); + const isReconcilingInstallResult = aiToolsStatus.statusWarningCode === "reconciling_install_state"; const requiresExternalPython = aiToolsStatus.requiresExternalPython; const buildRuntimeMode = aiToolsStatus.buildRuntimeMode ?? "downloaded-runtime"; const isDownloadedRuntimeFlow = buildRuntimeMode === "downloaded-runtime" || (aiToolsStatus.installSource === "downloadedRuntime" && !requiresExternalPython); - const isModelFailure = aiToolsStatus.errorCode === "model_download_failed"; + const isModelFailure = (aiToolsStatus.errorCode ?? "").startsWith("model_"); const isUnsupportedPlatform = aiToolsStatus.errorCode === "runtime_platform_unsupported"; const isRuntimeManifestFailure = aiToolsStatus.errorCode === "runtime_manifest_missing" || @@ -81,17 +167,41 @@ export default function AiToolsSetupModal() { aiToolsStatus.errorCode === "installer_exited_incomplete" || aiToolsStatus.errorCode === "installer_output_timeout" || aiToolsStatus.errorCode === "model_preparation_incomplete"; + const terminalReason = aiToolsStatus.terminalReason ?? ""; + const isWindowsRuntimeLockFailure = + IS_WINDOWS && + (terminalReason === "runtime_locked_rebuild_failed" || + terminalReason === "runtime_rebuild_remove_failed"); const installLogPath = aiToolsStatus.detailLogPath; const activityLines = aiToolsStatus.activityLines ?? []; const hasActivityConsole = aiToolsStatus.installInProgress || activityLines.length > 0; + const showLatestInstallerActivity = !isInstallComplete && hasActivityConsole && !aiToolsStatus.installInProgress; + const hasByteProgress = (aiToolsStatus.bytesTotal ?? 0) > 0 && (aiToolsStatus.bytesDownloaded ?? 0) >= 0; + const byteProgressRatio = hasByteProgress + ? Math.max(0, Math.min((aiToolsStatus.bytesDownloaded ?? 0) / Math.max(aiToolsStatus.bytesTotal ?? 1, 1), 1)) + : 0; + const visualProgressRatio = hasByteProgress ? byteProgressRatio : Math.max(0, Math.min(aiToolsStatus.progress ?? 0, 1)); + const progressPercent = Math.round(visualProgressRatio * 100); + const transferText = hasByteProgress + ? `${formatBytes(aiToolsStatus.bytesDownloaded)} / ${formatBytes(aiToolsStatus.bytesTotal)}` + : ""; + const availableProfiles = aiToolsStatus.musicGenerationAvailableProfiles ?? []; + const unavailableProfiles = aiToolsStatus.musicGenerationUnavailableProfiles ?? []; + const defaultProfile = aiToolsStatus.musicGenerationDefaultProfile ?? ""; + const unavailableProfileDetails = getUnavailableProfileDetails( + aiToolsStatus.musicGenerationRuntimeProfiles as Record<string, unknown> | undefined, + unavailableProfiles, + ); const errorTitle = isModelFailure ? "Model download needs attention" : isUnsupportedPlatform ? "AI Tools are not available on this Mac" - : isRuntimeManifestFailure - ? "AI runtime download info is unavailable" - : isRuntimeArchiveFailure + : isWindowsRuntimeLockFailure + ? "Windows blocked the AI runtime update" + : isRuntimeManifestFailure + ? "AI runtime download info is unavailable" + : isRuntimeArchiveFailure ? "AI runtime setup failed" : aiToolsStatus.state === "cancelled" ? "AI tools setup was cancelled" @@ -101,12 +211,16 @@ export default function AiToolsSetupModal() { const recommendationText = isDownloadedRuntimeFlow ? "This release downloads the OpenStudio AI runtime the first time you use AI Tools, verifies it, and then downloads the stem model. You can keep using the app while that setup runs." - : "This dev build needs Python 3.10 through 3.13 on your machine first. Once Python is installed, OpenStudio will continue the rest of the AI setup automatically."; + : IS_WINDOWS + ? "This dev build needs Python 3.11 on your machine first for the Windows ACE-Step runtime. Once Python is installed, OpenStudio will continue the rest of the AI setup automatically." + : "This dev build needs Python 3.10 through 3.12 on your machine first. Once Python is installed, OpenStudio will continue the rest of the AI setup automatically."; const retryGuidance = isModelFailure ? "The runtime is already in place. Retry after checking your internet connection, VPN, firewall, or antivirus if the download keeps failing." : isUnsupportedPlatform ? "This release currently supports AI Tools on Apple Silicon Macs only. The base app can still be used normally on Intel Macs." + : isWindowsRuntimeLockFailure + ? "OpenStudio already attempted a runtime-only rebuild of stem-runtime after Windows denied access to a managed runtime file. Close any remaining helper processes, Python workers, or antivirus scanners that may still be touching the runtime, then retry. Normal retry preserves downloaded models and music-generation checkpoints. Use Reset AI Tools only for a full cleanup, which also removes the downloaded models and checkpoints." : isRuntimeManifestFailure ? "Retry once in case the release metadata service was temporarily unavailable. If the same message appears again, OpenStudio may not be able to reach the published AI runtime metadata from this machine." : isDownloadedRuntimeFlow @@ -125,11 +239,21 @@ export default function AiToolsSetupModal() { }; const handleRetry = async () => { - await installAiTools(); + await installAiTools({ userConfirmedDownload: true }); + }; + + const handleReset = async () => { + await resetAiTools(); }; return ( - <Modal isOpen={showAiToolsSetup} onClose={closeAiToolsSetup} size="md" fullHeight className="max-h-[80vh]"> + <Modal + isOpen={showAiToolsSetup} + onClose={closeAiToolsSetup} + size="xl" + fullHeight + className="max-h-[80vh] w-[min(96vw,1100px)]" + > <ModalHeader title="AI Tools Setup" /> <ModalContent className="space-y-5"> <div className="space-y-5"> @@ -137,29 +261,113 @@ export default function AiToolsSetupModal() { <p className="text-sm font-medium text-daw-text">What is this?</p> <p className="text-xs text-daw-text-secondary leading-relaxed"> AI Tools enables <span className="text-daw-text">Stem Separation</span> - splitting a - clip into individual tracks like Vocals, Drums, Bass, Guitar, and more. + clip into individual tracks like Vocals, Drums, Bass, Guitar, and more. It also powers + <span className="text-daw-text"> AI Track music generation</span> when ACE-Step is installed. </p> </div> - <div className="rounded border border-neutral-800 bg-neutral-950/60 p-3 space-y-1"> - <p className="text-sm font-medium text-daw-text"> - {isDownloadedRuntimeFlow ? "OpenStudio-managed runtime setup" : "Python-based setup"} - </p> - <p className="text-xs text-daw-text-secondary leading-relaxed">{recommendationText}</p> - </div> + {showSetupModeCard ? ( + <div className="rounded border border-neutral-800 bg-neutral-950/60 p-3 space-y-1"> + <p className="text-sm font-medium text-daw-text"> + {isDownloadedRuntimeFlow ? "OpenStudio-managed runtime setup" : "Python-based setup"} + </p> + <p className="text-xs text-daw-text-secondary leading-relaxed">{recommendationText}</p> + {!isInstallComplete ? ( + <p className="text-xs text-daw-text-secondary leading-relaxed"> + Nothing downloads until you click <span className="text-daw-text font-medium">Download and Install</span>. + </p> + ) : null} + </div> + ) : null} {isInstallComplete ? ( <div className="rounded border border-green-600/40 bg-green-950/30 p-3 space-y-2"> <p className="text-sm font-semibold text-green-400">AI Tools are ready</p> <p className="text-xs text-daw-text-secondary leading-relaxed"> - The runtime and stem-separation model are installed for this OpenStudio session. - You can continue straight into stem separation now. + The runtime, stem-separation model, and pinned ACE-Step files are installed for this OpenStudio session. + You can continue straight into stem separation or music generation now. </p> {aiToolsStatus.selectedBackend ? ( <p className="text-xs text-daw-text-secondary leading-relaxed"> Active backend: <span className="text-daw-text">{aiToolsStatus.selectedBackend}</span> </p> ) : null} + <p className="text-xs text-daw-text-secondary leading-relaxed"> + Music generation:{" "} + <span className={isMusicGenerationFullyReady ? "text-green-400" : "text-yellow-300"}> + {isMusicGenerationFullyReady ? "Ready" : "Installed, but degraded"} + </span> + {aiToolsStatus.aceStepVersion ? ` (ACE-Step ${aiToolsStatus.aceStepVersion})` : ""} + </p> + {aiToolsStatus.musicGenerationModelId ? ( + <p className="text-xs text-daw-text-secondary leading-relaxed"> + Pinned model: <span className="text-daw-text">{aiToolsStatus.musicGenerationModelId}</span> + </p> + ) : null} + {aiToolsStatus.musicGenerationCheckpointRoot ? ( + <p className="text-xs text-daw-text-secondary leading-relaxed break-all"> + Checkpoint root: <span className="text-daw-text">{aiToolsStatus.musicGenerationCheckpointRoot}</span> + </p> + ) : null} + <p className="text-xs text-daw-text-secondary leading-relaxed"> + Checkpoint layout:{" "} + <span className={aiToolsStatus.musicGenerationLayoutValid ? "text-green-400" : "text-yellow-300"}> + {aiToolsStatus.musicGenerationLayoutValid ? "Valid" : "Missing files"} + </span> + </p> + {availableProfiles.length > 0 ? ( + <p className="text-xs text-daw-text-secondary leading-relaxed"> + Available profiles: <span className="text-daw-text">{availableProfiles.map(formatRuntimeProfileLabel).join(", ")}</span> + {defaultProfile ? <span className="text-daw-text-secondary"> (default: {formatRuntimeProfileLabel(defaultProfile)})</span> : null} + </p> + ) : null} + </div> + ) : isPartiallyReady ? ( + <div className="rounded border border-yellow-600/40 bg-yellow-950/30 p-3 space-y-2"> + <p className="text-sm font-semibold text-yellow-300">Stem separation is ready</p> + <p className="text-xs text-daw-text-secondary leading-relaxed"> + {musicGenerationBlockedMessage} + </p> + {aiToolsStatus.selectedBackend ? ( + <p className="text-xs text-daw-text-secondary leading-relaxed"> + Active backend: <span className="text-daw-text">{aiToolsStatus.selectedBackend}</span> + </p> + ) : null} + <p className="text-xs text-daw-text-secondary leading-relaxed"> + Music generation: <span className="text-yellow-300">{isMusicGenerationInstalled ? "Installed, but degraded" : "Not ready yet"}</span> + {aiToolsStatus.aceStepVersion ? ` (ACE-Step ${aiToolsStatus.aceStepVersion})` : ""} + </p> + {aiToolsStatus.musicGenerationModelId ? ( + <p className="text-xs text-daw-text-secondary leading-relaxed"> + Pinned model: <span className="text-daw-text">{aiToolsStatus.musicGenerationModelId}</span> + </p> + ) : null} + {aiToolsStatus.musicGenerationCheckpointRoot ? ( + <p className="text-xs text-daw-text-secondary leading-relaxed break-all"> + Checkpoint root: <span className="text-daw-text">{aiToolsStatus.musicGenerationCheckpointRoot}</span> + </p> + ) : null} + <p className="text-xs text-daw-text-secondary leading-relaxed"> + Checkpoint layout:{" "} + <span className={aiToolsStatus.musicGenerationLayoutValid ? "text-green-400" : "text-yellow-300"}> + {aiToolsStatus.musicGenerationLayoutValid ? "Valid" : "Missing files"} + </span> + {aiToolsStatus.musicGenerationLayoutValid ? <span className="text-daw-text-secondary"> (the pinned files are present, but the OpenStudio ACE split backend is still unavailable)</span> : null} + </p> + {availableProfiles.length > 0 ? ( + <p className="text-xs text-daw-text-secondary leading-relaxed"> + Available profiles: <span className="text-daw-text">{availableProfiles.map(formatRuntimeProfileLabel).join(", ")}</span> + {defaultProfile ? <span className="text-daw-text-secondary"> (default: {formatRuntimeProfileLabel(defaultProfile)})</span> : null} + </p> + ) : null} + {unavailableProfileDetails.length > 0 ? ( + <p className="text-xs text-daw-text-secondary leading-relaxed"> + Missing profile assets:{" "} + <span className="text-daw-text"> + {unavailableProfileDetails.join("; ")} + </span> + </p> + ) : null} </div> ) : aiToolsStatus.installInProgress ? ( <div className="rounded border border-daw-accent/40 bg-neutral-950/70 p-3 space-y-3"> @@ -167,7 +375,7 @@ export default function AiToolsSetupModal() { <div className="w-full bg-neutral-900 rounded-full h-2"> <div className="bg-daw-accent h-2 rounded-full transition-all duration-200" - style={{ width: `${Math.max(4, Math.round(aiToolsStatus.progress * 100))}%` }} + style={{ width: `${Math.max(4, progressPercent)}%` }} /> </div> <p className="text-xs text-daw-text-secondary leading-relaxed"> @@ -186,9 +394,14 @@ export default function AiToolsSetupModal() { <p> {aiToolsStatus.stepCount && aiToolsStatus.stepIndex ? <>Stage: <span className="text-daw-text">{aiToolsStatus.stepIndex} / {aiToolsStatus.stepCount}</span></> - : <>Progress: <span className="text-daw-text">{Math.max(0, Math.round((aiToolsStatus.progress ?? 0) * 100))}%</span></>} + : <>Progress: <span className="text-daw-text">{progressPercent}%</span></>} </p> </div> + {hasByteProgress ? ( + <p className="text-xs text-daw-text-secondary leading-relaxed"> + Transfer: <span className="text-daw-text">{transferText}</span> + </p> + ) : null} {aiToolsStatus.lastPhase ? ( <p className="text-xs text-daw-text-secondary leading-relaxed"> Current phase: <span className="text-daw-text">{aiToolsStatus.lastPhase}</span> @@ -228,12 +441,34 @@ export default function AiToolsSetupModal() { </div> </div> </div> + ) : isReconcilingInstallResult ? ( + <div className="rounded border border-yellow-600/40 bg-yellow-950/30 p-3 space-y-3"> + <p className="text-sm font-semibold text-yellow-300">Confirming the install result</p> + <p className="text-xs text-daw-text-secondary leading-relaxed"> + OpenStudio temporarily lost contact with the installer, so it is now probing the installed runtime on disk before deciding whether setup succeeded. + </p> + {aiToolsStatus.statusWarning ? ( + <p className="rounded border border-yellow-600/30 bg-yellow-950/20 px-3 py-2 text-xs leading-relaxed text-yellow-200"> + {aiToolsStatus.statusWarning} + </p> + ) : null} + {aiToolsStatus.lastPhase ? ( + <p className="text-xs text-daw-text-secondary leading-relaxed"> + Last observed phase: <span className="text-daw-text">{aiToolsStatus.lastPhase}</span> + </p> + ) : null} + {aiToolsStatus.installSessionId ? ( + <p className="text-xs text-daw-text-secondary leading-relaxed break-all"> + Install session: <span className="text-daw-text">{aiToolsStatus.installSessionId}</span> + </p> + ) : null} + </div> ) : isPythonMissing ? ( <div className="rounded border border-yellow-600/40 bg-yellow-950/30 p-3 space-y-2"> <p className="text-sm font-semibold text-yellow-400">Python is required</p> <p className="text-xs text-daw-text-secondary leading-relaxed"> OpenStudio could not find a supported Python version on this machine. This dev - build needs Python 3.10 through 3.13 before AI Tools can be installed. + build needs {IS_WINDOWS ? "Python 3.11" : "Python 3.10 through 3.12"} before AI Tools can be installed. </p> </div> ) : hasInstallError ? ( @@ -244,6 +479,8 @@ export default function AiToolsSetupModal() { ? "OpenStudio prepared the AI runtime, but the stem model download did not complete." : isUnsupportedPlatform ? "This OpenStudio release does not currently publish an AI runtime for this Mac architecture." + : isWindowsRuntimeLockFailure + ? "OpenStudio detected a locked file inside the managed Windows runtime and already attempted a runtime-only rebuild of stem-runtime." : isRuntimeManifestFailure ? "OpenStudio could not fetch the published AI runtime metadata needed for this setup." : isRuntimeArchiveFailure @@ -292,6 +529,12 @@ export default function AiToolsSetupModal() { Terminal reason: <span className="text-daw-text">{aiToolsStatus.terminalReason}</span> </p> ) : null} + {isWindowsRuntimeLockFailure ? ( + <p className="text-xs text-daw-text-secondary leading-relaxed"> + Retry keeps the downloaded stem models and ACE-Step checkpoints in place.{" "} + <span className="text-daw-text">Reset AI Tools</span> is the full cleanup option and removes those downloads too. + </p> + ) : null} {requiresExternalPython ? ( <p className="text-xs text-daw-text-secondary leading-relaxed"> Python detected:{" "} @@ -309,7 +552,7 @@ export default function AiToolsSetupModal() { </div> ) : null} - {requiresExternalPython ? ( + {showSetupSteps && requiresExternalPython ? ( <div className="space-y-1"> <p className="text-xs font-medium text-daw-text-secondary uppercase tracking-wide"> Python setup steps @@ -320,7 +563,7 @@ export default function AiToolsSetupModal() { <Step number={1}> Click <span className="text-daw-text font-medium">Download Python</span> below. Your browser will open the official Python website. Download the latest{" "} - <span className="text-daw-text font-medium">Python 3.10, 3.11, 3.12, or 3.13</span> installer for{" "} + <span className="text-daw-text font-medium">Python 3.11</span> installer for{" "} <span className="text-daw-text font-medium">Windows 64-bit</span>. </Step> <Step number={2}> @@ -350,7 +593,7 @@ export default function AiToolsSetupModal() { <Step number={1}> Click <span className="text-daw-text font-medium">Download Python</span> below. This opens the official Python website. Download the latest{" "} - <span className="text-daw-text font-medium">Python 3.10, 3.11, 3.12, or 3.13</span> installer for + <span className="text-daw-text font-medium">Python 3.10, 3.11, or 3.12</span> installer for macOS. </Step> <Step number={2}> @@ -363,7 +606,7 @@ export default function AiToolsSetupModal() { </Step> <Step number={4}> Advanced option: if you already use Homebrew, you can install Python from - Terminal with <CodeSnip>brew install python@3.13</CodeSnip>, then restart + Terminal with <CodeSnip>brew install python@3.12</CodeSnip>, then restart OpenStudio and retry from this window. </Step> <Step number={5}> @@ -374,7 +617,7 @@ export default function AiToolsSetupModal() { )} </div> </div> - ) : ( + ) : showSetupSteps ? ( <div className="space-y-1"> <p className="text-xs font-medium text-daw-text-secondary uppercase tracking-wide"> Runtime download steps @@ -403,9 +646,9 @@ export default function AiToolsSetupModal() { </Step> </div> </div> - )} + ) : null} - {hasActivityConsole && !aiToolsStatus.installInProgress ? ( + {showLatestInstallerActivity ? ( <div className="rounded-xl border border-neutral-800 bg-black px-3 py-3 font-mono text-[11px] text-green-300"> <div className="mb-2 flex items-center justify-between text-[10px] uppercase tracking-wide text-neutral-500"> <span>Latest Installer Activity</span> @@ -425,37 +668,57 @@ export default function AiToolsSetupModal() { <p className="text-xs text-daw-text-secondary leading-relaxed"> {isInstallComplete ? "AI Tools finished installing in this session. You can close this window and continue working." - : "OpenStudio keeps the app responsive while setup runs. This window now shows the live install activity, current phase, and long-download hints."} + : isPartiallyReady + ? `Stem separation is ready in this session. ${musicGenerationBlockedMessage} Retry install to finish music generation, or close this window if you only need stem separation right now.` + : "OpenStudio keeps the app responsive while setup runs. This window now shows the live install activity, current phase, and long-download hints."} </p> </div> </div> </ModalContent> - <ModalFooter> - <Button variant="ghost" onClick={closeAiToolsSetup}> + <ModalFooter className="flex-wrap justify-end gap-3 sm:flex-nowrap"> + <Button variant="ghost" onClick={closeAiToolsSetup} className="whitespace-nowrap"> Close </Button> {installLogPath ? ( - <Button variant="ghost" onClick={() => void handleOpenInstallLog()}> + <Button + variant="ghost" + onClick={() => void handleOpenInstallLog()} + className="whitespace-nowrap" + > Open Install Log </Button> ) : null} - {requiresExternalPython ? ( - <Button variant="secondary" onClick={() => void handleDownloadPython()}> + {!aiToolsStatus.installInProgress ? ( + <Button variant="ghost" onClick={() => void handleReset()} className="whitespace-nowrap"> + Reset AI Tools + </Button> + ) : null} + {requiresExternalPython && !isInstallComplete ? ( + <Button + variant="secondary" + onClick={() => void handleDownloadPython()} + className="whitespace-nowrap" + > Download Python </Button> ) : null} <Button variant="primary" onClick={() => void (isInstallComplete ? closeAiToolsSetup() : handleRetry())} - disabled={aiToolsStatus.installInProgress} + disabled={aiToolsStatus.installInProgress || isReconcilingInstallResult} + className="whitespace-nowrap" > {isInstallComplete ? "Continue" - : aiToolsStatus.installInProgress + : isPartiallyReady + ? "Retry Install" + : aiToolsStatus.installInProgress ? "Installing..." + : isReconcilingInstallResult + ? "Checking Result..." : requiresExternalPython && isPythonMissing ? "Retry After Python Install" - : "Retry Install"} + : "Download and Install"} </Button> </ModalFooter> </Modal> diff --git a/frontend/src/components/KeyboardShortcutsModal.tsx b/frontend/src/components/KeyboardShortcutsModal.tsx index adcebfe..ea6c3bf 100644 --- a/frontend/src/components/KeyboardShortcutsModal.tsx +++ b/frontend/src/components/KeyboardShortcutsModal.tsx @@ -5,47 +5,13 @@ import { getActionShortcutScopeLabel, getRegisteredActions } from "../store/acti import { useDAWStore } from "../store/useDAWStore"; import { Button, Input } from "./ui"; import { Modal } from "./ui/Modal/Modal"; +import { formatShortcut, keyEventToCanonicalShortcut } from "../utils/platform"; interface KeyboardShortcutsModalProps { isOpen: boolean; onClose: () => void; } -/** - * Convert a KeyboardEvent into a human-readable shortcut string - * matching the format used in actionRegistry (e.g. "Ctrl+Shift+Z", "Alt+B", "F1"). - */ -function keyEventToShortcutString(e: KeyboardEvent): string { - const parts: string[] = []; - - if (e.ctrlKey || e.metaKey) parts.push("Ctrl"); - if (e.shiftKey) parts.push("Shift"); - if (e.altKey) parts.push("Alt"); - - // Map the key to a display name - let key = e.key; - - // Skip standalone modifier keys - if (["Control", "Shift", "Alt", "Meta"].includes(key)) return ""; - - // Normalize keys - if (key === " ") key = "Space"; - else if (key === "ArrowLeft") key = "Left"; - else if (key === "ArrowRight") key = "Right"; - else if (key === "ArrowUp") key = "Up"; - else if (key === "ArrowDown") key = "Down"; - else if (key === "Escape") key = "Esc"; - else if (key.startsWith("F") && key.length <= 3 && /^F\d+$/.test(key)) { - // Function keys: keep as-is (F1, F2, etc.) - } else if (key.length === 1) { - // Single character: uppercase it - key = key.toUpperCase(); - } - - parts.push(key); - return parts.join("+"); -} - /** * KeyboardShortcutsModal - Searchable, categorized keyboard shortcuts reference * with rebinding support. @@ -126,7 +92,7 @@ export function KeyboardShortcutsModal({ return; } - const shortcut = keyEventToShortcutString(e); + const shortcut = keyEventToCanonicalShortcut(e); if (!shortcut) return; // Ignore standalone modifier keys setCapturedShortcut(shortcut); @@ -179,7 +145,7 @@ export function KeyboardShortcutsModal({ if (!printGroups[action.category]) printGroups[action.category] = []; printGroups[action.category].push({ name: action.name, - shortcut: effectiveShortcut, + shortcut: formatShortcut(effectiveShortcut), }); } @@ -386,11 +352,11 @@ export function KeyboardShortcutsModal({ }`} title={ isCustom - ? `Custom (default: ${action.shortcut || "none"})` + ? `Custom (default: ${formatShortcut(action.shortcut) || "none"})` : undefined } > - {effectiveShortcut} + {formatShortcut(effectiveShortcut)} </span> <span className="text-[10px] uppercase tracking-wide text-daw-text-muted/70"> {shortcutScopeLabel} diff --git a/frontend/src/components/MainToolbar.tsx b/frontend/src/components/MainToolbar.tsx index 1eb56ca..9ff21a8 100644 --- a/frontend/src/components/MainToolbar.tsx +++ b/frontend/src/components/MainToolbar.tsx @@ -16,7 +16,7 @@ import { Cpu, } from "lucide-react"; import { usePitchEditorStore } from "../store/pitchEditorStore"; -import { getActionShortcut, getActionShortcutScopeLabel } from "../store/actionRegistry"; +import { getDisplayShortcut, getActionShortcutScopeLabel } from "../store/actionRegistry"; import { useDAWStore } from "../store/useDAWStore"; import { useShallow } from "zustand/shallow"; import { Button } from "./ui"; @@ -32,15 +32,15 @@ export function MainToolbar({ onToggleMixer, showMixer, }: MainToolbarProps) { - const mixerShortcut = getActionShortcut("view.toggleMixer") ?? "Ctrl+M"; - const loopShortcut = getActionShortcut("transport.loop") ?? "L"; - const recordShortcut = getActionShortcut("transport.record") ?? "Ctrl+R"; - const undoShortcut = getActionShortcut("edit.undo") ?? "Ctrl+Z"; - const redoShortcut = getActionShortcut("edit.redo") ?? "Ctrl+Shift+Z"; - const selectToolShortcut = getActionShortcut("tools.selectTool") ?? "V"; - const splitToolShortcut = getActionShortcut("tools.splitTool") ?? "B"; - const muteToolShortcut = getActionShortcut("tools.muteTool") ?? "X"; - const smartToolShortcut = getActionShortcut("tools.smartTool") ?? "Y"; + const mixerShortcut = getDisplayShortcut("view.toggleMixer") ?? "Ctrl+M"; + const loopShortcut = getDisplayShortcut("transport.loop") ?? "L"; + const recordShortcut = getDisplayShortcut("transport.record") ?? "Ctrl+R"; + const undoShortcut = getDisplayShortcut("edit.undo") ?? "Ctrl+Z"; + const redoShortcut = getDisplayShortcut("edit.redo") ?? "Ctrl+Shift+Z"; + const selectToolShortcut = getDisplayShortcut("tools.selectTool") ?? "V"; + const splitToolShortcut = getDisplayShortcut("tools.splitTool") ?? "B"; + const muteToolShortcut = getDisplayShortcut("tools.muteTool") ?? "X"; + const smartToolShortcut = getDisplayShortcut("tools.smartTool") ?? "Y"; const timelineScopeLabel = getActionShortcutScopeLabel("timeline"); const { isPlaying, diff --git a/frontend/src/components/MenuBar.tsx b/frontend/src/components/MenuBar.tsx index 2b910ec..86abbd5 100644 --- a/frontend/src/components/MenuBar.tsx +++ b/frontend/src/components/MenuBar.tsx @@ -2,11 +2,12 @@ import { useState, useEffect, useCallback } from "react"; import { Minus, Square, X, Copy } from "lucide-react"; import { EditMenu } from "./menus/EditMenu"; import { MenuDropdown, MenuItemProps } from "./menus/MenuDropdown"; -import { getEffectiveActionShortcut } from "../store/actionRegistry"; +import { getDisplayEffectiveShortcut } from "../store/actionRegistry"; import { useDAWStore, THEME_PRESETS } from "../store/useDAWStore"; import { useShallow } from "zustand/shallow"; import { nativeBridge } from "../services/NativeBridge"; import { usesNativeWindowChrome } from "../utils/windowEnvironment"; +import { createTrackOfType } from "../utils/trackCreation"; /** * Main Menu Bar Component @@ -14,7 +15,7 @@ import { usesNativeWindowChrome } from "../utils/windowEnvironment"; */ export function MenuBar() { const shortcut = (actionId: string, fallback: string) => - getEffectiveActionShortcut(actionId) ?? fallback; + getDisplayEffectiveShortcut(actionId) ?? fallback; const { toggleMixer, showMixer, @@ -711,6 +712,13 @@ export function MenuBar() { }, dividerAfter: true, }, + { + label: "New AI Track", + shortcut: shortcut("insert.aiTrack", "Ctrl+Alt+T"), + onClick: () => { + void createTrackOfType("ai"); + }, + }, { label: "New Bus/Group Track", onClick: () => { diff --git a/frontend/src/components/NoteInspector.tsx b/frontend/src/components/NoteInspector.tsx index 99aeb42..701f4c8 100644 --- a/frontend/src/components/NoteInspector.tsx +++ b/frontend/src/components/NoteInspector.tsx @@ -1,6 +1,6 @@ import React, { useCallback } from "react"; import { useShallow } from "zustand/shallow"; -import { usePitchEditorStore } from "../store/pitchEditorStore"; +import { PITCH_EDITOR_FORMANT_EDITING_ENABLED, usePitchEditorStore } from "../store/pitchEditorStore"; import type { PitchNoteData } from "../services/NativeBridge"; const NOTE_NAMES = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"]; @@ -128,15 +128,16 @@ export function NoteInspector() { }} /> - {/* Formant (cents) */} - <InspectorRow - label="Formant" - value={formantDisplay} - suffix="ct" - disabled={!note} - min={-386} max={386} step={1} - onChange={(v) => { if (note) setNoteFormant(note.id, v / 100); }} - /> + {PITCH_EDITOR_FORMANT_EDITING_ENABLED && ( + <InspectorRow + label="Formant" + value={formantDisplay} + suffix="ct" + disabled={!note} + min={-386} max={386} step={1} + onChange={(v) => { if (note) setNoteFormant(note.id, v / 100); }} + /> + )} {/* Volume */} <InspectorRow diff --git a/frontend/src/components/PitchEditorCanvas.ts b/frontend/src/components/PitchEditorCanvas.ts index f726fae..ac97563 100644 --- a/frontend/src/components/PitchEditorCanvas.ts +++ b/frontend/src/components/PitchEditorCanvas.ts @@ -1,24 +1,24 @@ /** - * PitchEditorCanvas — Imperative Canvas 2D rendering for the pitch editor lower zone. + * PitchEditorCanvas — Imperative Canvas 2D rendering for the monophonic pitch editor. * - * Handles drawing the grid, notes, contour, playhead, selection, and hover effects. - * Called from PitchEditorLowerZone via requestAnimationFrame. + * The editor is intentionally monophonic even for stereo vocal clips: + * analysis happens on a mono sum, while correction still renders against the + * full multichannel clip in the backend. */ -import type { PitchNoteData, PitchContourData, PolyNoteData, UnifiedNoteData } from "../services/NativeBridge"; -import type { PitchRenderCoverageRange } from "../store/pitchEditorStore"; -import { polyToUnified, monoToUnified } from "../services/NativeBridge"; +import type { PitchNoteData, PitchContourData } from "../services/NativeBridge"; +import { PITCH_EDITOR_FORMANT_EDITING_ENABLED, type PitchRenderCoverageRange } from "../store/pitchEditorStore"; const NOTE_NAMES = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"]; const BLACK_KEYS = new Set([1, 3, 6, 8, 10]); -export const PIANO_WIDTH = 30; // small piano reference strip on left edge of canvas +export const PIANO_WIDTH = 30; export interface PitchEditorViewport { - scrollX: number; // seconds (horizontal offset, same as dawScrollX / pps) - scrollY: number; // MIDI note (bottom of viewport) - pixelsPerSecond: number; // from DAW store (synced with timeline) - pixelsPerSemitone: number; // vertical zoom - clipStartTime: number; // clip offset in project timeline + scrollX: number; + scrollY: number; + pixelsPerSecond: number; + pixelsPerSemitone: number; + clipStartTime: number; clipDuration: number; } @@ -27,53 +27,35 @@ export interface PitchEditorRenderState { contour: PitchContourData | null; selectedNoteIds: string[]; hoveredNoteId: string | null; - currentTime: number; // transport position (seconds) + currentTime: number; isPlaying: boolean; bpm: number; timeSignature: [number, number]; - - // Scale overlay - scaleNotes: boolean[]; // 12 booleans - scaleKey: number; // 0-11 - - // Polyphonic mode - polyMode: boolean; - polyNotes: PolyNoteData[]; - showPitchSalience: boolean; - pitchSalience: number[][] | null; // downsampled [T][264] - salienceDownsampleFactor: number; - salienceHopSize: number; - salienceSampleRate: number; + scaleNotes: boolean[]; + scaleKey: number; renderCoverage: PitchRenderCoverageRange[]; } -// Convert a MIDI note to Y position on canvas function midiToY(midi: number, viewport: PitchEditorViewport, canvasHeight: number): number { const visibleSemitones = canvasHeight / viewport.pixelsPerSemitone; const topMidi = viewport.scrollY + visibleSemitones; return (topMidi - midi) * viewport.pixelsPerSemitone; } -// Convert clip-relative time (seconds) to X position on canvas. -// Note/contour times are relative to clip start (0 = start of clip). -// viewport.scrollX is in project-seconds, so we add clipStartTime. function timeToX(clipTime: number, viewport: PitchEditorViewport): number { const projectTime = viewport.clipStartTime + clipTime; return PIANO_WIDTH + (projectTime - viewport.scrollX) * viewport.pixelsPerSecond; } -// Convert project-absolute time to X (for playhead, beat grid — already in project time) function projectTimeToX(projectTime: number, viewport: PitchEditorViewport): number { return PIANO_WIDTH + (projectTime - viewport.scrollX) * viewport.pixelsPerSecond; } -// Convert X position to clip-relative time export function xToTime(x: number, viewport: PitchEditorViewport): number { const projectTime = (x - PIANO_WIDTH) / viewport.pixelsPerSecond + viewport.scrollX; return projectTime - viewport.clipStartTime; } -// Convert Y position to MIDI note export function yToMidi(y: number, viewport: PitchEditorViewport, canvasHeight: number): number { const visibleSemitones = canvasHeight / viewport.pixelsPerSemitone; const topMidi = viewport.scrollY + visibleSemitones; @@ -82,61 +64,110 @@ export function yToMidi(y: number, viewport: PitchEditorViewport, canvasHeight: export type NoteHitZone = "body" | "left" | "right" | "top-center" | "bottom-left" | "bottom-right"; -/** Hit-test a point against note rectangles. Returns note ID and zone. */ +function getNoteEffectiveStart(note: PitchNoteData) { + return note.effectiveStartTime ?? note.startTime; +} + +function getNoteEffectiveEnd(note: PitchNoteData) { + return note.effectiveEndTime ?? note.endTime; +} + +function getEffectiveTransitionMs(note: PitchNoteData, edge: "in" | "out") { + const explicitMs = edge === "in" ? note.transitionIn : note.transitionOut; + return explicitMs > 0 ? explicitMs : 0; +} + export function hitTestNote( - x: number, y: number, + x: number, + y: number, notes: PitchNoteData[], viewport: PitchEditorViewport, - canvasHeight: number + canvasHeight: number, ): { noteId: string; edge: NoteHitZone } | null { - const EDGE_PX = 6; - const HANDLE_SIZE = 8; + const edgePx = 6; + const handleSize = 8; + for (let i = notes.length - 1; i >= 0; i--) { - const n = notes[i]; - const nx = timeToX(n.startTime, viewport); - const nx2 = timeToX(n.endTime, viewport); - const ny = midiToY(n.correctedPitch + 0.5, viewport, canvasHeight); - const nh = viewport.pixelsPerSemitone; - if (x >= nx && x <= nx2 && y >= ny && y <= ny + nh) { - const w = nx2 - nx; - // Smart control handles (only if note is wide enough) - if (w > 30 && nh > 8) { - const midX = nx + w / 2; - // Top-center handle: straighten/vibrato - if (Math.abs(x - midX) < HANDLE_SIZE && y - ny < HANDLE_SIZE) { - return { noteId: n.id, edge: "top-center" }; - } - // Bottom-left handle: formant - if (x - nx < HANDLE_SIZE * 1.5 && ny + nh - y < HANDLE_SIZE) { - return { noteId: n.id, edge: "bottom-left" }; - } - // Bottom-right handle: gain - if (nx2 - x < HANDLE_SIZE * 1.5 && ny + nh - y < HANDLE_SIZE) { - return { noteId: n.id, edge: "bottom-right" }; - } + const note = notes[i]; + const x1 = timeToX(note.startTime, viewport); + const x2 = timeToX(note.endTime, viewport); + const noteY = midiToY(note.correctedPitch + 0.5, viewport, canvasHeight); + const noteH = viewport.pixelsPerSemitone; + + if (x < x1 || x > x2 || y < noteY || y > noteY + noteH) continue; + + const width = x2 - x1; + if (width > 30 && noteH > 8) { + const midX = x1 + width / 2; + if (Math.abs(x - midX) < handleSize && y - noteY < handleSize) { + return { noteId: note.id, edge: "top-center" }; + } + if (PITCH_EDITOR_FORMANT_EDITING_ENABLED && x - x1 < handleSize * 1.5 && noteY + noteH - y < handleSize) { + return { noteId: note.id, edge: "bottom-left" }; + } + if (x2 - x < handleSize * 1.5 && noteY + noteH - y < handleSize) { + return { noteId: note.id, edge: "bottom-right" }; } - // Edge resize - if (x - nx < EDGE_PX) return { noteId: n.id, edge: "left" }; - if (nx2 - x < EDGE_PX) return { noteId: n.id, edge: "right" }; - return { noteId: n.id, edge: "body" }; } + + if (x - x1 < edgePx) return { noteId: note.id, edge: "left" }; + if (x2 - x < edgePx) return { noteId: note.id, edge: "right" }; + return { noteId: note.id, edge: "body" }; } + return null; } -/** Main render function — draws everything on the canvas. */ +function drawPitchContour( + ctx: CanvasRenderingContext2D, + width: number, + height: number, + viewport: PitchEditorViewport, + contour: PitchContourData, + strokeStyle: string, + lineWidth: number, +) { + const { times, midi, confidence } = contour.frames; + ctx.beginPath(); + ctx.strokeStyle = strokeStyle; + ctx.lineWidth = lineWidth; + + let started = false; + for (let i = 0; i < times.length; i++) { + if (midi[i] <= 0 || confidence[i] < 0.3) { + started = false; + continue; + } + + const x = timeToX(times[i], viewport); + const y = midiToY(midi[i], viewport, height); + if (x < PIANO_WIDTH - 10 || x > width + 10) { + started = false; + continue; + } + + if (!started) { + ctx.moveTo(x, y); + started = true; + } else { + ctx.lineTo(x, y); + } + } + + ctx.stroke(); +} + export function renderPitchEditor( ctx: CanvasRenderingContext2D, width: number, height: number, viewport: PitchEditorViewport, - state: PitchEditorRenderState + state: PitchEditorRenderState, ) { const dpr = window.devicePixelRatio || 1; ctx.save(); ctx.scale(dpr, dpr); - // Clear ctx.fillStyle = "#1a1a1a"; ctx.fillRect(0, 0, width, height); @@ -145,11 +176,8 @@ export function renderPitchEditor( const topMidi = bottomMidi + visibleSemitones; const minMidi = Math.floor(bottomMidi); const maxMidi = Math.ceil(topMidi); + const hasScale = state.scaleNotes.some((inScale) => !inScale); - // Check if a non-chromatic scale is active - const hasScale = state.scaleNotes.some((v) => !v) || false; - - // --- 1. Piano keys + semitone bands --- for (let midi = minMidi; midi <= maxMidi; midi++) { const y = midiToY(midi + 0.5, viewport, height); const h = viewport.pixelsPerSemitone; @@ -158,25 +186,18 @@ export function renderPitchEditor( const inScale = state.scaleNotes[noteClass]; const isTonic = noteClass === state.scaleKey; - // Semitone band — dim out-of-scale notes - if (hasScale && !inScale) { - ctx.fillStyle = "#111111"; - } else if (hasScale && isTonic) { - ctx.fillStyle = "#252525"; - } else { - ctx.fillStyle = isBlack ? "#161616" : "#1e1e1e"; - } + if (hasScale && !inScale) ctx.fillStyle = "#111111"; + else if (hasScale && isTonic) ctx.fillStyle = "#252525"; + else ctx.fillStyle = isBlack ? "#161616" : "#1e1e1e"; ctx.fillRect(PIANO_WIDTH, y, width - PIANO_WIDTH, h); - // Grid line ctx.strokeStyle = "#2a2a2a"; - ctx.lineWidth = noteClass === 0 ? 1 : 0.5; // C notes get thicker line + ctx.lineWidth = noteClass === 0 ? 1 : 0.5; ctx.beginPath(); ctx.moveTo(PIANO_WIDTH, y + h); ctx.lineTo(width, y + h); ctx.stroke(); - // Piano key label if (y >= -h && y <= height + h) { ctx.fillStyle = "#1a1a1a"; ctx.fillRect(0, y, PIANO_WIDTH, h); @@ -189,21 +210,19 @@ export function renderPitchEditor( ctx.fillRect(0, y, PIANO_WIDTH * 0.65, h); } - // Label for C notes or every few semitones if (noteClass === 0 || viewport.pixelsPerSemitone > 14) { const octave = Math.floor(midi / 12) - 1; - const name = NOTE_NAMES[noteClass] + octave; - const keyBrightness = hasScale ? (inScale ? (isTonic ? "#bbb" : "#999") : "#555") : (isBlack ? "#888" : "#999"); - ctx.fillStyle = keyBrightness; + const label = NOTE_NAMES[noteClass] + octave; + const brightness = hasScale ? (inScale ? (isTonic ? "#bbb" : "#999") : "#555") : (isBlack ? "#888" : "#999"); + ctx.fillStyle = brightness; ctx.font = "9px Inter, system-ui, sans-serif"; ctx.textAlign = "right"; ctx.textBaseline = "middle"; - ctx.fillText(name, PIANO_WIDTH - 3, y + h / 2); + ctx.fillText(label, PIANO_WIDTH - 3, y + h / 2); } } } - // Piano key border ctx.strokeStyle = "#333"; ctx.lineWidth = 1; ctx.beginPath(); @@ -211,7 +230,6 @@ export function renderPitchEditor( ctx.lineTo(PIANO_WIDTH, height); ctx.stroke(); - // --- 2. Vertical grid lines (beats) --- if (state.bpm > 0) { const beatDuration = 60 / state.bpm; const beatsPerBar = state.timeSignature[0]; @@ -220,11 +238,10 @@ export function renderPitchEditor( const firstBeat = Math.floor(startTime / beatDuration); const lastBeat = Math.ceil(endTime / beatDuration); - for (let b = firstBeat; b <= lastBeat; b++) { - const t = b * beatDuration; - const x = projectTimeToX(t, viewport); + for (let beat = firstBeat; beat <= lastBeat; beat++) { + const x = projectTimeToX(beat * beatDuration, viewport); if (x < PIANO_WIDTH || x > width) continue; - const isBar = b % beatsPerBar === 0; + const isBar = beat % beatsPerBar === 0; ctx.strokeStyle = isBar ? "#3a3a3a" : "#262626"; ctx.lineWidth = isBar ? 1 : 0.5; ctx.beginPath(); @@ -234,79 +251,88 @@ export function renderPitchEditor( } } - // --- 2.5. Waveform RMS overlay (amplitude envelope behind notes) --- - if (state.renderCoverage.length > 0) { - for (const range of state.renderCoverage) { - if (range.state === "hq_ready") continue; - const x1 = timeToX(range.startTime, viewport); - const x2 = timeToX(range.endTime, viewport); - if (x2 < PIANO_WIDTH || x1 > width) continue; - ctx.fillStyle = range.state === "preview_ready" - ? "rgba(56, 189, 248, 0.12)" - : "rgba(245, 158, 11, 0.10)"; - ctx.fillRect(Math.max(PIANO_WIDTH, x1), 0, Math.max(0, Math.min(width, x2) - Math.max(PIANO_WIDTH, x1)), height); - - ctx.strokeStyle = range.state === "preview_ready" - ? "rgba(56, 189, 248, 0.22)" - : "rgba(245, 158, 11, 0.18)"; - ctx.lineWidth = 1; - ctx.beginPath(); - ctx.moveTo(x1, 0); - ctx.lineTo(x1, height); - ctx.moveTo(x2, 0); - ctx.lineTo(x2, height); - ctx.stroke(); - } + for (const range of state.renderCoverage) { + if (range.state === "hq_ready") continue; + const x1 = timeToX(range.startTime, viewport); + const x2 = timeToX(range.endTime, viewport); + if (x2 < PIANO_WIDTH || x1 > width) continue; + + ctx.fillStyle = range.state === "preview_ready" + ? "rgba(56, 189, 248, 0.12)" + : "rgba(245, 158, 11, 0.10)"; + ctx.fillRect(Math.max(PIANO_WIDTH, x1), 0, Math.max(0, Math.min(width, x2) - Math.max(PIANO_WIDTH, x1)), height); + + ctx.strokeStyle = range.state === "preview_ready" + ? "rgba(56, 189, 248, 0.22)" + : "rgba(245, 158, 11, 0.18)"; + ctx.lineWidth = 1; + ctx.beginPath(); + ctx.moveTo(x1, 0); + ctx.lineTo(x1, height); + ctx.moveTo(x2, 0); + ctx.lineTo(x2, height); + ctx.stroke(); } if (state.contour?.frames.rms && state.contour.frames.rms.length > 0) { - const { times, rms, midi: rmsMidi, confidence: rmsConf } = state.contour.frames; - const startTime = viewport.scrollX - viewport.clipStartTime; - const endTime = startTime + (width - PIANO_WIDTH) / viewport.pixelsPerSecond; - // Find the max RMS for normalization + const { times, rms, midi, confidence } = state.contour.frames; let maxRms = 0; - for (const val of rms) { - if (val > maxRms) maxRms = val; - } + for (const value of rms) maxRms = Math.max(maxRms, value); + if (maxRms > 0) { ctx.save(); ctx.globalAlpha = 0.12; - ctx.fillStyle = "#60a5fa"; // blue tint - // Draw RMS bars aligned to pitch position + ctx.fillStyle = "#60a5fa"; + for (let i = 0; i < times.length; i++) { - const t = times[i]; - if (t < startTime - 0.1 || t > endTime + 0.1) continue; - if (rmsConf[i] < 0.1 || rmsMidi[i] <= 0) continue; // skip unvoiced/silent - const x = timeToX(t, viewport); + if (confidence[i] < 0.1 || midi[i] <= 0) continue; + const x = timeToX(times[i], viewport); if (x < PIANO_WIDTH || x > width) continue; + const normalizedRms = rms[i] / maxRms; - const barHeight = normalizedRms * viewport.pixelsPerSemitone * 3; // scale to ~3 semitones max height - const y = midiToY(rmsMidi[i], viewport, height); - const nextT = i + 1 < times.length ? times[i + 1] : t + 0.01; - const barWidth = Math.max(1, (nextT - t) * viewport.pixelsPerSecond); + const barHeight = normalizedRms * viewport.pixelsPerSemitone * 3; + const y = midiToY(midi[i], viewport, height); + const nextTime = i + 1 < times.length ? times[i + 1] : times[i] + 0.01; + const barWidth = Math.max(1, (nextTime - times[i]) * viewport.pixelsPerSecond); ctx.fillRect(x, y - barHeight / 2, barWidth, barHeight); } + ctx.restore(); } } - // --- 3. Pitch contour (raw detected pitch) --- if (state.contour && state.contour.frames.times.length > 0) { - const { times, midi: midiArr, confidence: confArr } = state.contour.frames; + drawPitchContour(ctx, width, height, viewport, state.contour, "rgba(245, 158, 11, 0.35)", 1.5); + } + + if (state.notes.length > 0 && state.contour && state.contour.frames.times.length > 0) { + const { times, midi, confidence } = state.contour.frames; ctx.beginPath(); - ctx.strokeStyle = "rgba(245, 158, 11, 0.35)"; // amber, semi-transparent + ctx.strokeStyle = "rgba(255, 255, 255, 0.65)"; ctx.lineWidth = 1.5; let started = false; + for (let i = 0; i < times.length; i++) { - const midiVal = midiArr[i]; - const conf = confArr[i]; - if (midiVal <= 0 || conf < 0.3) { + if (midi[i] <= 0 || confidence[i] < 0.3) { + started = false; + continue; + } + + const frameTime = times[i]; + const note = state.notes.find((candidate) => frameTime >= getNoteEffectiveStart(candidate) && frameTime <= getNoteEffectiveEnd(candidate)); + if (!note) { started = false; continue; } - const x = timeToX(times[i], viewport); - const y = midiToY(midiVal, viewport, height); - if (x < PIANO_WIDTH - 10 || x > width + 10) { started = false; continue; } + + const correctedMidi = midi[i] + (note.correctedPitch - note.detectedPitch); + const x = timeToX(frameTime, viewport); + const y = midiToY(correctedMidi, viewport, height); + if (x < PIANO_WIDTH - 10 || x > width + 10) { + started = false; + continue; + } + if (!started) { ctx.moveTo(x, y); started = true; @@ -314,97 +340,14 @@ export function renderPitchEditor( ctx.lineTo(x, y); } } - ctx.stroke(); - } - // --- 3.5. Corrected pitch overlay (white line — shows where notes will actually play) --- - if (!state.polyMode && state.notes.length > 0 && state.contour && state.contour.frames.times.length > 0) { - const { times, midi: midiArr, confidence: confArr } = state.contour.frames; - ctx.beginPath(); - ctx.strokeStyle = "rgba(255, 255, 255, 0.65)"; - ctx.lineWidth = 1.5; - let corrStarted = false; - for (let i = 0; i < times.length; i++) { - const rawMidi = midiArr[i]; - const conf = confArr[i]; - if (rawMidi <= 0 || conf < 0.3) { corrStarted = false; continue; } - const frameTime = times[i]; - // Find which note this frame falls within - const note = state.notes.find(n => frameTime >= n.startTime && frameTime <= n.endTime); - if (note === undefined) { corrStarted = false; continue; } - // Shift the raw pitch by the correction applied to this note - const correctedMidi = rawMidi + (note.correctedPitch - note.detectedPitch); - const x = timeToX(frameTime, viewport); - const y = midiToY(correctedMidi, viewport, height); - if (x < PIANO_WIDTH - 10 || x > width + 10) { corrStarted = false; continue; } - if (corrStarted) { ctx.lineTo(x, y); } else { ctx.moveTo(x, y); corrStarted = true; } - } ctx.stroke(); } - // --- 3.6. Pitch salience heatmap (polyphonic mode) --- - if (state.polyMode && state.showPitchSalience && state.pitchSalience && state.pitchSalience.length > 0) { - const salience = state.pitchSalience; - const dsFactor = state.salienceDownsampleFactor || 1; - const hopSec = (state.salienceHopSize || 256) / (state.salienceSampleRate || 22050); - const frameTimeSec = hopSec * dsFactor; - const SALIENCE_BINS = salience[0]?.length || 264; - // 264 bins = 88 notes * 3 (1/3 semitone), MIDI 21-108 - const MIDI_LOW = 21; - const binsPerSemitone = 3; - - ctx.globalAlpha = 0.3; - for (let f = 0; f < salience.length; f++) { - const t = f * frameTimeSec; - const x = timeToX(t, viewport); - const xNext = timeToX(t + frameTimeSec, viewport); - if (xNext < PIANO_WIDTH || x > width) continue; - const fw = Math.max(1, xNext - x); - - for (let b = 0; b < SALIENCE_BINS; b++) { - const val = salience[f][b]; - if (val < 20) continue; // skip low energy - const midi = MIDI_LOW + b / binsPerSemitone; - const y = midiToY(midi + 0.5 / binsPerSemitone, viewport, height); - const bh = viewport.pixelsPerSemitone / binsPerSemitone; - if (y + bh < 0 || y > height) continue; - // Color: yellow-orange gradient based on intensity - const intensity = val / 255; - const r2 = Math.round(255 * intensity); - const g = Math.round(180 * intensity); - ctx.fillStyle = `rgb(${r2},${g},0)`; - ctx.fillRect(x, y, fw, Math.max(1, bh)); - } - } - ctx.globalAlpha = 1; - } - - // --- 4. Note blocks (monophonic or polyphonic) --- - // 12 hue-based colors for pitch classes in poly mode (rainbow mapping) - const POLY_NOTE_COLORS = [ - [220, 50, 50], // C — red - [220, 100, 50], // C# — red-orange - [220, 160, 30], // D — orange - [180, 180, 30], // D# — yellow - [80, 180, 50], // E — yellow-green - [40, 180, 80], // F — green - [30, 180, 160], // F# — teal - [40, 140, 200], // G — blue - [60, 100, 220], // G# — indigo - [120, 70, 220], // A — purple - [180, 50, 200], // A# — magenta - [220, 50, 150], // B — pink - ]; - - const notesToRender: UnifiedNoteData[] = - state.polyMode - ? state.polyNotes.map(polyToUnified) - : state.notes.map(n => monoToUnified(n)); - - for (const note of notesToRender) { + for (const note of state.notes) { const x1 = timeToX(note.startTime, viewport); const x2 = timeToX(note.endTime, viewport); - if (x2 < PIANO_WIDTH || x1 > width) continue; // off-screen + if (x2 < PIANO_WIDTH || x1 > width) continue; const y = midiToY(note.correctedPitch + 0.5, viewport, height); const h = viewport.pixelsPerSemitone; @@ -413,135 +356,108 @@ export function renderPitchEditor( const isHovered = state.hoveredNoteId === note.id; const isShifted = Math.abs(note.correctedPitch - note.detectedPitch) > 0.1; const isUnvoiced = note.voiced === false; + const radius = Math.min(3, h / 4, w / 4); - // Note fill — poly mode uses pitch-class rainbow colors with confidence-based opacity - if (note.isPoly) { - const pitchClass = ((Math.round(note.detectedPitch) % 12) + 12) % 12; - const [cr, cg, cb] = POLY_NOTE_COLORS[pitchClass]; - const confAlpha = 0.3 + (note.confidence ?? 0.5) * 0.55; // 0.3–0.85 based on confidence - if (isSelected) { - ctx.fillStyle = `rgba(${cr}, ${cg}, ${cb}, ${Math.min(1, confAlpha + 0.2)})`; - } else if (isShifted) { - // Slightly brighter when shifted - ctx.fillStyle = `rgba(${Math.min(255, cr + 30)}, ${Math.min(255, cg + 30)}, ${Math.min(255, cb + 30)}, ${confAlpha})`; - } else { - ctx.fillStyle = `rgba(${cr}, ${cg}, ${cb}, ${confAlpha})`; - } - } else { - const alpha = 0.75; - if (isUnvoiced) { - ctx.fillStyle = `rgba(100, 100, 100, 0.4)`; // gray for unvoiced - } else if (isSelected) { - ctx.fillStyle = `rgba(59, 130, 246, ${alpha})`; // blue - } else if (isShifted) { - ctx.fillStyle = `rgba(245, 158, 11, ${alpha})`; // amber - } else { - ctx.fillStyle = `rgba(34, 197, 94, ${alpha})`; // green - } - } + const alpha = 0.75; + if (isUnvoiced) ctx.fillStyle = "rgba(100, 100, 100, 0.4)"; + else if (isSelected) ctx.fillStyle = `rgba(59, 130, 246, ${alpha})`; + else if (isShifted) ctx.fillStyle = `rgba(245, 158, 11, ${alpha})`; + else ctx.fillStyle = `rgba(34, 197, 94, ${alpha})`; - // Rounded rect - const r = Math.min(3, h / 4, w / 4); ctx.beginPath(); - ctx.moveTo(x1 + r, y); - ctx.lineTo(x2 - r, y); - ctx.quadraticCurveTo(x2, y, x2, y + r); - ctx.lineTo(x2, y + h - r); - ctx.quadraticCurveTo(x2, y + h, x2 - r, y + h); - ctx.lineTo(x1 + r, y + h); - ctx.quadraticCurveTo(x1, y + h, x1, y + h - r); - ctx.lineTo(x1, y + r); - ctx.quadraticCurveTo(x1, y, x1 + r, y); + ctx.moveTo(x1 + radius, y); + ctx.lineTo(x2 - radius, y); + ctx.quadraticCurveTo(x2, y, x2, y + radius); + ctx.lineTo(x2, y + h - radius); + ctx.quadraticCurveTo(x2, y + h, x2 - radius, y + h); + ctx.lineTo(x1 + radius, y + h); + ctx.quadraticCurveTo(x1, y + h, x1, y + h - radius); + ctx.lineTo(x1, y + radius); + ctx.quadraticCurveTo(x1, y, x1 + radius, y); ctx.closePath(); ctx.fill(); - // Diagonal hatching for unvoiced notes (Melodyne-style) if (isUnvoiced && w > 4) { ctx.save(); - ctx.clip(); // clip to the rounded rect path + ctx.clip(); ctx.strokeStyle = "rgba(150, 150, 150, 0.3)"; ctx.lineWidth = 0.5; - const spacing = 6; - for (let hx = -h; hx < w + h; hx += spacing) { + for (let hatchX = -h; hatchX < w + h; hatchX += 6) { ctx.beginPath(); - ctx.moveTo(x1 + hx, y); - ctx.lineTo(x1 + hx + h, y + h); + ctx.moveTo(x1 + hatchX, y); + ctx.lineTo(x1 + hatchX + h, y + h); ctx.stroke(); } ctx.restore(); } - // Border — poly notes always get a thin border for overlap visibility if (isSelected || isHovered) { ctx.strokeStyle = isSelected ? "#60a5fa" : "#888"; ctx.lineWidth = isSelected ? 1.5 : 1; ctx.stroke(); - } else if (note.isPoly) { - const pitchClass = ((Math.round(note.detectedPitch) % 12) + 12) % 12; - const [br, bg, bb] = POLY_NOTE_COLORS[pitchClass]; - ctx.strokeStyle = `rgba(${br}, ${bg}, ${bb}, 0.6)`; - ctx.lineWidth = 0.75; - ctx.stroke(); } - // Pitch curves inside note blob (clipped to blob bounds) if (w > 10 && !isUnvoiced) { ctx.save(); ctx.beginPath(); ctx.rect(x1, y, w, h); ctx.clip(); - // Draw original pitch contour from frame data (faint) - if (state.contour && state.contour.frames.times.length > 0 && !state.polyMode) { - const { times, midi: midiArr, confidence: confArr } = state.contour.frames; + if (state.contour && state.contour.frames.times.length > 0) { + const { times, midi, confidence } = state.contour.frames; + ctx.beginPath(); ctx.strokeStyle = "rgba(255,255,255,0.15)"; ctx.lineWidth = 1; let started = false; - for (let fi = 0; fi < times.length; fi++) { - const ft = times[fi]; - if (ft < note.startTime - 0.01 || ft > note.endTime + 0.01) continue; - const rawMidi = midiArr[fi]; - if (rawMidi <= 0 || confArr[fi] < 0.3) { started = false; continue; } - const px = timeToX(ft, viewport); - const py = midiToY(rawMidi, viewport, height); - if (started) ctx.lineTo(px, py); else { ctx.moveTo(px, py); started = true; } + for (let i = 0; i < times.length; i++) { + if (times[i] < note.startTime - 0.01 || times[i] > note.endTime + 0.01) continue; + if (midi[i] <= 0 || confidence[i] < 0.3) { + started = false; + continue; + } + const px = timeToX(times[i], viewport); + const py = midiToY(midi[i], viewport, height); + if (!started) { + ctx.moveTo(px, py); + started = true; + } else { + ctx.lineTo(px, py); + } } ctx.stroke(); - } - // Draw corrected pitch contour (bold white — shows where audio will play) - if (state.contour && state.contour.frames.times.length > 0 && !state.polyMode) { - const { times, midi: midiArr, confidence: confArr } = state.contour.frames; ctx.beginPath(); ctx.strokeStyle = isSelected ? "rgba(255,255,255,0.6)" : "rgba(255,255,255,0.35)"; ctx.lineWidth = 1.5; - let started = false; + started = false; const pitchShift = note.correctedPitch - note.detectedPitch; - for (let fi = 0; fi < times.length; fi++) { - const ft = times[fi]; - if (ft < note.startTime - 0.01 || ft > note.endTime + 0.01) continue; - const rawMidi = midiArr[fi]; - if (rawMidi <= 0 || confArr[fi] < 0.3) { started = false; continue; } - const corrMidi = rawMidi + pitchShift; - const px = timeToX(ft, viewport); - const py = midiToY(corrMidi, viewport, height); - if (started) ctx.lineTo(px, py); else { ctx.moveTo(px, py); started = true; } + for (let i = 0; i < times.length; i++) { + if (times[i] < note.startTime - 0.01 || times[i] > note.endTime + 0.01) continue; + if (midi[i] <= 0 || confidence[i] < 0.3) { + started = false; + continue; + } + const px = timeToX(times[i], viewport); + const py = midiToY(midi[i] + pitchShift, viewport, height); + if (!started) { + ctx.moveTo(px, py); + started = true; + } else { + ctx.lineTo(px, py); + } } ctx.stroke(); - } - - // Fallback: draw drift curve if no contour data available - if ((state.contour === null || state.polyMode) && note.pitchDrift && note.pitchDrift.length > 1) { + } else if (note.pitchDrift && note.pitchDrift.length > 1) { ctx.beginPath(); ctx.strokeStyle = isSelected ? "rgba(255,255,255,0.4)" : "rgba(255,255,255,0.2)"; ctx.lineWidth = 1; for (let i = 0; i < note.pitchDrift.length; i++) { const t = note.startTime + (i / (note.pitchDrift.length - 1)) * (note.endTime - note.startTime); const px = timeToX(t, viewport); - const drift = note.pitchDrift[i]; - const py = midiToY(note.correctedPitch + drift, viewport, height); - if (i === 0) ctx.moveTo(px, py); else ctx.lineTo(px, py); + const py = midiToY(note.correctedPitch + note.pitchDrift[i], viewport, height); + if (i === 0) ctx.moveTo(px, py); + else ctx.lineTo(px, py); } ctx.stroke(); } @@ -549,7 +465,23 @@ export function renderPitchEditor( ctx.restore(); } - // Note name label inside block (if wide enough) + const transitionStart = getNoteEffectiveStart(note); + const transitionEnd = getNoteEffectiveEnd(note); + if (!isUnvoiced && (transitionStart < note.startTime || transitionEnd > note.endTime)) { + ctx.save(); + const inX = timeToX(transitionStart, viewport); + const outX = timeToX(transitionEnd, viewport); + if (transitionStart < note.startTime) { + ctx.fillStyle = "rgba(245, 158, 11, 0.18)"; + ctx.fillRect(inX, y, Math.max(0, x1 - inX), h); + } + if (transitionEnd > note.endTime) { + ctx.fillStyle = "rgba(245, 158, 11, 0.18)"; + ctx.fillRect(x2, y, Math.max(0, outX - x2), h); + } + ctx.restore(); + } + if (w > 30 && h > 10) { const noteClass = Math.round(note.correctedPitch) % 12; const octave = Math.floor(Math.round(note.correctedPitch) / 12) - 1; @@ -561,78 +493,67 @@ export function renderPitchEditor( ctx.fillText(label, x1 + 4, y + h / 2); } - // Smart control handles on hover (VariAudio-style) if (isHovered && w > 30 && h > 8 && !isUnvoiced) { - const hs = 5; // handle size + const handleSize = 5; const handleColor = "rgba(255,255,255,0.6)"; const midX = x1 + w / 2; - // Top-center: straighten/vibrato (triangle up) ctx.fillStyle = handleColor; ctx.beginPath(); - ctx.moveTo(midX - hs, y + hs + 1); + ctx.moveTo(midX - handleSize, y + handleSize + 1); ctx.lineTo(midX, y + 1); - ctx.lineTo(midX + hs, y + hs + 1); + ctx.lineTo(midX + handleSize, y + handleSize + 1); ctx.closePath(); ctx.fill(); - // Bottom-left: formant (small diamond) - ctx.beginPath(); - const blx = x1 + hs + 2; - const bly = y + h - hs - 1; - ctx.moveTo(blx, bly - hs / 2); - ctx.lineTo(blx + hs / 2, bly); - ctx.lineTo(blx, bly + hs / 2); - ctx.lineTo(blx - hs / 2, bly); - ctx.closePath(); - ctx.fill(); + if (PITCH_EDITOR_FORMANT_EDITING_ENABLED) { + const blx = x1 + handleSize + 2; + const bly = y + h - handleSize - 1; + ctx.beginPath(); + ctx.moveTo(blx, bly - handleSize / 2); + ctx.lineTo(blx + handleSize / 2, bly); + ctx.lineTo(blx, bly + handleSize / 2); + ctx.lineTo(blx - handleSize / 2, bly); + ctx.closePath(); + ctx.fill(); + } - // Bottom-right: gain (small square) - const brx = x2 - hs - 4; - const bry = y + h - hs - 2; - ctx.fillRect(brx, bry, hs, hs); + const brx = x2 - handleSize - 4; + const bry = y + h - handleSize - 2; + ctx.fillRect(brx, bry, handleSize, handleSize); } } - // --- 4.5. Transition curves between adjacent notes --- - if (!state.polyMode) { - const monoNotes = state.notes; - for (let i = 0; i < monoNotes.length - 1; i++) { - const n1 = monoNotes[i]; - const n2 = monoNotes[i + 1]; - const gap = n2.startTime - n1.endTime; - if (gap > 0.1) continue; // too far apart - const hasTransition = n1.transitionOut > 0 || n2.transitionIn > 0; - if (!hasTransition) continue; - - const transMs = Math.max(n1.transitionOut, n2.transitionIn); - const transSec = transMs / 1000; - const x1 = timeToX(n1.endTime - transSec / 2, viewport); - const x2 = timeToX(n2.startTime + transSec / 2, viewport); - const y1 = midiToY(n1.correctedPitch, viewport, height); - const y2 = midiToY(n2.correctedPitch, viewport, height); - - if (x2 < PIANO_WIDTH || x1 > width) continue; + for (let i = 0; i < state.notes.length - 1; i++) { + const left = state.notes[i]; + const right = state.notes[i + 1]; + const gap = getNoteEffectiveStart(right) - getNoteEffectiveEnd(left); + if (gap > 0.1) continue; + if (!(getEffectiveTransitionMs(left, "out") > 0 || getEffectiveTransitionMs(right, "in") > 0)) continue; - ctx.beginPath(); - ctx.strokeStyle = "rgba(245, 158, 11, 0.6)"; - ctx.lineWidth = 2; - const midX = (x1 + x2) / 2; - ctx.moveTo(x1, y1); - ctx.bezierCurveTo(midX, y1, midX, y2, x2, y2); - ctx.stroke(); - } + const x1 = timeToX(getNoteEffectiveEnd(left), viewport); + const x2 = timeToX(getNoteEffectiveStart(right), viewport); + const y1 = midiToY(left.correctedPitch, viewport, height); + const y2 = midiToY(right.correctedPitch, viewport, height); + if (x2 < PIANO_WIDTH || x1 > width) continue; + + ctx.beginPath(); + ctx.strokeStyle = "rgba(245, 158, 11, 0.6)"; + ctx.lineWidth = 2; + const midX = (x1 + x2) / 2; + ctx.moveTo(x1, y1); + ctx.bezierCurveTo(midX, y1, midX, y2, x2, y2); + ctx.stroke(); } - // --- 5. Playhead (currentTime is in project time, not clip-relative) --- if (state.currentTime >= 0) { - const px = projectTimeToX(state.currentTime, viewport); - if (px >= PIANO_WIDTH && px <= width) { + const playheadX = projectTimeToX(state.currentTime, viewport); + if (playheadX >= PIANO_WIDTH && playheadX <= width) { ctx.strokeStyle = "#fff"; ctx.lineWidth = 1; ctx.beginPath(); - ctx.moveTo(px, 0); - ctx.lineTo(px, height); + ctx.moveTo(playheadX, 0); + ctx.lineTo(playheadX, height); ctx.stroke(); } } @@ -640,11 +561,6 @@ export function renderPitchEditor( ctx.restore(); } -/** - * Lightweight playhead-only render for the RAF loop during playback. - * Blits a pre-rendered static canvas (grid, notes, contour) and draws - * only the playhead line on top — avoids redrawing the entire canvas at 60fps. - */ export function renderPlayheadOverlay( ctx: CanvasRenderingContext2D, width: number, @@ -655,19 +571,18 @@ export function renderPlayheadOverlay( ) { const dpr = globalThis.window?.devicePixelRatio || 1; ctx.clearRect(0, 0, width * dpr, height * dpr); - ctx.drawImage(staticCanvas as any, 0, 0); + ctx.drawImage(staticCanvas as CanvasImageSource, 0, 0); - // Draw playhead if (currentTime >= 0) { ctx.save(); ctx.scale(dpr, dpr); - const px = projectTimeToX(currentTime, viewport); - if (px >= PIANO_WIDTH && px <= width) { + const playheadX = projectTimeToX(currentTime, viewport); + if (playheadX >= PIANO_WIDTH && playheadX <= width) { ctx.strokeStyle = "#fff"; ctx.lineWidth = 1; ctx.beginPath(); - ctx.moveTo(px, 0); - ctx.lineTo(px, height); + ctx.moveTo(playheadX, 0); + ctx.lineTo(playheadX, height); ctx.stroke(); } ctx.restore(); diff --git a/frontend/src/components/PitchEditorLowerZone.tsx b/frontend/src/components/PitchEditorLowerZone.tsx index 6a70241..02a60e6 100644 --- a/frontend/src/components/PitchEditorLowerZone.tsx +++ b/frontend/src/components/PitchEditorLowerZone.tsx @@ -1,7 +1,8 @@ import React, { useRef, useEffect, useCallback, useState, useMemo } from "react"; import { useShallow } from "zustand/shallow"; import { useDAWStore } from "../store/useDAWStore"; -import { usePitchEditorStore, PitchEditorTool, PitchSnapMode } from "../store/pitchEditorStore"; +import { isPrimaryModifier, formatShortcut } from "../utils/platform"; +import { usePitchEditorStore, PitchEditorTool, PitchSnapMode, PITCH_EDITOR_FORMANT_EDITING_ENABLED } from "../store/pitchEditorStore"; import { GripHorizontal, X, Scissors, MousePointer, Activity, Waves, ChevronRight, Pencil } from "lucide-react"; import { NoteInspector } from "./NoteInspector"; import { CorrectPitchModal } from "./CorrectPitchModal"; @@ -45,6 +46,71 @@ function formatNoteName(midi: number): string { return `${name}${octave}${cents !== 0 ? ` ${cents > 0 ? "+" : ""}${cents}¢` : ""}`; } +function getNoteEffectiveStart(note: { startTime: number; effectiveStartTime?: number }) { + return note.effectiveStartTime ?? note.startTime; +} + +function getNoteEffectiveEnd(note: { endTime: number; effectiveEndTime?: number }) { + return note.effectiveEndTime ?? note.endTime; +} + +function getOpeningViewportWindow( + clipInfo: { startTime: number; duration: number }, + currentTime: number, +) { + const playheadInsideClip = + currentTime >= clipInfo.startTime && + currentTime <= clipInfo.startTime + clipInfo.duration; + const leftEdgeProjectTime = playheadInsideClip ? currentTime : clipInfo.startTime; + const clipStart = Math.max(0, leftEdgeProjectTime - clipInfo.startTime); + const clipEnd = Math.min(clipInfo.duration, clipStart + 5); + return { + leftEdgeProjectTime, + clipStart, + clipEnd, + }; +} + +function isVoicedAtClipTime(contour: ReturnType<typeof usePitchEditorStore.getState>["contour"], clipTime: number) { + const frames = contour?.frames; + if (!frames || frames.times.length === 0) return true; + + let closestIndex = -1; + let closestDt = Number.POSITIVE_INFINITY; + for (let i = 0; i < frames.times.length; i++) { + const dt = Math.abs(frames.times[i] - clipTime); + if (dt < closestDt) { + closestDt = dt; + closestIndex = i; + } + } + + if (closestIndex < 0 || closestDt > 0.04) return false; + return Boolean(frames.voiced[closestIndex]) && (frames.midi[closestIndex] ?? 0) > 0 && (frames.confidence[closestIndex] ?? 0) >= 0.15; +} + +function findDrawTargetNote( + notes: ReturnType<typeof usePitchEditorStore.getState>["notes"], + selectedNoteIds: string[], + clipTime: number, + midi: number, +) { + const selectedSet = new Set(selectedNoteIds); + const candidates = notes.filter((note) => clipTime >= getNoteEffectiveStart(note) && clipTime <= getNoteEffectiveEnd(note)); + if (candidates.length === 0) return null; + const scored = candidates + .map((note) => ({ + note, + selected: selectedSet.has(note.id), + pitchDistance: Math.abs(note.correctedPitch - midi), + })) + .sort((a, b) => { + if (a.selected !== b.selected) return a.selected ? -1 : 1; + return a.pitchDistance - b.pitchDistance; + }); + return scored[0]?.note ?? null; +} + // Click-to-audition: play a short sine tone at a MIDI pitch via Web Audio API let _auditionCtx: AudioContext | null = null; function auditionNote(midiPitch: number, durationMs = 200) { @@ -121,31 +187,26 @@ export function PitchEditorLowerZone() { const timeSignature = useDAWStore((s) => s.timeSignature); const { - contour, notes, isAnalyzing, selectedNoteIds, tool, snapMode, progressPercent, progressLabel, + contour, notes, isAnalyzing, analysisPhase, selectedNoteIds, tool, snapMode, progressPercent, progressLabel, applyState, applyMessage, scrollY, zoomY, clipStartTime: pitchClipStartTime, clipDuration: pitchClipDuration, analyze, setTool, setSnapMode, selectNote, selectAll, deselectAll, updateNote, commitNoteEdit, moveSelectedPitch, splitNote, + beginInteractivePreview, endInteractivePreview, correctSelectedToScale, correctAllToScale, undo, redo, pushUndo, setScrollY, undoStack, redoStack, - polyMode, polyNotes, polyAnalysisResult, showPitchSalience, - togglePolyMode, analyzePolyphonic, moveSelectedPolyPitch, - applyPolyCorrection, togglePitchSalience, - polyNoteThreshold, polyOnsetThreshold, polyMinDuration, - setPolyNoteThreshold, setPolyOnsetThreshold, setPolyMinDuration, scaleKey, scaleType, scaleNotes, setScale, autoDetectScale, mergeNotes, toggleCorrectPitchModal, abCompareMode, toggleABCompare, drawPitchOnNote, beginDrawPitch, commitDrawPitch, - globalFormantCents, setGlobalFormantCents, renderCoverage, } = usePitchEditorStore( useShallow((s) => ({ - contour: s.contour, notes: s.notes, isAnalyzing: s.isAnalyzing, progressPercent: s.progressPercent, progressLabel: s.progressLabel, + contour: s.contour, notes: s.notes, isAnalyzing: s.isAnalyzing, analysisPhase: s.analysisPhase, progressPercent: s.progressPercent, progressLabel: s.progressLabel, applyState: s.applyState, applyMessage: s.applyMessage, selectedNoteIds: s.selectedNoteIds, tool: s.tool, snapMode: s.snapMode, scrollY: s.scrollY, zoomY: s.zoomY, @@ -153,26 +214,17 @@ export function PitchEditorLowerZone() { analyze: s.analyze, setTool: s.setTool, setSnapMode: s.setSnapMode, selectNote: s.selectNote, selectAll: s.selectAll, deselectAll: s.deselectAll, updateNote: s.updateNote, commitNoteEdit: s.commitNoteEdit, moveSelectedPitch: s.moveSelectedPitch, + beginInteractivePreview: s.beginInteractivePreview, endInteractivePreview: s.endInteractivePreview, splitNote: s.splitNote, correctSelectedToScale: s.correctSelectedToScale, correctAllToScale: s.correctAllToScale, undo: s.undo, redo: s.redo, pushUndo: s.pushUndo, setScrollY: s.setScrollY, undoStack: s.undoStack, redoStack: s.redoStack, - polyMode: s.polyMode, polyNotes: s.polyNotes, - polyAnalysisResult: s.polyAnalysisResult, showPitchSalience: s.showPitchSalience, - togglePolyMode: s.togglePolyMode, analyzePolyphonic: s.analyzePolyphonic, - moveSelectedPolyPitch: s.moveSelectedPolyPitch, - applyPolyCorrection: s.applyPolyCorrection, togglePitchSalience: s.togglePitchSalience, - polyNoteThreshold: s.polyNoteThreshold, polyOnsetThreshold: s.polyOnsetThreshold, - polyMinDuration: s.polyMinDuration, - setPolyNoteThreshold: s.setPolyNoteThreshold, setPolyOnsetThreshold: s.setPolyOnsetThreshold, - setPolyMinDuration: s.setPolyMinDuration, scaleKey: s.scaleKey, scaleType: s.scaleType, scaleNotes: s.scaleNotes, setScale: s.setScale, autoDetectScale: s.autoDetectScale, mergeNotes: s.mergeNotes, toggleCorrectPitchModal: s.toggleCorrectPitchModal, abCompareMode: s.abCompareMode, toggleABCompare: s.toggleABCompare, drawPitchOnNote: s.drawPitchOnNote, beginDrawPitch: s.beginDrawPitch, commitDrawPitch: s.commitDrawPitch, - globalFormantCents: s.globalFormantCents, setGlobalFormantCents: s.setGlobalFormantCents, renderCoverage: s.renderCoverage, })) ); @@ -184,16 +236,6 @@ export function PitchEditorLowerZone() { : applyState === "queued" || applyState === "processing" || applyState === "preview_processing" || applyState === "final_processing" ? "text-daw-accent bg-daw-accent/10 border-daw-accent/30" : "text-neutral-500 bg-neutral-800/80 border-neutral-700"; - const formantStatusHint = globalFormantCents !== 0 - ? ( - applyState === "preview_processing" - ? "Rendering preview near playhead..." - : applyState === "preview_ready" || applyState === "final_processing" || applyState === "done" - ? "Previewing rendered formant" - : "Rendered formant preview" - ) - : "Clip-wide timbre shift"; - const [hoveredNoteId, setHoveredNoteId] = useState<string | null>(null); const [dragState, setDragState] = useState<{ type: "move" | "pitch" | "resize-left" | "resize-right" | "straighten" | "formant" | "gain" | "drift" | "transition-in" | "transition-out" | "transition-both" | "draw"; @@ -206,18 +248,52 @@ export function PitchEditorLowerZone() { } | null>(null); // Clear analyzed range tracking when clip changes const analyzedRangesRef = useRef<Array<[number, number]>>([]); + const openViewportKeyRef = useRef<string | null>(null); useEffect(() => { analyzedRangesRef.current = []; + openViewportKeyRef.current = null; }, [pitchEditorClipId]); + // On open, snap the timeline to a 5-second viewport. + // If the playhead is inside the clip, anchor to the playhead. + // If not, anchor to the clip start so the editor opens on visible content. + useEffect(() => { + if (!pitchEditorTrackId || !pitchEditorClipId || !clipInfo || canvasSize.width <= 0) return; + + const openKey = `${pitchEditorTrackId}:${pitchEditorClipId}`; + if (openViewportKeyRef.current === openKey) return; + + const targetPixelsPerSecond = Math.max(MIN_PPS, Math.min(MAX_PPS, canvasSize.width / 5)); + const { leftEdgeProjectTime } = getOpeningViewportWindow(clipInfo, currentTime); + const daw = useDAWStore.getState(); + + setZoom(targetPixelsPerSecond); + setScroll(Math.max(0, leftEdgeProjectTime * targetPixelsPerSecond), daw.scrollY); + openViewportKeyRef.current = openKey; + }, [pitchEditorTrackId, pitchEditorClipId, clipInfo, canvasSize.width, currentTime, setScroll, setZoom]); + // Auto-analyze on mount useEffect(() => { if (pitchEditorTrackId && pitchEditorClipId && !contour && !isAnalyzing) { - // Record initial range so scroll-based re-analyze won't duplicate it - analyzedRangesRef.current = [[0, 30]]; - analyze(); + if (clipInfo) { + const openingWindow = getOpeningViewportWindow(clipInfo, currentTime); + const clipStart = openingWindow.clipStart; + const clipEnd = openingWindow.clipEnd; + if (clipEnd > clipStart) { + analyzedRangesRef.current = [[clipStart, clipEnd]]; + analyze(clipStart, clipEnd); + } else { + const fallbackStart = Math.max(0, Math.min(openingWindow.clipStart, Math.max(0, clipInfo.duration - 5))); + const fallbackEnd = Math.min(clipInfo.duration, fallbackStart + 5); + analyzedRangesRef.current = [[fallbackStart, fallbackEnd]]; + analyze(fallbackStart, fallbackEnd); + } + } else { + analyzedRangesRef.current = [[0, 5]]; + analyze(); + } } - }, [pitchEditorTrackId, pitchEditorClipId, contour, isAnalyzing, analyze]); + }, [pitchEditorTrackId, pitchEditorClipId, contour, isAnalyzing, analyze, clipInfo, dawScrollX, pixelsPerSecond, canvasSize.width, currentTime]); // Re-analyze when scrolling into unanalyzed territory (debounced) const analyzeTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null); @@ -251,6 +327,66 @@ export function PitchEditorLowerZone() { }; }, [dawScrollX, pixelsPerSecond, canvasSize.width, pitchEditorTrackId, pitchEditorClipId, isAnalyzing, contour, clipInfo, analyze, undoStack]); + // Continue filling only the viewport/playhead neighborhood in the background. + useEffect(() => { + if (!pitchEditorTrackId || !pitchEditorClipId || isAnalyzing || !contour || !clipInfo) return; + if (undoStack.length > 0) return; + + const visibleStartProject = dawScrollX / pixelsPerSecond; + const visibleStart = visibleStartProject - clipInfo.startTime; + const visibleEnd = visibleStart + canvasSize.width / pixelsPerSecond; + const clipViewportStart = Math.max(0, visibleStart); + const clipViewportEnd = Math.min(clipInfo.duration, visibleEnd); + const clipPlayhead = Math.max(0, Math.min(clipInfo.duration, currentTime - clipInfo.startTime)); + const targetStart = Math.max(0, Math.min(clipViewportStart, clipPlayhead)); + const targetEnd = Math.min(clipInfo.duration, Math.max(clipViewportEnd, clipPlayhead + 30)); + if (targetEnd <= targetStart) return; + + const merged = [...analyzedRangesRef.current] + .sort((a, b) => a[0] - b[0]) + .reduce<Array<[number, number]>>((acc, range) => { + if (acc.length === 0) { + acc.push([...range] as [number, number]); + return acc; + } + const last = acc[acc.length - 1]; + if (range[0] <= last[1] + 0.25) { + last[1] = Math.max(last[1], range[1]); + } else { + acc.push([...range] as [number, number]); + } + return acc; + }, []); + + let nextGap: [number, number] | null = null; + let cursor = targetStart; + for (const [start, end] of merged) { + if (end <= targetStart || start >= targetEnd) continue; + const clippedStart = Math.max(start, targetStart); + const clippedEnd = Math.min(end, targetEnd); + if (clippedStart > cursor + 0.5) { + nextGap = [cursor, clippedStart]; + break; + } + cursor = Math.max(cursor, clippedEnd); + } + + if (!nextGap && cursor < targetEnd - 0.25) + nextGap = [cursor, targetEnd]; + + if (!nextGap) return; + + const nextStart = nextGap[0]; + const nextEnd = Math.min(nextGap[1], nextStart + 5); + if (nextEnd <= nextStart + 0.1) return; + const timer = setTimeout(() => { + analyzedRangesRef.current.push([nextStart, nextEnd]); + analyze(nextStart, nextEnd); + }, 250); + + return () => clearTimeout(timer); + }, [pitchEditorTrackId, pitchEditorClipId, isAnalyzing, contour, clipInfo, analyze, undoStack, dawScrollX, pixelsPerSecond, canvasSize.width, currentTime]); + // Resize observer useEffect(() => { const el = containerRef.current; @@ -289,13 +425,6 @@ export function PitchEditorLowerZone() { timeSignature: [daw.timeSignature.numerator, daw.timeSignature.denominator], scaleNotes: st.scaleNotes, scaleKey: st.scaleKey, - polyMode: st.polyMode, - polyNotes: st.polyNotes, - showPitchSalience: st.showPitchSalience, - pitchSalience: st.polyAnalysisResult?.pitchSalience ?? null, - salienceDownsampleFactor: st.polyAnalysisResult?.salienceDownsampleFactor ?? 1, - salienceHopSize: st.polyAnalysisResult?.hopSize ?? 256, - salienceSampleRate: st.polyAnalysisResult?.sampleRate ?? 22050, renderCoverage: st.renderCoverage, }), []); @@ -320,11 +449,6 @@ export function PitchEditorLowerZone() { currentTime, isPlaying, bpm, timeSignature: [timeSignature.numerator, timeSignature.denominator], scaleNotes, scaleKey, - polyMode, polyNotes, showPitchSalience, - pitchSalience: polyAnalysisResult?.pitchSalience ?? null, - salienceDownsampleFactor: polyAnalysisResult?.salienceDownsampleFactor ?? 1, - salienceHopSize: polyAnalysisResult?.hopSize ?? 256, - salienceSampleRate: polyAnalysisResult?.sampleRate ?? 22050, renderCoverage, }; renderPitchEditor(ctx, canvasSize.width, canvasSize.height, viewport, renderState); @@ -336,7 +460,7 @@ export function PitchEditorLowerZone() { offCtx.drawImage(canvas, 0, 0); } staticCanvasRef.current = offscreen; - }, [canvasSize, viewport, notes, contour, selectedNoteIds, hoveredNoteId, currentTime, isPlaying, bpm, timeSignature, scaleNotes, scaleKey, polyMode, polyNotes, showPitchSalience, polyAnalysisResult, renderCoverage]); + }, [canvasSize, viewport, notes, contour, selectedNoteIds, hoveredNoteId, currentTime, isPlaying, bpm, timeSignature, scaleNotes, scaleKey, renderCoverage]); // Playhead RAF — during playback, blit cached static canvas + draw playhead only. // This avoids a full renderPitchEditor() (grid, notes, contour) at 60fps. @@ -461,6 +585,11 @@ export function PitchEditorLowerZone() { }, [setZoom, setScroll, setScrollY]); // Mouse handlers + const startInteractivePreviewForHit = useCallback((noteId: string, edge: "body" | "left" | "right" | "top-center" | "bottom-left" | "bottom-right") => { + if (edge === "left" || edge === "right") return; + beginInteractivePreview(noteId); + }, [beginInteractivePreview]); + const handleMouseDown = useCallback((e: React.MouseEvent<HTMLCanvasElement>) => { const rect = canvasRef.current?.getBoundingClientRect(); if (!rect) return; @@ -497,6 +626,23 @@ export function PitchEditorLowerZone() { return; } + const clipTime = xToTime(x, freshViewport); + const midi = yToMidi(y, freshViewport, canvasSize.height); + if (!hit && pitchState.tool === "draw" && isVoicedAtClipTime(pitchState.contour, clipTime)) { + const drawTarget = findDrawTargetNote(currentNotes, pitchState.selectedNoteIds, clipTime, midi); + if (drawTarget) { + selectNote(drawTarget.id, e.ctrlKey || e.metaKey); + beginDrawPitch(); + drawPitchOnNote(drawTarget.id, clipTime, midi); + setDragState({ + type: "draw", noteId: drawTarget.id, startMouseY: y, + origPitch: drawTarget.correctedPitch, origStart: drawTarget.startTime, origEnd: drawTarget.endTime, + origValue: 0, + }); + return; + } + } + if (hit) { selectNote(hit.noteId, e.ctrlKey || e.metaKey); const note = currentNotes.find((n) => n.id === hit.noteId); @@ -504,13 +650,15 @@ export function PitchEditorLowerZone() { if (hit.edge === "top-center") { pushUndo("Straighten vibrato"); + startInteractivePreviewForHit(hit.noteId, hit.edge); setDragState({ type: "straighten", noteId: hit.noteId, startMouseY: y, origPitch: note.correctedPitch, origStart: note.startTime, origEnd: note.endTime, origValue: note.vibratoDepth, }); - } else if (hit.edge === "bottom-left") { + } else if (hit.edge === "bottom-left" && PITCH_EDITOR_FORMANT_EDITING_ENABLED) { pushUndo("Change formant"); + startInteractivePreviewForHit(hit.noteId, hit.edge); setDragState({ type: "formant", noteId: hit.noteId, startMouseY: y, origPitch: note.correctedPitch, origStart: note.startTime, origEnd: note.endTime, @@ -518,6 +666,7 @@ export function PitchEditorLowerZone() { }); } else if (hit.edge === "bottom-right") { pushUndo("Change gain"); + startInteractivePreviewForHit(hit.noteId, hit.edge); setDragState({ type: "gain", noteId: hit.noteId, startMouseY: y, origPitch: note.correctedPitch, origStart: note.startTime, origEnd: note.endTime, @@ -528,6 +677,7 @@ export function PitchEditorLowerZone() { const currentTool = usePitchEditorStore.getState().tool; if (currentTool === "vibrato") { pushUndo("Adjust vibrato"); + startInteractivePreviewForHit(hit.noteId, hit.edge); setDragState({ type: "straighten", noteId: hit.noteId, startMouseY: y, origPitch: note.correctedPitch, origStart: note.startTime, origEnd: note.endTime, @@ -535,6 +685,7 @@ export function PitchEditorLowerZone() { }); } else if (currentTool === "drift") { pushUndo("Adjust drift"); + startInteractivePreviewForHit(hit.noteId, hit.edge); setDragState({ type: "drift", noteId: hit.noteId, startMouseY: y, origPitch: note.correctedPitch, origStart: note.startTime, origEnd: note.endTime, @@ -543,6 +694,7 @@ export function PitchEditorLowerZone() { } else if (currentTool === "pitch") { // Pitch tool: same as move but with finer control (no snap) and visual feedback pushUndo("Adjust pitch"); + startInteractivePreviewForHit(hit.noteId, hit.edge); setDragState({ type: "pitch", noteId: hit.noteId, startMouseY: y, origPitch: note.correctedPitch, origStart: note.startTime, origEnd: note.endTime, @@ -551,6 +703,7 @@ export function PitchEditorLowerZone() { } else if (currentTool === "transition") { // Transition tool on body: adjust both transitionIn and transitionOut simultaneously pushUndo("Adjust transitions"); + startInteractivePreviewForHit(hit.noteId, hit.edge); setDragState({ type: "transition-both", noteId: hit.noteId, startMouseY: y, origPitch: note.correctedPitch, origStart: note.startTime, origEnd: note.endTime, @@ -562,6 +715,7 @@ export function PitchEditorLowerZone() { return; } else if (currentTool === "draw") { beginDrawPitch(); + startInteractivePreviewForHit(hit.noteId, hit.edge); const clipTime = xToTime(x, freshViewport); const midi = yToMidi(y, freshViewport, canvasSize.height); drawPitchOnNote(hit.noteId, clipTime, midi); @@ -572,6 +726,7 @@ export function PitchEditorLowerZone() { }); } else { pushUndo("Move note"); + startInteractivePreviewForHit(hit.noteId, hit.edge); setDragState({ type: "move", noteId: hit.noteId, startMouseY: y, origPitch: note.correctedPitch, origStart: note.startTime, origEnd: note.endTime, @@ -582,6 +737,7 @@ export function PitchEditorLowerZone() { const currentTool = usePitchEditorStore.getState().tool; if (currentTool === "transition") { pushUndo("Adjust transition"); + startInteractivePreviewForHit(hit.noteId, hit.edge); setDragState({ type: hit.edge === "left" ? "transition-in" : "transition-out", noteId: hit.noteId, startMouseY: y, @@ -601,7 +757,7 @@ export function PitchEditorLowerZone() { } else { deselectAll(); } - }, [canvasSize.height, tool, selectNote, deselectAll, splitNote, pushUndo, beginDrawPitch, drawPitchOnNote]); + }, [canvasSize.height, tool, selectNote, deselectAll, splitNote, pushUndo, beginDrawPitch, drawPitchOnNote, startInteractivePreviewForHit]); const handleMouseMove = useCallback((e: React.MouseEvent<HTMLCanvasElement>) => { const rect = canvasRef.current?.getBoundingClientRect(); @@ -624,9 +780,17 @@ export function PitchEditorLowerZone() { if (dragState) { const deltaY = -(y - dragState.startMouseY); if (dragState.type === "move") { - const deltaSemitones = deltaY / freshViewport.pixelsPerSemitone; + // Alt = fine control (10x slower), regular = normal speed + const sensitivity = e.altKey ? 10 : 1; + const deltaSemitones = deltaY / (freshViewport.pixelsPerSemitone * sensitivity); let newPitch = dragState.origPitch + deltaSemitones; - if (snapMode === "chromatic") newPitch = Math.round(newPitch); + // Ctrl/Cmd bypasses chromatic snap for between-semitone placement + if (snapMode === "chromatic" && !e.ctrlKey && !e.metaKey) { + newPitch = Math.round(newPitch); + } else { + // Round to 1-cent precision + newPitch = Math.round(newPitch * 100) / 100; + } updateNote(dragState.noteId, { correctedPitch: newPitch }); } else if (dragState.type === "resize-left") { const t = xToTime(x, freshViewport); @@ -652,10 +816,11 @@ export function PitchEditorLowerZone() { const newDrift = Math.max(0, Math.min(1, dragState.origValue - deltaY / 100)); updateNote(dragState.noteId, { driftCorrectionAmount: newDrift }); } else if (dragState.type === "pitch") { - // Pitch tool: finer control (0.1 semitone per 10px), no chromatic snap - const deltaSemitones = deltaY / (freshViewport.pixelsPerSemitone * 1.5); + // Pitch tool: fine control, no chromatic snap, 1-cent precision + const sensitivity = e.altKey ? 20 : 2; + const deltaSemitones = deltaY / (freshViewport.pixelsPerSemitone * sensitivity); const newPitch = dragState.origPitch + deltaSemitones; - updateNote(dragState.noteId, { correctedPitch: Math.round(newPitch * 10) / 10 }); + updateNote(dragState.noteId, { correctedPitch: Math.round(newPitch * 100) / 100 }); } else if (dragState.type === "transition-in") { const newMs = Math.max(0, Math.min(200, dragState.origValue + deltaY / 2)); updateNote(dragState.noteId, { transitionIn: Math.round(newMs) }); @@ -693,7 +858,16 @@ export function PitchEditorLowerZone() { : "grab"; } else { const currentTool = usePitchEditorStore.getState().tool; - canvas.style.cursor = currentTool === "draw" ? "not-allowed" : "default"; + if (currentTool === "draw") { + const clipTime = xToTime(x, freshViewport); + const midi = yToMidi(y, freshViewport, canvasSize.height); + const drawTarget = isVoicedAtClipTime(pitchState.contour, clipTime) + ? findDrawTargetNote(currentNotes, pitchState.selectedNoteIds, clipTime, midi) + : null; + canvas.style.cursor = drawTarget ? "crosshair" : "not-allowed"; + } else { + canvas.style.cursor = "default"; + } } } }, [dragState, canvasSize.height, snapMode, updateNote, drawPitchOnNote]); @@ -705,32 +879,47 @@ export function PitchEditorLowerZone() { } else { commitNoteEdit(); } + } else { + endInteractivePreview(); } setDragState(null); - }, [dragState, commitNoteEdit, commitDrawPitch]); + }, [dragState, commitNoteEdit, commitDrawPitch, endInteractivePreview]); // Keyboard shortcuts useEffect(() => { const handler = (e: KeyboardEvent) => { if (!pitchEditorTrackId) return; - if (e.ctrlKey && e.key === "z" && !e.shiftKey) { e.preventDefault(); undo(); return; } - if (e.ctrlKey && (e.key === "y" || (e.key === "z" && e.shiftKey))) { e.preventDefault(); redo(); return; } - if (e.ctrlKey && e.key === "a") { e.preventDefault(); selectAll(); return; } + const key = e.key.toLowerCase(); + if (isPrimaryModifier(e) && key === "z" && !e.shiftKey) { + if (usePitchEditorStore.getState().undoStack.length > 0) { + e.preventDefault(); + e.stopImmediatePropagation(); + undo(); + } + return; + } + if (isPrimaryModifier(e) && (key === "y" || (key === "z" && e.shiftKey))) { + if (usePitchEditorStore.getState().redoStack.length > 0) { + e.preventDefault(); + e.stopImmediatePropagation(); + redo(); + } + return; + } + if (isPrimaryModifier(e) && key === "a") { e.preventDefault(); selectAll(); return; } if (e.key === "Escape") { closePitchEditor(); return; } - if (e.key === "ArrowUp" && !e.ctrlKey) { + if (e.key === "ArrowUp" && !isPrimaryModifier(e)) { e.preventDefault(); - if (polyMode) moveSelectedPolyPitch(e.shiftKey ? 0.1 : 1); - else moveSelectedPitch(e.shiftKey ? 0.1 : 1); + moveSelectedPitch(e.shiftKey ? 0.1 : 1); return; } - if (e.key === "ArrowDown" && !e.ctrlKey) { + if (e.key === "ArrowDown" && !isPrimaryModifier(e)) { e.preventDefault(); - if (polyMode) moveSelectedPolyPitch(e.shiftKey ? -0.1 : -1); - else moveSelectedPitch(e.shiftKey ? -0.1 : -1); + moveSelectedPitch(e.shiftKey ? -0.1 : -1); return; } - if (e.key === "q" || e.key === "Q") { correctSelectedToScale(); return; } - if (e.ctrlKey && e.key === "j") { + if (key === "q") { correctSelectedToScale(); return; } + if (isPrimaryModifier(e) && key === "j") { e.preventDefault(); const sIds = usePitchEditorStore.getState().selectedNoteIds; if (sIds.length >= 2) mergeNotes(sIds); @@ -739,9 +928,9 @@ export function PitchEditorLowerZone() { const toolKey = Number.parseInt(e.key); if (toolKey >= 1 && toolKey <= TOOL_DEFS.length) setTool(TOOL_DEFS[toolKey - 1].id); }; - window.addEventListener("keydown", handler); - return () => window.removeEventListener("keydown", handler); - }, [pitchEditorTrackId, undo, redo, selectAll, closePitchEditor, moveSelectedPitch, moveSelectedPolyPitch, polyMode, correctSelectedToScale, setTool, mergeNotes]); + window.addEventListener("keydown", handler, true); + return () => window.removeEventListener("keydown", handler, true); + }, [pitchEditorTrackId, undo, redo, selectAll, closePitchEditor, moveSelectedPitch, correctSelectedToScale, setTool, mergeNotes]); // Panel resize const isDragging = useRef(false); @@ -835,7 +1024,7 @@ export function PitchEditorLowerZone() { ))} </div> <p className="mt-1 text-[9px] text-neutral-600 leading-tight"> - Double-click or use Split tool to split notes. Draw modifies pitch on existing notes. + Double-click or use Split tool to split notes. Draw follows voiced contour and can start from note shoulders. </p> </div> @@ -931,7 +1120,7 @@ export function PitchEditorLowerZone() { <button onClick={() => mergeNotes(selectedNoteIds)} className="w-full px-2 py-1 text-[10px] rounded bg-neutral-800 text-neutral-300 hover:bg-neutral-700 transition-colors text-left" - title="Merge selected notes (Ctrl+J)" + title={`Merge selected notes (${formatShortcut("Ctrl+J")})`} > Merge notes </button> @@ -939,33 +1128,21 @@ export function PitchEditorLowerZone() { </div> </div> - {/* Global Formant */} <div className="px-2 py-1.5 border-b border-neutral-800/60 shrink-0"> - <div className="text-[9px] text-neutral-600 uppercase tracking-wider mb-1">Formant</div> - <div className="flex items-center gap-2"> - <span className="text-[9px] text-neutral-500 shrink-0">Global</span> - <input - type="range" - min={-386} - max={386} - step={1} - value={globalFormantCents} - onChange={(e) => setGlobalFormantCents(Number(e.target.value))} - className="flex-1 h-1 accent-daw-accent cursor-pointer" - /> - <span className="w-12 text-right text-[10px] font-mono text-neutral-300 shrink-0"> - {globalFormantCents > 0 ? "+" : ""} - {globalFormantCents}c - </span> + <div className="text-[9px] text-neutral-600 uppercase tracking-wider mb-1">Pitch Engine</div> + <div className="text-[10px] text-neutral-400 leading-snug"> + Formant editing is temporarily disabled while the pitch-only engine is being rebuilt around the research renderer. + </div> + <div className="mt-1 text-[10px] text-neutral-500 leading-snug"> + Pitch editing is monophonic only. Stereo vocal clips are still supported and analyzed from a mono sum while correction keeps the clip's multichannel audio. </div> - <div className="mt-1 flex items-center justify-between gap-2"> - <span className="text-[8px] text-neutral-600">{formantStatusHint}</span> - {applyState !== "idle" && applyMessage && ( - <span className={`px-1.5 py-0.5 rounded border text-[8px] uppercase tracking-wider ${applyStateClassName}`}> + {applyState !== "idle" && applyMessage && ( + <div className="mt-1"> + <span className={`inline-flex px-1.5 py-0.5 rounded border text-[8px] uppercase tracking-wider ${applyStateClassName}`}> {applyMessage} </span> - )} - </div> + </div> + )} </div> {/* Note Inspector */} @@ -978,7 +1155,7 @@ export function PitchEditorLowerZone() { onClick={undo} disabled={undoStack.length === 0} className="flex-1 px-2 py-1 text-[10px] rounded bg-neutral-800 text-neutral-300 hover:bg-neutral-700 disabled:opacity-30 transition-colors" - title="Undo (Ctrl+Z)" + title={`Undo (${formatShortcut("Ctrl+Z")})`} > ← Undo </button> @@ -986,89 +1163,13 @@ export function PitchEditorLowerZone() { onClick={redo} disabled={redoStack.length === 0} className="flex-1 px-2 py-1 text-[10px] rounded bg-neutral-800 text-neutral-300 hover:bg-neutral-700 disabled:opacity-30 transition-colors" - title="Redo (Ctrl+Y)" + title={`Redo (${formatShortcut("Ctrl+Y")})`} > Redo → </button> </div> </div> - {/* Poly mode */} - <div className="px-2 py-1.5 border-b border-neutral-800/60 shrink-0"> - <div className="text-[9px] text-neutral-600 uppercase tracking-wider mb-1">Mode</div> - <button - onClick={togglePolyMode} - className={`w-full px-2 py-1 text-[10px] rounded mb-0.5 transition-colors ${ - polyMode - ? "bg-purple-700/80 text-purple-100 ring-1 ring-purple-500/40" - : "bg-neutral-800 text-neutral-400 hover:bg-neutral-700 hover:text-neutral-200" - }`} - title="Toggle polyphonic mode" - > - {polyMode ? "Polyphonic ✓" : "Polyphonic"} - </button> - {polyMode && ( - <div className="flex flex-col gap-0.5 mt-0.5"> - <button - onClick={analyzePolyphonic} - disabled={isAnalyzing} - className="w-full px-2 py-1 text-[10px] rounded bg-neutral-800 text-amber-400 hover:bg-neutral-700 disabled:opacity-30 transition-colors" - title="Run polyphonic analysis" - > - {isAnalyzing ? "Analyzing…" : "Run Poly Analysis"} - </button> - <button - onClick={togglePitchSalience} - className={`w-full px-2 py-0.5 text-[9px] rounded transition-colors ${ - showPitchSalience ? "bg-amber-800/60 text-amber-300" : "bg-neutral-800 text-neutral-500 hover:bg-neutral-700" - }`} - > - Salience heatmap - </button> - - {/* Poly detection tuning */} - <div className="mt-1 pt-1 border-t border-neutral-800/40"> - <div className="text-[8px] text-neutral-600 uppercase tracking-wider mb-1">Detection Tuning</div> - <div className="flex flex-col gap-1"> - <div> - <div className="flex items-center justify-between"> - <span className="text-[8px] text-neutral-500">Note thresh</span> - <span className="text-[8px] font-mono text-neutral-400">{polyNoteThreshold.toFixed(2)}</span> - </div> - <input - type="range" min={0.01} max={0.5} step={0.01} value={polyNoteThreshold} - onChange={(e) => setPolyNoteThreshold(Number(e.target.value))} - className="w-full h-0.5 bg-neutral-700 rounded-full appearance-none cursor-pointer accent-purple-500" - /> - </div> - <div> - <div className="flex items-center justify-between"> - <span className="text-[8px] text-neutral-500">Onset thresh</span> - <span className="text-[8px] font-mono text-neutral-400">{polyOnsetThreshold.toFixed(2)}</span> - </div> - <input - type="range" min={0.05} max={0.8} step={0.01} value={polyOnsetThreshold} - onChange={(e) => setPolyOnsetThreshold(Number(e.target.value))} - className="w-full h-0.5 bg-neutral-700 rounded-full appearance-none cursor-pointer accent-purple-500" - /> - </div> - <div> - <div className="flex items-center justify-between"> - <span className="text-[8px] text-neutral-500">Min duration</span> - <span className="text-[8px] font-mono text-neutral-400">{polyMinDuration}ms</span> - </div> - <input - type="range" min={10} max={500} step={10} value={polyMinDuration} - onChange={(e) => setPolyMinDuration(Number(e.target.value))} - className="w-full h-0.5 bg-neutral-700 rounded-full appearance-none cursor-pointer accent-purple-500" - /> - </div> - </div> - </div> - </div> - )} - </div> - {/* A/B Compare */} <div className="px-2 py-1.5 border-b border-neutral-800/60 shrink-0"> <button @@ -1078,23 +1179,11 @@ export function PitchEditorLowerZone() { ? "bg-amber-700/80 text-amber-100 ring-1 ring-amber-500/40" : "bg-neutral-800 text-neutral-400 hover:bg-neutral-700 hover:text-neutral-200" }`} - title="Toggle A/B comparison (hear original vs corrected)" + title="A = corrected state, B = original/raw clip" > - {abCompareMode ? "B (Original)" : "A/B Compare"} + {abCompareMode ? "B (Original)" : "A (Corrected)"} </button> </div> - - {polyMode && ( - <div className="px-2 py-1.5 shrink-0"> - <button - onClick={applyPolyCorrection} - className="w-full px-2 py-1.5 text-[11px] rounded bg-daw-accent text-white hover:bg-daw-accent/80 font-semibold transition-colors shadow-sm" - title="Apply polyphonic correction" - > - Apply Poly - </button> - </div> - )} </div> {/* Status footer */} @@ -1119,8 +1208,10 @@ export function PitchEditorLowerZone() { ) : ( <div className="text-[9px] text-neutral-700">Hover or click a note</div> )} - {(isAnalyzing || progressLabel) && ( - <div className="text-[9px] text-daw-accent animate-pulse">{progressLabel || "Analyzing…"}</div> + {(analysisPhase !== "idle" || progressLabel) && ( + <div className="text-[9px] text-daw-accent animate-pulse"> + {progressLabel || (analysisPhase === "loading" ? "Loading clip…" : "Analyzing pitch…")} + </div> )} {!isAnalyzing && applyMessage && ( <div className={`text-[9px] ${applyState === "error" ? "text-red-400" : applyState === "done" ? "text-emerald-300" : applyState === "preview_ready" ? "text-sky-300" : "text-daw-accent"} ${applyState === "queued" || applyState === "processing" || applyState === "preview_processing" || applyState === "final_processing" ? "animate-pulse" : ""}`}> @@ -1132,11 +1223,13 @@ export function PitchEditorLowerZone() { {/* ── Canvas (aligned with timeline) ── */} <div ref={containerRef} className="flex-1 min-w-0 relative bg-daw-panel"> - {isAnalyzing && !contour && ( + {(analysisPhase !== "idle" || isAnalyzing) && !contour && ( <div className="absolute inset-0 flex items-center justify-center z-10 bg-black/50"> <div className="flex flex-col items-center gap-2 w-48"> <div className="w-5 h-5 border-2 border-daw-accent border-t-transparent rounded-full animate-spin" /> - <div className="text-sm text-neutral-400">{progressLabel || "Analyzing pitch…"}</div> + <div className="text-sm text-neutral-400"> + {progressLabel || (analysisPhase === "loading" ? "Loading clip…" : "Analyzing pitch…")} + </div> {progressPercent > 0 && progressPercent < 100 && ( <div className="w-full h-1.5 bg-neutral-800 rounded-full overflow-hidden"> <div diff --git a/frontend/src/components/PluginBrowser.tsx b/frontend/src/components/PluginBrowser.tsx index b880d1e..cd72896 100644 --- a/frontend/src/components/PluginBrowser.tsx +++ b/frontend/src/components/PluginBrowser.tsx @@ -275,7 +275,7 @@ function getPluginGroupId(plugin: { category: string; isInstrument: boolean }): interface PluginBrowserProps { trackId: string; targetChain: "input" | "track" | "master" | "instrument"; - trackType?: "audio" | "midi" | "instrument" | "bus"; + trackType?: "audio" | "midi" | "instrument" | "bus" | "ai"; onClose: () => void; embedded?: boolean; } diff --git a/frontend/src/components/RegionRenderMatrix.tsx b/frontend/src/components/RegionRenderMatrix.tsx index 534432d..548f26a 100644 --- a/frontend/src/components/RegionRenderMatrix.tsx +++ b/frontend/src/components/RegionRenderMatrix.tsx @@ -2,6 +2,7 @@ import { useState } from "react"; import { useDAWStore } from "../store/useDAWStore"; import { useShallow } from "zustand/react/shallow"; import { nativeBridge } from "../services/NativeBridge"; +import { prepareForManualRender } from "../utils/renderPreparation"; import { Button, Input, @@ -97,7 +98,7 @@ export function RegionRenderMatrix({ isOpen, onClose }: RegionRenderMatrixProps) setProgress(0); try { - await syncClipsWithBackend(); + await prepareForManualRender(syncClipsWithBackend, "region-render-matrix"); for (let i = 0; i < jobs.length; i++) { const job = jobs[i]; @@ -117,6 +118,7 @@ export function RegionRenderMatrix({ isOpen, onClose }: RegionRenderMatrixProps) normalize: false, addTail: false, tailLength: 0, + includeMetronome: false, }); setProgress(Math.round(((i + 1) / jobs.length) * 100)); diff --git a/frontend/src/components/RenderModal.tsx b/frontend/src/components/RenderModal.tsx index 718482c..aabdd08 100644 --- a/frontend/src/components/RenderModal.tsx +++ b/frontend/src/components/RenderModal.tsx @@ -1,7 +1,16 @@ import { useState, useEffect } from "react"; -import { useDAWStore } from "../store/useDAWStore"; +import { + useDAWStore, + type RenderDialogOptions, + type RenderSource, + type RenderBounds, + type AudioFormat, + type SampleRate, + type BitDepth, +} from "../store/useDAWStore"; import { useShallow } from "zustand/shallow"; import { nativeBridge } from "../services/NativeBridge"; +import { prepareForManualRender } from "../utils/renderPreparation"; import { Button, Input, @@ -18,31 +27,6 @@ interface RenderModalProps { onClose: () => void; } -type RenderSource = "master" | "selected_tracks" | "stems" | "selected_items" | "selected_items_master" | "razor"; -type RenderBounds = "entire" | "custom" | "time_selection" | "project_regions" | "selected_regions"; -type AudioFormat = "wav" | "aiff" | "flac" | "mp3" | "ogg" | "raw"; -type SampleRate = 44100 | 48000 | 88200 | 96000 | 192000; -type BitDepth = 16 | 24 | 32; - -interface RenderOptions { - source: RenderSource; - bounds: RenderBounds; - startTime: number; - endTime: number; - tailLength: number; - addTail: boolean; - directory: string; - fileName: string; - format: AudioFormat; - sampleRate: SampleRate; - bitDepth: BitDepth; - channels: "stereo" | "mono"; - normalize: boolean; - dither: boolean; - mp3Bitrate: number; - oggQuality: number; -} - const isLossyFormat = (format: AudioFormat) => format === "mp3" || format === "ogg"; /** @@ -81,15 +65,18 @@ function resolveWildcards( */ export function RenderModal({ isOpen, onClose }: RenderModalProps) { const { - tracks, timeSelection, projectPath, projectRange, syncClipsWithBackend, + tracks, timeSelection, projectRange, projectPath, syncClipsWithBackend, selectedTrackIds, regions, selectedRegionIds, selectedClipIds, razorEdits, projectName, renderMetadata, secondaryOutputEnabled, secondaryOutputFormat, secondaryOutputBitDepth, onlineRender, addToProjectAfterRender, + loopEnabled, loopStart, loopEnd, + renderDialogOptions, setRenderDialogOptions, lastRenderDirectory, setLastRenderDirectory, + showToast, } = useDAWStore(useShallow((s) => ({ tracks: s.tracks, timeSelection: s.timeSelection, - projectPath: s.projectPath, projectRange: s.projectRange, + projectPath: s.projectPath, syncClipsWithBackend: s.syncClipsWithBackend, selectedTrackIds: s.selectedTrackIds, regions: s.regions, @@ -98,67 +85,63 @@ export function RenderModal({ isOpen, onClose }: RenderModalProps) { razorEdits: s.razorEdits, projectName: s.projectName, renderMetadata: s.renderMetadata, + loopEnabled: s.transport.loopEnabled, + loopStart: s.transport.loopStart, + loopEnd: s.transport.loopEnd, + renderDialogOptions: s.renderDialogOptions, + setRenderDialogOptions: s.setRenderDialogOptions, + lastRenderDirectory: s.lastRenderDirectory, + setLastRenderDirectory: s.setLastRenderDirectory, secondaryOutputEnabled: s.secondaryOutputEnabled, secondaryOutputFormat: s.secondaryOutputFormat, secondaryOutputBitDepth: s.secondaryOutputBitDepth, onlineRender: s.onlineRender, addToProjectAfterRender: s.addToProjectAfterRender, + showToast: s.showToast, }))); const [isRendering, setIsRendering] = useState(false); const [renderProgress, setRenderProgress] = useState(0); const [renderStatus, setRenderStatus] = useState(""); - const [options, setOptions] = useState<RenderOptions>(() => { - // Compute initial extent from clips if projectRange is empty - let extent = projectRange; - if (extent.end <= extent.start) { - let minStart = Infinity; - let maxEnd = 0; - for (const track of tracks) { - for (const clip of track.clips) { - minStart = Math.min(minStart, clip.startTime); - maxEnd = Math.max(maxEnd, clip.startTime + clip.duration); - } - } - if (maxEnd > 0) extent = { start: Math.min(minStart, 0), end: maxEnd }; + const options = renderDialogOptions; + const setOptions = ( + next: + | RenderDialogOptions + | ((prev: RenderDialogOptions) => RenderDialogOptions) + ) => { + const resolved = typeof next === "function" ? next(renderDialogOptions) : next; + setRenderDialogOptions(resolved); + }; + + const getActiveTimeSelection = () => { + const liveState = useDAWStore.getState(); + const liveTimeSelection = liveState.timeSelection; + if (liveTimeSelection && liveTimeSelection.end > liveTimeSelection.start) { + return liveTimeSelection; } - return { - source: "master", - bounds: "entire", - startTime: extent.start, - endTime: extent.end, - tailLength: 1000, - addTail: true, - directory: "", - fileName: "$project", - format: "wav", - sampleRate: 44100, - bitDepth: 24, - channels: "stereo", - normalize: false, - dither: false, - mp3Bitrate: 320, - oggQuality: 6, - }; - }); + if (liveState.projectRange.end > liveState.projectRange.start) { + return { + start: liveState.projectRange.start, + end: liveState.projectRange.end, + }; + } + if (liveState.transport.loopEnabled && liveState.transport.loopEnd > liveState.transport.loopStart) { + return { + start: liveState.transport.loopStart, + end: liveState.transport.loopEnd, + }; + } + return null; + }; - // Compute actual project extent from clips if projectRange is empty + // Compute project extent from clips: start=0, end=last clip end const getProjectExtent = () => { - if (projectRange.end > projectRange.start) { - return projectRange; - } - // Fall back: compute from clip positions - let minStart = Infinity; let maxEnd = 0; for (const track of tracks) { for (const clip of track.clips) { - minStart = Math.min(minStart, clip.startTime); maxEnd = Math.max(maxEnd, clip.startTime + clip.duration); } } - if (maxEnd > 0) { - return { start: Math.min(minStart, 0), end: maxEnd }; - } - return { start: 0, end: 0 }; + return { start: 0, end: maxEnd }; }; const resolveBoundsRange = (bounds: RenderBounds, fallbackStart: number, fallbackEnd: number) => { @@ -167,8 +150,9 @@ export function RenderModal({ isOpen, onClose }: RenderModalProps) { return { startTime: extent.start, endTime: extent.end }; } if (bounds === "time_selection") { - if (timeSelection) { - return { startTime: timeSelection.start, endTime: timeSelection.end }; + const selection = getActiveTimeSelection(); + if (selection) { + return { startTime: selection.start, endTime: selection.end }; } return { startTime: fallbackStart, endTime: fallbackEnd }; } @@ -202,22 +186,49 @@ export function RenderModal({ isOpen, onClose }: RenderModalProps) { } return { ...prev, ...resolved }; }); - }, [options.bounds, projectRange, tracks, timeSelection, regions, selectedRegionIds]); + }, [options.bounds, tracks, timeSelection, projectRange, loopEnabled, loopStart, loopEnd, regions, selectedRegionIds]); - // Set default directory from project path + // On open: always refresh time bounds from current state, handle missing time selection, + // and set default directory. useEffect(() => { - if (isOpen && projectPath && !options.directory) { - const dir = projectPath.substring(0, projectPath.lastIndexOf("\\")); - setOptions((prev) => ({ ...prev, directory: dir })); + if (!isOpen) return; + const activeSelection = getActiveTimeSelection(); + const effectiveBounds = + renderDialogOptions.bounds === "time_selection" && !activeSelection + ? ("entire" as const) + : renderDialogOptions.bounds; + let next: RenderDialogOptions = { ...renderDialogOptions, bounds: effectiveBounds }; + if (effectiveBounds !== "custom") { + const resolved = resolveBoundsRange( + effectiveBounds, + renderDialogOptions.startTime, + renderDialogOptions.endTime + ); + next = { ...next, startTime: resolved.startTime, endTime: resolved.endTime }; } - }, [isOpen, projectPath]); + if (!renderDialogOptions.directory) { + const dir = + lastRenderDirectory || + (projectPath + ? projectPath.substring(0, Math.max(projectPath.lastIndexOf("\\"), projectPath.lastIndexOf("/"))) + : ""); + if (dir) { + next = { ...next, directory: dir }; + } + } + setOptions(next); + }, [isOpen]); const handleBrowseDirectory = async () => { try { const ext = options.format; + const initialDir = options.directory || + lastRenderDirectory || + (projectPath ? projectPath.substring(0, Math.max(projectPath.lastIndexOf("\\"), projectPath.lastIndexOf("/"))) : ""); const path = await nativeBridge.showRenderSaveDialog( resolveWildcards(options.fileName, { projectName }) || "untitled", - ext + ext, + initialDir ); if (path) { const lastSlash = Math.max(path.lastIndexOf("\\"), path.lastIndexOf("/")); @@ -226,6 +237,7 @@ export function RenderModal({ isOpen, onClose }: RenderModalProps) { // Strip extension from filename if present const dotIdx = name.lastIndexOf("."); if (dotIdx > 0) name = name.substring(0, dotIdx); + if (dir) setLastRenderDirectory(dir); setOptions((prev) => ({ ...prev, directory: dir, fileName: name })); } } catch (error) { @@ -287,6 +299,7 @@ export function RenderModal({ isOpen, onClose }: RenderModalProps) { normalize: options.normalize, addTail: options.addTail, tailLength: options.tailLength, + includeMetronome: false, }; }; @@ -374,11 +387,12 @@ export function RenderModal({ isOpen, onClose }: RenderModalProps) { setIsRendering(true); setRenderProgress(0); - setRenderStatus("Syncing clips..."); + setRenderStatus("Checking pitch renders..."); const renderedFiles: string[] = []; try { - await syncClipsWithBackend(); + setRenderStatus("Syncing clips..."); + await prepareForManualRender(syncClipsWithBackend, "render-modal"); const ranges = getRenderRanges(); const totalFiles = getFileCount(); let rendered = 0; @@ -511,6 +525,7 @@ export function RenderModal({ isOpen, onClose }: RenderModalProps) { isOpen={isOpen} onClose={handleCancel} size="lg" + fullHeight closeOnOverlayClick={!isRendering} closeOnEscape={!isRendering} > @@ -555,6 +570,11 @@ export function RenderModal({ isOpen, onClose }: RenderModalProps) { value={options.bounds} onChange={(val) => { const newBounds = val as RenderBounds; + const activeSelection = getActiveTimeSelection(); + if (newBounds === "time_selection" && !activeSelection) { + showToast("No time selection active. Make a selection on the timeline first.", "info"); + return; + } const resolved = resolveBoundsRange(newBounds, options.startTime, options.endTime); setOptions({ ...options, diff --git a/frontend/src/components/SortableTrackHeader.tsx b/frontend/src/components/SortableTrackHeader.tsx index a6a2cda..68774c1 100644 --- a/frontend/src/components/SortableTrackHeader.tsx +++ b/frontend/src/components/SortableTrackHeader.tsx @@ -5,6 +5,7 @@ import { ChevronRight, ChevronDown } from "lucide-react"; import { Track, useDAWStore } from "../store/useDAWStore"; import { useShallow } from "zustand/react/shallow"; import { TrackHeader } from "./TrackHeader"; +import { AITrackHeader } from "./AITrackHeader"; import { TRACK_COLORS } from "./ColorPicker"; import { useContextMenu, MenuItem } from "./ContextMenu"; @@ -371,7 +372,11 @@ export function SortableTrackHeader({ track }: SortableTrackHeaderProps) { </button> )} <div className="flex-1 min-w-0"> - <TrackHeader track={track} isSelected={isSelected} /> + {track.type === "ai" ? ( + <AITrackHeader track={track} isSelected={isSelected} /> + ) : ( + <TrackHeader track={track} isSelected={isSelected} /> + )} </div> </div> </div> diff --git a/frontend/src/components/StemSeparationModal.tsx b/frontend/src/components/StemSeparationModal.tsx index 10389b6..667f036 100644 --- a/frontend/src/components/StemSeparationModal.tsx +++ b/frontend/src/components/StemSeparationModal.tsx @@ -253,6 +253,33 @@ export default function StemSeparationModal() { <div className="rounded bg-neutral-900/80 p-3 text-xs text-daw-text-secondary"> {aiToolsStatus.message || "Install AI Tools to enable stem separation."} + <p className="mt-2"> + Music generation:{" "} + <span className="text-daw-text"> + {( + aiToolsStatus.musicGenerationReady + && aiToolsStatus.musicGenerationLayoutValid + && (aiToolsStatus.musicGenerationPerformanceReady ?? true) + ) ? "ready" : "not ready yet"} + </span> + {aiToolsStatus.aceStepVersion ? ` (ACE-Step ${aiToolsStatus.aceStepVersion})` : ""} + </p> + {aiToolsStatus.musicGenerationModelId ? ( + <p className="mt-1"> + Pinned model: <span className="text-daw-text">{aiToolsStatus.musicGenerationModelId}</span> + </p> + ) : null} + {aiToolsStatus.musicGenerationCheckpointRoot ? ( + <p className="mt-1 break-all"> + Checkpoint root: <span className="text-daw-text">{aiToolsStatus.musicGenerationCheckpointRoot}</span> + </p> + ) : null} + <p className="mt-1"> + Checkpoint layout:{" "} + <span className="text-daw-text"> + {aiToolsStatus.musicGenerationLayoutValid ? "valid" : "missing files"} + </span> + </p> {aiToolsStatus.error ? ( <p className="mt-2 text-daw-record">{aiToolsStatus.error}</p> ) : null} diff --git a/frontend/src/components/Timeline.tsx b/frontend/src/components/Timeline.tsx index e4d64f3..46a3024 100644 --- a/frontend/src/components/Timeline.tsx +++ b/frontend/src/components/Timeline.tsx @@ -28,6 +28,7 @@ import { snapToGrid, calculateGridInterval, } from "../utils/snapToGrid"; +import { getRulerClickSnapTime } from "../utils/rulerClickSnap"; import { fadeInCurvePoints, fadeOutCurvePoints, @@ -319,7 +320,7 @@ export function Timeline({ // Ruler interaction state (ref-based to avoid stale closures in global listeners) const rulerDragRef = useRef<{ - type: "handle" | "range-create" | "pending"; // pending = mousedown, not yet determined + type: "handle-pending" | "handle-drag" | "range-create" | "pending"; // pending = mousedown, not yet determined handle?: "start" | "end"; startX: number; // pixel X at mousedown (relative to ruler canvas) startTime: number; // time at mousedown @@ -361,6 +362,7 @@ export function Timeline({ gridSize, timeSelection, setTimeSelection, + clearTimeSelection, openPianoRoll, addMIDIClip, addEmptyClip, @@ -416,6 +418,7 @@ export function Timeline({ gridSize: state.gridSize, timeSelection: state.timeSelection, setTimeSelection: state.setTimeSelection, + clearTimeSelection: state.clearTimeSelection, openPianoRoll: state.openPianoRoll, addMIDIClip: state.addMIDIClip, addEmptyClip: state.addEmptyClip, @@ -577,6 +580,13 @@ export function Timeline({ // Refs for ruler drag to avoid stale closures in global listeners const projectRangeRef = useRef(projectRange); projectRangeRef.current = projectRange; + const shouldUseSnapRef = useRef(snapEnabled); + shouldUseSnapRef.current = snapEnabled; + + const isSnapActive = useCallback((ctrlBypass = false) => { + return shouldUseSnapRef.current && !ctrlBypass; + }, []); + const snapEnabledRef = useRef(snapEnabled); snapEnabledRef.current = snapEnabled; const gridSizeRef = useRef(gridSize); @@ -930,6 +940,7 @@ export function Timeline({ // ── Ruler interaction: click = seek, drag = set range, drag handle = adjust range ── const RANGE_HANDLE_HIT_PX = 8; + const RANGE_HANDLE_VISUAL_HEIGHT_PX = 8; const DRAG_THRESHOLD_PX = 4; // Movement needed to distinguish drag from click const handleRulerMouseDown = (e: KonvaEvent) => { @@ -937,20 +948,30 @@ export function Timeline({ const pointerPos = stage.getPointerPosition(); if (!pointerPos) return; - const clickedTime = Math.max(0, (pointerPos.x + scrollX) / pixelsPerSecond); + const rawClickedTime = Math.max(0, (pointerPos.x + scrollX) / pixelsPerSecond); + const clickedTime = getRulerClickSnapTime({ + time: rawClickedTime, + pixelsPerSecond: pixelsPerSecondRef.current, + tempo: tempoRef.current, + timeSignature: timeSignatureRef.current, + gridSize: gridSizeRef.current, + snapEnabled: shouldUseSnapRef.current, + ctrlBypass: Boolean(e.evt?.ctrlKey || e.evt?.metaKey), + }); - // Check if clicking near a range handle (anywhere in ruler height) + // Check if clicking near a range handle inside the visible marker zone. const startX = projectRange.start * pixelsPerSecond - scrollX; const endX = projectRange.end * pixelsPerSecond - scrollX; + const inHandleZone = pointerPos.y <= RANGE_HANDLE_VISUAL_HEIGHT_PX; // Prefer end handle when both overlap (both at 0) - if (Math.abs(pointerPos.x - endX) < RANGE_HANDLE_HIT_PX) { - rulerDragRef.current = { type: "handle", handle: "end", startX: pointerPos.x, startTime: clickedTime }; + if (inHandleZone && Math.abs(pointerPos.x - endX) < RANGE_HANDLE_HIT_PX) { + rulerDragRef.current = { type: "handle-pending", handle: "end", startX: pointerPos.x, startTime: clickedTime }; setRulerDragging(true); return; } - if (Math.abs(pointerPos.x - startX) < RANGE_HANDLE_HIT_PX) { - rulerDragRef.current = { type: "handle", handle: "start", startX: pointerPos.x, startTime: clickedTime }; + if (inHandleZone && Math.abs(pointerPos.x - startX) < RANGE_HANDLE_HIT_PX) { + rulerDragRef.current = { type: "handle-pending", handle: "start", startX: pointerPos.x, startTime: clickedTime }; setRulerDragging(true); return; } @@ -975,7 +996,10 @@ export function Timeline({ return; } - // Not on a handle — record as pending (will become seek or range-create on mouseup/move) + // Not on a handle — plain click clears any existing time selection and seeks + if (useDAWStore.getState().timeSelection) { + clearTimeSelection(); + } rulerDragRef.current = { type: "pending", startX: pointerPos.x, startTime: clickedTime }; }; @@ -1062,11 +1086,18 @@ export function Timeline({ const range = projectRangeRef.current; // Apply snap-to-grid if enabled - if (snapEnabledRef.current) { + if (isSnapActive(Boolean(e.ctrlKey || e.metaKey))) { time = snapToGrid(time, tempoRef.current, timeSignatureRef.current, gridSizeRef.current); } - if (drag.type === "handle") { + if (drag.type === "handle-pending") { + if (Math.abs(pointerX - drag.startX) <= DRAG_THRESHOLD_PX) { + return; + } + drag.type = "handle-drag"; + } + + if (drag.type === "handle-drag") { // Dragging an existing handle if (drag.handle === "start") { setProjectRange(Math.min(time, range.end), range.end); @@ -1080,7 +1111,7 @@ export function Timeline({ drag.type = "range-create"; // Snap the drag start time too let startTime = drag.startTime; - if (snapEnabledRef.current) { + if (isSnapActive(Boolean(e.ctrlKey || e.metaKey))) { startTime = snapToGrid(startTime, tempoRef.current, timeSignatureRef.current, gridSizeRef.current); drag.startTime = startTime; } @@ -1101,6 +1132,11 @@ export function Timeline({ if (drag.type === "pending") { // No significant movement — this was a click, so seek seekTo(drag.startTime); + } else if (drag.type === "handle-pending") { + if (useDAWStore.getState().timeSelection) { + clearTimeSelection(); + } + seekTo(drag.startTime); } rulerDragRef.current = null; @@ -1113,7 +1149,7 @@ export function Timeline({ window.removeEventListener("mousemove", handleGlobalMouseMove); window.removeEventListener("mouseup", handleGlobalMouseUp); }; - }, [seekTo, setProjectRange]); + }, [clearTimeSelection, seekTo, setProjectRange]); // Handle mouse move for time selection / razor edit dragging (on main stage) const handleStageMouseMove = (e: KonvaEvent) => { @@ -2242,7 +2278,7 @@ export function Timeline({ let newStartTime = rawStartTime; // Apply snap-to-grid if enabled - if (snapEnabled) { + if (isSnapActive(Boolean(e.evt?.ctrlKey))) { newStartTime = snapToGrid(newStartTime, tempo, timeSignature, gridSize); } @@ -2252,7 +2288,7 @@ export function Timeline({ const targetTY = trackYs[Math.max(0, targetTrackIdx)] ?? 0; // Update snap ghost preview: show semi-transparent rect at snapped position - if (snapEnabled && Math.abs(newStartTime - rawStartTime) > 0.001) { + if (isSnapActive(Boolean(e.evt?.ctrlKey)) && Math.abs(newStartTime - rawStartTime) > 0.001) { const ghostScreenX = newStartTime * pixelsPerSecond - scrollX; const targetTrack = tracks[Math.max(0, Math.min(targetTrackIdx, tracks.length - 1))]; const targetMetrics = targetTrack @@ -2479,7 +2515,7 @@ export function Timeline({ const stage = e.target.getStage(); const pointerPos = stage.getPointerPosition(); let splitTime = (pointerPos.x + scrollX) / pixelsPerSecond; - if (snapEnabled) { + if (isSnapActive(Boolean(e.evt?.ctrlKey))) { splitTime = snapToGrid(splitTime, tempo, timeSignature, gridSize); } splitClipAtPosition(clip.id, splitTime); @@ -2637,7 +2673,7 @@ export function Timeline({ ); // Apply snap-to-grid if enabled - if (snapEnabled) { + if (isSnapActive(Boolean(e.evt?.ctrlKey))) { newStartTime = snapToGrid(newStartTime, tempo, timeSignature, gridSize); } @@ -2661,7 +2697,7 @@ export function Timeline({ } // Apply snap-to-grid to end time (start + duration) if enabled - if (snapEnabled) { + if (isSnapActive(Boolean(e.evt?.ctrlKey))) { const endTime = clip.startTime + newDuration; const snappedEndTime = snapToGrid( endTime, @@ -3377,7 +3413,7 @@ export function Timeline({ const stage = e.target.getStage(); const pointerPos = stage.getPointerPosition(); let splitTime = (pointerPos.x + scrollX) / pixelsPerSecond; - if (snapEnabled) { + if (isSnapActive(Boolean(e.evt?.ctrlKey))) { splitTime = snapToGrid(splitTime, tempo, timeSignature, gridSize); } splitMIDIClipAtPosition(clip.id, splitTime); @@ -4794,6 +4830,12 @@ export function Timeline({ void createTrackOfType("midi"); }, }, + { + label: "Add AI Track", + onClick: () => { + void createTrackOfType("ai"); + }, + }, ] : []), ...((backgroundContextMenu.trackType === "midi" || diff --git a/frontend/src/components/TimelineRuler.tsx b/frontend/src/components/TimelineRuler.tsx index 5330152..9a35cc4 100644 --- a/frontend/src/components/TimelineRuler.tsx +++ b/frontend/src/components/TimelineRuler.tsx @@ -4,6 +4,7 @@ import { useShallow } from "zustand/shallow"; import { useDAWStore } from "../store/useDAWStore"; import { MemoizedPlayhead as Playhead } from "./Playhead"; import { snapToGrid } from "../utils/snapToGrid"; +import { getRulerClickSnapTime } from "../utils/rulerClickSnap"; // eslint-disable-next-line @typescript-eslint/no-explicit-any type KonvaEvent = any; @@ -11,13 +12,14 @@ type KonvaEvent = any; export const TIMELINE_RULER_HEIGHT = 30; const RANGE_HANDLE_HIT_PX = 8; +const RANGE_HANDLE_VISUAL_HEIGHT_PX = 8; const DRAG_THRESHOLD_PX = 4; export function TimelineRuler() { const containerRef = useRef<HTMLDivElement>(null); const [width, setWidth] = useState(800); const rulerDragRef = useRef<{ - type: "handle" | "range-create" | "pending"; + type: "handle-pending" | "handle-drag" | "range-create" | "pending"; handle?: "start" | "end"; startX: number; startTime: number; @@ -91,14 +93,24 @@ export function TimelineRuler() { const pointerPos = stage.getPointerPosition(); if (!pointerPos) return; - const clickedTime = Math.max(0, (pointerPos.x + scrollX) / pixelsPerSecond); + const rawClickedTime = Math.max(0, (pointerPos.x + scrollX) / pixelsPerSecond); + const clickedTime = getRulerClickSnapTime({ + time: rawClickedTime, + pixelsPerSecond: pixelsPerSecondRef.current, + tempo: tempoRef.current, + timeSignature: timeSignatureRef.current, + gridSize: gridSizeRef.current, + snapEnabled: snapEnabledRef.current, + ctrlBypass: Boolean(e.evt?.ctrlKey || e.evt?.metaKey), + }); const startX = projectRange.start * pixelsPerSecond - scrollX; const endX = projectRange.end * pixelsPerSecond - scrollX; + const inHandleZone = pointerPos.y <= RANGE_HANDLE_VISUAL_HEIGHT_PX; - if (Math.abs(pointerPos.x - endX) < RANGE_HANDLE_HIT_PX) { + if (inHandleZone && Math.abs(pointerPos.x - endX) < RANGE_HANDLE_HIT_PX) { rulerDragRef.current = { - type: "handle", + type: "handle-pending", handle: "end", startX: pointerPos.x, startTime: clickedTime, @@ -106,9 +118,9 @@ export function TimelineRuler() { setRulerDragging(true); return; } - if (Math.abs(pointerPos.x - startX) < RANGE_HANDLE_HIT_PX) { + if (inHandleZone && Math.abs(pointerPos.x - startX) < RANGE_HANDLE_HIT_PX) { rulerDragRef.current = { - type: "handle", + type: "handle-pending", handle: "start", startX: pointerPos.x, startTime: clickedTime, @@ -201,7 +213,14 @@ export function TimelineRuler() { ); } - if (drag.type === "handle") { + if (drag.type === "handle-pending") { + if (Math.abs(pointerX - drag.startX) <= DRAG_THRESHOLD_PX) { + return; + } + drag.type = "handle-drag"; + } + + if (drag.type === "handle-drag") { if (drag.handle === "start") { setProjectRange(Math.min(time, range.end), range.end); } else { @@ -238,6 +257,8 @@ export function TimelineRuler() { if (drag.type === "pending") { seekTo(drag.startTime); + } else if (drag.type === "handle-pending") { + seekTo(drag.startTime); } rulerDragRef.current = null; diff --git a/frontend/src/components/TransportBar.tsx b/frontend/src/components/TransportBar.tsx index 5049869..ba29790 100644 --- a/frontend/src/components/TransportBar.tsx +++ b/frontend/src/components/TransportBar.tsx @@ -14,6 +14,7 @@ import { useDAWStore } from "../store/useDAWStore"; import { MetronomeSettings } from "./MetronomeSettings"; import { MetronomeIcon } from "./icons"; import { Button, Input, TimeSignatureInput } from "./ui"; +import { formatShortcut } from "../utils/platform"; function formatSMPTE(seconds: number, frameRate: number): string { const totalFrames = Math.floor(seconds * frameRate); @@ -254,7 +255,7 @@ export function TransportBar() { active={isRecording} disabled={!hasArmedTracks} onClick={handleRecord} - title={hasArmedTracks ? "Record (Ctrl+R)" : "Arm a track to record"} + title={hasArmedTracks ? `Record (${formatShortcut("Ctrl+R")})` : "Arm a track to record"} aria-label={hasArmedTracks ? "Record" : "Arm a track to record"} > <Circle size={16} fill="currentColor" /> diff --git a/frontend/src/components/UnsavedChangesDialog.tsx b/frontend/src/components/UnsavedChangesDialog.tsx index d2c9103..d934312 100644 --- a/frontend/src/components/UnsavedChangesDialog.tsx +++ b/frontend/src/components/UnsavedChangesDialog.tsx @@ -7,6 +7,7 @@ export function UnsavedChangesDialog() { showUnsavedChangesDialog, pendingProjectActionLabel, projectName, + projectPath, resolveUnsavedChanges, dismissUnsavedChangesDialog, } = useDAWStore( @@ -14,12 +15,21 @@ export function UnsavedChangesDialog() { showUnsavedChangesDialog: state.showUnsavedChangesDialog, pendingProjectActionLabel: state.pendingProjectActionLabel, projectName: state.projectName, + projectPath: state.projectPath, resolveUnsavedChanges: state.resolveUnsavedChanges, dismissUnsavedChangesDialog: state.dismissUnsavedChangesDialog, })), ); - const displayName = projectName?.trim() || "Untitled Project"; + // Prefer the explicit project name only if it's not the generic default placeholder. + // A saved project with no explicit name will have projectName = "Untitled Project" in the store, + // so we fall through to derive the name from the file path instead. + const trimmedName = projectName?.trim(); + const hasRealName = trimmedName && trimmedName !== "Untitled Project"; + const displayName = + (hasRealName ? trimmedName : null) || + (projectPath ? projectPath.split(/[\\/]/).pop()?.replace(/\.osproj$/i, "") : null) || + "Untitled Project"; return ( <Modal diff --git a/frontend/src/components/icons.tsx b/frontend/src/components/icons.tsx index b7cce3f..c029dd7 100644 --- a/frontend/src/components/icons.tsx +++ b/frontend/src/components/icons.tsx @@ -153,6 +153,34 @@ export function FolderIcon({ size = 16, className }: IconProps) { ); } +export function AIIcon({ size = 16, className }: IconProps) { + return ( + <svg + width={size} + height={size} + viewBox="0 0 24 24" + fill="none" + stroke="currentColor" + strokeWidth="1.5" + strokeLinecap="round" + strokeLinejoin="round" + className={className} + > + <path d="M12 2v4" /> + <path d="M12 18v4" /> + <path d="M4.9 4.9l2.8 2.8" /> + <path d="M16.3 16.3l2.8 2.8" /> + <path d="M2 12h4" /> + <path d="M18 12h4" /> + <path d="M4.9 19.1l2.8-2.8" /> + <path d="M16.3 7.7l2.8-2.8" /> + <circle cx="12" cy="12" r="4.5" /> + <path d="M10.8 10.5h2.4l1.6 4" /> + <path d="M9.6 14.5l2.4-6 2.4 6" /> + </svg> + ); +} + /** All available track icons, keyed by icon ID */ export const TRACK_ICONS: Record<string, React.FC<IconProps>> = { microphone: MicrophoneIcon, @@ -164,6 +192,7 @@ export const TRACK_ICONS: Record<string, React.FC<IconProps>> = { midi: MIDIIcon, folder: FolderIcon, piano: PianoIcon, + ai: AIIcon, }; /** Icon ID labels for UI display */ @@ -177,4 +206,5 @@ export const TRACK_ICON_LABELS: Record<string, string> = { midi: "MIDI", folder: "Folder", piano: "Piano", + ai: "AI", }; diff --git a/frontend/src/components/ui/Modal/Modal.tsx b/frontend/src/components/ui/Modal/Modal.tsx index aefc3be..332d877 100644 --- a/frontend/src/components/ui/Modal/Modal.tsx +++ b/frontend/src/components/ui/Modal/Modal.tsx @@ -111,10 +111,11 @@ export function Modal({ leaveTo="opacity-0 scale-95" > <DialogPanel + onKeyDown={(e) => e.stopPropagation()} className={classNames( 'bg-daw-panel border border-daw-border rounded-lg shadow-xl', modalSizeStyles[size], - fullHeight ? 'flex max-h-[90vh] flex-col overflow-hidden' : 'overflow-y-auto', + fullHeight ? 'flex max-h-[90vh] flex-col overflow-hidden' : 'max-h-[90vh] flex flex-col overflow-hidden', className )} > @@ -139,7 +140,7 @@ export function Modal({ )} {/* Content */} - <div className={classNames(fullHeight ? 'flex min-h-0 flex-1 flex-col' : 'p-4')}> + <div className={classNames('flex min-h-0 flex-1 flex-col overflow-y-auto', !fullHeight && 'p-4')}> {children} </div> diff --git a/frontend/src/components/ui/Modal/ModalHeader.tsx b/frontend/src/components/ui/Modal/ModalHeader.tsx index 49c2964..061ac02 100644 --- a/frontend/src/components/ui/Modal/ModalHeader.tsx +++ b/frontend/src/components/ui/Modal/ModalHeader.tsx @@ -21,7 +21,7 @@ export function ModalHeader({ showCloseButton = true, }: ModalHeaderProps) { return ( - <div className="flex items-center justify-between p-4 border-b border-daw-border"> + <div className="flex shrink-0 items-center justify-between p-4 border-b border-daw-border"> <h2 className="text-lg font-semibold text-daw-text">{title}</h2> {showCloseButton && onClose && ( <Button diff --git a/frontend/src/data/aiWorkflows.ts b/frontend/src/data/aiWorkflows.ts new file mode 100644 index 0000000..59c597a --- /dev/null +++ b/frontend/src/data/aiWorkflows.ts @@ -0,0 +1,374 @@ +export type AIWorkflowParamType = + | "text" + | "textarea" + | "number" + | "slider" + | "select" + | "toggle"; + +export type AIWorkflowSection = + | "prompt" + | "music" + | "sampling" + | "generation"; + +export interface AIWorkflowParam { + key: string; + label: string; + type: AIWorkflowParamType; + default: unknown; + section: AIWorkflowSection; + min?: number; + max?: number; + step?: number; + options?: string[]; + placeholder?: string; + description?: string; +} + +export interface AIWorkflow { + id: string; + label: string; + description: string; + params: AIWorkflowParam[]; + available?: boolean; + availabilityNote?: string; +} + +export const AI_WORKFLOW_SECTION_LABELS: Record<AIWorkflowSection, string> = { + prompt: "Prompt and Lyrics", + music: "Musical Controls", + sampling: "Sampling Controls", + generation: "Generation", +}; + +const LANGUAGE_OPTIONS = ["en", "es", "fr", "de", "it", "pt", "ja", "ko", "zh"]; +const TIME_SIGNATURE_OPTIONS = ["4/4", "3/4", "6/8", "5/4", "7/8"]; +const KEY_SCALE_OPTIONS = [ + "C major", + "C minor", + "C# major", + "C# minor", + "D major", + "D minor", + "D# major", + "D# minor", + "E major", + "E minor", + "F major", + "F minor", + "F# major", + "F# minor", + "G major", + "G minor", + "G# major", + "G# minor", + "A major", + "A minor", + "A# major", + "A# minor", + "B major", + "B minor", +]; + +const FIXED_ACE_STEP_PARAMS: AIWorkflowParam[] = [ + { + key: "prompt", + label: "Prompt", + type: "textarea", + section: "prompt", + placeholder: + "mellow melodic rock, soft acoustic guitar intro, deep groovy bass, harmonic female and male vocals", + default: "", + }, + { + key: "lyrics", + label: "Lyrics", + type: "textarea", + section: "prompt", + placeholder: "[verse]\nLine one\nLine two\n[chorus]\n...", + default: "", + }, + { + key: "seed", + label: "Seed", + type: "number", + section: "sampling", + default: -1, + }, + { + key: "bpm", + label: "BPM", + type: "number", + section: "music", + min: 40, + max: 240, + step: 1, + default: 120, + }, + { + key: "duration", + label: "Duration (seconds)", + type: "slider", + section: "music", + min: 5, + max: 240, + step: 1, + default: 30, + }, + { + key: "timesignature", + label: "Time Signature", + type: "select", + section: "music", + options: TIME_SIGNATURE_OPTIONS, + default: "4/4", + }, + { + key: "language", + label: "Language", + type: "select", + section: "music", + options: LANGUAGE_OPTIONS, + default: "en", + }, + { + key: "keyscale", + label: "Key / Scale", + type: "select", + section: "music", + options: KEY_SCALE_OPTIONS, + default: "C major", + }, + { + key: "generate_audio_codes", + label: "Generate Audio Codes", + type: "toggle", + section: "generation", + default: true, + description: + "Matches the OpenStudio ACE split-graph workflow. Disable only for manual direct DiT troubleshooting.", + }, + { + key: "inferenceSteps", + label: "Diffusion Steps", + type: "slider", + section: "generation", + min: 4, + max: 24, + step: 1, + default: 8, + }, + { + key: "cfg_scale", + label: "Text Encoder CFG", + type: "slider", + section: "sampling", + min: 0, + max: 10, + step: 0.05, + default: 2, + }, + { + key: "guidance_scale", + label: "Sampler CFG", + type: "slider", + section: "generation", + min: 0, + max: 20, + step: 0.5, + default: 1, + }, + { + key: "shift", + label: "Turbo Shift", + type: "slider", + section: "generation", + min: 1, + max: 5, + step: 0.05, + default: 3, + }, + { + key: "temperature", + label: "Temperature", + type: "slider", + section: "sampling", + min: 0, + max: 2, + step: 0.01, + default: 0.85, + }, + { + key: "top_p", + label: "Top P", + type: "slider", + section: "sampling", + min: 0, + max: 1, + step: 0.01, + default: 0.9, + }, + { + key: "top_k", + label: "Top K", + type: "number", + section: "sampling", + min: 0, + max: 200, + step: 1, + default: 0, + }, + { + key: "min_p", + label: "Min P", + type: "slider", + section: "sampling", + min: 0, + max: 1, + step: 0.001, + default: 0, + }, +]; + +export const AI_WORKFLOWS: AIWorkflow[] = [ + { + id: "text-to-music", + label: "Text to Music", + description: "Generate a fresh music clip from a detailed style and arrangement prompt.", + params: FIXED_ACE_STEP_PARAMS, + available: true, + }, + { + id: "lyrics+style", + label: "Lyrics + Style", + description: "Generate a song guided by both prompt text and structured lyrics.", + params: FIXED_ACE_STEP_PARAMS, + available: true, + }, + { + id: "continuation", + label: "Audio Continuation", + description: "Extend an existing clip with matching style and motion.", + params: [], + available: false, + availabilityNote: "Continuation is not implemented yet in the OpenStudio ACE-Step bridge.", + }, +]; + +function normalizeText(value: unknown, fallback = "") { + return typeof value === "string" ? value : String(value ?? fallback); +} + +function normalizeNumber( + value: unknown, + fallback: number, + min?: number, + max?: number, +) { + const parsed = + typeof value === "number" + ? value + : value === "" || value == null + ? fallback + : Number(value); + const safeValue = Number.isFinite(parsed) ? parsed : fallback; + if (typeof min === "number" && typeof max === "number") { + return Math.min(max, Math.max(min, safeValue)); + } + if (typeof min === "number") { + return Math.max(min, safeValue); + } + if (typeof max === "number") { + return Math.min(max, safeValue); + } + return safeValue; +} + +function normalizeToggle(value: unknown, fallback: boolean) { + if (typeof value === "boolean") return value; + if (typeof value === "string") { + const lowered = value.trim().toLowerCase(); + if (["true", "1", "yes", "on"].includes(lowered)) return true; + if (["false", "0", "no", "off"].includes(lowered)) return false; + } + if (typeof value === "number") return value !== 0; + return fallback; +} + +function normalizeSelectValue(param: AIWorkflowParam, value: unknown) { + const normalized = normalizeText(value, String(param.default ?? "")); + if (param.options?.includes(normalized)) { + return normalized; + } + if (param.key === "timesignature") { + if (/^\d+\/\d+$/.test(normalized)) return normalized; + const numeric = Number(normalized); + if (Number.isFinite(numeric) && numeric > 0) { + return `${Math.round(numeric)}/4`; + } + } + return String(param.default ?? ""); +} + +export function getAIWorkflow(workflowId?: string | null): AIWorkflow { + return ( + AI_WORKFLOWS.find((workflow) => workflow.id === workflowId) ?? AI_WORKFLOWS[0] + ); +} + +export function getDefaultWorkflowParams( + workflowId?: string | null, +): Record<string, unknown> { + const workflow = getAIWorkflow(workflowId); + return Object.fromEntries( + workflow.params.map((param) => [param.key, param.default]), + ); +} + +export function normalizeWorkflowParams( + workflowId?: string | null, + params?: Record<string, unknown>, +): Record<string, unknown> { + const workflow = getAIWorkflow(workflowId); + const source = params ?? {}; + const normalized = Object.fromEntries( + workflow.params.map((param) => { + const value = source[param.key]; + + if (param.type === "textarea" || param.type === "text") { + return [param.key, normalizeText(value, String(param.default ?? ""))]; + } + + if (param.type === "number" || param.type === "slider") { + return [ + param.key, + normalizeNumber( + value, + Number(param.default ?? 0), + param.min, + param.max, + ), + ]; + } + + if (param.type === "toggle") { + return [param.key, normalizeToggle(value, Boolean(param.default))]; + } + + return [param.key, normalizeSelectValue(param, value)]; + }), + ); + + return normalized; +} + +export function mergeWorkflowParams( + workflowId?: string | null, + params?: Record<string, unknown>, +): Record<string, unknown> { + return normalizeWorkflowParams(workflowId, { + ...getDefaultWorkflowParams(workflowId), + ...(params ?? {}), + }); +} diff --git a/frontend/src/services/NativeBridge.ts b/frontend/src/services/NativeBridge.ts index 4e366f3..fc6c9d7 100644 --- a/frontend/src/services/NativeBridge.ts +++ b/frontend/src/services/NativeBridge.ts @@ -1,3 +1,5 @@ +import { normalizeWorkflowParams } from "../data/aiWorkflows"; + // Type definitions for the JUCE backend const FORMANT_LOG_PREFIX = "[pitchEditor.formant]"; @@ -39,6 +41,8 @@ export interface PitchNoteData { id: string; startTime: number; endTime: number; + effectiveStartTime?: number; // note body + rendered shoulder span, clip-relative seconds + effectiveEndTime?: number; // note body + rendered shoulder span, clip-relative seconds detectedPitch: number; // MIDI note (fractional) correctedPitch: number; // MIDI note (fractional) driftCorrectionAmount: number; // 0-1 @@ -49,9 +53,26 @@ export interface PitchNoteData { formantShift: number; // semitones gain: number; // dB voiced: boolean; // true = pitched vocal, false = unvoiced (sibilant/breath) + wordGroupId?: string; // editable word/phrase group; fragments in the same group move together + entryBoundaryKind?: "unknown" | "hard_word_like" | "soft_legato" | "internal_bend" | "internal_vibrato" | string; + exitBoundaryKind?: "unknown" | "hard_word_like" | "soft_legato" | "internal_bend" | "internal_vibrato" | string; + entryBoundaryReason?: string; + exitBoundaryReason?: string; + entryBoundaryScore?: number; + exitBoundaryScore?: number; pitchDrift: number[]; // per-frame deviation from note center } +export interface PitchBoundaryCandidateData { + id: string; + sourceNoteId?: string; + time: number; + kind: "unknown" | "hard_word_like" | "soft_legato" | "internal_bend" | "internal_vibrato" | string; + reason?: string; + score?: number; + destructiveSplitAllowed?: boolean; +} + export interface PitchContourData { clipId: string; sampleRate: number; @@ -64,6 +85,7 @@ export interface PitchContourData { voiced: boolean[]; }; notes: PitchNoteData[]; + boundaryCandidates?: PitchBoundaryCandidateData[]; } export interface ClipPitchPreviewPayload { @@ -75,9 +97,403 @@ export interface ClipPitchPreviewPayload { globalFormantSemitones?: number; previewStartSec?: number; previewEndSec?: number; + allowReplacingCorrectedSource?: boolean; +} + +export type PitchCorrectionRenderMode = "single" | "preview_segment" | "full_clip_hq" | "note_hq"; +export type PitchRegressionJobType = "render" | "analysis" | "scrub" | "export" | "clean_export" | "audition"; + +export interface PitchScrubPreviewStatus { + active: boolean; + releasePending?: boolean; + audible?: boolean; + previewArmed?: boolean; + firstCallbackServiced?: boolean; + firstDragAudible?: boolean; + trackId?: string; + clipId?: string; + pitchRatio?: number; + basePitchHz?: number; + currentGain?: number; + targetGain?: number; + repeatStability?: number; + lastPeak?: number; + loopDurationMs?: number; + lastRenderWallTimeMs?: number; + mixedCallbackCount?: number; + mixedSampleCount?: number; + renderMethod?: string; +} + +export interface PitchPreviewRoutingStatus { + scrubPreviewActive: boolean; + clipLivePreviewActive: boolean; + renderedSegmentActive: boolean; + correctedSourceActive: boolean; + monitorMode: "none" | "scrub" | "clip_live_preview" | "rendered_segment" | "corrected_source" | string; +} + +export interface PitchPlaybackRouteTraceEntry { + blockIndex?: number; + projectTimeSec?: number; + sourceType?: "none" | "original" | "corrected_source" | "rendered_segment" | string; + audioFile?: string; + clipTimeSec?: number; + playbackOffsetSec?: number; + renderedSegmentActiveAtTime?: boolean; + correctedSourceActiveAtTime?: boolean; +} + +export interface PitchRegressionJob { + jobType?: PitchRegressionJobType; + projectFixturePath?: string; + sourceAudioPath: string; + referenceAudioPath?: string; + trackId?: string; + clipId: string; + renderMode?: PitchCorrectionRenderMode; + globalFormantSemitones?: number; + targetShiftSemitones?: number | null; + notes?: PitchNoteData[]; + frames?: PitchContourData["frames"]; + windowStartSec?: number; + windowEndSec?: number; + noteBodyStartSec?: number; + noteBodyEndSec?: number; + entryWindowSec?: number; + exitWindowSec?: number; + neighborWindowSec?: number; + analysisOffsetSec?: number; + analysisDurationSec?: number; + expectedNotes?: PitchNoteData[]; + scrubUpdatePitchRatio?: number; + scrubWaitMs?: number; + scrubRepeatCount?: number; + scrubTransportCycle?: boolean; + scrubSelectionChange?: boolean; + exportOutputPath?: string; + auditionStartSec?: number; + auditionDurationSec?: number; + auditionPlaybackOutputPath?: string; + auditionExportOutputPath?: string; + resultJsonPath: string; + label?: string; +} + +export interface PitchAuditionCaptureResult { + success: boolean; + trackId?: string; + clipId?: string; + filePath?: string; + startSec?: number; + durationSec?: number; + sampleRate?: number; + peak?: number; + rms?: number; + source?: string; + offlineRenderMode?: boolean; + routeTrace?: PitchPlaybackRouteTraceEntry[]; + error?: string; +} + +export interface PitchAppFinalParityReport { + parityPassed?: boolean; + residualRelativeDb?: number | null; + entryResidualRelativeDb?: number | null; + shortRmsEnvelopeCorrelation?: number | null; + activeStartDeltaMs?: number | null; + bakedContextPath?: string; + appPlaybackContextPath?: string; + [key: string]: unknown; +} + +export interface PitchAppFinalCaptureResult { + success: boolean; + trackId?: string; + clipId?: string; + capture?: PitchAuditionCaptureResult; + livePlaybackCapture?: PitchAuditionCaptureResult; + offlineRenderCapture?: PitchAuditionCaptureResult; + bakedCorrectedCapture?: PitchAuditionCaptureResult; + bakedCorrectedPath?: string; + livePlaybackPath?: string; + offlineRenderPath?: string; + comparisonReportPath?: string; + bakedVsLiveParityReport?: PitchAppFinalParityReport; + bakedVsOfflineParityReport?: PitchAppFinalParityReport; + liveVsOfflineParityReport?: PitchAppFinalParityReport; + routeBefore?: PitchPreviewRoutingStatus; + routeAfter?: PitchPreviewRoutingStatus; + routeReportPath?: string; + routeReportWritten?: boolean; + capturedAt?: string; + metadata?: unknown; +} + +export interface PitchRegressionResult { + success: boolean; + jobType?: PitchRegressionJobType; + outputFile?: string; + pitchCorrectionOutputFile?: string; + exportOutputFile?: string; + auditionPlaybackOutputFile?: string; + auditionExportOutputFile?: string; + auditionStartSec?: number; + auditionDurationSec?: number; + auditionCapture?: PitchAuditionCaptureResult; + appFinalCapture?: PitchAppFinalCaptureResult; + appFinalBakedCapture?: PitchAuditionCaptureResult; + appFinalParityReport?: PitchAppFinalParityReport; + appFinalRouteReportPath?: string; + appFinalBakedContextPath?: string; + appFinalPlaybackContextPath?: string; + appFinalParityReportPath?: string; + clipId?: string; + requestId?: string; + renderMode?: PitchCorrectionRenderMode; + targetShiftSemitones?: number | null; + actualRequestedShiftSemitones?: number | null; + requestedShiftErrorCents?: number | null; + referenceVsOriginalCents?: number | null; + chromaticSnapBypassed?: boolean; + postApplyRouteStatus?: PitchPreviewRoutingStatus; + requestedRendererBranch?: string; + actualRendererBranch?: string; + pitchOnlyRecoveryPath?: string; + pitchOnlyNeutralFormantUsed?: boolean; + processingMode?: string; + formantCurveUsed?: boolean; + explicitFormantRequested?: boolean; + pitchOnlyFormantSuppressed?: boolean; + usedFallback?: boolean; + fallbackReason?: string; + hardFailReason?: string; + pitchRenderStrategy?: string; + pitchRenderProductPath?: string; + pitchRenderBackendId?: string; + pitchRenderBackendVersion?: string; + pitchRenderBackendFailureCode?: string; + pitchRenderBackendCapabilities?: unknown; + pitchRenderBackendDiagnostics?: unknown; + pitchRenderCommitPolicy?: string; + pitchRenderDryProtectedSamples?: number; + pitchRenderContextDurationSec?: number; + pitchRenderCommitDurationSec?: number; + pitchRenderJobStartDelayMs?: number; + pitchRenderDirection?: string; + downshiftFormantGuardUsed?: boolean; + downshiftFormantGuardAlpha?: number; + noteHqEffectiveStartSec?: number; + noteHqEffectiveEndSec?: number; + noteHqContextStartSec?: number; + noteHqContextEndSec?: number; + noteHqAudibleCommitStartSec?: number; + noteHqAudibleCommitEndSec?: number; + noteHqPreBodyDryProtectedSamples?: number; + noteHqEntryInsideBodyFadeMs?: number; + noteHqExitLeadInMs?: number; + noteHqEntryBridgeStartSec?: number; + noteHqEntryBridgeEndSec?: number; + noteHqEntryBridgeWetLagMs?: number; + noteHqEntryBridgeEnvelopeGainDb?: number; + noteHqEntryBridgeUsed?: boolean; + noteHqEntryTransientDryPreservedMs?: number; + pitchOnlyEntrySimpleHandoffUsed?: boolean; + pitchOnlyEntrySafeHandoffUsed?: boolean; + pitchOnlyEntryDryHoldMs?: number; + pitchOnlyEntrySafeBridgeMs?: number; + pitchOnlyEntryWetAlignmentMs?: number; + pitchOnlyEntryWetGainDb?: number; + pitchOnlyEntryWetVsDryRmsDb?: number; + pitchOnlyEntryEqualPowerBlendUsed?: boolean; + pitchOnlyEntryRmsContinuityUsed?: boolean; + pitchOnlyEntryRmsContinuityGainDb?: number; + pitchOnlyEntryRmsContinuityMs?: number; + pitchOnlyEntryPhaseSafeUsed?: boolean; + pitchOnlyEntryWetAlignmentAccepted?: boolean; + pitchOnlyEntryFirstCycleCorrelation?: number; + pitchOnlyEntryZeroCrossOffsetMs?: number; + pitchOnlyEntryBridgeGainRampDb?: number; + pitchOnlyDownshiftCoreEnvelopePassUsed?: boolean; + pitchOnlyDownshiftCoreRmsTrimDb?: number; + pitchOnlyDownshiftCoreEnvelopeMaxDb?: number; + pitchOnlyDownshiftCoreEnvelopeFrames?: number; + pitchOnlyEntryWetLagMs?: number; + pitchOnlyEntryBridgeDurationMs?: number; + pitchOnlyExitDryRestoreUsed?: boolean; + pitchOnlyExitDryRestoreStartSec?: number; + pitchOnlyExitDryRestoreEndSec?: number; + noteHqEntryBoundaryKind?: string; + noteHqExitBoundaryKind?: string; + noteHqEntryBoundaryScore?: number; + noteHqExitBoundaryScore?: number; + noteHqRendererEntryBoundaryKind?: string; + noteHqRendererExitBoundaryKind?: string; + noteHqEditIslandCount?: number; + noteHqEditedNoteCount?: number; + noteHqEntryPitchHandoffUsed?: boolean; + noteHqEntryPitchHandoffStartSec?: number; + noteHqEntryPitchHandoffEndSec?: number; + noteHqEntryPitchHandoffPreMs?: number; + noteHqEntryPitchHandoffBodyMs?: number; + noteHqEntryPitchSlopeJumpStPerSec?: number; + noteHqEntryPitchAccelerationLimited?: boolean; + phraseHqRenderUsed?: boolean; + phraseHqExpandedToFullClip?: boolean; + phraseHqStartSec?: number; + phraseHqEndSec?: number; + bridgeUsed?: boolean; + bridgeFallbackUsed?: boolean; + bridgeStartSec?: number; + bridgeLengthMs?: number; + bridgeAlignmentLagSamples?: number; + bridgeCorrelationScore?: number; + bridgeGainDeltaDb?: number; + bodyReplacementUsed?: boolean; + bodyReplacementFallbackUsed?: boolean; + entryLockStartSec?: number; + entryLockLengthMs?: number; + exitLockStartSec?: number; + renderedBodyStartSec?: number; + renderedBodyEndSec?: number; + islandNativeUsed?: boolean; + islandNativeFallbackUsed?: boolean; + islandRenderStartSec?: number; + islandRenderEndSec?: number; + transientMaskPeak?: number; + voicedCoreMaskPeak?: number; + hpssUsed?: boolean; + hpssFallbackUsed?: boolean; + harmonicMaskPeak?: number; + aperiodicMaskPeak?: number; + spectralEnvelopeCorrectionUsed?: boolean; + pitchOnlyCoreTimbreCorrectionUsed?: boolean; + pitchOnlyCoreEnvelopeMix?: number; + pitchOnlyCoreRmsTrimDb?: number; + pitchOnlyCoreEnvelopeLifter?: number; + pitchOnlyEntryTimbreCorrectionUsed?: boolean; + pitchOnlyEntryRmsTrimDb?: number; + pitchOnlyEntryTiltDb?: number; + pitchOnlyEntryHandoffUsed?: boolean; + pitchOnlyExitHandoffUsed?: boolean; + vocalSourceFilterUsed?: boolean; + vocalSourceFilterVoicedCoverage?: number; + vocalSourceFilterResidualMix?: number; + vocalSourceFilterFallbackUsed?: boolean; + vocalSourceFilterFallbackReason?: string; + vocalSourceFilterEntryDryMs?: number; + vocalSourceFilterExitDryMs?: number; + wsolaUsed?: boolean; + wsolaFallbackUsed?: boolean; + wsolaEntryLagSamples?: number; + wsolaExitLagSamples?: number; + wsolaCorrelationScore?: number; + phaseLockUsed?: boolean; + phaseLockFallbackUsed?: boolean; + phaseAlignedEntry?: boolean; + phaseAlignedExit?: boolean; + phasePeakCount?: number; + transitionHqUsed?: boolean; + transitionHqFallbackUsed?: boolean; + transitionStartSec?: number; + transitionEndSec?: number; + transitionTransientPeak?: number; + transitionVoicedCorePeak?: number; + transitionResidualPeak?: number; + transitionEnvelopeCorrectionUsed?: boolean; + engineV2Used?: boolean; + engineV2FallbackUsed?: boolean; + engineV2TransitionCount?: number; + engineV2TransitionStartSec?: number; + engineV2TransitionEndSec?: number; + engineV2HarmonicSupportPeak?: number; + engineV2ResidualSupportPeak?: number; + engineV2EnvelopeSupportPeak?: number; + transientBypassUsed?: boolean; + residualCarryUsed?: boolean; + cepstralCutoffUsed?: number; + fftSizeUsed?: number; + hopSizeUsed?: number; + immediateLeftNeighborUsed?: boolean; + immediateRightNeighborUsed?: boolean; + leftNeighborSamplesRendered?: number; + rightNeighborSamplesRendered?: number; + leftNeighborSmoothMs?: number; + rightNeighborSmoothMs?: number; + nonImmediateNeighborTouched?: boolean; + entryAlignmentOffsetMs?: number; + exitAlignmentOffsetMs?: number; + firstVoicedCyclesEntryUsed?: boolean; + firstVoicedCyclesExitUsed?: boolean; + v3TransitionPairUsed?: boolean; + v3ContinuousRenderUsed?: boolean; + v3EntryAnchorMs?: number; + v3ExitAnchorMs?: number; + v3FirstCyclesEntryCount?: number; + v3FirstCyclesExitCount?: number; + v3ShellDurationMs?: number; + v3BodyDurationMs?: number; + v3ResidualMix?: number; + v3FormantMode?: string; + v3NeighborLeftOverlapMs?: number; + v3NeighborRightOverlapMs?: number; + elapsedMs?: number; + outputDurationSec?: number; + error?: string; + comparison?: unknown; + sourceAudioPath?: string; + referenceAudioPath?: string; + windowStartSec?: number; + windowEndSec?: number; + noteBodyStartSec?: number; + noteBodyEndSec?: number; + entryWindowSec?: number; + exitWindowSec?: number; + neighborWindowSec?: number; + previewCoverageStartSec?: number; + previewCoverageEndSec?: number; + candidateCoverageStartSec?: number; + candidateCoverageEndSec?: number; + label?: string; + analysisSummary?: unknown; + analysisResult?: PitchContourData; + scrubPreviewAudible?: boolean; + scrubPreviewStartLatencyMs?: number | null; + scrubPreviewStopLatencyMs?: number | null; + scrubPreviewMixedCallbackCount?: number; + scrubPreviewMixedSampleCount?: number; + scrubPreviewLastPeak?: number | null; + scrubPreviewFirstDragAudible?: boolean; + scrubPreviewRepeatStability?: number | null; + scrubPreviewScenarioCount?: number; + scrubPreviewScenarioPassCount?: number; + scrubPreviewRepeatedDragAudible?: boolean; + scrubPreviewAfterTransportCycleAudible?: boolean; + scrubPreviewSelectionChangeAudible?: boolean; + scrubPreviewSelectionChangeRequested?: boolean; + scrubPreviewInputNoteCount?: number; + scrubPreviewContourNoteCount?: number; + scrubPreviewAlternateNoteFound?: boolean; + scrubPreviewScenarioNames?: string[]; + scrubPreviewScenarioResults?: Array<{ + name: string; + audible: boolean; + firstDragAudible?: boolean; + startLatencyMs?: number | null; + stopLatencyMs?: number | null; + repeatStability?: number | null; + lastPeak?: number | null; + }>; + scrubPreviewStatus?: PitchScrubPreviewStatus; } -export type PitchCorrectionRenderMode = "single" | "preview_segment" | "full_clip_hq"; +export interface PitchCorrectionCompletionData extends PitchRegressionResult { + clipId: string; + success: boolean; + restored?: boolean; + cancelled?: boolean; + swapDeferred?: boolean; +} // Polyphonic pitch detection types (Phase 6) export interface PolyNoteData { @@ -264,8 +680,65 @@ export interface AiToolsStatus { lastPhase?: string; terminalReason?: string; fallbackAttempted?: boolean; - restartRequired?: boolean; - } + restartRequired?: boolean; + aceStepVersion?: string | null; + musicGenerationReady?: boolean; + musicGenerationLayoutValid?: boolean; + musicGenerationModelId?: string | null; + musicGenerationModelRepoId?: string | null; + musicGenerationSharedRepoId?: string | null; + musicGenerationCheckpointRoot?: string | null; + musicGenerationStatusMessage?: string; + musicGenerationFailureCode?: string; + musicGenerationPerformanceReady?: boolean; + musicGenerationPerformanceStatusMessage?: string; + musicGenerationRuntimeProfiles?: Record<string, unknown>; + musicGenerationAvailableProfiles?: string[]; + musicGenerationUnavailableProfiles?: Array<Record<string, unknown>>; + musicGenerationDefaultProfile?: string; + musicGenerationWarmSessionCapable?: boolean; + } + +export interface AIGenerationProgress { + state: + | "idle" + | "loading" + | "generating" + | "writing" + | "done" + | "error" + | "cancelled"; + progress: number; + phase?: string; + message?: string; + backend?: string; + outputFile?: string; + error?: string; + elapsedMs?: number; + heartbeatTs?: number; + phaseProgress?: number; + etaMs?: number; + runMode?: "cold" | "warm"; + runtimeProfile?: string; + lmModel?: string; + statusNote?: string; + failureKind?: string; + sessionMode?: "persistent" | "oneshot" | "oneshot-fallback" | string; + workerExitCode?: number; + lastStdoutLine?: string; + lastStderrLine?: string; + attemptMode?: "lm_dit" | "dit_only" | "dit_only_retry" | string; + attemptIndex?: number; + protocolVersion?: number; + scriptVersion?: string; + requestId?: string; + priorFailure?: string; + lastProgressAgeMs?: number; + tracePath?: string; + failureDetail?: string; + lmBackend?: string; + lmStage?: string; +} export interface InstallAiToolsResponse { started: boolean; @@ -274,6 +747,18 @@ export interface InstallAiToolsResponse { status?: AiToolsStatus; } +export interface InstallAiToolsOptions { + userConfirmedDownload?: boolean; +} + +export interface ResetAiToolsResponse { + success: boolean; + error?: string; + message?: string; + status?: AiToolsStatus; + deletedPaths?: string[]; +} + export interface ARAPluginInfo { name: string; manufacturer: string; @@ -385,6 +870,14 @@ export interface AudioDebugSnapshot { blockSize: number; playbackClipCount: number; activeOutputChannels: number; + lastAudioCallbackProcessMs?: number; + maxAudioCallbackProcessMs?: number; + audioCallbackDeadlineMissCount?: number; + audioCallbackTrackBufferResizeCount?: number; + audioCallbackPitchScrubBufferResizeCount?: number; + audioCallbackSidechainBufferResizeCount?: number; + spectrumFftPublishCount?: number; + spectrumFftLockMissCount?: number; postTrackPlaybackPeak: number; postMonitoringInputPeak: number; postMasterFxPeak: number; @@ -724,7 +1217,7 @@ declare global { // Track Type Management (Phase 2) setTrackType?: ( trackId: string, - type: "audio" | "midi" | "instrument", + type: "audio" | "midi" | "instrument" | "ai", ) => Promise<boolean>; setTrackMIDIInput?: ( trackId: string, @@ -749,6 +1242,8 @@ declare global { ) => Promise<boolean>; loadProjectFromFile?: (filePath: string) => Promise<string>; consumePendingLaunchProjectPath?: () => Promise<string>; + getPitchRegressionJob?: () => Promise<PitchRegressionJob | null>; + completePitchRegressionJob?: (result: PitchRegressionResult) => Promise<boolean>; // Plugin State Serialization (F2) getPluginState?: ( @@ -828,18 +1323,26 @@ declare global { saveDroppedFile?: (fileName: string, base64Data: string) => Promise<string>; // Render/Export (F3) - showRenderSaveDialog?: (defaultFileName: string, formatExtension: string) => Promise<string>; + showRenderSaveDialog?: (defaultFileName: string, formatExtension: string, initialDirectory?: string) => Promise<string>; renderProject?: ( source: string, startTime: number, endTime: number, filePath: string, format: string, sampleRate: number, bitDepth: number, channels: number, normalize: boolean, - addTail: boolean, tailLength: number, + addTail: boolean, tailLength: number, includeMetronome?: boolean, ) => Promise<boolean>; + capturePitchAuditionPlayback?: ( + trackId: string, clipId: string, startTime: number, duration: number, + filePath: string, sampleRate?: number, offlineRenderMode?: boolean, + ) => Promise<PitchAuditionCaptureResult>; + capturePitchAppFinalContext?: ( + trackId: string, clipId: string, startTime: number, duration: number, + wavPath: string, routeJsonPath: string, sampleRate?: number, metadata?: unknown, + ) => Promise<PitchAppFinalCaptureResult>; renderProjectWithDither?: ( source: string, startTime: number, endTime: number, filePath: string, format: string, sampleRate: number, bitDepth: number, channels: number, normalize: boolean, - addTail: boolean, tailLength: number, ditherType: string, + addTail: boolean, tailLength: number, ditherType: string, includeMetronome?: boolean, ) => Promise<boolean>; // Phase 9: Audio Engine Enhancements @@ -1042,20 +1545,31 @@ declare global { setPitchCorrectionBypass?: (trackId: string, clipId: string, bypass: boolean) => Promise<void>; // Real-time pitch preview (Phase 7.5) + startPitchScrubPreview?: (trackId: string, clipId: string, noteJson: string, framesJson?: string) => Promise<{ success: boolean }>; + updatePitchScrubPreview?: (clipId: string, pitchRatio: number) => Promise<boolean>; + stopPitchScrubPreview?: (clipId: string) => Promise<boolean>; + getPitchScrubPreviewStatus?: (clipId?: string) => Promise<PitchScrubPreviewStatus>; + getPitchPreviewRoutingStatus?: (clipId?: string) => Promise<PitchPreviewRoutingStatus>; setClipPitchPreview?: (clipId: string, payload: ClipPitchPreviewPayload) => Promise<boolean>; clearClipPitchPreview?: (clipId: string) => Promise<boolean>; clearClipRenderedPreviewSegments?: (clipId: string) => Promise<boolean>; + clearAllPitchPreviewRoutes?: (clipId?: string) => Promise<boolean>; + clearPitchPreviewRoutesForCorrectedSources?: () => Promise<number>; // Source Separation (Phase 8 + Phase 10) separateStems?: (trackId: string, clipId: string) => Promise<StemSeparationResult>; isStemSeparationAvailable?: () => Promise<boolean>; getAiToolsStatus?: () => Promise<AiToolsStatus>; refreshAiToolsStatus?: () => Promise<AiToolsStatus>; - installAiTools?: () => Promise<InstallAiToolsResponse>; + installAiTools?: (userConfirmedDownload?: boolean) => Promise<InstallAiToolsResponse>; + resetAiTools?: () => Promise<ResetAiToolsResponse>; separateStemsAsync?: (trackId: string, clipId: string, optionsJSON: string) => Promise<{ started: boolean; error?: string; cached?: boolean }>; getStemSeparationProgress?: () => Promise<StemSepProgress>; cancelStemSeparation?: () => Promise<void>; cancelAiToolsInstall?: () => Promise<void>; + startAIGeneration?: (trackId: string, workflowId: string, paramsJSON: string) => Promise<{ started: boolean; error?: string }>; + getAIGenerationProgress?: () => Promise<AIGenerationProgress>; + cancelAIGeneration?: () => Promise<void>; // ARA Plugin Hosting (Phase 9) initializeARA?: (trackId: string, fxIndex: number) => Promise<{ success: boolean; error?: string }>; @@ -1189,7 +1703,7 @@ class NativeBridge { // Subscribe to pitch correction completion events (emitted when WORLD vocoder finishes). // Returns an unsubscribe function (or no-op in dev mode). - onPitchCorrectionComplete(callback: (data: { clipId: string; success: boolean; outputFile?: string; requestId?: string; restored?: boolean; renderMode?: PitchCorrectionRenderMode; cancelled?: boolean; swapDeferred?: boolean }) => void): () => void { + onPitchCorrectionComplete(callback: (data: PitchCorrectionCompletionData) => void): () => void { const backend = this.getBackend(); if (this.isNative && backend?.addEventListener) { const listener = backend.addEventListener( @@ -1663,6 +2177,14 @@ class NativeBridge { blockSize: 512, playbackClipCount: 0, activeOutputChannels: 0, + lastAudioCallbackProcessMs: 0, + maxAudioCallbackProcessMs: 0, + audioCallbackDeadlineMissCount: 0, + audioCallbackTrackBufferResizeCount: 0, + audioCallbackPitchScrubBufferResizeCount: 0, + audioCallbackSidechainBufferResizeCount: 0, + spectrumFftPublishCount: 0, + spectrumFftLockMissCount: 0, postTrackPlaybackPeak: 0, postMonitoringInputPeak: 0, postMasterFxPeak: 0, @@ -2442,7 +2964,7 @@ class NativeBridge { // Track Type Management (Phase 2) async setTrackType( trackId: string, - type: "audio" | "midi" | "instrument", + type: "audio" | "midi" | "instrument" | "ai", ): Promise<boolean> { if (this.isNative && window.__JUCE__?.backend.setTrackType) { return await window.__JUCE__.backend.setTrackType(trackId, type); @@ -2585,10 +3107,31 @@ class NativeBridge { return ""; } + async getPitchRegressionJob(): Promise<PitchRegressionJob | null> { + if (this.isNative && window.__JUCE__?.backend.getPitchRegressionJob) { + const raw = await window.__JUCE__.backend.getPitchRegressionJob(); + if (!raw) { + return null; + } + if (typeof raw === "string") { + return JSON.parse(raw) as PitchRegressionJob; + } + return raw as PitchRegressionJob; + } + return null; + } + + async completePitchRegressionJob(result: PitchRegressionResult): Promise<boolean> { + if (this.isNative && window.__JUCE__?.backend.completePitchRegressionJob) { + return await window.__JUCE__.backend.completePitchRegressionJob(result); + } + return false; + } + // Render/Export (F3) - async showRenderSaveDialog(defaultFileName: string, formatExtension: string): Promise<string> { + async showRenderSaveDialog(defaultFileName: string, formatExtension: string, initialDirectory?: string): Promise<string> { if (this.isNative && window.__JUCE__?.backend.showRenderSaveDialog) { - return await window.__JUCE__.backend.showRenderSaveDialog(defaultFileName, formatExtension); + return await window.__JUCE__.backend.showRenderSaveDialog(defaultFileName, formatExtension, initialDirectory); } console.log("[NativeBridge] Mock showRenderSaveDialog"); return ""; @@ -2606,9 +3149,10 @@ class NativeBridge { normalize: boolean; addTail: boolean; tailLength: number; + includeMetronome?: boolean; }): Promise<boolean> { if (this.isNative && window.__JUCE__?.backend.renderProject) { - return await window.__JUCE__.backend.renderProject( + const success = await window.__JUCE__.backend.renderProject( options.source, options.startTime, options.endTime, @@ -2619,8 +3163,15 @@ class NativeBridge { options.channels, options.normalize, options.addTail, - options.tailLength + options.tailLength, + options.includeMetronome ?? false, ); + if (success) { + const dot = options.filePath.lastIndexOf("."); + const reportPath = `${dot > 0 ? options.filePath.slice(0, dot) : options.filePath}_render_route_report.json`; + console.log(`[NativeBridge] Render route report: ${reportPath}`); + } + return success; } console.log( `[NativeBridge] Mock renderProject: ${options.filePath} (${options.format}, ${options.startTime}-${options.endTime}s)`, @@ -2631,6 +3182,54 @@ class NativeBridge { }); } + async capturePitchAuditionPlayback(options: { + trackId: string; + clipId: string; + startTime: number; + duration: number; + filePath: string; + sampleRate?: number; + offlineRenderMode?: boolean; + }): Promise<PitchAuditionCaptureResult | null> { + if (this.isNative && window.__JUCE__?.backend.capturePitchAuditionPlayback) { + return await window.__JUCE__.backend.capturePitchAuditionPlayback( + options.trackId, + options.clipId, + options.startTime, + options.duration, + options.filePath, + options.sampleRate ?? 44100, + options.offlineRenderMode ?? true, + ); + } + return { success: false, error: "Pitch audition capture is only available in the native app." }; + } + + async capturePitchAppFinalContext(options: { + trackId: string; + clipId: string; + startTime: number; + duration: number; + wavPath: string; + routeJsonPath: string; + sampleRate?: number; + metadata?: unknown; + }): Promise<PitchAppFinalCaptureResult | null> { + if (this.isNative && window.__JUCE__?.backend.capturePitchAppFinalContext) { + return await window.__JUCE__.backend.capturePitchAppFinalContext( + options.trackId, + options.clipId, + options.startTime, + options.duration, + options.wavPath, + options.routeJsonPath, + options.sampleRate ?? 44100, + options.metadata ?? {}, + ); + } + return { success: false }; + } + async renderProjectWithDither(options: { source: string; startTime: number; @@ -2644,9 +3243,10 @@ class NativeBridge { addTail: boolean; tailLength: number; ditherType: string; + includeMetronome?: boolean; }): Promise<boolean> { if (this.isNative && window.__JUCE__?.backend.renderProjectWithDither) { - return await window.__JUCE__.backend.renderProjectWithDither( + const success = await window.__JUCE__.backend.renderProjectWithDither( options.source, options.startTime, options.endTime, @@ -2659,7 +3259,14 @@ class NativeBridge { options.addTail, options.tailLength, options.ditherType, + options.includeMetronome ?? false, ); + if (success) { + const dot = options.filePath.lastIndexOf("."); + const reportPath = `${dot > 0 ? options.filePath.slice(0, dot) : options.filePath}_render_route_report.json`; + console.log(`[NativeBridge] Render route report: ${reportPath}`); + } + return success; } // Fallback to non-dither render return this.renderProject(options); @@ -4076,6 +4683,18 @@ class NativeBridge { return false; } + async clearAllPitchPreviewRoutes(clipId?: string): Promise<boolean> { + if (this.isNative && window.__JUCE__?.backend.clearAllPitchPreviewRoutes) + return await window.__JUCE__.backend.clearAllPitchPreviewRoutes(clipId ?? ""); + return false; + } + + async clearPitchPreviewRoutesForCorrectedSources(): Promise<number> { + if (this.isNative && window.__JUCE__?.backend.clearPitchPreviewRoutesForCorrectedSources) + return await window.__JUCE__.backend.clearPitchPreviewRoutesForCorrectedSources(); + return 0; + } + // ==================== Polyphonic Pitch Detection (Phase 6) ==================== async analyzePolyphonic(trackId: string, clipId: string, options?: { noteThreshold?: number; onsetThreshold?: number; minDurationMs?: number }): Promise<PolyAnalysisResult | null> { @@ -4116,6 +4735,41 @@ class NativeBridge { await window.__JUCE__.backend.setPitchCorrectionBypass(trackId, clipId, bypass); } + async startPitchScrubPreview(trackId: string, clipId: string, note: PitchNoteData, frames?: PitchContourData["frames"]): Promise<{ success: boolean } | null> { + if (this.isNative && window.__JUCE__?.backend.startPitchScrubPreview) + return await window.__JUCE__.backend.startPitchScrubPreview( + trackId, + clipId, + JSON.stringify(note), + frames ? JSON.stringify(frames) : undefined, + ); + return null; + } + + async updatePitchScrubPreview(clipId: string, pitchRatio: number): Promise<boolean> { + if (this.isNative && window.__JUCE__?.backend.updatePitchScrubPreview) + return await window.__JUCE__.backend.updatePitchScrubPreview(clipId, pitchRatio); + return false; + } + + async stopPitchScrubPreview(clipId: string): Promise<boolean> { + if (this.isNative && window.__JUCE__?.backend.stopPitchScrubPreview) + return await window.__JUCE__.backend.stopPitchScrubPreview(clipId); + return false; + } + + async getPitchScrubPreviewStatus(clipId?: string): Promise<PitchScrubPreviewStatus | null> { + if (this.isNative && window.__JUCE__?.backend.getPitchScrubPreviewStatus) + return await window.__JUCE__.backend.getPitchScrubPreviewStatus(clipId); + return null; + } + + async getPitchPreviewRoutingStatus(clipId?: string): Promise<PitchPreviewRoutingStatus | null> { + if (this.isNative && window.__JUCE__?.backend.getPitchPreviewRoutingStatus) + return await window.__JUCE__.backend.getPitchPreviewRoutingStatus(clipId); + return null; + } + /** Set real-time pitch/formant preview state for a clip */ async setClipPitchPreview(clipId: string, payload: ClipPitchPreviewPayload): Promise<boolean> { if (this.isNative && window.__JUCE__?.backend.setClipPitchPreview) @@ -4168,6 +4822,15 @@ class NativeBridge { selectedBackend: "cpu", lastPhase: "runtimeMissing", restartRequired: false, + aceStepVersion: null, + musicGenerationReady: false, + musicGenerationLayoutValid: false, + musicGenerationPerformanceReady: true, + musicGenerationModelId: "acestep-v15-xl-turbo", + musicGenerationModelRepoId: "ACE-Step/acestep-v15-xl-turbo", + musicGenerationSharedRepoId: "ACE-Step/Ace-Step1.5", + musicGenerationCheckpointRoot: null, + musicGenerationPerformanceStatusMessage: "", }; } @@ -4179,9 +4842,9 @@ class NativeBridge { return this.getAiToolsStatus(); } - async installAiTools(): Promise<InstallAiToolsResponse> { + async installAiTools(options: InstallAiToolsOptions = {}): Promise<InstallAiToolsResponse> { if (this.isNative && window.__JUCE__?.backend.installAiTools) { - return await window.__JUCE__.backend.installAiTools(); + return await window.__JUCE__.backend.installAiTools(Boolean(options.userConfirmedDownload)); } return { @@ -4191,6 +4854,18 @@ class NativeBridge { }; } + async resetAiTools(): Promise<ResetAiToolsResponse> { + if (this.isNative && window.__JUCE__?.backend.resetAiTools) { + return await window.__JUCE__.backend.resetAiTools(); + } + + return { + success: false, + error: "AI tools reset is unavailable in the web preview.", + status: await this.getAiToolsStatus(), + }; + } + // ==================== Phase 10: Stem Separation Workflow ==================== async separateStemsAsync(trackId: string, clipId: string, options: { stems: string[]; filePath?: string; accelerationMode?: "auto" | "cpu-only" }): Promise<{ started: boolean; error?: string; cached?: boolean }> { @@ -4216,6 +4891,36 @@ class NativeBridge { return await window.__JUCE__.backend.cancelAiToolsInstall(); } + async startAIGeneration( + trackId: string, + workflowId: string, + params: Record<string, unknown>, + ): Promise<{ started: boolean; error?: string }> { + const normalizedParams = normalizeWorkflowParams(workflowId, params); + if (this.isNative && window.__JUCE__?.backend.startAIGeneration) { + return await window.__JUCE__.backend.startAIGeneration( + trackId, + workflowId, + JSON.stringify(normalizedParams), + ); + } + console.log("[NativeBridge] Mock startAIGeneration:", trackId, workflowId, normalizedParams); + return { started: false, error: "AI generation is only available in the native app." }; + } + + async getAIGenerationProgress(): Promise<AIGenerationProgress> { + if (this.isNative && window.__JUCE__?.backend.getAIGenerationProgress) { + return await window.__JUCE__.backend.getAIGenerationProgress(); + } + return { state: "idle", progress: 0 }; + } + + async cancelAIGeneration(): Promise<void> { + if (this.isNative && window.__JUCE__?.backend.cancelAIGeneration) { + return await window.__JUCE__.backend.cancelAIGeneration(); + } + } + // ==================== Phase 9: ARA Plugin Hosting ==================== async initializeARA(trackId: string, fxIndex: number): Promise<{ success: boolean; error?: string }> { diff --git a/frontend/src/store/actionRegistry.ts b/frontend/src/store/actionRegistry.ts index 96711db..16613e2 100644 --- a/frontend/src/store/actionRegistry.ts +++ b/frontend/src/store/actionRegistry.ts @@ -5,6 +5,8 @@ import { nativeBridge } from "../services/NativeBridge"; import { useDAWStore } from "./useDAWStore"; +import { formatShortcut } from "../utils/platform"; +import { createTrackOfType } from "../utils/trackCreation"; export type ActionShortcutScope = | "global" @@ -117,6 +119,9 @@ export function getRegisteredActions(): ActionDef[] { state.addTrack({ id: trackId, name: `Instrument ${state.tracks.filter((t) => t.type === "instrument").length + 1}`, type: "instrument", armed: true, monitorEnabled: true }); state.openPluginBrowser(trackId); }}, + { id: "insert.aiTrack", name: "Insert AI Track", category: "Insert", shortcut: "Ctrl+Alt+T", execute: () => { + void createTrackOfType("ai"); + }}, { id: "insert.quickAddInstrument", name: "Quick Add Instrument Track", category: "Insert", shortcut: "Ctrl+Shift+I", execute: () => { const state = s(); const trackId = crypto.randomUUID(); @@ -389,6 +394,18 @@ export function getEffectiveActionShortcut(actionId: string): string | undefined return getActionShortcut(actionId); } +/** Platform-formatted shortcut for display (e.g. "Cmd+Z" on Mac, "Ctrl+Z" on Windows). */ +export function getDisplayShortcut(actionId: string): string | undefined { + const s = getActionShortcut(actionId); + return s ? formatShortcut(s) : undefined; +} + +/** Platform-formatted effective shortcut (custom override + platform formatting) for display. */ +export function getDisplayEffectiveShortcut(actionId: string): string | undefined { + const s = getEffectiveActionShortcut(actionId); + return s ? formatShortcut(s) : undefined; +} + export function getActionShortcutScope(actionId: string): ActionShortcutScope | undefined { return getRegisteredAction(actionId)?.shortcutScope; } diff --git a/frontend/src/store/actions/project.ts b/frontend/src/store/actions/project.ts index 6a7eb54..73cb829 100644 --- a/frontend/src/store/actions/project.ts +++ b/frontend/src/store/actions/project.ts @@ -14,6 +14,7 @@ import { logBridgeError } from "../../utils/bridgeErrorHandler"; import { resetSyncCache } from "./clips"; import { createFreshProjectDocumentState } from "../useDAWStore"; import { syncAutomationLaneToBackend, syncTempoMarkersToBackend } from "./storeHelpers"; +import { getDefaultWorkflowParams, normalizeWorkflowParams } from "../../data/aiWorkflows"; const TRANSIENT_STATE_KEYS: ReadonlySet<string> = new Set([ "meterLevels", "peakLevels", "masterLevel", "automatedParamValues", @@ -125,6 +126,7 @@ function buildSerializedProjectData( trackGroups: state.trackGroups, clipLauncher: state.clipLauncher, renderMetadata: state.renderMetadata, + renderDialogOptions: state.renderDialogOptions, secondaryOutputEnabled: state.secondaryOutputEnabled, secondaryOutputFormat: state.secondaryOutputFormat, secondaryOutputBitDepth: state.secondaryOutputBitDepth, @@ -375,6 +377,9 @@ export const projectActions = (set: SetFn, get: GetFn) => ({ inputChannel: track.inputChannel, clips: track.clips, midiClips: track.midiClips, + icon: track.icon, + aiWorkflow: track.aiWorkflow, + aiWorkflowParams: track.aiWorkflowParams, inputFXPaths, inputFXStates, trackFXPaths, @@ -516,6 +521,10 @@ export const projectActions = (set: SetFn, get: GetFn) => ({ ...freshProjectState.renderMetadata, ...(data.renderMetadata || {}), }; + const loadedRenderDialogOptions = { + ...freshProjectState.renderDialogOptions, + ...(data.renderDialogOptions || {}), + }; const loadedClipLauncher = { ...freshProjectState.clipLauncher, ...(data.clipLauncher || {}), @@ -567,6 +576,7 @@ export const projectActions = (set: SetFn, get: GetFn) => ({ trackGroups: Array.isArray(data.trackGroups) ? data.trackGroups : [], clipLauncher: loadedClipLauncher, renderMetadata: loadedRenderMetadata, + renderDialogOptions: loadedRenderDialogOptions, secondaryOutputEnabled: Boolean(data.secondaryOutputEnabled), secondaryOutputFormat: data.secondaryOutputFormat || freshProjectState.secondaryOutputFormat, @@ -673,6 +683,121 @@ export const projectActions = (set: SetFn, get: GetFn) => ({ const frontendTrack: Track = { ...trackData, + aiWorkflow: + trackData.type === "ai" + ? trackData.aiWorkflow || "text-to-music" + : trackData.aiWorkflow, + aiWorkflowParams: + trackData.type === "ai" + ? normalizeWorkflowParams( + trackData.aiWorkflow || "text-to-music", + trackData.aiWorkflowParams || getDefaultWorkflowParams(trackData.aiWorkflow || "text-to-music"), + ) + : trackData.aiWorkflowParams, + aiGenerationState: + trackData.type === "ai" + ? "idle" + : trackData.aiGenerationState, + aiGenerationProgress: + trackData.type === "ai" + ? 0 + : trackData.aiGenerationProgress, + aiGenerationError: + trackData.type === "ai" + ? "" + : trackData.aiGenerationError, + aiGenerationPhase: + trackData.type === "ai" + ? "" + : trackData.aiGenerationPhase, + aiGenerationMessage: + trackData.type === "ai" + ? "" + : trackData.aiGenerationMessage, + aiGenerationBackend: + trackData.type === "ai" + ? "" + : trackData.aiGenerationBackend, + aiGenerationElapsedMs: + trackData.type === "ai" + ? 0 + : trackData.aiGenerationElapsedMs, + aiGenerationHeartbeatTs: + trackData.type === "ai" + ? 0 + : trackData.aiGenerationHeartbeatTs, + aiGenerationPhaseProgress: + trackData.type === "ai" + ? undefined + : trackData.aiGenerationPhaseProgress, + aiGenerationEtaMs: + trackData.type === "ai" + ? undefined + : trackData.aiGenerationEtaMs, + aiGenerationRunMode: + trackData.type === "ai" + ? undefined + : trackData.aiGenerationRunMode, + aiGenerationRuntimeProfile: + trackData.type === "ai" + ? "" + : trackData.aiGenerationRuntimeProfile, + aiGenerationLmModel: + trackData.type === "ai" + ? "" + : trackData.aiGenerationLmModel, + aiGenerationStatusNote: + trackData.type === "ai" + ? "" + : trackData.aiGenerationStatusNote, + aiGenerationFailureKind: + trackData.type === "ai" + ? "" + : trackData.aiGenerationFailureKind, + aiGenerationSessionMode: + trackData.type === "ai" + ? "" + : trackData.aiGenerationSessionMode, + aiGenerationWorkerExitCode: + trackData.type === "ai" + ? 0 + : trackData.aiGenerationWorkerExitCode, + aiGenerationLastStdoutLine: + trackData.type === "ai" + ? "" + : trackData.aiGenerationLastStdoutLine, + aiGenerationLastStderrLine: + trackData.type === "ai" + ? "" + : trackData.aiGenerationLastStderrLine, + aiGenerationAttemptMode: + trackData.type === "ai" + ? "" + : trackData.aiGenerationAttemptMode, + aiGenerationAttemptIndex: + trackData.type === "ai" + ? 0 + : trackData.aiGenerationAttemptIndex, + aiGenerationProtocolVersion: + trackData.type === "ai" + ? 0 + : trackData.aiGenerationProtocolVersion, + aiGenerationScriptVersion: + trackData.type === "ai" + ? "" + : trackData.aiGenerationScriptVersion, + aiGenerationRequestId: + trackData.type === "ai" + ? "" + : trackData.aiGenerationRequestId, + aiGenerationPriorFailure: + trackData.type === "ai" + ? "" + : trackData.aiGenerationPriorFailure, + aiGenerationLastProgressAgeMs: + trackData.type === "ai" + ? 0 + : trackData.aiGenerationLastProgressAgeMs, clips: trackData.clips || [], midiClips: trackData.midiClips || [], automationLanes: trackData.automationLanes || [], diff --git a/frontend/src/store/actions/renderQueue.ts b/frontend/src/store/actions/renderQueue.ts index 42ce422..d52d463 100644 --- a/frontend/src/store/actions/renderQueue.ts +++ b/frontend/src/store/actions/renderQueue.ts @@ -1,5 +1,6 @@ // @ts-nocheck import { nativeBridge } from "../../services/NativeBridge"; +import { prepareForManualRender } from "../../utils/renderPreparation"; // @ts-nocheck // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -31,7 +32,7 @@ export const renderQueueActions = (set: SetFn, get: GetFn) => ({ ), })); try { - await get().syncClipsWithBackend(); + await prepareForManualRender(get().syncClipsWithBackend, "render-queue"); await nativeBridge.renderProject({ source: job.options.source, startTime: job.options.startTime, @@ -44,6 +45,7 @@ export const renderQueueActions = (set: SetFn, get: GetFn) => ({ normalize: job.options.normalize, addTail: job.options.addTail, tailLength: job.options.tailLength, + includeMetronome: false, }); set((s) => ({ renderQueue: s.renderQueue.map((j) => diff --git a/frontend/src/store/actions/rendering.ts b/frontend/src/store/actions/rendering.ts index 05650ea..f68af7a 100644 --- a/frontend/src/store/actions/rendering.ts +++ b/frontend/src/store/actions/rendering.ts @@ -1,5 +1,5 @@ // @ts-nocheck -import { applyTheme } from "../useDAWStore"; +import { applyTheme, createDefaultRenderDialogOptions } from "../useDAWStore"; import { usePitchEditorStore } from "../pitchEditorStore"; // eslint-disable-next-line @typescript-eslint/no-explicit-any type SetFn = (...args: any[]) => void; @@ -13,6 +13,7 @@ type GetFn = () => any; import { nativeBridge } from "../../services/NativeBridge"; import { commandManager } from "../commands"; import { logBridgeError } from "../../utils/bridgeErrorHandler"; +import { prepareForManualRender } from "../../utils/renderPreparation"; export const renderingActions = (set: SetFn, get: GetFn) => ({ @@ -337,6 +338,11 @@ export const renderingActions = (set: SetFn, get: GetFn) => ({ deselectAllRegions: () => set({ selectedRegionIds: [] }), setRenderMetadata: (metadata) => set((s) => ({ renderMetadata: { ...s.renderMetadata, ...metadata } })), + setRenderDialogOptions: (options) => + set((s) => ({ renderDialogOptions: { ...s.renderDialogOptions, ...options } })), + resetRenderDialogOptions: () => + set({ renderDialogOptions: createDefaultRenderDialogOptions() }), + setLastRenderDirectory: (dir) => set({ lastRenderDirectory: dir }), setSecondaryOutputEnabled: (enabled) => set({ secondaryOutputEnabled: enabled }), setSecondaryOutputFormat: (format) => @@ -383,6 +389,7 @@ export const renderingActions = (set: SetFn, get: GetFn) => ({ const fileName = `${track.name.replace(/[^a-zA-Z0-9_-]/g, "_")}_consolidated.wav`; const filePath = await nativeBridge.showRenderSaveDialog(fileName, "wav"); if (!filePath) return null; + await prepareForManualRender(get().syncClipsWithBackend, "consolidate-track"); const success = await nativeBridge.renderProject({ source: `stem:${trackId}`, startTime: earliest, @@ -395,6 +402,7 @@ export const renderingActions = (set: SetFn, get: GetFn) => ({ normalize: false, addTail: false, tailLength: 0, + includeMetronome: false, }); if (success) { // Replace track clips with single consolidated clip @@ -439,6 +447,7 @@ export const renderingActions = (set: SetFn, get: GetFn) => ({ const safeName = sourceClip.name.replace(/[^a-zA-Z0-9_-]/g, "_"); const filePath = await nativeBridge.showRenderSaveDialog(`${safeName}_rendered.wav`, "wav"); if (!filePath) return; + await prepareForManualRender(get().syncClipsWithBackend, "render-clip-in-place"); const success = await nativeBridge.renderProject({ source: `stem:${sourceTrack.id}`, @@ -452,6 +461,7 @@ export const renderingActions = (set: SetFn, get: GetFn) => ({ normalize: false, addTail: true, tailLength: 1000, + includeMetronome: false, }); if (!success) return; @@ -506,6 +516,7 @@ export const renderingActions = (set: SetFn, get: GetFn) => ({ const safeName = track.name.replace(/[^a-zA-Z0-9_-]/g, "_"); const filePath = await nativeBridge.showRenderSaveDialog(`${safeName}_rendered.wav`, "wav"); if (!filePath) return; + await prepareForManualRender(get().syncClipsWithBackend, "render-track-in-place"); const success = await nativeBridge.renderProject({ source: `stem:${trackId}`, @@ -519,6 +530,7 @@ export const renderingActions = (set: SetFn, get: GetFn) => ({ normalize: false, addTail: true, tailLength: 1000, + includeMetronome: false, }); if (!success) return; diff --git a/frontend/src/store/actions/transport.ts b/frontend/src/store/actions/transport.ts index 9cc56e7..4dda9ea 100644 --- a/frontend/src/store/actions/transport.ts +++ b/frontend/src/store/actions/transport.ts @@ -21,6 +21,17 @@ const stringifyForDebug = (value: unknown) => { } }; +async function clearPitchRoutesForCorrectedSourcesBeforePlayback(reason: string) { + try { + const cleared = await nativeBridge.clearPitchPreviewRoutesForCorrectedSources(); + if (cleared > 0) { + console.warn(`${AUDIO_TRANSPORT_LOG_PREFIX} ${reason}:clearedPitchPreviewRoutes`, { correctedSourceClipCount: cleared }); + } + } catch (error) { + console.warn(`${AUDIO_TRANSPORT_LOG_PREFIX} ${reason}:clearPitchPreviewRoutesForCorrectedSources failed`, error); + } +} + // eslint-disable-next-line @typescript-eslint/no-explicit-any type SetFn = (...args: any[]) => void; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -80,6 +91,7 @@ export const transportActions = (set: SetFn, get: GetFn) => ({ console.log(`${AUDIO_TRANSPORT_LOG_PREFIX} play:ara`, { araActive, startTime }); if (araActive) { + await clearPitchRoutesForCorrectedSourcesBeforePlayback("play:ara"); const positionResult = await nativeBridge.setTransportPosition(startTime); console.log(`${AUDIO_TRANSPORT_LOG_PREFIX} play:setTransportPosition`, { startTime, positionResult }); set((state) => ({ @@ -96,6 +108,7 @@ export const transportActions = (set: SetFn, get: GetFn) => ({ console.log(`${AUDIO_TRANSPORT_LOG_PREFIX} play:setTransportPlaying`, { playingResult, mode: "ara" }); } else { await syncClipsWithBackend(); + await clearPitchRoutesForCorrectedSourcesBeforePlayback("play:standard"); const positionResult = await nativeBridge.setTransportPosition(startTime); console.log(`${AUDIO_TRANSPORT_LOG_PREFIX} play:setTransportPosition`, { startTime, positionResult }); set((state) => ({ diff --git a/frontend/src/store/pitchEditorStore.ts b/frontend/src/store/pitchEditorStore.ts index 2c5a7c8..5dbeb56 100644 --- a/frontend/src/store/pitchEditorStore.ts +++ b/frontend/src/store/pitchEditorStore.ts @@ -1,10 +1,12 @@ import { create } from "zustand"; -import { nativeBridge, PitchNoteData, PitchContourData, PolyNoteData, PolyAnalysisResult, UnifiedNoteData, polyToUnified, monoToUnified, type ClipPitchPreviewPayload, type PitchCorrectionRenderMode } from "../services/NativeBridge"; +import { nativeBridge, PitchNoteData, PitchContourData, type PitchCorrectionCompletionData, type PitchCorrectionRenderMode, type PitchPreviewRoutingStatus } from "../services/NativeBridge"; import { useDAWStore } from "./useDAWStore"; import { logBridgeError } from "../utils/bridgeErrorHandler"; export type PitchEditorTool = "select" | "pitch" | "drift" | "vibrato" | "transition" | "draw" | "split"; export type PitchSnapMode = "off" | "chromatic" | "scale"; +// Keep disabled until the active final-render backend declares explicit formant control. +export const PITCH_EDITOR_FORMANT_EDITING_ENABLED = false; export type PitchEditorApplyState = | "idle" | "queued" @@ -15,12 +17,18 @@ export type PitchEditorApplyState = | "done" | "error"; +export type PitchAnalysisPhase = "idle" | "loading" | "analyzing"; + export type PitchRenderCoverageState = "pending" | "preview_ready" | "hq_ready"; +export type PitchRenderCoverageKind = "edited" | "edit_island" | "left_neighbor_tail" | "right_neighbor_head"; + export interface PitchRenderCoverageRange { startTime: number; endTime: number; state: PitchRenderCoverageState; + kind?: PitchRenderCoverageKind; + noteId?: string; } // Scale interval templates @@ -54,6 +62,21 @@ function buildScaleNotes(key: number, type: string): boolean[] { interface UndoEntry { description: string; notes: PitchNoteData[]; + selectedNoteIds: string[]; +} + +function clonePitchNotes(notes: PitchNoteData[]): PitchNoteData[] { + return JSON.parse(JSON.stringify(notes)); +} + +function filterSelectedNoteIdsForNotes(selectedNoteIds: string[], notes: PitchNoteData[]): string[] { + if (selectedNoteIds.length === 0 || notes.length === 0) return []; + const validIds = new Set(notes.map((note) => note.id)); + return selectedNoteIds.filter((id) => validIds.has(id)); +} + +function pitchNotesEqual(a: PitchNoteData[], b: PitchNoteData[]): boolean { + return JSON.stringify(a) === JSON.stringify(b); } const REFERENCE_COLORS = ["#f59e0b", "#ec4899", "#06b6d4", "#a855f7", "#ef4444", "#14b8a6"]; @@ -89,6 +112,7 @@ interface PitchEditorState { contour: PitchContourData | null; notes: PitchNoteData[]; isAnalyzing: boolean; + analysisPhase: PitchAnalysisPhase; isApplying: boolean; progressPercent: number; // 0-100, for analysis/correction progress display progressLabel: string; // e.g. "Analyzing..." or "Applying correction..." @@ -125,6 +149,8 @@ interface PitchEditorState { setSelectedNoteIds: (ids: string[]) => void; // Note editing (with undo) + beginInteractivePreview: (noteId: string) => void; + endInteractivePreview: () => void; updateNote: (noteId: string, changes: Partial<PitchNoteData>) => void; commitNoteEdit: () => void; // Call on mouseup/drag-end to trigger auto-apply updateSelectedNotes: (changes: Partial<PitchNoteData>) => void; @@ -193,39 +219,9 @@ interface PitchEditorState { beginDrawPitch: () => void; commitDrawPitch: () => void; - // Polyphonic mode (Phase 7) - polyMode: boolean; - polyNotes: PolyNoteData[]; - polyAnalysisResult: PolyAnalysisResult | null; - showPitchSalience: boolean; - soloNoteId: string | null; - - // Poly detection tuning - polyNoteThreshold: number; // 0-1, default 0.15 - polyOnsetThreshold: number; // 0-1, default 0.3 - polyMinDuration: number; // ms, default 80 - - togglePolyMode: () => void; - analyzePolyphonic: () => Promise<void>; - updatePolyNote: (noteId: string, changes: Partial<PolyNoteData>) => void; - movePolyNotePitch: (noteId: string, semitones: number) => void; - moveSelectedPolyPitch: (semitones: number) => void; - soloPolyNote: (noteId: string | null) => void; - applyPolyCorrection: () => Promise<void>; - togglePitchSalience: () => void; - setPolyNoteThreshold: (v: number) => void; - setPolyOnsetThreshold: (v: number) => void; - setPolyMinDuration: (v: number) => void; - - // Auto-detect mono vs poly - autoDetectMonoPoly: () => void; - // A/B comparison abCompareMode: boolean; // true = playing original (bypass correction) toggleABCompare: () => void; - - // Unified note accessor — returns mono or poly notes as UnifiedNoteData[] - getUnifiedNotes: () => UnifiedNoteData[]; } // Tracks the active pitchAnalysisComplete listener so stale listeners from @@ -233,6 +229,9 @@ interface PitchEditorState { // Without this, a stale listener fires for the next analysis result, // doubling the notes and corrupting pitch correction data. let _pitchAnalysisUnsubscribe: (() => void) | null = null; +let _deferredAnalysisRange: { start?: number; end?: number } | null = null; +let _noteHqApplyInFlight = false; +let _analysisRunSeq = 0; let _pitchCorrectionRequestSeq = 0; let _logicalApplySeq = 0; @@ -244,8 +243,13 @@ let _dawTransportSubscriptionInitialized = false; let _editRevision = 0; let _requestedApplyRevision = 0; let _appliedRevision = 0; +type PitchApplyRenderStage = "single" | "preview_segment" | "full_clip_hq" | "note_hq"; +type PitchPreviewMonitorMode = "none" | "scrub" | "clip_live_preview"; +let _interactivePreviewNoteId: string | null = null; +let _interactivePreviewActive = false; +let _activePitchPreviewMonitorMode: PitchPreviewMonitorMode = "none"; +let _dirtyPitchNoteIds = new Set<string>(); const _requestRevisionById = new Map<string, number>(); -type PitchApplyRenderStage = "single" | "preview_segment" | "full_clip_hq"; interface PendingPreviewSegment { startSec: number; endSec: number; @@ -279,24 +283,23 @@ interface PitchCorrectionRequestMeta { summary: ApplyRequestSummary; windowStartSec?: number; windowEndSec?: number; + coverageRanges?: PitchRenderCoverageRange[]; requestGroupId: string; segmentIndex?: number; } const _requestMetaById = new Map<string, PitchCorrectionRequestMeta>(); -// Auto-apply debounce — fires 400ms after the last note mutation so playback -// always reflects the current state without a manual Apply button. +// Auto-apply debounce — fires 300ms after the last note mutation so playback +// swaps to a fresh note-local HQ render without requiring a manual Apply click. let _autoApplyTimer: ReturnType<typeof setTimeout> | null = null; // Throttle timer for real-time pitch preview during drag (max ~12fps, fast enough to feel live). let _dragPreviewThrottle: ReturnType<typeof setTimeout> | null = null; const FORMANT_LOG_PREFIX = "[pitchEditor.formant]"; const PREVIEW_SEGMENT_DURATION_SEC = 10; const MAX_PREVIEW_SEGMENT_CONCURRENCY = 2; -const PREVIEW_LOOKBEHIND_SEC = 0.2; -const PREVIEW_LOOKAHEAD_PLAYING_SEC = 3.0; -const PREVIEW_LOOKAHEAD_STOPPED_SEC = 1.5; const AUTO_BAKE_LOOKBEHIND_SEC = 0.25; const AUTO_BAKE_LOOKAHEAD_SEC = 2.5; +const MAX_NEIGHBOR_LINK_GAP_SEC = 0.18; let _activePreviewRequestGroupId: string | null = null; let _pendingPreviewSegments: PendingPreviewSegment[] = []; let _runningPreviewSegmentJobs = 0; @@ -308,6 +311,24 @@ function shouldLogPitchEditorFormant() { return win.__S13_DEBUG_FORMANT__ === true || host === "localhost" || host === "127.0.0.1"; } +function shouldCaptureAppFinalPitchContext(routeSuspect = false) { + if (routeSuspect) return true; + const win = window as Window & { + __JUCE__?: { backend?: { capturePitchAppFinalContext?: unknown } }; + __S13_CAPTURE_APP_FINAL_PITCH__?: boolean; + }; + if (win.__S13_CAPTURE_APP_FINAL_PITCH__ === false) return false; + if (win.__S13_CAPTURE_APP_FINAL_PITCH__ === true) return true; + try { + const stored = window.localStorage?.getItem("s13.pitch.captureAppFinal"); + if (stored === "0") return false; + if (stored === "1") return true; + } catch { + // Ignore storage failures and fall through to native capability detection. + } + return typeof win.__JUCE__?.backend?.capturePitchAppFinalContext === "function"; +} + function logPitchEditorFormant(message: string, extra?: Record<string, unknown>) { if (!shouldLogPitchEditorFormant()) return; if (extra) console.log(FORMANT_LOG_PREFIX, message, extra); @@ -353,10 +374,61 @@ function clearStagedPreviewQueue() { clearFullClipHqTimer(); } +function deferPitchAnalysis(start?: number, end?: number, reason = "hq_priority") { + _deferredAnalysisRange = { start, end }; + logPitchEditorFormant("pitch analysis deferred", { start: start ?? null, end: end ?? null, reason }); +} + +function cancelActivePitchAnalysisForHq() { + _analysisRunSeq += 1; + if (_pitchAnalysisUnsubscribe) { + _pitchAnalysisUnsubscribe(); + _pitchAnalysisUnsubscribe = null; + } + usePitchEditorStore.setState((prev) => ( + prev.isAnalyzing + ? { ...prev, isAnalyzing: false, analysisPhase: "idle", progressPercent: 0, progressLabel: "" } + : prev + )); +} + +function retryDeferredPitchAnalysis() { + if (_noteHqApplyInFlight || !_deferredAnalysisRange) return; + const deferred = _deferredAnalysisRange; + _deferredAnalysisRange = null; + window.setTimeout(() => { + usePitchEditorStore.getState().analyze(deferred.start, deferred.end); + }, 0); +} + +function finishNoteHqPriority(logicalRequestId?: string) { + if (logicalRequestId && _activePitchCorrectionRequestId && _activePitchCorrectionRequestId !== logicalRequestId) { + return; + } + if (!_noteHqApplyInFlight) return; + _noteHqApplyInFlight = false; + retryDeferredPitchAnalysis(); +} + +function markDirtyPitchNotes(noteIds: Iterable<string>) { + for (const noteId of noteIds) { + if (noteId) _dirtyPitchNoteIds.add(noteId); + } +} + +function replaceDirtyPitchNotes(noteIds: Iterable<string>) { + _dirtyPitchNoteIds.clear(); + markDirtyPitchNotes(noteIds); +} + function resetApplyRevisions() { _editRevision = 0; _requestedApplyRevision = 0; _appliedRevision = 0; + _interactivePreviewNoteId = null; + _interactivePreviewActive = false; + _activePitchPreviewMonitorMode = "none"; + _dirtyPitchNoteIds.clear(); _requestRevisionById.clear(); _requestMetaById.clear(); _activePitchCorrectionRequestId = null; @@ -416,20 +488,42 @@ function setApplyStatus( } } +function getEffectivePitchEditorGlobalFormantSt(globalFormantSt: number) { + return PITCH_EDITOR_FORMANT_EDITING_ENABLED ? globalFormantSt : 0; +} + +function getPitchCorrectionFailureMessage(data?: { + hardFailReason?: string; + fallbackReason?: string; + pitchRenderBackendFailureCode?: string; +}) { + const reason = (data?.hardFailReason || data?.fallbackReason || "").trim(); + return reason || "Pitch/formant apply failed"; +} + +function sanitizePitchEditorNotesForApply(notes: PitchNoteData[]) { + const formantSanitizedNotes = PITCH_EDITOR_FORMANT_EDITING_ENABLED + ? cloneNotesSnapshot(notes) + : notes.map((note) => ({ ...note, formantShift: 0 })); + return formantSanitizedNotes.map(applySampleMatchDownshiftDefaults); +} + function summarizeApplyRequest(notes: PitchNoteData[], globalFormantSt: number): ApplyRequestSummary { + const effectiveGlobalFormantSt = getEffectivePitchEditorGlobalFormantSt(globalFormantSt); + const effectiveNotes = PITCH_EDITOR_FORMANT_EDITING_ENABLED ? notes : sanitizePitchEditorNotesForApply(notes); let pitchEdits = 0; let noteFormantEdits = 0; let gainEdits = 0; let driftEdits = 0; let vibratoEdits = 0; - for (const note of notes) { + for (const note of effectiveNotes) { if (Math.abs(note.correctedPitch - note.detectedPitch) > 0.01) pitchEdits++; if (Math.abs(note.formantShift) > 0.01) noteFormantEdits++; if (Math.abs(note.gain) > 0.01) gainEdits++; if (note.driftCorrectionAmount > 0.01) driftEdits++; if (Math.abs(note.vibratoDepth - 1.0) > 0.01) vibratoEdits++; } - const hasGlobalFormant = Math.abs(globalFormantSt) > 0.01; + const hasGlobalFormant = Math.abs(effectiveGlobalFormantSt) > 0.01; let mode: "none" | "pitch-only" | "formant-only" | "mixed" = "none"; const hasPitch = pitchEdits > 0; const hasFormant = hasGlobalFormant || noteFormantEdits > 0; @@ -447,8 +541,8 @@ function summarizeApplyRequest(notes: PitchNoteData[], globalFormantSt: number): hasPitch, hasFormant, hasOther, - globalFormantCents: Math.round(globalFormantSt * 100), - globalFormantSemitones: globalFormantSt, + globalFormantCents: Math.round(effectiveGlobalFormantSt * 100), + globalFormantSemitones: effectiveGlobalFormantSt, mode, }; } @@ -462,6 +556,163 @@ function cloneFramesSnapshot(frames?: PitchContourData["frames"]) { return JSON.parse(JSON.stringify(frames)) as PitchContourData["frames"]; } +const DRAW_VOICED_TOLERANCE_SEC = 0.04; +const IMMEDIATE_NEIGHBOR_SMOOTH_MS = 10; +const NOTE_HQ_DEFAULT_TRANSITION_IN_MS = 40; +const NOTE_HQ_DEFAULT_TRANSITION_OUT_MS = 60; +const NOTE_HQ_HARD_BOUNDARY_TRANSITION_IN_MS = 24; +const NOTE_HQ_HARD_BOUNDARY_TRANSITION_OUT_MS = 40; +const NOTE_HQ_SOFT_BOUNDARY_TRANSITION_MS = 80; +const NOTE_HQ_SAMPLE_MATCH_DOWNSHIFT_THRESHOLD_ST = -2.0; +const NOTE_HQ_SAMPLE_MATCH_DOWNSHIFT_VIBRATO_DEPTH = 0.70; +const NOTE_HQ_SAMPLE_MATCH_DOWNSHIFT_TRANSITION_IN_MS = 80; +const NOTE_HQ_SAMPLE_MATCH_DOWNSHIFT_TRANSITION_OUT_MS = 100; +const NOTE_HQ_NEXT_HEAD_OWNERSHIP_MS = 40; +const MAX_NOTE_HQ_TRANSITION_MS = 140; +const MAX_EDIT_ISLAND_GAP_SEC = 0.08; +const NOTE_HQ_INTERNAL_ISLAND_RAMP_MS = 32; + +function noteHasPitchStyleEdit(note: PitchNoteData) { + return Math.abs(note.correctedPitch - note.detectedPitch) > 0.01 + || note.driftCorrectionAmount > 0.01 + || Math.abs(note.vibratoDepth - 1.0) > 0.01 + || (Array.isArray(note.pitchDrift) && note.pitchDrift.some((v) => Math.abs(v) > 0.01)); +} + +function noteHasRenderableEdit(note: PitchNoteData) { + return noteHasPitchStyleEdit(note) + || Math.abs(note.gain) > 0.01 + || (PITCH_EDITOR_FORMANT_EDITING_ENABLED && Math.abs(note.formantShift) > 0.01); +} + +function getNoteEffectiveTransitionMs(note: PitchNoteData, edge: "in" | "out") { + const explicitMs = edge === "in" ? note.transitionIn : note.transitionOut; + return explicitMs > 0 ? explicitMs : 0; +} + +function getNoteHqBoundaryTransitionFloorMs(note: PitchNoteData, edge: "in" | "out") { + if (isSampleMatchDownshift(note)) { + return edge === "in" + ? NOTE_HQ_SAMPLE_MATCH_DOWNSHIFT_TRANSITION_IN_MS + : NOTE_HQ_SAMPLE_MATCH_DOWNSHIFT_TRANSITION_OUT_MS; + } + + const kind = edge === "in" ? note.entryBoundaryKind : note.exitBoundaryKind; + if (kind === "hard_word_like") { + return edge === "in" ? NOTE_HQ_HARD_BOUNDARY_TRANSITION_IN_MS : NOTE_HQ_HARD_BOUNDARY_TRANSITION_OUT_MS; + } + if (kind === "soft_legato") { + return NOTE_HQ_SOFT_BOUNDARY_TRANSITION_MS; + } + return edge === "in" ? NOTE_HQ_DEFAULT_TRANSITION_IN_MS : NOTE_HQ_DEFAULT_TRANSITION_OUT_MS; +} + +function getNoteEffectiveStart(note: Pick<PitchNoteData, "startTime" | "effectiveStartTime">) { + return note.effectiveStartTime ?? note.startTime; +} + +function getNoteEffectiveEnd(note: Pick<PitchNoteData, "endTime" | "effectiveEndTime">) { + return note.effectiveEndTime ?? note.endTime; +} + +function getNoteWordGroupId(note: Pick<PitchNoteData, "id" | "wordGroupId">) { + return note.wordGroupId && note.wordGroupId.trim().length > 0 ? note.wordGroupId : note.id; +} + +function notesShareWordGroup( + left: Pick<PitchNoteData, "id" | "wordGroupId">, + right: Pick<PitchNoteData, "id" | "wordGroupId">, +) { + return getNoteWordGroupId(left) === getNoteWordGroupId(right); +} + +function getPitchShiftSemitones(note: Pick<PitchNoteData, "correctedPitch" | "detectedPitch">) { + return note.correctedPitch - note.detectedPitch; +} + +function isSampleMatchDownshift(note: Pick<PitchNoteData, "correctedPitch" | "detectedPitch">) { + return getPitchShiftSemitones(note) <= NOTE_HQ_SAMPLE_MATCH_DOWNSHIFT_THRESHOLD_ST; +} + +function applySampleMatchDownshiftDefaults(note: PitchNoteData): PitchNoteData { + if (!isSampleMatchDownshift(note) || Math.abs(note.vibratoDepth - 1.0) > 0.01) { + return note; + } + return { + ...note, + vibratoDepth: NOTE_HQ_SAMPLE_MATCH_DOWNSHIFT_VIBRATO_DEPTH, + }; +} + +function sharesTransitionPair( + left: Pick<PitchNoteData, "startTime" | "endTime" | "effectiveStartTime" | "effectiveEndTime" | "correctedPitch" | "detectedPitch" | "transitionIn" | "transitionOut">, + right: Pick<PitchNoteData, "startTime" | "endTime" | "effectiveStartTime" | "effectiveEndTime" | "correctedPitch" | "detectedPitch" | "transitionIn" | "transitionOut">, +) { + const gapSec = right.startTime - left.endTime; + return gapSec <= MAX_NEIGHBOR_LINK_GAP_SEC + || getNoteEffectiveStart(right) <= getNoteEffectiveEnd(left) + MAX_NEIGHBOR_LINK_GAP_SEC; +} + +function normalizePitchNote(note: PitchNoteData): PitchNoteData { + const effectiveStartTime = Math.max(0, note.startTime - getNoteEffectiveTransitionMs(note, "in") / 1000); + const effectiveEndTime = Math.max(note.endTime, note.endTime + getNoteEffectiveTransitionMs(note, "out") / 1000); + return { + ...note, + wordGroupId: getNoteWordGroupId(note), + effectiveStartTime, + effectiveEndTime, + }; +} + +function normalizePitchNotes(notes: PitchNoteData[]) { + return notes + .map((note) => normalizePitchNote(note)) + .sort((a, b) => a.startTime - b.startTime); +} + +function expandDirtyPitchNoteIdsWithNeighbors(noteIds: Iterable<string>, notes: PitchNoteData[]) { + const sortedNotes = [...notes].sort((a, b) => a.startTime - b.startTime); + const dirtyIds = new Set<string>(); + for (const noteId of noteIds) { + if (noteId) dirtyIds.add(noteId); + } + + for (let index = 0; index < sortedNotes.length; index += 1) { + const note = sortedNotes[index]; + if (!dirtyIds.has(note.id)) continue; + + const prev = sortedNotes[index - 1]; + if (prev && sharesTransitionPair(prev, note)) { + dirtyIds.add(prev.id); + } + + const next = sortedNotes[index + 1]; + if (next && sharesTransitionPair(note, next)) { + dirtyIds.add(next.id); + } + } + + return dirtyIds; +} + +function isVoicedAtClipTime(contour: PitchContourData | null, clipTime: number) { + const frames = contour?.frames; + if (!frames || frames.times.length === 0) return true; + + let closestIndex = -1; + let closestDt = Number.POSITIVE_INFINITY; + for (let i = 0; i < frames.times.length; i++) { + const dt = Math.abs(frames.times[i] - clipTime); + if (dt < closestDt) { + closestDt = dt; + closestIndex = i; + } + } + + if (closestIndex < 0 || closestDt > DRAW_VOICED_TOLERANCE_SEC) return false; + return Boolean(frames.voiced[closestIndex]) && (frames.midi[closestIndex] ?? 0) > 0 && (frames.confidence[closestIndex] ?? 0) >= 0.15; +} + function buildLogicalApplyRequestId(clipId: string) { return `${clipId}:apply:${++_logicalApplySeq}`; } @@ -526,6 +777,7 @@ function consumeRequestMeta(requestId?: string | null) { } function hasRequestedFormantWork(notes: PitchNoteData[], globalFormantSt: number) { + if (!PITCH_EDITOR_FORMANT_EDITING_ENABLED) return false; return Math.abs(globalFormantSt) > 0.01 || notes.some((n) => Math.abs(n.formantShift) > 0.01); } @@ -535,27 +787,464 @@ function hasRealtimePitchPreviewWork(notes: PitchNoteData[], globalFormantSt: nu return notes.some((n) => Math.abs(n.correctedPitch - n.detectedPitch) > 0.01); } -function getPlaybackPreviewStatusMessage(notes: PitchNoteData[], globalFormantSt: number) { - if (hasRequestedFormantWork(notes, globalFormantSt) && hasRealtimePitchPreviewWork(notes, globalFormantSt)) { - return "Rendered formant + live pitch preview"; +function buildEditedNoteWindow(notes: PitchNoteData[], clipDuration: number) { + const edited = notes.filter((note) => noteHasRenderableEdit(note)); + if (edited.length === 0) return null; + + const sortedNotes = [...notes].sort((a, b) => a.startTime - b.startTime); + const noteIndexById = new Map(sortedNotes.map((note, index) => [note.id, index] as const)); + const editedIdSet = new Set(edited.map((note) => note.id)); + const dirtyEditedIds = [..._dirtyPitchNoteIds].filter((id) => editedIdSet.has(id) && noteIndexById.has(id)); + const focusIdSet = new Set(dirtyEditedIds.length > 0 ? dirtyEditedIds : edited.map((note) => note.id)); + const focusIndices = [...focusIdSet] + .map((id) => noteIndexById.get(id) ?? -1) + .filter((index) => index >= 0); + + if (focusIndices.length === 0) return null; + + const coverageRanges: PitchRenderCoverageRange[] = []; + const requestNotes = new Map<string, PitchNoteData>(); + + const upsertRequestNote = (note: PitchNoteData, patch?: Partial<PitchNoteData>) => { + const existing = requestNotes.get(note.id) ?? { ...note }; + const merged = applySampleMatchDownshiftDefaults({ ...existing, ...patch }); + requestNotes.set(note.id, merged); + return merged; + }; + + const addCoverageRange = ( + startTime: number, + endTime: number, + kind: PitchRenderCoverageKind, + noteId: string, + ) => { + const clampedStart = Math.max(0, Math.min(clipDuration, startTime)); + const clampedEnd = Math.max(clampedStart, Math.min(clipDuration, endTime)); + if (clampedEnd - clampedStart < 0.0005) return; + coverageRanges.push({ + startTime: clampedStart, + endTime: clampedEnd, + state: "pending", + kind, + noteId, + }); + }; + + const sortedFocusIndices = [...new Set(focusIndices)].sort((a, b) => a - b); + const islands: number[][] = []; + for (const index of sortedFocusIndices) { + const note = sortedNotes[index]; + if (!note || !noteHasRenderableEdit(note)) continue; + + const previousIsland = islands[islands.length - 1]; + const previousIndex = previousIsland?.[previousIsland.length - 1]; + const previousNote = previousIndex !== undefined ? sortedNotes[previousIndex] : null; + const joinsPrevious = Boolean(previousNote) + && ( + notesShareWordGroup(previousNote!, note) + || index === previousIndex! + 1 + || note.startTime - previousNote!.endTime <= MAX_EDIT_ISLAND_GAP_SEC + ); + + if (!previousIsland || !joinsPrevious) { + islands.push([index]); + } else { + previousIsland.push(index); + } + } + + for (const island of islands) { + const firstIndex = island[0]; + const lastIndex = island[island.length - 1]; + const firstNote = sortedNotes[firstIndex]; + const lastNote = sortedNotes[lastIndex]; + if (!firstNote || !lastNote) continue; + + const prev = sortedNotes[firstIndex - 1]; + const next = sortedNotes[lastIndex + 1]; + const usePrev = Boolean(prev && sharesTransitionPair(prev, firstNote)); + const useNext = Boolean(next && sharesTransitionPair(lastNote, next)); + + const entryTransitionMs = Math.min(MAX_NOTE_HQ_TRANSITION_MS, Math.max( + getNoteEffectiveTransitionMs(firstNote, "in"), + getNoteHqBoundaryTransitionFloorMs(firstNote, "in"), + usePrev ? IMMEDIATE_NEIGHBOR_SMOOTH_MS : 0, + )); + const exitTransitionMs = Math.min(MAX_NOTE_HQ_TRANSITION_MS, Math.max( + getNoteEffectiveTransitionMs(lastNote, "out"), + getNoteHqBoundaryTransitionFloorMs(lastNote, "out"), + useNext ? Math.max(IMMEDIATE_NEIGHBOR_SMOOTH_MS, NOTE_HQ_NEXT_HEAD_OWNERSHIP_MS) : 0, + )); + + for (let localIndex = 0; localIndex < island.length; localIndex += 1) { + const noteIndex = island[localIndex]; + const editedNote = sortedNotes[noteIndex]; + const previousEdited = localIndex > 0 ? sortedNotes[island[localIndex - 1]] : null; + const nextEdited = localIndex + 1 < island.length ? sortedNotes[island[localIndex + 1]] : null; + const previousShiftDiff = previousEdited + ? Math.abs(getPitchShiftSemitones(previousEdited) - getPitchShiftSemitones(editedNote)) + : 0; + const nextShiftDiff = nextEdited + ? Math.abs(getPitchShiftSemitones(nextEdited) - getPitchShiftSemitones(editedNote)) + : 0; + const internalInMs = previousEdited && previousShiftDiff > 0.01 ? NOTE_HQ_INTERNAL_ISLAND_RAMP_MS : 0; + const internalOutMs = nextEdited && nextShiftDiff > 0.01 ? NOTE_HQ_INTERNAL_ISLAND_RAMP_MS : 0; + const transitionInMs = localIndex === 0 ? entryTransitionMs : internalInMs; + const transitionOutMs = localIndex === island.length - 1 ? exitTransitionMs : internalOutMs; + const effectiveStartTime = Math.max(0, editedNote.startTime - transitionInMs / 1000); + const effectiveEndTime = Math.min(clipDuration, editedNote.endTime + transitionOutMs / 1000); + + upsertRequestNote(editedNote, { + transitionIn: transitionInMs, + transitionOut: transitionOutMs, + effectiveStartTime, + effectiveEndTime, + }); + } + + const islandStart = Math.max(0, firstNote.startTime - entryTransitionMs / 1000); + const islandEnd = Math.min(clipDuration, lastNote.endTime + exitTransitionMs / 1000); + addCoverageRange(islandStart, islandEnd, "edit_island", firstNote.id); + + if (usePrev && prev) { + upsertRequestNote(prev, { + transitionIn: 0, + transitionOut: 0, + effectiveStartTime: prev.startTime, + effectiveEndTime: prev.endTime, + }); + addCoverageRange(islandStart, firstNote.startTime, "left_neighbor_tail", prev.id); + } + + if (useNext && next) { + upsertRequestNote(next, { + transitionIn: 0, + transitionOut: 0, + effectiveStartTime: next.startTime, + effectiveEndTime: next.endTime, + }); + addCoverageRange(lastNote.endTime, islandEnd, "right_neighbor_head", next.id); + } + } + + if (coverageRanges.length === 0 || requestNotes.size === 0) { + return null; + } + + const mergedCoverage = coverageRanges + .slice() + .sort((a, b) => a.startTime - b.startTime || a.endTime - b.endTime); + let windowStartSec = mergedCoverage[0].startTime; + let windowEndSec = mergedCoverage[0].endTime; + for (const range of mergedCoverage) { + windowStartSec = Math.min(windowStartSec, range.startTime); + windowEndSec = Math.max(windowEndSec, range.endTime); } - if (hasRequestedFormantWork(notes, globalFormantSt)) return "Formant after apply"; - if (hasRealtimePitchPreviewWork(notes, globalFormantSt)) return "Live pitch preview"; - return "No preview changes"; + + return { + requestNotes: [...requestNotes.values()].sort((a, b) => a.startTime - b.startTime), + windowStartSec: Math.max(0, windowStartSec), + windowEndSec: Math.min(clipDuration, windowEndSec), + coverageRanges: mergedCoverage, + }; } -function buildPreviewWindow() { - const pitchState = usePitchEditorStore.getState(); - const transport = useDAWStore.getState().transport; - const clipDuration = pitchState.clipDuration || 0; - const clipTime = Math.max(0, Math.min(clipDuration, transport.currentTime - pitchState.clipStartTime)); - const lookAhead = transport.isPlaying ? PREVIEW_LOOKAHEAD_PLAYING_SEC : PREVIEW_LOOKAHEAD_STOPPED_SEC; +function buildInteractivePreviewPayload(note: PitchNoteData, globalFormantSt: number) { + const semitoneDelta = note.correctedPitch - note.detectedPitch; + const pitchRatio = Math.pow(2, semitoneDelta / 12); + const previewStartSec = Math.max(0, getNoteEffectiveStart(note) - 0.04); + const previewEndSec = Math.max(previewStartSec + 0.05, getNoteEffectiveEnd(note) + 0.04); return { - previewStartSec: Math.max(0, clipTime - PREVIEW_LOOKBEHIND_SEC), - previewEndSec: Math.min(clipDuration, clipTime + lookAhead), - clipTime, - isPlaying: transport.isPlaying, + pitchSegments: [{ + startTime: note.startTime, + endTime: note.endTime, + pitchRatio, + }], + globalFormantSemitones: getEffectivePitchEditorGlobalFormantSt(globalFormantSt), + previewStartSec, + previewEndSec, + allowReplacingCorrectedSource: _interactivePreviewActive === true, + }; +} + +function startPitchScrubPreviewForNote(noteId: string) { + const { trackId, clipId, notes, contour } = usePitchEditorStore.getState(); + if (!trackId || !clipId) return; + const note = notes.find((candidate) => candidate.id === noteId); + if (!note) return; + nativeBridge.startPitchScrubPreview(trackId, clipId, note, contour?.frames) + .catch(logBridgeError("startPitchScrubPreview")); +} + +function clearTransientPitchPreview(clipId: string, reason: string) { + nativeBridge.stopPitchScrubPreview(clipId).catch(logBridgeError("stopPitchScrubPreview")); + nativeBridge.clearClipPitchPreview(clipId).catch(logBridgeError("clearClipPitchPreview")); + _activePitchPreviewMonitorMode = "none"; + logPitchEditorFormant("cleared transient pitch preview", { clipId, reason }); +} + +function clearRenderedPreviewForInteractiveEdit(clipId: string, reason: string) { + nativeBridge.clearClipRenderedPreviewSegments(clipId).catch(logBridgeError("clearClipRenderedPreviewSegments")); + logPitchEditorFormant("cleared rendered pitch preview for interactive edit", { clipId, reason }); +} + +function isCleanNoteHqFinalRoute(status: PitchPreviewRoutingStatus | null) { + return Boolean(status) + && status?.monitorMode === "corrected_source" + && status.correctedSourceActive === true + && status.renderedSegmentActive === false + && status.clipLivePreviewActive === false + && status.scrubPreviewActive === false; +} + +function routeSummary(status: PitchPreviewRoutingStatus | null) { + if (!status) return null; + return { + monitorMode: status.monitorMode, + correctedSourceActive: status.correctedSourceActive, + renderedSegmentActive: status.renderedSegmentActive, + clipLivePreviewActive: status.clipLivePreviewActive, + scrubPreviewActive: status.scrubPreviewActive, + }; +} + +async function clearAllPitchPreviewRoutes(clipId: string, reason: string) { + _activePitchPreviewMonitorMode = "none"; + await nativeBridge.clearAllPitchPreviewRoutes(clipId).catch(logBridgeError("clearAllPitchPreviewRoutes")); + logPitchEditorFormant("cleared all pitch preview routes", { clipId, reason }); +} + +async function clearCorrectedSourcePreviewRoutesBeforePlayback(clipId: string) { + let status: PitchPreviewRoutingStatus | null = null; + try { + status = await nativeBridge.getPitchPreviewRoutingStatus(clipId); + } catch (err) { + warnPitchEditorFormant("transport-start route query failed", { + clipId, + error: err instanceof Error ? err.message : String(err), + }); + return; + } + + if (!status?.correctedSourceActive) return; + const previewRouteActive = status.renderedSegmentActive + || status.clipLivePreviewActive + || status.scrubPreviewActive; + if (!previewRouteActive) return; + + warnPitchEditorFormant("transport start found corrected source with stale pitch preview route; clearing", { + clipId, + route: routeSummary(status), + }); + await clearAllPitchPreviewRoutes(clipId, "transport_start_corrected_source"); +} + +function pathWithSuffix(filePath: string, suffix: string, extension: string) { + const slashIndex = Math.max(filePath.lastIndexOf("/"), filePath.lastIndexOf("\\")); + const dotIndex = filePath.lastIndexOf("."); + const base = dotIndex > slashIndex ? filePath.slice(0, dotIndex) : filePath; + return `${base}${suffix}${extension}`; +} + +function safeCaptureToken(value: string | undefined | null) { + const raw = value && value.trim().length > 0 ? value : `${Date.now()}`; + return raw.replace(/[^a-zA-Z0-9_.-]+/g, "_").slice(-72); +} + +async function captureAppFinalPitchContext( + meta: PitchCorrectionRequestMeta, + data: PitchCorrectionCompletionData, + routeBeforeRepair: PitchPreviewRoutingStatus | null, + routeAfterRepair: PitchPreviewRoutingStatus | null, + routeSuspect: boolean, +) { + if (!data.outputFile || !shouldCaptureAppFinalPitchContext(routeSuspect)) return; + + const editedNotes = meta.notes.filter((note) => Math.abs(note.correctedPitch - note.detectedPitch) > 0.01); + if (editedNotes.length === 0) return; + + const state = usePitchEditorStore.getState(); + const clipDuration = state.clipDuration || Math.max(...editedNotes.map(getNoteEffectiveEnd)); + const noteStartSec = Math.max(0, Math.min(...editedNotes.map(getNoteEffectiveStart))); + const noteEndSec = Math.max(noteStartSec, Math.max(...editedNotes.map(getNoteEffectiveEnd))); + const contextStartSec = Math.max(0, noteStartSec - 0.5); + const contextEndSec = Math.min(clipDuration, noteEndSec + 0.6); + const durationSec = Math.max(0.25, contextEndSec - contextStartSec); + const projectStartSec = state.clipStartTime + contextStartSec; + const token = safeCaptureToken(meta.logicalRequestId || meta.requestId); + const wavPath = pathWithSuffix(data.outputFile, `_appfinal_${token}_context`, ".wav"); + const routeJsonPath = pathWithSuffix(data.outputFile, `_appfinal_${token}_route`, ".json"); + const shifts = editedNotes.map((note) => note.correctedPitch - note.detectedPitch); + const editedNoteDiagnostics = editedNotes.map((note) => ({ + id: note.id, + startTime: note.startTime, + endTime: note.endTime, + effectiveStartTime: getNoteEffectiveStart(note), + effectiveEndTime: getNoteEffectiveEnd(note), + detectedPitch: note.detectedPitch, + correctedPitch: note.correctedPitch, + requestedShiftSemitones: note.correctedPitch - note.detectedPitch, + pitchDrift: note.pitchDrift ?? null, + vibratoDepth: note.vibratoDepth ?? null, + vibratoRate: note.vibratoRate ?? null, + formantShift: note.formantShift ?? null, + gainDb: note.gain ?? null, + })); + + const capture = await nativeBridge.capturePitchAppFinalContext({ + trackId: meta.trackId, + clipId: meta.clipId, + startTime: projectStartSec, + duration: durationSec, + wavPath, + routeJsonPath, + sampleRate: 44100, + metadata: { + requestId: meta.requestId, + logicalRequestId: meta.logicalRequestId, + renderMode: data.renderMode ?? meta.renderMode, + outputFile: data.outputFile, + routeSuspect, + routeBeforeRepair: routeSummary(routeBeforeRepair), + routeAfterRepair: routeSummary(routeAfterRepair), + postApplyRouteStatus: routeSummary(data.postApplyRouteStatus ?? null), + clipContextStartSec: contextStartSec, + clipContextEndSec: contextEndSec, + projectStartSec, + noteStartSec, + noteEndSec, + snapMode: state.snapMode, + scaleType: state.scaleType, + chromaticSnapActive: state.snapMode === "chromatic", + exactRelativeShiftRequested: data.targetShiftSemitones ?? null, + editedNoteCount: editedNotes.length, + editedNotes: editedNoteDiagnostics, + requestedShiftSemitonesMin: Math.min(...shifts), + requestedShiftSemitonesMax: Math.max(...shifts), + actualRequestedShiftSemitones: data.actualRequestedShiftSemitones ?? null, + requestedShiftErrorCents: data.requestedShiftErrorCents ?? null, + actualRendererBranch: data.actualRendererBranch ?? null, + formantCurveUsed: data.formantCurveUsed ?? null, + backendAppFinalRouteReportPath: data.appFinalRouteReportPath ?? null, + backendAppFinalBakedContextPath: data.appFinalBakedContextPath ?? null, + backendAppFinalPlaybackContextPath: data.appFinalPlaybackContextPath ?? null, + backendAppFinalParityReportPath: data.appFinalParityReportPath ?? null, + backendAppFinalParityReport: data.appFinalParityReport ?? null, + }, + }); + + const debugPaths = { + backendBakedContextPath: data.appFinalBakedContextPath ?? data.appFinalBakedCapture?.filePath ?? null, + backendPlaybackContextPath: data.appFinalPlaybackContextPath ?? data.appFinalCapture?.capture?.filePath ?? null, + backendParityReportPath: data.appFinalParityReportPath ?? null, + backendRouteReportPath: data.appFinalRouteReportPath ?? null, + frontendBakedCorrectedPath: capture?.bakedCorrectedPath ?? null, + frontendLivePlaybackContextPath: capture?.livePlaybackPath ?? capture?.capture?.filePath ?? wavPath, + frontendOfflineRenderContextPath: capture?.offlineRenderPath ?? null, + frontendComparisonReportPath: capture?.comparisonReportPath ?? null, + frontendRouteReportPath: capture?.routeReportPath ?? routeJsonPath, }; + + logPitchEditorFormant("app-final pitch context capture finished", { + clipId: meta.clipId, + requestId: meta.requestId, + success: capture?.success ?? false, + ...debugPaths, + routeBefore: routeSummary(capture?.routeBefore ?? null), + routeAfter: routeSummary(capture?.routeAfter ?? null), + }); + console.info("[pitchEditor] Last app-final pitch debug capture", debugPaths); + useDAWStore.getState().showToast("Pitch debug capture written; paths are in the console", "success"); +} + +async function verifyNoteHqFinalRoute( + meta: PitchCorrectionRequestMeta, + data: PitchCorrectionCompletionData, + reason: string, + captureIfEnabled: boolean, +) { + let routeBeforeRepair: PitchPreviewRoutingStatus | null = null; + try { + routeBeforeRepair = await nativeBridge.getPitchPreviewRoutingStatus(meta.clipId); + } catch (err) { + warnPitchEditorFormant("note-HQ final route query failed", { + clipId: meta.clipId, + requestId: meta.requestId, + reason, + error: err instanceof Error ? err.message : String(err), + }); + if (reason === "immediate") { + setApplyStatus("error", "HQ note route verification failed", meta.logicalRequestId, { autoClearMs: 2600 }); + useDAWStore.getState().showToast("Pitch note route verification failed", "error"); + } + return; + } + + const wasClean = isCleanNoteHqFinalRoute(routeBeforeRepair); + let routeAfterRepair = routeBeforeRepair; + if (!wasClean) { + warnPitchEditorFormant("note-HQ final route was not clean; clearing stale pitch preview routes", { + clipId: meta.clipId, + requestId: meta.requestId, + logicalRequestId: meta.logicalRequestId, + reason, + route: routeSummary(routeBeforeRepair), + }); + await clearAllPitchPreviewRoutes(meta.clipId, `note_hq_route_repair:${reason}`); + routeAfterRepair = await nativeBridge.getPitchPreviewRoutingStatus(meta.clipId).catch(() => null); + const repaired = isCleanNoteHqFinalRoute(routeAfterRepair); + setApplyStatus( + repaired ? "done" : "error", + repaired ? "HQ note render ready (route repaired)" : "HQ note render route suspect", + meta.logicalRequestId, + { autoClearMs: repaired ? 1800 : 3200 }, + ); + if (reason === "immediate") { + useDAWStore.getState().showToast( + repaired ? "Pitch note render ready" : "Pitch note route still suspect", + repaired ? "success" : "error", + ); + } + logPitchEditorFormant("note-HQ final route repair result", { + clipId: meta.clipId, + requestId: meta.requestId, + reason, + repaired, + routeAfter: routeSummary(routeAfterRepair), + }); + } else { + logPitchEditorFormant("note-HQ final route verified", { + clipId: meta.clipId, + requestId: meta.requestId, + reason, + route: routeSummary(routeBeforeRepair), + }); + if (reason === "immediate") { + setApplyStatus("done", "HQ note render ready", meta.logicalRequestId, { autoClearMs: 1800 }); + useDAWStore.getState().showToast("Pitch note render ready", "success"); + } + } + + if (captureIfEnabled || !wasClean) { + await captureAppFinalPitchContext(meta, data, routeBeforeRepair, routeAfterRepair, !wasClean); + } +} + +function scheduleNoteHqFinalRouteVerification(meta: PitchCorrectionRequestMeta, data: PitchCorrectionCompletionData) { + void verifyNoteHqFinalRoute(meta, data, "immediate", true); + window.setTimeout(() => { + void verifyNoteHqFinalRoute(meta, data, "late_250ms", false); + }, 250); + window.setTimeout(() => { + void verifyNoteHqFinalRoute(meta, data, "late_1000ms", false); + }, 1000); +} + +function resolvePitchPreviewMonitorMode(): PitchPreviewMonitorMode { + if (!_interactivePreviewActive || !_interactivePreviewNoteId) { + return "none"; + } + return useDAWStore.getState().transport.isPlaying ? "clip_live_preview" : "scrub"; } function buildAutoBakeWindow() { @@ -571,52 +1260,61 @@ function buildAutoBakeWindow() { }; } -/** Build pitch correction segments and send pitch+global-formant preview to backend. */ +/** Legacy rolling preview path. Pitch/formant preview now prefers rendered segments. */ function sendPitchPreviewMap(reason: "edit" | "transport" | "sync" = "edit") { const { clipId, notes, globalFormantCents } = usePitchEditorStore.getState(); if (!clipId) return; - const globalFormantSt = globalFormantCents / 100; - const pitchSegments = notes - .filter(n => Math.abs(n.correctedPitch - n.detectedPitch) > 0.01) - .map(n => ({ - startTime: n.startTime, - endTime: n.endTime, - pitchRatio: Math.pow(2, (n.correctedPitch - n.detectedPitch) / 12), - })); - - if (!hasRealtimePitchPreviewWork(notes, globalFormantSt)) { + const globalFormantSt = getEffectivePitchEditorGlobalFormantSt(globalFormantCents / 100); + const monitorMode = resolvePitchPreviewMonitorMode(); + const previewNote = _interactivePreviewActive && _interactivePreviewNoteId + ? notes.find((note) => note.id === _interactivePreviewNoteId) ?? null + : null; + if (!previewNote || Math.abs(previewNote.correctedPitch - previewNote.detectedPitch) <= 0.01) { + clearTransientPitchPreview(clipId, reason); + logPitchEditorFormant("cleared live drag preview", { + clipId, + reason, + requestedGlobalFormantSemitones: globalFormantSt, + activeInteractivePreview: _interactivePreviewActive, + previewNoteId: _interactivePreviewNoteId, + }); + return; + } + + if (monitorMode === "scrub") { + const pitchRatio = Math.pow(2, (previewNote.correctedPitch - previewNote.detectedPitch) / 12); + if (_activePitchPreviewMonitorMode !== "scrub") { + startPitchScrubPreviewForNote(previewNote.id); + } + nativeBridge.updatePitchScrubPreview(clipId, pitchRatio).catch(logBridgeError("updatePitchScrubPreview")); nativeBridge.clearClipPitchPreview(clipId).catch(logBridgeError("clearClipPitchPreview")); - logPitchEditorFormant( - hasRequestedFormantWork(notes, globalFormantSt) - ? "suppressed realtime preview while formant edits are active" - : "cleared rolling preview", - { clipId, reason, globalFormantSemitones: globalFormantSt }, - ); + _activePitchPreviewMonitorMode = "scrub"; + logPitchEditorFormant("updated dedicated scrub preview", { + clipId, + reason, + previewNoteId: previewNote.id, + pitchRatio, + }); return; } - const window = buildPreviewWindow(); - const payload: ClipPitchPreviewPayload = { - pitchSegments, - globalFormantSemitones: 0, - previewStartSec: window.previewStartSec, - previewEndSec: window.previewEndSec, - }; + if (monitorMode !== "clip_live_preview") { + clearTransientPitchPreview(clipId, reason); + return; + } - logPitchEditorFormant("sending rolling preview", { + nativeBridge.stopPitchScrubPreview(clipId).catch(logBridgeError("stopPitchScrubPreview")); + const payload = buildInteractivePreviewPayload(previewNote, globalFormantSt); + nativeBridge.setClipPitchPreview(clipId, payload).catch(logBridgeError("setClipPitchPreview")); + _activePitchPreviewMonitorMode = "clip_live_preview"; + logPitchEditorFormant("updated live drag preview", { clipId, reason, - pitchSegments: pitchSegments.length, - requestedGlobalFormantSemitones: globalFormantSt, - livePreviewPitchOnly: true, + previewNoteId: previewNote.id, previewStartSec: payload.previewStartSec, previewEndSec: payload.previewEndSec, - transportPlaying: window.isPlaying, + pitchRatio: payload.pitchSegments[0]?.pitchRatio ?? 1, }); - - nativeBridge.setClipPitchPreview(clipId, payload).catch(err => - console.warn("[pitchEditor] setClipPitchPreview failed:", err) - ); } function scheduleRollingPreviewRefresh() { @@ -644,8 +1342,18 @@ function ensureDAWTransportSubscription() { if (!hasRealtimePitchPreviewWork(pitchState.notes, globalFormantSt)) return; if (transport.isPlaying) { + if (!prevTransport?.isPlaying && !_interactivePreviewActive) { + void clearCorrectedSourcePreviewRoutesBeforePlayback(pitchState.clipId); + } + if (_interactivePreviewActive) { + sendPitchPreviewMap("transport"); + } scheduleRollingPreviewRefresh(); } else if (prevTransport?.isPlaying) { + if (_interactivePreviewActive) { + sendPitchPreviewMap("transport"); + return; + } // Transport just stopped. If edits were skipped during playback (preview-only // mode for pitch-only edits), bake them now so the corrected file is up to date // before the user renders or exports. @@ -709,16 +1417,25 @@ function isCurrentRevision(revision: number) { } function dispatchNativePitchCorrection(meta: PitchCorrectionRequestMeta) { + const effectiveNotes = sanitizePitchEditorNotesForApply(meta.notes); + const effectiveGlobalFormantSt = getEffectivePitchEditorGlobalFormantSt(meta.globalFormantSt); registerRequestMeta(meta); const statusRequestId = meta.logicalRequestId; _activePitchCorrectionRequestId = statusRequestId; + if (meta.stage === "note_hq" || meta.stage === "full_clip_hq" || meta.stage === "single") { + clearTransientPitchPreview(meta.clipId, `dispatch:${meta.stage}`); + } + if (meta.stage === "note_hq") { + clearStagedPreviewQueue(); + clearRenderedPreviewForInteractiveEdit(meta.clipId, "dispatch:note_hq"); + } if (meta.stage === "preview_segment") { if (usePitchEditorStore.getState().applyState !== "final_processing") { - setApplyStatus("preview_processing", getFormantPreviewStatusMessage("preview_processing", false), statusRequestId); + setApplyStatus("preview_processing", getRenderedPreviewStatusMessage("preview_processing", false), statusRequestId); } } else if (meta.stage === "full_clip_hq") { - setApplyStatus("final_processing", getFormantPreviewStatusMessage("final_processing", false), statusRequestId); + setApplyStatus("final_processing", getRenderedPreviewStatusMessage("final_processing", false), statusRequestId); } else { setApplyStatus("processing", "Applying...", statusRequestId); } @@ -738,10 +1455,10 @@ function dispatchNativePitchCorrection(meta: PitchCorrectionRequestMeta) { nativeBridge.applyPitchCorrection( meta.trackId, meta.clipId, - meta.notes, + effectiveNotes, meta.frames, meta.requestId, - meta.globalFormantSt, + effectiveGlobalFormantSt, meta.windowStartSec, meta.windowEndSec, meta.renderMode, @@ -754,6 +1471,9 @@ function dispatchNativePitchCorrection(meta: PitchCorrectionRequestMeta) { if (_activePitchCorrectionRequestId === meta.logicalRequestId) { _activePitchCorrectionRequestId = null; } + if (meta.stage === "note_hq") { + finishNoteHqPriority(meta.logicalRequestId); + } setApplyStatus("error", "Failed", meta.logicalRequestId); useDAWStore.getState().showToast("Pitch/formant apply failed", "error"); warnPitchEditorFormant("apply request was not queued", { @@ -768,6 +1488,9 @@ function dispatchNativePitchCorrection(meta: PitchCorrectionRequestMeta) { if (_activePitchCorrectionRequestId === meta.logicalRequestId) { _activePitchCorrectionRequestId = null; } + if (meta.stage === "note_hq") { + finishNoteHqPriority(meta.logicalRequestId); + } setApplyStatus("error", "Failed", meta.logicalRequestId); useDAWStore.getState().showToast("Pitch/formant apply failed", "error"); errorPitchEditorFormant("applyPitchCorrection call failed", { @@ -811,6 +1534,52 @@ function dispatchSingleApplyRequest( }); } +function dispatchNoteHqApplyRequest( + trackId: string, + clipId: string, + notes: PitchNoteData[], + frames: PitchContourData["frames"] | undefined, + globalFormantSt: number, + requestRevision: number, + summary: ApplyRequestSummary, + logicalRequestId: string, + windowStartSec?: number, + windowEndSec?: number, + coverageRanges?: PitchRenderCoverageRange[], +) { + const requestId = buildStageRequestId(logicalRequestId, "note_hq"); + _noteHqApplyInFlight = true; + cancelActivePitchAnalysisForHq(); + if (coverageRanges && coverageRanges.length > 0) { + setRenderCoverage(coverageRanges.map((range) => ({ ...range, state: "pending" })), logicalRequestId); + } else if (windowStartSec !== undefined && windowEndSec !== undefined) { + setRenderCoverage([{ + startTime: windowStartSec, + endTime: windowEndSec, + state: "pending", + }], logicalRequestId); + } else { + setRenderCoverage([], logicalRequestId); + } + dispatchNativePitchCorrection({ + clipId, + trackId, + requestId, + logicalRequestId, + revision: requestRevision, + stage: "note_hq", + renderMode: "note_hq", + notes: cloneNotesSnapshot(notes), + frames: cloneFramesSnapshot(frames), + globalFormantSt, + summary, + windowStartSec, + windowEndSec, + coverageRanges: coverageRanges?.map((range) => ({ ...range })), + requestGroupId: logicalRequestId, + }); +} + function dispatchNextPreviewSegments() { if (!_stagedPreviewBase || _activePreviewRequestGroupId !== _stagedPreviewBase.logicalRequestId) return; while (_runningPreviewSegmentJobs < MAX_PREVIEW_SEGMENT_CONCURRENCY && _pendingPreviewSegments.length > 0) { @@ -830,22 +1599,22 @@ function dispatchNextPreviewSegments() { } } -function getFormantPreviewStatusMessage(state: PitchEditorApplyState, previewCoverageComplete: boolean) { +function getRenderedPreviewStatusMessage(state: PitchEditorApplyState, previewCoverageComplete: boolean) { switch (state) { case "preview_processing": return "Rendering preview near playhead..."; case "preview_ready": - return previewCoverageComplete ? "Rendered formant preview ready" : "Previewing rendered formant"; + return previewCoverageComplete ? "Rendered preview ready" : "Previewing rendered audio"; case "final_processing": return "Refining full clip render..."; case "done": - return "Previewing rendered formant"; + return "Previewing rendered audio"; default: - return "Rendered formant preview"; + return "Rendered preview"; } } -function dispatchStagedFormantApply( +function dispatchStagedRenderedApply( trackId: string, clipId: string, notes: PitchNoteData[], @@ -894,7 +1663,14 @@ function dispatchStagedFormantApply( }, 1000); } -function scheduleAutoApply(delayMs = 400) { +const _deferredPitchEditorHelpers = [ + hasRequestedFormantWork, + buildAutoBakeWindow, + dispatchStagedRenderedApply, +]; +void _deferredPitchEditorHelpers; + +function scheduleAutoApply(delayMs = 300) { if (_autoApplyTimer) clearTimeout(_autoApplyTimer); const { notes, globalFormantCents, clipId } = usePitchEditorStore.getState(); const globalFormantSt = globalFormantCents / 100; @@ -912,7 +1688,7 @@ function scheduleAutoApply(delayMs = 400) { } _autoApplyTimer = setTimeout(async () => { _autoApplyTimer = null; - const { trackId, clipId, notes, contour, globalFormantCents } = usePitchEditorStore.getState(); + const { trackId, clipId, notes, contour, globalFormantCents, clipDuration } = usePitchEditorStore.getState(); if (!trackId || !clipId) return; if (_editRevision <= _appliedRevision) { logPitchEditorFormant("skipping auto-apply because edit revision is already applied", { @@ -933,22 +1709,10 @@ function scheduleAutoApply(delayMs = 400) { } const globalFormantSt = globalFormantCents / 100; const requestSummary = summarizeApplyRequest(notes, globalFormantSt); - const bakeWindow = buildAutoBakeWindow(); - const requiresStagedFormantBake = hasRequestedFormantWork(notes, globalFormantSt); // Ensure the clip is registered in the playback engine before applying. - // Re-establish the real-time preview after sync (clearAllClips destroys it). + // Clear the legacy live stretcher preview path. We now prefer rendered preview + // segments so playback matches the offline/native engine more closely. sendPitchPreviewMap("sync"); - if (bakeWindow.isPlaying && !requiresStagedFormantBake) { - setApplyStatus("done", getPlaybackPreviewStatusMessage(notes, globalFormantSt), null, { autoClearMs: 1400 }); - logPitchEditorFormant("skipping auto-apply during playback; preview-only mode active", { - clipId, - editRevision: _editRevision, - appliedRevision: _appliedRevision, - realtimePitchPreview: hasRealtimePitchPreviewWork(notes, globalFormantSt), - formantDeferred: hasRequestedFormantWork(notes, globalFormantSt), - }); - return; - } try { await useDAWStore.getState().syncClipsWithBackend(); } catch (e) { @@ -961,18 +1725,11 @@ function scheduleAutoApply(delayMs = 400) { const requestRevision = _editRevision; _requestedApplyRevision = requestRevision; const logicalRequestId = buildLogicalApplyRequestId(clipId); - logPitchEditorFormant("dispatching auto-bake request", { - clipId, - logicalRequestId, - requestRevision, - stagedFormant: requiresStagedFormantBake, - windowStartSec: requiresStagedFormantBake ? null : bakeWindow.windowStartSec, - windowEndSec: requiresStagedFormantBake ? null : bakeWindow.windowEndSec, - ...requestSummary, - }); - - if (requiresStagedFormantBake) { - dispatchStagedFormantApply( + const editedNoteWindow = buildEditedNoteWindow(notes, clipDuration || 0); + if (requestSummary.mode === "none") { + setRenderCoverage([], null); + nativeBridge.clearClipRenderedPreviewSegments(clipId).catch(logBridgeError("clearClipRenderedPreviewSegments")); + dispatchSingleApplyRequest( trackId, clipId, notes, @@ -984,26 +1741,29 @@ function scheduleAutoApply(delayMs = 400) { ); return; } + logPitchEditorFormant("dispatching auto-bake request", { + clipId, + logicalRequestId, + requestRevision, + renderMode: "note_hq", + windowStartSec: editedNoteWindow?.windowStartSec ?? null, + windowEndSec: editedNoteWindow?.windowEndSec ?? null, + ...requestSummary, + }); clearStagedPreviewQueue(); - setRenderCoverage([], null); - nativeBridge.clearClipRenderedPreviewSegments(clipId).catch(logBridgeError("clearClipRenderedPreviewSegments")); - // Do NOT pass a playhead-based window override for single bakes. - // When the playhead is inside a note, the playhead window ([clipTime-0.25, clipTime+2.5]) - // starts mid-note, clamping the note's startSample to 0 and eliminating its pre-roll. - // The Signalsmith stretcher then has no warm-up before the note (analysis window = 120ms), - // producing edge artifacts at the splice crossfade that sound abrupt ("not smoothened"). - // Without an override, the backend computes the window from note boundaries with 1.0s - // of padding on each side — always giving the stretcher time to fully initialize. - dispatchSingleApplyRequest( + dispatchNoteHqApplyRequest( trackId, clipId, - notes, + editedNoteWindow?.requestNotes ?? notes, contour?.frames, globalFormantSt, requestRevision, requestSummary, logicalRequestId, + editedNoteWindow?.windowStartSec, + editedNoteWindow?.windowEndSec, + editedNoteWindow?.coverageRanges, ); }, delayMs); } @@ -1026,6 +1786,7 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ contour: null, notes: [], isAnalyzing: false, + analysisPhase: "idle", isApplying: false, progressPercent: 0, progressLabel: "", @@ -1040,7 +1801,7 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ scrollX: 0, scrollY: 48, // C3 zoomX: 200, // pixels per second - zoomY: 24, // pixels per semitone + zoomY: 32, // pixels per semitone (increased for finer drag control) undoStack: [], redoStack: [], referenceTracks: [], @@ -1050,14 +1811,6 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ inspectorExpanded: true, globalFormantCents: 0, showCorrectPitchModal: false, - polyMode: false, - polyNotes: [], - polyAnalysisResult: null, - showPitchSalience: false, - soloNoteId: null, - polyNoteThreshold: 0.15, - polyOnsetThreshold: 0.3, - polyMinDuration: 80, abCompareMode: false, open: (trackId, clipId, fxIndex) => { @@ -1087,7 +1840,10 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ redoStack: [], referenceTracks: [], isAnalyzing: false, + analysisPhase: "loading", isApplying: false, + progressPercent: 0, + progressLabel: "Loading clip...", applyState: "idle", applyMessage: "", lastApplyRequestId: null, @@ -1095,11 +1851,6 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ activeLogicalRequestId: null, scrollY: 48, // Reset to middle C, will be auto-fit after analysis globalFormantCents: 0, - polyMode: false, - polyNotes: [], - polyAnalysisResult: null, - showPitchSalience: false, - soloNoteId: null, }); }, @@ -1107,7 +1858,7 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ // Clear real-time pitch preview when closing the editor const { clipId } = get(); if (clipId) { - nativeBridge.clearClipPitchPreview(clipId).catch(logBridgeError("clearClipPitchPreview")); + clearTransientPitchPreview(clipId, "close"); nativeBridge.clearClipRenderedPreviewSegments(clipId).catch(logBridgeError("clearClipRenderedPreviewSegments")); } set({ @@ -1119,11 +1870,10 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ undoStack: [], redoStack: [], referenceTracks: [], - polyMode: false, - polyNotes: [], - polyAnalysisResult: null, - showPitchSalience: false, - soloNoteId: null, + isAnalyzing: false, + analysisPhase: "idle", + progressPercent: 0, + progressLabel: "", applyState: "idle", applyMessage: "", lastApplyRequestId: null, @@ -1136,11 +1886,15 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ }, // Analyze a specific time window of the clip (viewport-based analysis). - // Analyzes a chunk (max ~30s) around the given time range. + // Analyzes a small chunk around the given time range and builds coverage incrementally. // Merges results with existing notes to build up the full picture as user scrolls. analyze: async (viewStartTime?: number, viewEndTime?: number) => { const { trackId, clipId, isAnalyzing, originalClipFilePath, originalClipOffset } = get(); if (!trackId || !clipId || isAnalyzing) return; + if (_noteHqApplyInFlight && get().contour) { + deferPitchAnalysis(viewStartTime, viewEndTime, "note_hq_in_flight"); + return; + } const dawState = useDAWStore.getState(); const track = dawState.tracks.find((t) => t.id === trackId); @@ -1153,8 +1907,8 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ const analysisFilePath = originalClipFilePath ?? clip.filePath; if (!analysisFilePath) return; - // Determine analysis window: max 30s chunk centered on viewport - const MAX_CHUNK = 30; // seconds + // Determine analysis window: keep first-open note load as light as possible + const MAX_CHUNK = 5; // seconds // Use original offset — after correction clip.offset is reset to 0 (corrected file // starts at 0), but the original file needs its original seek position. const clipOffset = originalClipOffset; @@ -1166,17 +1920,19 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ if (viewStartTime !== undefined && viewEndTime !== undefined) { // Analyze around viewport with padding const viewDuration = viewEndTime - viewStartTime; - const padding = Math.min(5, viewDuration * 0.5); // 5s padding or 50% of view + const padding = Math.min(2, viewDuration * 0.35); // tighter padding for faster viewport fills analyzeStart = Math.max(0, viewStartTime - padding); const analyzeEnd = Math.min(clipDuration, viewEndTime + padding); analyzeDuration = Math.min(MAX_CHUNK, analyzeEnd - analyzeStart); } else { - // Initial analysis: first 30s of the clip + // Initial analysis: first 5s of the clip analyzeStart = 0; analyzeDuration = Math.min(MAX_CHUNK, clipDuration); } - set({ isAnalyzing: true, progressPercent: 0, progressLabel: "Analyzing pitch..." }); + const analysisRevision = _editRevision; + const analysisSeq = ++_analysisRunSeq; + set({ isAnalyzing: true, analysisPhase: "analyzing", progressPercent: 0, progressLabel: "Analyzing pitch..." }); console.log(`[PitchEditor] Analyzing ${analyzeStart.toFixed(1)}s - ${(analyzeStart + analyzeDuration).toFixed(1)}s of ${clipDuration.toFixed(1)}s clip`); // Cancel any stale listener before registering a new one. @@ -1197,10 +1953,26 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ unsubscribe(); // notification contains noteCount and ready flag only (no heavy JSON) + if (notification?.cancelled || analysisSeq !== _analysisRunSeq) { + set({ isAnalyzing: false, analysisPhase: "idle", progressPercent: 0, progressLabel: "" }); + if (_noteHqApplyInFlight) { + deferPitchAnalysis(viewStartTime, viewEndTime, "native_analysis_cancelled"); + } + return; + } + if (notification?.ready && notification?.noteCount >= 0) { try { const fullResult = await nativeBridge.getLastPitchAnalysisResult(); if (fullResult?.notes) { + if (analysisRevision !== _editRevision && get().notes.length > 0) { + logPitchEditorFormant("ignored stale pitch analysis after edit revision changed", { + analysisRevision, + currentEditRevision: _editRevision, + }); + set({ isAnalyzing: false, analysisPhase: "idle", progressPercent: 0, progressLabel: "" }); + return; + } // Offset note/frame times: C++ returns times from 0, but we need // them relative to clip start (analyzeStart offset). // Prefix IDs with the window start so notes from different analysis @@ -1212,6 +1984,8 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ id: idPrefix + n.id, startTime: n.startTime + analyzeStart, endTime: n.endTime + analyzeStart, + effectiveStartTime: (n.effectiveStartTime ?? n.startTime) + analyzeStart, + effectiveEndTime: (n.effectiveEndTime ?? n.endTime) + analyzeStart, })); if (fullResult.frames) { fullResult.frames.times = fullResult.frames.times.map( @@ -1219,7 +1993,13 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ ); } - const { notes: existingNotes } = get(); + const { + notes: existingNotes, + selectedNoteIds: existingSelectedNoteIds, + undoStack: existingUndoStack, + redoStack: existingRedoStack, + } = get(); + const hadPitchHistory = existingUndoStack.length > 0 || existingRedoStack.length > 0; // Merge new notes with existing (replace overlapping range) let mergedNotes: PitchNoteData[]; @@ -1230,9 +2010,9 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ (n) => n.endTime < rangeStart || n.startTime > rangeEnd ); mergedNotes.push(...offsetNotes); - mergedNotes.sort((a, b) => a.startTime - b.startTime); + mergedNotes = normalizePitchNotes(mergedNotes); } else { - mergedNotes = offsetNotes; + mergedNotes = normalizePitchNotes(offsetNotes); } console.log("[PitchEditor] Applied: " + offsetNotes.length + " new notes, " + mergedNotes.length + " total"); @@ -1291,14 +2071,19 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ set({ isAnalyzing: false, + analysisPhase: "idle", progressPercent: 100, progressLabel: "", contour: mergedContour, notes: mergedNotes, - selectedNoteIds: [], - undoStack: [], - redoStack: [], + selectedNoteIds: filterSelectedNoteIdsForNotes(existingSelectedNoteIds, mergedNotes), }); + if (hadPitchHistory) { + logPitchEditorFormant("preserved pitch undo history across analysis refresh", { + undoDepth: existingUndoStack.length, + redoDepth: existingRedoStack.length, + }); + } // Auto-fit viewport on first analysis (no existing notes) if (existingNotes.length === 0 && mergedNotes.length > 0) { @@ -1308,20 +2093,18 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ maxMidi = Math.max(maxMidi, Math.ceil(n.detectedPitch) + 2); } set({ scrollY: minMidi, scrollX: 0 }); - // Auto-detect mono vs poly on first analysis - get().autoDetectMonoPoly(); } } else { console.warn("[PitchEditor] No notes in analysis result"); - set({ isAnalyzing: false }); + set({ isAnalyzing: false, analysisPhase: "idle" }); } } catch (err) { console.error("[PitchEditor] Failed to fetch analysis result:", err); - set({ isAnalyzing: false }); + set({ isAnalyzing: false, analysisPhase: "idle" }); } } else { console.warn("[PitchEditor] Analysis completed with 0 notes or failed"); - set({ isAnalyzing: false }); + set({ isAnalyzing: false, analysisPhase: "idle" }); } }); _pitchAnalysisUnsubscribe = unsubscribe; @@ -1334,14 +2117,17 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ ); const started = !!(response as any)?.started; if (!started) { + if ((response as any)?.deferred) { + deferPitchAnalysis(viewStartTime, viewEndTime, "native_deferred"); + } _pitchAnalysisUnsubscribe = null; unsubscribe(); - set({ isAnalyzing: false }); + set({ isAnalyzing: false, analysisPhase: "idle" }); } } catch { _pitchAnalysisUnsubscribe = null; unsubscribe(); - set({ isAnalyzing: false }); + set({ isAnalyzing: false, analysisPhase: "idle" }); } }, @@ -1369,10 +2155,41 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ setSelectedNoteIds: (ids) => set({ selectedNoteIds: ids }), + beginInteractivePreview: (noteId) => { + replaceDirtyPitchNotes(expandDirtyPitchNoteIdsWithNeighbors([noteId], get().notes)); + _interactivePreviewNoteId = noteId; + _interactivePreviewActive = true; + if (get().clipId) { + clearRenderedPreviewForInteractiveEdit(get().clipId!, "beginInteractivePreview"); + } + sendPitchPreviewMap("edit"); + logPitchEditorFormant("interactive preview started", { + clipId: get().clipId, + noteId, + }); + }, + + endInteractivePreview: () => { + const { clipId } = get(); + _interactivePreviewActive = false; + _interactivePreviewNoteId = null; + if (clipId) { + clearTransientPitchPreview(clipId, "endInteractivePreview"); + } + logPitchEditorFormant("interactive preview ended", { + clipId, + }); + }, + pushUndo: (description) => { - const { notes, undoStack } = get(); + const { notes, selectedNoteIds, undoStack } = get(); + const snapshot = clonePitchNotes(notes); + const last = undoStack[undoStack.length - 1]; + if (last && pitchNotesEqual(last.notes, snapshot)) { + return; + } set({ - undoStack: [...undoStack, { description, notes: JSON.parse(JSON.stringify(notes)) }], + undoStack: [...undoStack, { description, notes: snapshot, selectedNoteIds: [...selectedNoteIds] }], redoStack: [], }); }, @@ -1381,8 +2198,23 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ // Called many times per drag; auto-apply is deferred to commitNoteEdit() on mouseup. // Sends a throttled real-time preview so the user hears the new pitch while dragging. updateNote: (noteId, changes) => { + const currentNotes = get().notes; + const boundaryAffectingChange = + Object.prototype.hasOwnProperty.call(changes, "startTime") + || Object.prototype.hasOwnProperty.call(changes, "endTime") + || Object.prototype.hasOwnProperty.call(changes, "transitionIn") + || Object.prototype.hasOwnProperty.call(changes, "transitionOut") + || Object.prototype.hasOwnProperty.call(changes, "effectiveStartTime") + || Object.prototype.hasOwnProperty.call(changes, "effectiveEndTime"); + const targetIds = new Set([noteId]); + const updatedNotes = normalizePitchNotes(currentNotes.map(n => n.id === noteId ? normalizePitchNote({ ...n, ...changes }) : n)); + if (boundaryAffectingChange) { + replaceDirtyPitchNotes(expandDirtyPitchNoteIdsWithNeighbors(targetIds, updatedNotes)); + } else { + markDirtyPitchNotes(targetIds); + } set({ - notes: get().notes.map(n => n.id === noteId ? { ...n, ...changes } : n), + notes: updatedNotes, }); if (!_dragPreviewThrottle) { _dragPreviewThrottle = setTimeout(() => { @@ -1394,17 +2226,27 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ // Call once when an interactive drag/resize ends to schedule the debounced backend apply. commitNoteEdit: () => { + get().endInteractivePreview(); + const { undoStack, notes } = get(); + const last = undoStack[undoStack.length - 1]; + if (last && pitchNotesEqual(last.notes, notes)) { + set({ undoStack: undoStack.slice(0, -1) }); + return; + } onNotesChanged(); }, updateSelectedNotes: (changes) => { const { selectedNoteIds, notes } = get(); if (selectedNoteIds.length === 0) return; + const targetIds = new Set(selectedNoteIds); + replaceDirtyPitchNotes(targetIds); get().pushUndo("Edit selected notes"); set({ - notes: notes.map(n => - selectedNoteIds.includes(n.id) ? { ...n, ...changes } : n - ), + notes: normalizePitchNotes(notes.map(n => + targetIds.has(n.id) ? normalizePitchNote({ ...n, ...changes }) : n + )), + selectedNoteIds: [...targetIds], }); onNotesChanged(); }, @@ -1412,13 +2254,16 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ moveSelectedPitch: (semitones) => { const { selectedNoteIds, notes } = get(); if (selectedNoteIds.length === 0) return; + const targetIds = new Set(selectedNoteIds); + replaceDirtyPitchNotes(targetIds); get().pushUndo(`Move pitch ${semitones > 0 ? "up" : "down"}`); set({ - notes: notes.map(n => - selectedNoteIds.includes(n.id) - ? { ...n, correctedPitch: n.correctedPitch + semitones } + notes: normalizePitchNotes(notes.map(n => + targetIds.has(n.id) + ? normalizePitchNote({ ...n, correctedPitch: n.correctedPitch + semitones }) : n - ), + )), + selectedNoteIds: [...targetIds], }); onNotesChanged(); }, @@ -1440,14 +2285,18 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ const note1: PitchNoteData = { ...note, id: note.id + "_a", + wordGroupId: `${getNoteWordGroupId(note)}_${note.id}_a`, endTime: time, + effectiveEndTime: time, pitchDrift: drift.slice(0, splitIdx), transitionOut: 0, }; const note2: PitchNoteData = { ...note, id: note.id + "_b", + wordGroupId: `${getNoteWordGroupId(note)}_${note.id}_b`, startTime: time, + effectiveStartTime: time, pitchDrift: drift.slice(splitIdx), transitionIn: 0, }; @@ -1482,54 +2331,63 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ } set({ - notes: notes.flatMap(n => n.id === noteId ? [note1, note2] : [n]), + notes: normalizePitchNotes(notes.flatMap(n => n.id === noteId ? [note1, note2] : [n])), selectedNoteIds: [note1.id, note2.id], }); + replaceDirtyPitchNotes([note1.id, note2.id]); onNotesChanged(); }, correctSelectedToScale: () => { const { selectedNoteIds, notes } = get(); if (selectedNoteIds.length === 0) return; + const targetIds = new Set(selectedNoteIds); + replaceDirtyPitchNotes(targetIds); get().pushUndo("Correct to scale"); set({ - notes: notes.map(n => { - if (!selectedNoteIds.includes(n.id)) return n; - return { ...n, correctedPitch: Math.round(n.correctedPitch) }; - }), + notes: normalizePitchNotes(notes.map(n => { + if (!targetIds.has(n.id)) return n; + return normalizePitchNote({ ...n, correctedPitch: Math.round(n.correctedPitch) }); + })), + selectedNoteIds: [...targetIds], }); onNotesChanged(); }, correctAllToScale: () => { const { notes } = get(); + replaceDirtyPitchNotes(notes.map((note) => note.id)); get().pushUndo("Correct all to scale"); set({ - notes: notes.map(n => ({ ...n, correctedPitch: Math.round(n.correctedPitch) })), + notes: normalizePitchNotes(notes.map(n => normalizePitchNote({ ...n, correctedPitch: Math.round(n.correctedPitch) }))), }); onNotesChanged(); }, undo: () => { - const { undoStack, notes } = get(); + const { undoStack, notes, selectedNoteIds } = get(); if (undoStack.length === 0) return; const entry = undoStack[undoStack.length - 1]; + replaceDirtyPitchNotes(entry.notes.map((note) => note.id)); set({ undoStack: undoStack.slice(0, -1), - redoStack: [...get().redoStack, { description: entry.description, notes: JSON.parse(JSON.stringify(notes)) }], - notes: entry.notes, + redoStack: [...get().redoStack, { description: entry.description, notes: clonePitchNotes(notes), selectedNoteIds: [...selectedNoteIds] }], + notes: normalizePitchNotes(entry.notes), + selectedNoteIds: [...(entry.selectedNoteIds ?? [])], }); onNotesChanged(); }, redo: () => { - const { redoStack, notes } = get(); + const { redoStack, notes, selectedNoteIds } = get(); if (redoStack.length === 0) return; const entry = redoStack[redoStack.length - 1]; + replaceDirtyPitchNotes(entry.notes.map((note) => note.id)); set({ redoStack: redoStack.slice(0, -1), - undoStack: [...get().undoStack, { description: entry.description, notes: JSON.parse(JSON.stringify(notes)) }], - notes: entry.notes, + undoStack: [...get().undoStack, { description: entry.description, notes: clonePitchNotes(notes), selectedNoteIds: [...selectedNoteIds] }], + notes: normalizePitchNotes(entry.notes), + selectedNoteIds: [...(entry.selectedNoteIds ?? [])], }); onNotesChanged(); }, @@ -1537,10 +2395,10 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ setScrollX: (x) => set({ scrollX: Math.max(0, x) }), setScrollY: (y) => set({ scrollY: Math.max(0, Math.min(120, y)) }), setZoomX: (z) => set({ zoomX: Math.max(50, Math.min(2000, z)) }), - setZoomY: (z) => set({ zoomY: Math.max(4, Math.min(40, z)) }), + setZoomY: (z) => set({ zoomY: Math.max(4, Math.min(80, z)) }), applyCorrection: async () => { - const { trackId, clipId, notes, contour, globalFormantCents } = get(); + const { trackId, clipId, notes, contour, globalFormantCents, clipDuration } = get(); if (!trackId || !clipId) return; const globalFormantSt = globalFormantCents / 100; const summary = summarizeApplyRequest(notes, globalFormantSt); @@ -1549,8 +2407,10 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ const logicalRequestId = buildLogicalApplyRequestId(clipId); logPitchEditorFormant("manual apply requested", { clipId, logicalRequestId, requestRevision, ...summary }); - if (hasRequestedFormantWork(notes, globalFormantSt)) { - dispatchStagedFormantApply( + if (summary.mode === "none") { + setRenderCoverage([], null); + nativeBridge.clearClipRenderedPreviewSegments(clipId).catch(logBridgeError("clearClipRenderedPreviewSegments")); + dispatchSingleApplyRequest( trackId, clipId, notes, @@ -1563,15 +2423,19 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ return; } - dispatchSingleApplyRequest( + const editedNoteWindow = buildEditedNoteWindow(notes, clipDuration || 0); + dispatchNoteHqApplyRequest( trackId, clipId, - notes, + editedNoteWindow?.requestNotes ?? notes, contour?.frames, globalFormantSt, requestRevision, summary, logicalRequestId, + editedNoteWindow?.windowStartSec, + editedNoteWindow?.windowEndSec, + editedNoteWindow?.coverageRanges, ); }, @@ -1657,6 +2521,13 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ toggleCorrectPitchModal: () => set({ showCorrectPitchModal: !get().showCorrectPitchModal }), setGlobalFormantCents: (cents) => { + if (!PITCH_EDITOR_FORMANT_EDITING_ENABLED) { + logPitchEditorFormant("ignored global formant change because pitch editor is in pitch-only rebuild mode", { + clipId: get().clipId, + requestedGlobalFormantCents: Math.round(cents), + }); + return; + } const clamped = Math.max(-386, Math.min(386, Math.round(cents))); set({ globalFormantCents: clamped, @@ -1675,33 +2546,46 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ // Inspector editing actions setNoteFormant: (noteId, semitones) => { + if (!PITCH_EDITOR_FORMANT_EDITING_ENABLED) { + logPitchEditorFormant("ignored note formant change because pitch editor is in pitch-only rebuild mode", { + clipId: get().clipId, + noteId, + requestedSemitones: semitones, + }); + return; + } get().pushUndo("Change formant"); + replaceDirtyPitchNotes([noteId]); const clamped = Math.max(-3.86, Math.min(3.86, semitones)); - set({ notes: get().notes.map(n => n.id === noteId ? { ...n, formantShift: clamped } : n) }); + set({ notes: normalizePitchNotes(get().notes.map(n => n.id === noteId ? normalizePitchNote({ ...n, formantShift: clamped }) : n)) }); onNotesChanged(); }, setNoteGain: (noteId, dB) => { get().pushUndo("Change gain"); - set({ notes: get().notes.map(n => n.id === noteId ? { ...n, gain: dB } : n) }); + replaceDirtyPitchNotes([noteId]); + set({ notes: normalizePitchNotes(get().notes.map(n => n.id === noteId ? normalizePitchNote({ ...n, gain: dB }) : n)) }); onNotesChanged(); }, setNoteModulation: (noteId, percent) => { get().pushUndo("Change modulation"); - set({ notes: get().notes.map(n => n.id === noteId ? { ...n, vibratoDepth: percent / 100 } : n) }); + replaceDirtyPitchNotes([noteId]); + set({ notes: normalizePitchNotes(get().notes.map(n => n.id === noteId ? normalizePitchNote({ ...n, vibratoDepth: percent / 100 }) : n)) }); onNotesChanged(); }, setNoteDrift: (noteId, percent) => { get().pushUndo("Change drift"); - set({ notes: get().notes.map(n => n.id === noteId ? { ...n, driftCorrectionAmount: percent / 100 } : n) }); + replaceDirtyPitchNotes([noteId]); + set({ notes: normalizePitchNotes(get().notes.map(n => n.id === noteId ? normalizePitchNote({ ...n, driftCorrectionAmount: percent / 100 }) : n)) }); onNotesChanged(); }, setNoteTransition: (noteId, inMs, outMs) => { get().pushUndo("Change transition"); - set({ notes: get().notes.map(n => n.id === noteId ? { ...n, transitionIn: inMs, transitionOut: outMs } : n) }); + replaceDirtyPitchNotes([noteId]); + set({ notes: normalizePitchNotes(get().notes.map(n => n.id === noteId ? normalizePitchNote({ ...n, transitionIn: inMs, transitionOut: outMs }) : n)) }); onNotesChanged(); }, @@ -1710,8 +2594,9 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ const { notes, scaleNotes } = get(); if (notes.length === 0) return; get().pushUndo("Correct pitch macro"); + replaceDirtyPitchNotes(notes.map((note) => note.id)); set({ - notes: notes.map(n => { + notes: normalizePitchNotes(notes.map(n => { // Find nearest target (semitone or scale degree) let nearestTarget: number; if (useScale) { @@ -1740,12 +2625,12 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ correctedPitch = n.detectedPitch - offset * pitchCenter; } - return { + return normalizePitchNote({ ...n, correctedPitch, driftCorrectionAmount: pitchDriftAmount, - }; - }), + }); + })), }); onNotesChanged(); }, @@ -1772,6 +2657,8 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ ...toMerge[0], id: toMerge[0].id + "_merged", endTime: toMerge[toMerge.length - 1].endTime, + effectiveStartTime: toMerge[0].effectiveStartTime ?? toMerge[0].startTime, + effectiveEndTime: toMerge[toMerge.length - 1].effectiveEndTime ?? toMerge[toMerge.length - 1].endTime, detectedPitch: weightedPitch / totalDuration, correctedPitch: weightedPitch / totalDuration, pitchDrift: allDrift, @@ -1781,7 +2668,7 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ const mergeIds = new Set(noteIds); set({ - notes: [...notes.filter(n => !mergeIds.has(n.id)), merged].sort((a, b) => a.startTime - b.startTime), + notes: normalizePitchNotes([...notes.filter(n => !mergeIds.has(n.id)), merged]), selectedNoteIds: [merged.id], }); onNotesChanged(); @@ -1793,12 +2680,18 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ }, drawPitchOnNote: (noteId, clipTime, midiPitch) => { - const { notes } = get(); + const { notes, contour } = get(); const note = notes.find(n => n.id === noteId); if (!note) return; const noteDuration = note.endTime - note.startTime; if (noteDuration <= 0) return; + if (!isVoicedAtClipTime(contour, clipTime)) return; + + const effectiveStart = note.effectiveStartTime ?? note.startTime; + const effectiveEnd = note.effectiveEndTime ?? note.endTime; + if (clipTime < effectiveStart || clipTime > effectiveEnd) return; + const clampedClipTime = Math.max(note.startTime, Math.min(note.endTime, clipTime)); // Ensure pitchDrift array has adequate resolution (at least 100 points per note) const minLen = Math.max(100, Math.ceil(noteDuration * 100)); // ~100 points/sec @@ -1820,7 +2713,7 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ } // Map clipTime to index in drift array - const t = (clipTime - note.startTime) / noteDuration; + const t = (clampedClipTime - note.startTime) / noteDuration; if (t < 0 || t > 1) return; const idx = Math.round(t * (drift.length - 1)); @@ -1837,152 +2730,18 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ } set({ - notes: notes.map(n => n.id === noteId ? { ...n, pitchDrift: drift } : n), + notes: normalizePitchNotes(notes.map(n => n.id === noteId ? normalizePitchNote({ ...n, pitchDrift: drift }) : n)), }); }, commitDrawPitch: () => { - onNotesChanged(); - }, - - // Polyphonic mode actions - togglePolyMode: () => set({ polyMode: !get().polyMode }), - togglePitchSalience: () => set({ showPitchSalience: !get().showPitchSalience }), - - analyzePolyphonic: async () => { - const { trackId, clipId, polyNoteThreshold, polyOnsetThreshold, polyMinDuration } = get(); - if (!trackId || !clipId) return; - set({ isAnalyzing: true }); - try { - const result = await nativeBridge.analyzePolyphonic(trackId, clipId, { - noteThreshold: polyNoteThreshold, - onsetThreshold: polyOnsetThreshold, - minDurationMs: polyMinDuration, - }); - if (result && !result.error) { - const polyNotes: PolyNoteData[] = result.notes.map(n => ({ - ...n, - correctedPitch: n.correctedPitch ?? n.midiPitch, - formantShift: n.formantShift ?? 0, - gain: n.gain ?? 0, - })); - set({ - polyAnalysisResult: result, - polyNotes, - polyMode: true, - selectedNoteIds: [], - }); - // Auto-fit viewport to content - if (polyNotes.length > 0) { - let minMidi = 127, maxMidi = 0; - for (const n of polyNotes) { - minMidi = Math.min(minMidi, n.midiPitch - 2); - maxMidi = Math.max(maxMidi, n.midiPitch + 2); - } - set({ scrollY: minMidi }); - } - } else if (result?.error) { - console.error("Polyphonic analysis error:", result.error); - } - } finally { - set({ isAnalyzing: false }); - } - }, - - updatePolyNote: (noteId, changes) => { - set({ - polyNotes: get().polyNotes.map(n => n.id === noteId ? { ...n, ...changes } : n), - }); - }, - - movePolyNotePitch: (noteId, semitones) => { - set({ - polyNotes: get().polyNotes.map(n => - n.id === noteId ? { ...n, correctedPitch: n.correctedPitch + semitones } : n - ), - }); - }, - - moveSelectedPolyPitch: (semitones) => { - const { selectedNoteIds, polyNotes } = get(); - if (selectedNoteIds.length === 0) return; - set({ - polyNotes: polyNotes.map(n => - selectedNoteIds.includes(n.id) - ? { ...n, correctedPitch: n.correctedPitch + semitones } - : n - ), - }); - }, - - soloPolyNote: (noteId) => set({ soloNoteId: noteId }), - - applyPolyCorrection: async () => { - const { trackId, clipId, polyNotes } = get(); - if (!trackId || !clipId) return; - set({ isApplying: true }); - try { - const editedNotes = polyNotes - .filter(n => Math.abs(n.correctedPitch - n.midiPitch) > 0.05 || Math.abs(n.gain) > 0.1) - .map(n => ({ - id: n.id, - originalPitch: n.midiPitch, - correctedPitch: n.correctedPitch, - formantShift: n.formantShift, - gain: n.gain, - })); - await nativeBridge.applyPolyPitchCorrection(trackId, clipId, editedNotes); - } finally { - set({ isApplying: false }); - } - }, - - // Poly detection tuning - setPolyNoteThreshold: (v) => set({ polyNoteThreshold: Math.max(0.01, Math.min(1, v)) }), - setPolyOnsetThreshold: (v) => set({ polyOnsetThreshold: Math.max(0.01, Math.min(1, v)) }), - setPolyMinDuration: (v) => set({ polyMinDuration: Math.max(10, Math.min(500, v)) }), - - // Auto-detect mono vs poly — heuristic based on mono analysis quality - autoDetectMonoPoly: () => { - const { contour, notes } = get(); - if (!contour || notes.length === 0) return; - - const frames = contour.frames; - if (!frames || frames.confidence.length === 0) return; - - // Heuristic 1: average confidence — mono YIN struggles with polyphonic content - let totalConf = 0; - let voicedCount = 0; - for (let i = 0; i < frames.confidence.length; i++) { - if (frames.midi[i] > 0) { - totalConf += frames.confidence[i]; - voicedCount++; - } - } - const avgConf = voicedCount > 0 ? totalConf / voicedCount : 1; - - // Heuristic 2: note fragmentation — poly content produces many short notes - const avgDuration = notes.reduce((s, n) => s + (n.endTime - n.startTime), 0) / notes.length; - - // Heuristic 3: pitch instability — large jumps between adjacent notes - let jumpCount = 0; - for (let i = 1; i < notes.length; i++) { - const gap = notes[i].startTime - notes[i - 1].endTime; - const pitchDiff = Math.abs(notes[i].detectedPitch - notes[i - 1].detectedPitch); - if (gap < 0.1 && pitchDiff > 7) jumpCount++; - } - const jumpRate = notes.length > 1 ? jumpCount / (notes.length - 1) : 0; - - // Score: higher = more likely polyphonic - const isLikelyPoly = avgConf < 0.5 || avgDuration < 0.12 || jumpRate > 0.3; - - if (isLikelyPoly) { - console.log(`[PitchEditor] Auto-detect: likely polyphonic (avgConf=${avgConf.toFixed(2)}, avgDur=${avgDuration.toFixed(2)}s, jumpRate=${jumpRate.toFixed(2)})`); - set({ polyMode: true }); - } else { - console.log(`[PitchEditor] Auto-detect: likely monophonic (avgConf=${avgConf.toFixed(2)}, avgDur=${avgDuration.toFixed(2)}s, jumpRate=${jumpRate.toFixed(2)})`); - set({ polyMode: false }); + const { undoStack, notes } = get(); + const last = undoStack[undoStack.length - 1]; + if (last && pitchNotesEqual(last.notes, notes)) { + set({ undoStack: undoStack.slice(0, -1) }); + return; } + onNotesChanged(); }, // A/B comparison @@ -1995,27 +2754,9 @@ export const usePitchEditorStore = create<PitchEditorState>()((set, get) => ({ nativeBridge.setPitchCorrectionBypass(trackId, clipId, newBypass); } }, - - // Unified note accessor - getUnifiedNotes: () => { - const { polyMode, notes, polyNotes } = get(); - if (polyMode) { - return polyNotes.map(polyToUnified); - } - return notes.map(n => monoToUnified(n)); - }, })); -nativeBridge.onPitchCorrectionComplete((data: { - clipId: string; - success: boolean; - outputFile?: string; - requestId?: string; - restored?: boolean; - renderMode?: PitchCorrectionRenderMode; - cancelled?: boolean; - swapDeferred?: boolean; -}) => { +nativeBridge.onPitchCorrectionComplete((data: PitchCorrectionCompletionData) => { const state = usePitchEditorStore.getState(); const meta = consumeRequestMeta(data.requestId); const completedRevision = consumeRequestRevision(data.requestId) ?? meta?.revision ?? null; @@ -2048,6 +2789,9 @@ nativeBridge.onPitchCorrectionComplete((data: { }); if (!currentClipMatches) { + if (meta.stage === "note_hq") { + finishNoteHqPriority(meta.logicalRequestId); + } return; } @@ -2058,21 +2802,30 @@ nativeBridge.onPitchCorrectionComplete((data: { logicalRequestId: meta.logicalRequestId, stage: meta.stage, }); + if (meta.stage === "note_hq") { + finishNoteHqPriority(meta.logicalRequestId); + } return; } if (!data.success || !data.outputFile) { - if (meta.stage === "preview_segment" || meta.stage === "single") { + const failureMessage = getPitchCorrectionFailureMessage(data); + if (meta.stage === "preview_segment" || meta.stage === "single" || meta.stage === "note_hq") { if (meta.stage === "preview_segment") { _runningPreviewSegmentJobs = Math.max(0, _runningPreviewSegmentJobs - 1); } - _activePitchCorrectionRequestId = null; - setApplyStatus("error", "Failed", meta.logicalRequestId); - useDAWStore.getState().showToast("Pitch/formant apply failed", "error"); + if (meta.stage === "note_hq") { + finishNoteHqPriority(meta.logicalRequestId); + } + if (_activePitchCorrectionRequestId === meta.logicalRequestId) { + _activePitchCorrectionRequestId = null; + } + setApplyStatus("error", failureMessage, meta.logicalRequestId); + useDAWStore.getState().showToast(failureMessage, "error"); } else if (meta.stage === "full_clip_hq") { _activePitchCorrectionRequestId = null; - setApplyStatus("error", "Preview ready, HQ failed", meta.logicalRequestId); - useDAWStore.getState().showToast("Pitch/formant HQ render failed", "error"); + setApplyStatus("error", failureMessage, meta.logicalRequestId); + useDAWStore.getState().showToast(failureMessage, "error"); } warnPitchEditorFormant("persistent completion listener received failed result", { clipId: data.clipId, @@ -2080,6 +2833,12 @@ nativeBridge.onPitchCorrectionComplete((data: { logicalRequestId: meta.logicalRequestId, requestRevision: completedRevision, stage: meta.stage, + hardFailReason: data.hardFailReason ?? null, + fallbackReason: data.fallbackReason ?? null, + pitchRenderStrategy: data.pitchRenderStrategy ?? null, + pitchRenderProductPath: data.pitchRenderProductPath ?? null, + pitchRenderBackendId: data.pitchRenderBackendId ?? null, + pitchRenderBackendFailureCode: data.pitchRenderBackendFailureCode ?? null, }); return; } @@ -2094,6 +2853,12 @@ nativeBridge.onPitchCorrectionComplete((data: { requestedApplyRevision: _requestedApplyRevision, stage: meta.stage, }); + if (meta.stage === "note_hq") { + finishNoteHqPriority(meta.logicalRequestId); + if (_activePitchCorrectionRequestId === meta.logicalRequestId) { + _activePitchCorrectionRequestId = null; + } + } return; } @@ -2112,19 +2877,59 @@ nativeBridge.onPitchCorrectionComplete((data: { updateRenderCoverageRange(meta.windowStartSec, meta.windowEndSec, "preview_ready"); } const coverageDone = usePitchEditorStore.getState().renderCoverage.every((range) => range.state !== "pending"); - setApplyStatus("preview_ready", getFormantPreviewStatusMessage("preview_ready", coverageDone), meta.logicalRequestId); + setApplyStatus("preview_ready", getRenderedPreviewStatusMessage("preview_ready", coverageDone), meta.logicalRequestId); if (_activePreviewRequestGroupId === meta.logicalRequestId) { dispatchNextPreviewSegments(); } return; } + if (meta.stage === "note_hq") { + clearTransientPitchPreview(data.clipId, "note_hq_complete"); + clearStagedPreviewQueue(); + clearRenderedPreviewForInteractiveEdit(data.clipId, "note_hq_complete"); + if (data.outputFile) { + applyPitchCorrectionResultToClip(data.clipId, data.outputFile, Boolean(data.restored)); + } else { + warnPitchEditorFormant("note-HQ completion did not include an output file", { + clipId: data.clipId, + requestId: data.requestId, + logicalRequestId: meta.logicalRequestId, + }); + } + if (meta.coverageRanges && meta.coverageRanges.length > 0) { + for (const range of meta.coverageRanges) { + updateRenderCoverageRange(range.startTime, range.endTime, "hq_ready"); + } + } else if (meta.windowStartSec !== undefined && meta.windowEndSec !== undefined) { + updateRenderCoverageRange(meta.windowStartSec, meta.windowEndSec, "hq_ready"); + } + _dirtyPitchNoteIds.clear(); + finishNoteHqPriority(meta.logicalRequestId); + _activePitchCorrectionRequestId = null; + setApplyStatus("final_processing", "Verifying HQ note route...", meta.logicalRequestId); + scheduleNoteHqFinalRouteVerification(meta, data); + logPitchEditorFormant("note-local HQ cache updated", { + clipId: data.clipId, + requestId: data.requestId, + logicalRequestId: meta.logicalRequestId, + outputFile: data.outputFile, + appFinalBakedContextPath: data.appFinalBakedContextPath, + appFinalPlaybackContextPath: data.appFinalPlaybackContextPath, + appFinalParityReportPath: data.appFinalParityReportPath, + windowStartSec: meta.windowStartSec, + windowEndSec: meta.windowEndSec, + }); + return; + } + applyPitchCorrectionResultToClip(data.clipId, data.outputFile, Boolean(data.restored)); + _dirtyPitchNoteIds.clear(); markAllRenderCoverage("hq_ready", meta.logicalRequestId); clearStagedPreviewQueue(); _activePitchCorrectionRequestId = null; setApplyStatus("done", meta.stage === "full_clip_hq" - ? (data.swapDeferred ? "HQ ready on stop/seek" : getFormantPreviewStatusMessage("done", true)) + ? (data.swapDeferred ? "HQ ready on stop/seek" : getRenderedPreviewStatusMessage("done", true)) : "Applied", meta.logicalRequestId, { autoClearMs: 1800 }); useDAWStore.getState().showToast( meta.stage === "full_clip_hq" ? "Pitch/formant HQ render ready" : "Pitch/formant changes applied", diff --git a/frontend/src/store/useDAWStore.ts b/frontend/src/store/useDAWStore.ts index ce286dc..02235be 100644 --- a/frontend/src/store/useDAWStore.ts +++ b/frontend/src/store/useDAWStore.ts @@ -24,6 +24,11 @@ import { screensetActions } from "./actions/screensets"; import { macroActions } from "./actions/macros"; import { renderQueueActions } from "./actions/renderQueue"; import { quantizeActions } from "./actions/quantize"; +import { getDefaultWorkflowParams, normalizeWorkflowParams } from "../data/aiWorkflows"; + +export interface InstallAiToolsOptions { + userConfirmedDownload?: boolean; +} // Module-level helpers moved to store/actions/: _editSnapshots → tracks.ts, @@ -74,6 +79,45 @@ function getStoredString(key: string, fallback = ""): string { } } +function isMusicGenerationFullyReady( + status: Pick< + AiToolsStatus, + | "musicGenerationReady" + | "musicGenerationLayoutValid" + | "musicGenerationPerformanceReady" + | "musicGenerationAvailableProfiles" + >, +): boolean { + const availableProfiles = status.musicGenerationAvailableProfiles ?? []; + const nativeProfileReady = + availableProfiles.length === 0 || availableProfiles.includes("native-xl-turbo"); + return Boolean( + status.musicGenerationReady + && status.musicGenerationLayoutValid + && (status.musicGenerationPerformanceReady ?? true) + && nativeProfileReady + ); +} + +function getAiToolsReadyMessage(status: AiToolsStatus): string { + if (isMusicGenerationFullyReady(status)) { + return status.message || "AI tools are ready."; + } + + if (status.musicGenerationReady && status.musicGenerationLayoutValid && status.musicGenerationPerformanceStatusMessage) { + return status.musicGenerationPerformanceStatusMessage; + } + + if ((status.musicGenerationAvailableProfiles ?? []).length > 0 + && !(status.musicGenerationAvailableProfiles ?? []).includes("native-xl-turbo")) + { + return "Stem separation is ready, but the OpenStudio ACE split profile is still missing required music-generation assets."; + } + + return status.musicGenerationStatusMessage + || "Stem separation is ready, but music generation still needs the OpenStudio ACE split backend."; +} + const DEFAULT_AI_TOOLS_STATUS: AiToolsStatus = { state: "checking", progress: 0, @@ -98,6 +142,8 @@ const DEFAULT_AI_TOOLS_STATUS: AiToolsStatus = { activityLines: [], supportedBackends: ["cpu"], selectedBackend: "cpu", + musicGenerationPerformanceReady: true, + musicGenerationPerformanceStatusMessage: "", lastPhase: "checking", fallbackAttempted: false, restartRequired: false, @@ -107,8 +153,9 @@ const DEFAULT_AI_TOOLS_STATUS: AiToolsStatus = { // Type Definitions // ============================================ -export type TrackType = "audio" | "midi" | "instrument" | "bus"; +export type TrackType = "audio" | "midi" | "instrument" | "bus" | "ai"; export type InputType = "mono" | "stereo" | "midi"; +export type AITrackGenerationState = "idle" | "loading" | "generating" | "error"; // MIDI Event structure export interface MIDIEvent { @@ -191,6 +238,7 @@ export function getEffectiveTrackHeight( function getTrackHeaderControlWidth(track: Pick<Track, "type">): number { const commonWidth = 30 + 20 + 126 + 50 + 50 + 50; if (track.type === "audio") return commonWidth + 96; + if (track.type === "ai") return commonWidth + 128; if (track.type === "instrument") return commonWidth + 92; if (track.type === "midi") return commonWidth + 74; return commonWidth; @@ -444,6 +492,38 @@ export interface Track { notes?: string; // Free-form track notes/comments waveformZoom?: number; // Waveform vertical zoom factor (0.1 to 5.0, default 1.0) spectralView?: boolean; // Show spectrogram instead of waveform + aiWorkflow?: string; + aiWorkflowParams?: Record<string, unknown>; + aiGenerationState?: AITrackGenerationState; + aiGenerationProgress?: number; + aiGenerationError?: string; + aiGenerationPhase?: string; + aiGenerationMessage?: string; + aiGenerationBackend?: string; + aiGenerationElapsedMs?: number; + aiGenerationHeartbeatTs?: number; + aiGenerationPhaseProgress?: number; + aiGenerationEtaMs?: number; + aiGenerationRunMode?: "cold" | "warm"; + aiGenerationRuntimeProfile?: string; + aiGenerationLmModel?: string; + aiGenerationStatusNote?: string; + aiGenerationFailureKind?: string; + aiGenerationSessionMode?: "persistent" | "oneshot" | "oneshot-fallback" | string; + aiGenerationWorkerExitCode?: number; + aiGenerationLastStdoutLine?: string; + aiGenerationLastStderrLine?: string; + aiGenerationAttemptMode?: "lm_dit" | "dit_only" | "dit_only_retry" | string; + aiGenerationAttemptIndex?: number; + aiGenerationProtocolVersion?: number; + aiGenerationScriptVersion?: string; + aiGenerationRequestId?: string; + aiGenerationPriorFailure?: string; + aiGenerationLastProgressAgeMs?: number; + aiGenerationTracePath?: string; + aiGenerationFailureDetail?: string; + aiGenerationLmBackend?: string; + aiGenerationLmStage?: string; // Folder tracks isFolder?: boolean; // True if this track is a folder track @@ -532,6 +612,31 @@ export interface RenderJob { error?: string; } +export type RenderSource = "master" | "selected_tracks" | "stems" | "selected_items" | "selected_items_master" | "razor"; +export type RenderBounds = "entire" | "custom" | "time_selection" | "project_regions" | "selected_regions"; +export type AudioFormat = "wav" | "aiff" | "flac" | "mp3" | "ogg" | "raw"; +export type SampleRate = 44100 | 48000 | 88200 | 96000 | 192000; +export type BitDepth = 16 | 24 | 32; + +export interface RenderDialogOptions { + source: RenderSource; + bounds: RenderBounds; + startTime: number; + endTime: number; + tailLength: number; + addTail: boolean; + directory: string; + fileName: string; + format: AudioFormat; + sampleRate: SampleRate; + bitDepth: BitDepth; + channels: "stereo" | "mono"; + normalize: boolean; + dither: boolean; + mp3Bitrate: number; + oggQuality: number; +} + // ============================================ // Mixer Snapshot // ============================================ @@ -810,6 +915,8 @@ interface DAWState { description: string; isrc: string; }; + renderDialogOptions: RenderDialogOptions; + lastRenderDirectory: string; secondaryOutputEnabled: boolean; secondaryOutputFormat: string; secondaryOutputBitDepth: number; @@ -1019,6 +1126,52 @@ interface DAWActions { // Track Notes setTrackNotes: (trackId: string, notes: string) => void; + setAITrackWorkflow: (trackId: string, workflowId: string) => void; + setAITrackParams: ( + trackId: string, + params: Record<string, unknown>, + ) => void; + setAITrackGenerationState: ( + trackId: string, + generationState: AITrackGenerationState, + updates?: { + progress?: number; + error?: string; + phase?: string; + message?: string; + backend?: string; + elapsedMs?: number; + heartbeatTs?: number; + phaseProgress?: number; + etaMs?: number; + runMode?: "cold" | "warm"; + runtimeProfile?: string; + lmModel?: string; + statusNote?: string; + failureKind?: string; + sessionMode?: "persistent" | "oneshot" | "oneshot-fallback" | string; + workerExitCode?: number; + lastStdoutLine?: string; + lastStderrLine?: string; + attemptMode?: "lm_dit" | "dit_only" | "dit_only_retry" | string; + attemptIndex?: number; + protocolVersion?: number; + scriptVersion?: string; + requestId?: string; + priorFailure?: string; + lastProgressAgeMs?: number; + tracePath?: string; + failureDetail?: string; + lmBackend?: string; + lmStage?: string; + }, + ) => void; + addGeneratedAudioClip: ( + trackId: string, + filePath: string, + startTime: number, + clipName?: string, + ) => Promise<void>; // Track Audio Controls setTrackVolume: (id: string, volumeDB: number) => Promise<void>; @@ -1266,7 +1419,8 @@ interface DAWActions { reopenStemSeparation: () => void; refreshAiToolsStatus: (force?: boolean) => Promise<AiToolsStatus>; applyAiToolsStatusUpdate: (status: AiToolsStatus) => void; - installAiTools: () => Promise<void>; + installAiTools: (options?: InstallAiToolsOptions) => Promise<void>; + resetAiTools: () => Promise<void>; cancelAiToolsInstall: () => Promise<void>; completeStemSeparation: (sourceTrackId: string, sourceClipId: string, clipName: string, stemFiles: Array<{ name: string; filePath: string; duration?: number; sampleRate?: number }>, @@ -1384,6 +1538,9 @@ interface DAWActions { selectRegion: (id: string, modifiers?: { ctrl?: boolean }) => void; deselectAllRegions: () => void; setRenderMetadata: (metadata: Partial<DAWState["renderMetadata"]>) => void; + setRenderDialogOptions: (options: Partial<RenderDialogOptions>) => void; + resetRenderDialogOptions: () => void; + setLastRenderDirectory: (dir: string) => void; setSecondaryOutputEnabled: (enabled: boolean) => void; setSecondaryOutputFormat: (format: string) => void; setSecondaryOutputBitDepth: (bitDepth: number) => void; @@ -1645,6 +1802,34 @@ export const createDefaultTrack = ( playbackOffsetMs: 0, trackChannelCount: 2, midiOutputDevice: "", + aiWorkflow: "text-to-music", + aiWorkflowParams: getDefaultWorkflowParams("text-to-music"), + aiGenerationState: "idle", + aiGenerationProgress: 0, + aiGenerationError: "", + aiGenerationPhase: "", + aiGenerationMessage: "", + aiGenerationBackend: "", + aiGenerationElapsedMs: 0, + aiGenerationHeartbeatTs: 0, + aiGenerationPhaseProgress: undefined, + aiGenerationEtaMs: undefined, + aiGenerationRunMode: undefined, + aiGenerationRuntimeProfile: "", + aiGenerationLmModel: "", + aiGenerationStatusNote: "", + aiGenerationFailureKind: "", + aiGenerationSessionMode: "", + aiGenerationWorkerExitCode: 0, + aiGenerationLastStdoutLine: "", + aiGenerationLastStderrLine: "", + aiGenerationAttemptMode: "", + aiGenerationAttemptIndex: 0, + aiGenerationProtocolVersion: 0, + aiGenerationScriptVersion: "", + aiGenerationRequestId: "", + aiGenerationPriorFailure: "", + aiGenerationLastProgressAgeMs: 0, }); export const initialTransport: TransportState = { @@ -1680,6 +1865,27 @@ export function createDefaultRenderMetadata() { }; } +export function createDefaultRenderDialogOptions(): RenderDialogOptions { + return { + source: "master", + bounds: "entire", + startTime: 0, + endTime: 0, + tailLength: 1000, + addTail: true, + directory: "", + fileName: "$project", + format: "wav", + sampleRate: 44100, + bitDepth: 24, + channels: "stereo", + normalize: false, + dither: false, + mp3Bitrate: 320, + oggQuality: 6, + }; +} + export function createDefaultClipLauncherState() { return { slots: [], @@ -1747,6 +1953,8 @@ export function createFreshProjectDocumentState(): Partial<DAWState> { canRedo: false, trackGroups: [], renderMetadata: createDefaultRenderMetadata(), + renderDialogOptions: createDefaultRenderDialogOptions(), + lastRenderDirectory: "", secondaryOutputEnabled: false, secondaryOutputFormat: "mp3", secondaryOutputBitDepth: 16, @@ -2086,6 +2294,8 @@ export const useDAWStore = create<DAWState & DAWActions>()( // Phase 10: Render Pipeline Expansion selectedRegionIds: [], renderMetadata: createDefaultRenderMetadata(), + renderDialogOptions: createDefaultRenderDialogOptions(), + lastRenderDirectory: "", secondaryOutputEnabled: false, secondaryOutputFormat: "mp3", secondaryOutputBitDepth: 16, @@ -2244,7 +2454,7 @@ export const useDAWStore = create<DAWState & DAWActions>()( state: "ready" as const, installInProgress: false, available: true, - message: nextStatus.message || "AI tools are ready.", + message: getAiToolsReadyMessage(nextStatus), error: undefined, errorCode: undefined, terminalReason: undefined, @@ -2307,7 +2517,7 @@ export const useDAWStore = create<DAWState & DAWActions>()( terminalReason: undefined, statusWarning: undefined, statusWarningCode: undefined, - message: status.message || "AI tools are ready.", + message: getAiToolsReadyMessage({ ...currentStatus, ...status }), } : { ...currentStatus, @@ -2320,15 +2530,28 @@ export const useDAWStore = create<DAWState & DAWActions>()( aiToolsStatusLastUpdatedAt: Date.now(), }); }, - installAiTools: async () => { + installAiTools: async (options = {}) => { const currentStatus = get().aiToolsStatus; - if (currentStatus.installInProgress || currentStatus.available) return; + const aiToolsFullyReady = currentStatus.available && isMusicGenerationFullyReady(currentStatus); + if (currentStatus.installInProgress || aiToolsFullyReady) return; + + if (currentStatus.statusWarningCode === "reconciling_install_state") { + get().openAiToolsSetup(); + get().showToast("OpenStudio is still confirming the previous AI tools install result.", "info"); + return; + } if (currentStatus.state === "pythonMissing") { get().openAiToolsSetup(); return; } + if (!options.userConfirmedDownload) { + get().openAiToolsSetup(); + get().showToast("Confirm the AI Tools download before setup starts.", "info"); + return; + } + get().showToast("AI tools are being installed in the background.", "info"); set({ @@ -2361,7 +2584,7 @@ export const useDAWStore = create<DAWState & DAWActions>()( }); try { - const result = await nativeBridge.installAiTools(); + const result = await nativeBridge.installAiTools({ userConfirmedDownload: true }); if (result.status) { get().applyAiToolsStatusUpdate(result.status); } else { @@ -2399,6 +2622,30 @@ export const useDAWStore = create<DAWState & DAWActions>()( get().showToast(message, "error"); } }, + resetAiTools: async () => { + const currentStatus = get().aiToolsStatus; + if (currentStatus.installInProgress) { + await nativeBridge.cancelAiToolsInstall(); + } + + try { + const result = await nativeBridge.resetAiTools(); + if (result.status) { + get().applyAiToolsStatusUpdate(result.status); + } else { + await get().refreshAiToolsStatus(true); + } + + if (result.success) { + get().showToast(result.message || "AI tools data was removed.", "success"); + } else if (result.error) { + get().showToast(result.error, "error"); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + get().showToast(message, "error"); + } + }, cancelAiToolsInstall: async () => { await nativeBridge.cancelAiToolsInstall(); await get().refreshAiToolsStatus(true); @@ -3562,6 +3809,281 @@ export const useDAWStore = create<DAWState & DAWActions>()( }); }, + setAITrackWorkflow: (trackId, workflowId) => { + const state = get(); + const track = state.tracks.find((entry) => entry.id === trackId); + if (!track || track.type !== "ai") return; + + const previousWorkflow = track.aiWorkflow ?? "text-to-music"; + const previousParams = { ...(track.aiWorkflowParams ?? {}) }; + const nextParams = getDefaultWorkflowParams(workflowId); + + set((store) => ({ + tracks: store.tracks.map((entry) => + entry.id === trackId + ? { + ...entry, + aiWorkflow: workflowId, + aiWorkflowParams: nextParams, + } + : entry, + ), + })); + + commandManager.push({ + type: "UPDATE_TRACK", + description: `Set AI workflow on "${track.name}"`, + timestamp: Date.now(), + execute: () => { + set((store) => ({ + tracks: store.tracks.map((entry) => + entry.id === trackId + ? { + ...entry, + aiWorkflow: workflowId, + aiWorkflowParams: nextParams, + } + : entry, + ), + })); + }, + undo: () => { + set((store) => ({ + tracks: store.tracks.map((entry) => + entry.id === trackId + ? { + ...entry, + aiWorkflow: previousWorkflow, + aiWorkflowParams: previousParams, + } + : entry, + ), + })); + }, + }); + set({ canUndo: commandManager.canUndo(), canRedo: commandManager.canRedo() }); + }, + + setAITrackParams: (trackId, params) => { + const state = get(); + const track = state.tracks.find((entry) => entry.id === trackId); + if (!track || track.type !== "ai") return; + + const previousParams = { ...(track.aiWorkflowParams ?? {}) }; + const nextParams = normalizeWorkflowParams( + track.aiWorkflow ?? "text-to-music", + params, + ); + + set((store) => ({ + tracks: store.tracks.map((entry) => + entry.id === trackId + ? { ...entry, aiWorkflowParams: nextParams } + : entry, + ), + })); + + commandManager.push({ + type: "UPDATE_TRACK", + description: `Update AI parameters on "${track.name}"`, + timestamp: Date.now(), + execute: () => { + set((store) => ({ + tracks: store.tracks.map((entry) => + entry.id === trackId + ? { ...entry, aiWorkflowParams: nextParams } + : entry, + ), + })); + }, + undo: () => { + set((store) => ({ + tracks: store.tracks.map((entry) => + entry.id === trackId + ? { ...entry, aiWorkflowParams: previousParams } + : entry, + ), + })); + }, + }); + set({ canUndo: commandManager.canUndo(), canRedo: commandManager.canRedo() }); + }, + + setAITrackGenerationState: (trackId, generationState, updates = {}) => { + set((state) => ({ + tracks: state.tracks.map((entry) => + entry.id === trackId + ? { + ...entry, + aiGenerationState: generationState, + aiGenerationProgress: + updates.progress + ?? (generationState === "idle" + ? 0 + : entry.aiGenerationProgress ?? 0), + aiGenerationError: + updates.error + ?? (generationState === "idle" ? "" : entry.aiGenerationError ?? ""), + aiGenerationPhase: + updates.phase + ?? (generationState === "idle" ? "" : entry.aiGenerationPhase ?? ""), + aiGenerationMessage: + updates.message + ?? (generationState === "idle" ? "" : entry.aiGenerationMessage ?? ""), + aiGenerationBackend: + updates.backend + ?? (generationState === "idle" ? "" : entry.aiGenerationBackend ?? ""), + aiGenerationElapsedMs: + updates.elapsedMs + ?? (generationState === "idle" ? 0 : entry.aiGenerationElapsedMs ?? 0), + aiGenerationHeartbeatTs: + updates.heartbeatTs + ?? (generationState === "idle" ? 0 : entry.aiGenerationHeartbeatTs ?? 0), + aiGenerationPhaseProgress: + updates.phaseProgress + ?? (generationState === "idle" ? undefined : entry.aiGenerationPhaseProgress), + aiGenerationEtaMs: + updates.etaMs + ?? (generationState === "idle" ? undefined : entry.aiGenerationEtaMs), + aiGenerationRunMode: + updates.runMode + ?? (generationState === "idle" ? undefined : entry.aiGenerationRunMode), + aiGenerationRuntimeProfile: + updates.runtimeProfile + ?? (generationState === "idle" ? "" : entry.aiGenerationRuntimeProfile ?? ""), + aiGenerationLmModel: + updates.lmModel + ?? (generationState === "idle" ? "" : entry.aiGenerationLmModel ?? ""), + aiGenerationStatusNote: + updates.statusNote + ?? (generationState === "idle" ? "" : entry.aiGenerationStatusNote ?? ""), + aiGenerationFailureKind: + updates.failureKind + ?? (generationState === "idle" ? "" : entry.aiGenerationFailureKind ?? ""), + aiGenerationSessionMode: + updates.sessionMode + ?? (generationState === "idle" ? "" : entry.aiGenerationSessionMode ?? ""), + aiGenerationWorkerExitCode: + updates.workerExitCode + ?? (generationState === "idle" ? 0 : entry.aiGenerationWorkerExitCode ?? 0), + aiGenerationLastStdoutLine: + updates.lastStdoutLine + ?? (generationState === "idle" ? "" : entry.aiGenerationLastStdoutLine ?? ""), + aiGenerationLastStderrLine: + updates.lastStderrLine + ?? (generationState === "idle" ? "" : entry.aiGenerationLastStderrLine ?? ""), + aiGenerationAttemptMode: + updates.attemptMode + ?? (generationState === "idle" ? "" : entry.aiGenerationAttemptMode ?? ""), + aiGenerationAttemptIndex: + updates.attemptIndex + ?? (generationState === "idle" ? 0 : entry.aiGenerationAttemptIndex ?? 0), + aiGenerationProtocolVersion: + updates.protocolVersion + ?? (generationState === "idle" ? 0 : entry.aiGenerationProtocolVersion ?? 0), + aiGenerationScriptVersion: + updates.scriptVersion + ?? (generationState === "idle" ? "" : entry.aiGenerationScriptVersion ?? ""), + aiGenerationRequestId: + updates.requestId + ?? (generationState === "idle" ? "" : entry.aiGenerationRequestId ?? ""), + aiGenerationPriorFailure: + updates.priorFailure + ?? (generationState === "idle" ? "" : entry.aiGenerationPriorFailure ?? ""), + aiGenerationLastProgressAgeMs: + updates.lastProgressAgeMs + ?? (generationState === "idle" ? 0 : entry.aiGenerationLastProgressAgeMs ?? 0), + aiGenerationTracePath: + updates.tracePath + ?? (generationState === "idle" ? "" : entry.aiGenerationTracePath ?? ""), + aiGenerationFailureDetail: + updates.failureDetail + ?? (generationState === "idle" ? "" : entry.aiGenerationFailureDetail ?? ""), + aiGenerationLmBackend: + updates.lmBackend + ?? (generationState === "idle" ? "" : entry.aiGenerationLmBackend ?? ""), + aiGenerationLmStage: + updates.lmStage + ?? (generationState === "idle" ? "" : entry.aiGenerationLmStage ?? ""), + } + : entry, + ), + })); + }, + + addGeneratedAudioClip: async (trackId, filePath, startTime, clipName) => { + const track = get().tracks.find((entry) => entry.id === trackId); + if (!track) { + throw new Error(`Track not found: ${trackId}`); + } + + const mediaInfo = await nativeBridge.importMediaFile(filePath); + if (!mediaInfo?.filePath || !mediaInfo.duration) { + throw new Error(`Failed to prepare generated audio: ${filePath}`); + } + + const newClip: AudioClip = { + id: crypto.randomUUID(), + filePath: mediaInfo.filePath, + name: + clipName + || mediaInfo.filePath.split(/[/\\]/).pop()?.replace(/\.[^.]+$/, "") + || "Generated Clip", + startTime, + duration: mediaInfo.duration, + offset: 0, + color: track.color || "#4cc9f0", + volumeDB: 0, + fadeIn: 0, + fadeOut: 0, + sampleRate: mediaInfo.sampleRate, + sourceLength: mediaInfo.duration, + }; + + get().executeCommand({ + type: "ADD_CLIP", + description: `Add generated clip to "${track.name}"`, + timestamp: Date.now(), + execute: () => { + set((state) => ({ + tracks: state.tracks.map((entry) => + entry.id === trackId + ? { ...entry, clips: [...entry.clips, newClip] } + : entry, + ), + isModified: true, + })); + void nativeBridge.addPlaybackClip( + trackId, + newClip.filePath, + newClip.startTime, + newClip.duration, + newClip.offset || 0, + newClip.volumeDB || 0, + newClip.fadeIn || 0, + newClip.fadeOut || 0, + newClip.id, + newClip.pitchCorrectionSourceFilePath, + newClip.pitchCorrectionSourceOffset, + ); + }, + undo: () => { + set((state) => ({ + tracks: state.tracks.map((entry) => + entry.id === trackId + ? { + ...entry, + clips: entry.clips.filter((clip) => clip.id !== newClip.id), + } + : entry, + ), + isModified: true, + })); + void nativeBridge.removePlaybackClip(trackId, newClip.filePath); + }, + }); + }, + // ========== Empty Item (silent clip) ========== addEmptyClip: (trackId, startTime, duration) => { const clip: AudioClip = { diff --git a/frontend/src/utils/globalShortcutDispatcher.ts b/frontend/src/utils/globalShortcutDispatcher.ts index 236c229..3234234 100644 --- a/frontend/src/utils/globalShortcutDispatcher.ts +++ b/frontend/src/utils/globalShortcutDispatcher.ts @@ -1,6 +1,7 @@ import { type NativeGlobalShortcutEvent } from "../services/NativeBridge"; import { getRegisteredActions, type ActionDef } from "../store/actionRegistry"; import { useDAWStore } from "../store/useDAWStore"; +import { isMac } from "./platform"; let _lastSpacebarMs = 0; @@ -9,11 +10,26 @@ export interface GlobalShortcutPayload extends NativeGlobalShortcutEvent { preventDefault?: () => void; } +/** + * Convert a key event payload into the canonical shortcut string used in actionRegistry. + * + * Canonical format uses Windows-style names: "Ctrl+Z", "Alt+Enter", "Ctrl+Shift+Z" + * + * Platform mapping: + * macOS — metaKey (Cmd) → "Ctrl", ctrlKey (^) → "Alt", altKey (Option) ignored + * Win — ctrlKey/metaKey → "Ctrl", altKey → "Alt" + */ function toPressedShortcut(payload: GlobalShortcutPayload): string | null { const parts: string[] = []; - if (payload.ctrlKey || payload.metaKey) parts.push("Ctrl"); + + if (isMac) { + if (payload.metaKey) parts.push("Ctrl"); // Cmd → Ctrl + if (payload.ctrlKey) parts.push("Alt"); // Ctrl → Alt + } else { + if (payload.ctrlKey || payload.metaKey) parts.push("Ctrl"); + if (payload.altKey) parts.push("Alt"); + } if (payload.shiftKey) parts.push("Shift"); - if (payload.altKey) parts.push("Alt"); let key = payload.key ?? ""; if (["Control", "Shift", "Alt", "Meta"].includes(key)) { diff --git a/frontend/src/utils/pitchRegressionDriver.ts b/frontend/src/utils/pitchRegressionDriver.ts new file mode 100644 index 0000000..2ccda51 --- /dev/null +++ b/frontend/src/utils/pitchRegressionDriver.ts @@ -0,0 +1,1196 @@ +import { + nativeBridge, + type PitchCorrectionCompletionData, + type PitchContourData, + type PitchNoteData, + type PitchRegressionJob, + type PitchRegressionResult, + type PitchScrubPreviewStatus, +} from "../services/NativeBridge"; +import { useDAWStore } from "../store/useDAWStore"; +import { usePitchEditorStore } from "../store/pitchEditorStore"; + +let started = false; + +function normalizeArray<T>(value: unknown): T[] { + if (Array.isArray(value)) { + return value as T[]; + } + if (value && typeof value === "object") { + const entries = Object.entries(value as Record<string, T>) + .filter(([key]) => /^\d+$/.test(key)) + .sort((a, b) => Number(a[0]) - Number(b[0])) + .map(([, item]) => item); + if (entries.length > 0) { + return entries; + } + } + return []; +} + +function baseNameFromPath(path: string) { + const normalized = path.replace(/\\/g, "/"); + const leaf = normalized.split("/").pop() ?? normalized; + return leaf.replace(/\.[^.]+$/, "") || "Regression Clip"; +} + +function replaceFixtureClipSource(job: PitchRegressionJob) { + let replaced = false; + + useDAWStore.setState((state) => ({ + tracks: state.tracks.map((track) => { + if (track.id !== job.trackId) { + return track; + } + + return { + ...track, + clips: track.clips.map((clip) => { + if (clip.id !== job.clipId) { + return clip; + } + + replaced = true; + return { + ...clip, + filePath: job.sourceAudioPath, + name: baseNameFromPath(job.sourceAudioPath), + offset: 0, + pitchCorrectionSourceFilePath: undefined, + pitchCorrectionSourceOffset: undefined, + }; + }), + }; + }), + })); + + if (!replaced) { + throw new Error(`Fixture clip not found for track=${job.trackId} clip=${job.clipId}`); + } +} + +function deriveExportOutputPath(job: PitchRegressionJob) { + if (job.exportOutputPath) { + return job.exportOutputPath; + } + return job.resultJsonPath.replace(/\.json$/i, "_export.wav"); +} + +function deriveAuditionPlaybackOutputPath(job: PitchRegressionJob) { + if (job.auditionPlaybackOutputPath) { + return job.auditionPlaybackOutputPath; + } + return job.resultJsonPath.replace(/\.json$/i, "_after_app_playback_4s.wav"); +} + +function deriveAuditionExportOutputPath(job: PitchRegressionJob) { + if (job.auditionExportOutputPath) { + return job.auditionExportOutputPath; + } + return job.resultJsonPath.replace(/\.json$/i, "_after_export_4s.wav"); +} + +function waitForPitchCorrectionCompletion(targetClipId: string, timeoutMs = 120000) { + return new Promise<PitchCorrectionCompletionData>((resolve, reject) => { + let settled = false; + let unsubscribe = () => {}; + + const timeout = window.setTimeout(() => { + if (settled) { + return; + } + settled = true; + unsubscribe(); + reject(new Error(`Timed out waiting for pitchCorrectionComplete for clip ${targetClipId}`)); + }, timeoutMs); + + unsubscribe = nativeBridge.onPitchCorrectionComplete((data) => { + if (settled || data.clipId !== targetClipId) { + return; + } + + settled = true; + window.clearTimeout(timeout); + unsubscribe(); + resolve(data); + }); + }); +} + +async function waitForContourReady(timeoutMs = 120000) { + const startedAt = performance.now(); + + while (performance.now() - startedAt < timeoutMs) { + const state = usePitchEditorStore.getState(); + if (state.contour) { + return state.contour; + } + await new Promise((resolve) => window.setTimeout(resolve, 50)); + } + + return null; +} + +async function reportRegressionResult(result: PitchRegressionResult) { + await nativeBridge.completePitchRegressionJob(result); +} + +async function waitForScrubPreviewState( + clipId: string, + predicate: (status: PitchScrubPreviewStatus | null) => boolean, + timeoutMs: number, +) { + const startedAt = performance.now(); + let lastStatus: PitchScrubPreviewStatus | null = null; + while (performance.now() - startedAt < timeoutMs) { + lastStatus = await nativeBridge.getPitchScrubPreviewStatus(clipId); + if (predicate(lastStatus)) { + return { status: lastStatus, elapsedMs: performance.now() - startedAt }; + } + await new Promise((resolve) => window.setTimeout(resolve, 25)); + } + return { status: lastStatus, elapsedMs: performance.now() - startedAt }; +} + +function normalizePitchNotes(value: PitchRegressionJob["notes"] | PitchRegressionJob["expectedNotes"]) { + return normalizeArray<PitchNoteData>(value); +} + +function finiteNumberOrNull(value: unknown) { + const numeric = typeof value === "number" ? value : Number(value); + return Number.isFinite(numeric) ? numeric : null; +} + +function applyTargetShiftSemitones( + notes: PitchNoteData[], + targetShiftSemitones: unknown, +) { + const targetShift = finiteNumberOrNull(targetShiftSemitones); + if (targetShift === null) { + return { + notes: notes.map((note) => ({ ...note })), + targetShiftSemitones: null, + actualRequestedShiftSemitones: null, + requestedShiftErrorCents: null, + chromaticSnapBypassed: false, + }; + } + + const shiftedNotes = notes.map((note) => { + const detectedPitch = finiteNumberOrNull(note.detectedPitch); + if (detectedPitch === null) { + throw new Error(`Cannot apply targetShiftSemitones=${targetShift}: note ${note.id} has invalid detectedPitch.`); + } + return { + ...note, + detectedPitch, + correctedPitch: detectedPitch + targetShift, + }; + }); + + const shiftDeltas = shiftedNotes.map((note) => note.correctedPitch - note.detectedPitch); + const averageShift = shiftDeltas.reduce((sum, delta) => sum + delta, 0) / Math.max(1, shiftDeltas.length); + const maxErrorSemitones = shiftDeltas.reduce( + (maxError, delta) => Math.max(maxError, Math.abs(delta - targetShift)), + 0, + ); + if (maxErrorSemitones > 0.01) { + throw new Error( + `Requested relative pitch shift is not exact: target=${targetShift.toFixed(4)} st, ` + + `actual=${averageShift.toFixed(4)} st, maxError=${(maxErrorSemitones * 100).toFixed(2)} cents.`, + ); + } + + return { + notes: shiftedNotes, + targetShiftSemitones: targetShift, + actualRequestedShiftSemitones: averageShift, + requestedShiftErrorCents: maxErrorSemitones * 100, + chromaticSnapBypassed: true, + }; +} + +function computeMedian(values: number[]) { + if (values.length === 0) { + return 0; + } + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) * 0.5 + : sorted[middle]; +} + +function overlapDuration(startA: number, endA: number, startB: number, endB: number) { + return Math.max(0, Math.min(endA, endB) - Math.max(startA, startB)); +} + +function computeAnalysisSummary( + contour: PitchContourData, + expectedNotes: PitchNoteData[], + job: PitchRegressionJob, +) { + const notes = normalizePitchNotes(contour.notes); + const frameTimes = normalizeArray<number>(contour.frames?.times); + const frameMidi = normalizeArray<number>(contour.frames?.midi); + const frameConfidence = normalizeArray<number>(contour.frames?.confidence); + const frameVoiced = normalizeArray<boolean>(contour.frames?.voiced); + + const voicedMidi = frameMidi.filter((_, index) => frameVoiced[index]); + const voicedConfidence = frameConfidence.filter((_, index) => frameVoiced[index]); + + let jumpCountOver2Semitones = 0; + let jumpCountOver5Semitones = 0; + let previousVoicedMidi: number | null = null; + for (let index = 0; index < frameMidi.length; index += 1) { + if (!frameVoiced[index]) { + continue; + } + const midi = frameMidi[index]; + if (previousVoicedMidi != null) { + const delta = Math.abs(midi - previousVoicedMidi); + if (delta > 2) { + jumpCountOver2Semitones += 1; + } + if (delta > 5) { + jumpCountOver5Semitones += 1; + } + } + previousVoicedMidi = midi; + } + + const analysisStartSec = job.analysisOffsetSec ?? 0; + const analysisDurationSec = + job.analysisDurationSec ?? + Math.max(0, (frameTimes[frameTimes.length - 1] ?? 0) - (frameTimes[0] ?? 0)); + + const expectedMatches = expectedNotes.map((expectedNote) => { + let bestDetected: PitchNoteData | null = null; + let bestOverlap = 0; + + for (const detectedNote of notes) { + const overlap = overlapDuration( + expectedNote.startTime, + expectedNote.endTime, + detectedNote.startTime, + detectedNote.endTime, + ); + if (overlap > bestOverlap) { + bestOverlap = overlap; + bestDetected = detectedNote; + } + } + + if (!bestDetected) { + return { + expectedId: expectedNote.id, + matched: false, + overlapSec: 0, + overlapRatio: 0, + startDeltaSec: null, + endDeltaSec: null, + pitchDeltaSemitones: null, + }; + } + + const expectedDuration = Math.max(0.001, expectedNote.endTime - expectedNote.startTime); + return { + expectedId: expectedNote.id, + matched: bestOverlap > 0, + overlapSec: bestOverlap, + overlapRatio: bestOverlap / expectedDuration, + startDeltaSec: bestDetected.startTime - expectedNote.startTime, + endDeltaSec: bestDetected.endTime - expectedNote.endTime, + pitchDeltaSemitones: bestDetected.detectedPitch - expectedNote.detectedPitch, + }; + }); + + const unmatchedDetectedNoteCount = notes.filter((detectedNote) => + !expectedNotes.some((expectedNote) => + overlapDuration( + expectedNote.startTime, + expectedNote.endTime, + detectedNote.startTime, + detectedNote.endTime, + ) > 0, + ), + ).length; + const boundaryKindCounts = notes.reduce<Record<string, number>>((acc, note) => { + for (const kind of [note.entryBoundaryKind ?? "unknown", note.exitBoundaryKind ?? "unknown"]) { + acc[kind] = (acc[kind] ?? 0) + 1; + } + return acc; + }, {}); + const cornerBoundaryCount = notes.filter((note) => + note.entryBoundaryReason?.includes("pitch_corner") + || note.exitBoundaryReason?.includes("pitch_corner") + || note.entryBoundaryKind === "soft_legato" + || note.exitBoundaryKind === "soft_legato", + ).length; + const boundaryCandidates = normalizeArray(contour.boundaryCandidates); + const cornerCandidateCount = boundaryCandidates.filter((candidate: any) => + String(candidate?.reason ?? "").includes("pitch_corner"), + ).length; + const destructiveCornerSplitCount = boundaryCandidates.filter((candidate: any) => + String(candidate?.reason ?? "").includes("pitch_corner") + && String(candidate?.kind ?? "") !== "hard_word_like" + && candidate?.destructiveSplitAllowed === true, + ).length; + const hardAcousticSplitCount = boundaryCandidates.filter((candidate: any) => + String(candidate?.kind ?? "") === "hard_word_like" && candidate?.destructiveSplitAllowed === true, + ).length; + const pitchDeviationCandidateCount = boundaryCandidates.filter((candidate: any) => + String(candidate?.reason ?? "").includes("pitch_hysteresis"), + ).length; + const destructivePitchJumpSplitCount = boundaryCandidates.filter((candidate: any) => + String(candidate?.reason ?? "").includes("pitch_hysteresis") && candidate?.destructiveSplitAllowed === true, + ).length; + const vibratoSuppressedCandidateCount = boundaryCandidates.filter((candidate: any) => + String(candidate?.kind ?? "") === "internal_vibrato" + || String(candidate?.reason ?? "").includes("vibrato_suppressed"), + ).length; + const wordGroups = new Map<string, PitchNoteData[]>(); + for (const note of notes) { + const groupId = note.wordGroupId && note.wordGroupId.length > 0 ? note.wordGroupId : note.id; + const group = wordGroups.get(groupId) ?? []; + group.push(note); + wordGroups.set(groupId, group); + } + const wordGroupMatches = expectedNotes.map((expectedNote) => { + let bestGroupId = ""; + let bestOverlap = 0; + let bestFragmentCount = 0; + let bestGroupStart = 0; + let bestGroupEnd = 0; + for (const [groupId, groupNotes] of wordGroups.entries()) { + const overlap = groupNotes.reduce((sum, detectedNote) => sum + overlapDuration( + expectedNote.startTime, + expectedNote.endTime, + detectedNote.startTime, + detectedNote.endTime, + ), 0); + if (overlap > bestOverlap) { + bestOverlap = overlap; + bestGroupId = groupId; + bestFragmentCount = groupNotes.length; + bestGroupStart = Math.min(...groupNotes.map((note) => note.startTime)); + bestGroupEnd = Math.max(...groupNotes.map((note) => note.endTime)); + } + } + const expectedDuration = Math.max(0.001, expectedNote.endTime - expectedNote.startTime); + const groupDuration = Math.max(0, bestGroupEnd - bestGroupStart); + return { + expectedId: expectedNote.id, + wordGroupId: bestGroupId, + overlapSec: bestOverlap, + overlapRatio: bestOverlap / expectedDuration, + fragmentCount: bestFragmentCount, + groupStartSec: bestGroupStart, + groupEndSec: bestGroupEnd, + groupDurationSec: groupDuration, + groupOverhangSec: Math.max(0, expectedNote.startTime - bestGroupStart) + + Math.max(0, bestGroupEnd - expectedNote.endTime), + }; + }); + + return { + analyzerWindowStartSec: analysisStartSec, + analyzerWindowEndSec: analysisStartSec + analysisDurationSec, + noteCount: notes.length, + expectedNoteCount: expectedNotes.length, + noteCountDelta: notes.length - expectedNotes.length, + frameCount: frameTimes.length, + voicedFrameCount: frameVoiced.filter(Boolean).length, + voicedFrameRatio: frameVoiced.length > 0 + ? frameVoiced.filter(Boolean).length / frameVoiced.length + : 0, + medianVoicedMidi: computeMedian(voicedMidi), + medianVoicedConfidence: computeMedian(voicedConfidence), + jumpCountOver2Semitones, + jumpCountOver5Semitones, + jumpRateOver2PerSecond: analysisDurationSec > 0 + ? jumpCountOver2Semitones / analysisDurationSec + : 0, + expectedMatches, + matchedExpectedCount: expectedMatches.filter((match) => match.matched).length, + unmatchedDetectedNoteCount, + boundaryKindCounts, + cornerBoundaryCount, + boundaryCandidateCount: boundaryCandidates.length, + cornerCandidateCount, + destructiveCornerSplitCount, + hardAcousticSplitCount, + pitchDeviationCandidateCount, + destructivePitchJumpSplitCount, + vibratoSuppressedCandidateCount, + wordGroupCount: wordGroups.size, + wordGroupMatches, + minWordGroupOverlapRatio: wordGroupMatches.length > 0 + ? Math.min(...wordGroupMatches.map((match) => match.overlapRatio)) + : 0, + maxWordGroupFragmentsPerExpected: wordGroupMatches.length > 0 + ? Math.max(...wordGroupMatches.map((match) => match.fragmentCount)) + : 0, + maxWordGroupOverhangSec: wordGroupMatches.length > 0 + ? Math.max(...wordGroupMatches.map((match) => match.groupOverhangSec)) + : 0, + hardWordLikeBoundaryCount: boundaryKindCounts.hard_word_like ?? 0, + softLegatoBoundaryCount: boundaryKindCounts.soft_legato ?? 0, + }; +} + +function waitForDirectPitchAnalysis(targetClipId: string, timeoutMs = 120000) { + return new Promise<PitchContourData>((resolve, reject) => { + let settled = false; + let unsubscribe = () => {}; + + const timeout = window.setTimeout(() => { + if (settled) { + return; + } + settled = true; + unsubscribe(); + reject(new Error(`Timed out waiting for pitch analysis for clip ${targetClipId}`)); + }, timeoutMs); + + unsubscribe = nativeBridge.onPitchAnalysisComplete(async (notification: any) => { + if (settled || notification?.clipId !== targetClipId || !notification?.ready) { + return; + } + + try { + const fullResult = await nativeBridge.getLastPitchAnalysisResult(); + if (!fullResult) { + return; + } + + settled = true; + window.clearTimeout(timeout); + unsubscribe(); + resolve(fullResult); + } catch (error) { + settled = true; + window.clearTimeout(timeout); + unsubscribe(); + reject(error instanceof Error ? error : new Error(String(error))); + } + }); + }); +} + +async function runAnalysisRegressionJob(job: PitchRegressionJob): Promise<PitchRegressionResult> { + const clipId = job.clipId || "pitch-analysis-clip"; + const analysisOffsetSec = job.analysisOffsetSec ?? 0; + const analysisDurationSec = job.analysisDurationSec ?? 30; + const expectedNotes = normalizePitchNotes(job.expectedNotes ?? job.notes); + + const startedAt = performance.now(); + const completionPromise = waitForDirectPitchAnalysis(clipId); + const response = await nativeBridge.analyzePitchContourDirect( + job.sourceAudioPath, + analysisOffsetSec, + analysisDurationSec, + clipId, + ); + const analysisStarted = !!(response as { started?: boolean } | null)?.started; + if (!analysisStarted) { + throw new Error("Direct pitch analysis request was not accepted."); + } + + const contour = await completionPromise; + const elapsedMs = performance.now() - startedAt; + const analysisSummary = computeAnalysisSummary(contour, expectedNotes, job); + + return { + success: true, + jobType: "analysis", + clipId, + elapsedMs, + sourceAudioPath: job.sourceAudioPath, + label: job.label, + analysisSummary, + analysisResult: contour, + }; +} + +async function runScrubRegressionJob(job: PitchRegressionJob): Promise<PitchRegressionResult> { + const clipId = job.clipId || "pitch-scrub-clip"; + if (!job.projectFixturePath || !job.trackId) { + throw new Error("Scrub regression job is missing required fixture project metadata."); + } + + const normalizedNotes = normalizePitchNotes(job.notes); + if (normalizedNotes.length === 0) { + throw new Error("Scrub regression job requires at least one note."); + } + + const note = { ...normalizedNotes[0] }; + const loadOk = await useDAWStore.getState().loadProject(job.projectFixturePath, { bypassFX: true }); + if (!loadOk) { + throw new Error(`Failed to load fixture project: ${job.projectFixturePath}`); + } + + replaceFixtureClipSource(job); + await useDAWStore.getState().syncClipsWithBackend(); + useDAWStore.getState().openPitchEditor(job.trackId, clipId, -1); + await usePitchEditorStore.getState().analyze(); + const contour = await waitForContourReady(); + if (!contour) { + throw new Error("Pitch analysis did not produce a contour for scrub regression."); + } + + const contourNotes = normalizePitchNotes(contour.notes); + + const scrubWaitMs = Math.max(150, job.scrubWaitMs ?? 800); + const scrubUpdatePitchRatio = job.scrubUpdatePitchRatio ?? Math.pow(2, (note.correctedPitch - note.detectedPitch) / 12); + const scrubFrames = job.frames ?? contour.frames; + const repeatCount = Math.max(1, job.scrubRepeatCount ?? 1); + const selectionChangeRequested = !!job.scrubSelectionChange; + + async function performScrubCycle( + cycleNote: PitchNoteData, + scenarioName: string, + ) { + await nativeBridge.startPitchScrubPreview(job.trackId!, clipId, cycleNote, scrubFrames); + const started = await waitForScrubPreviewState(clipId, (status) => !!status?.audible, scrubWaitMs); + + await nativeBridge.updatePitchScrubPreview(clipId, scrubUpdatePitchRatio); + const afterUpdate = await waitForScrubPreviewState( + clipId, + (status) => !!status?.audible && (status.pitchRatio ?? 0) > 0, + scrubWaitMs, + ); + const routingStatus = await nativeBridge.getPitchPreviewRoutingStatus(clipId); + if (routingStatus && (routingStatus.monitorMode !== "scrub" || routingStatus.clipLivePreviewActive)) { + throw new Error(`Unexpected scrub routing state: ${JSON.stringify(routingStatus)}`); + } + + await nativeBridge.stopPitchScrubPreview(clipId); + const stopped = await waitForScrubPreviewState( + clipId, + (status) => !!status && !status.active && !status.releasePending, + scrubWaitMs, + ); + + const finalStatus = stopped.status ?? afterUpdate.status ?? started.status ?? null; + return { + name: scenarioName, + audible: !!(started.status?.audible || started.status?.firstDragAudible || afterUpdate.status?.audible), + firstDragAudible: !!(started.status?.firstDragAudible || afterUpdate.status?.firstDragAudible || finalStatus?.firstDragAudible), + startLatencyMs: started.status?.audible ? started.elapsedMs : null, + stopLatencyMs: stopped.status && !stopped.status.active && !stopped.status.releasePending ? stopped.elapsedMs : null, + repeatStability: finalStatus?.repeatStability ?? null, + lastPeak: finalStatus?.lastPeak ?? null, + mixedCallbackCount: finalStatus?.mixedCallbackCount, + mixedSampleCount: finalStatus?.mixedSampleCount, + status: finalStatus ?? undefined, + routingStatus: routingStatus ?? undefined, + }; + } + + const scenarioResults: Array<{ + name: string; + audible: boolean; + firstDragAudible: boolean; + startLatencyMs: number | null; + stopLatencyMs: number | null; + repeatStability: number | null; + lastPeak: number | null; + mixedCallbackCount?: number; + mixedSampleCount?: number; + status?: PitchScrubPreviewStatus; + routingStatus?: Awaited<ReturnType<typeof nativeBridge.getPitchPreviewRoutingStatus>>; + }> = []; + + scenarioResults.push(await performScrubCycle(note, "first_drag")); + + for (let repeatIndex = 1; repeatIndex < repeatCount; repeatIndex += 1) { + scenarioResults.push(await performScrubCycle(note, `repeated_drag_${repeatIndex + 1}`)); + } + + if (job.scrubTransportCycle) { + const cycleStart = Math.max(0, note.startTime); + await useDAWStore.getState().seekTo(cycleStart); + await useDAWStore.getState().play(); + await new Promise((resolve) => window.setTimeout(resolve, 300)); + await useDAWStore.getState().stop(); + await new Promise((resolve) => window.setTimeout(resolve, 120)); + scenarioResults.push(await performScrubCycle(note, "after_transport_cycle")); + } + + let selectionChangeScenario: typeof scenarioResults[number] | undefined; + + if (job.scrubSelectionChange) { + const alternateNote = contourNotes.find((candidate) => candidate.id !== note.id) ?? normalizedNotes.find((candidate) => candidate.id !== note.id); + if (alternateNote) { + selectionChangeScenario = await performScrubCycle( + { + ...alternateNote, + transitionIn: alternateNote.transitionIn ?? note.transitionIn, + transitionOut: alternateNote.transitionOut ?? note.transitionOut, + }, + "selection_change", + ); + scenarioResults.push(selectionChangeScenario); + } + } + + const firstScenario = scenarioResults[0]; + const finalScenario = scenarioResults[scenarioResults.length - 1]; + const finalStatus = finalScenario?.status ?? null; + const scenarioPassCount = scenarioResults.filter((scenario) => scenario.audible).length; + const repeatedDragScenarios = scenarioResults.filter((scenario) => scenario.name.startsWith("repeated_drag_")); + const repeatedDragAudible = repeatedDragScenarios.length > 0 + ? repeatedDragScenarios.every((scenario) => scenario.audible) + : firstScenario?.audible ?? false; + const afterTransportScenario = scenarioResults.find((scenario) => scenario.name === "after_transport_cycle"); + + return { + success: scenarioPassCount > 0, + jobType: "scrub", + clipId, + sourceAudioPath: job.sourceAudioPath, + label: job.label, + scrubPreviewAudible: !!firstScenario?.audible, + scrubPreviewFirstDragAudible: !!firstScenario?.firstDragAudible, + scrubPreviewStartLatencyMs: firstScenario?.startLatencyMs ?? null, + scrubPreviewStopLatencyMs: finalScenario?.stopLatencyMs ?? null, + scrubPreviewMixedCallbackCount: finalScenario?.mixedCallbackCount, + scrubPreviewMixedSampleCount: finalScenario?.mixedSampleCount, + scrubPreviewLastPeak: finalScenario?.lastPeak ?? null, + scrubPreviewRepeatStability: finalScenario?.repeatStability ?? null, + scrubPreviewScenarioCount: scenarioResults.length, + scrubPreviewScenarioPassCount: scenarioPassCount, + scrubPreviewRepeatedDragAudible: repeatedDragAudible, + scrubPreviewAfterTransportCycleAudible: afterTransportScenario ? afterTransportScenario.audible : false, + scrubPreviewSelectionChangeAudible: selectionChangeScenario ? selectionChangeScenario.audible : false, + scrubPreviewSelectionChangeRequested: selectionChangeRequested, + scrubPreviewInputNoteCount: normalizedNotes.length, + scrubPreviewContourNoteCount: contourNotes.length, + scrubPreviewAlternateNoteFound: !!selectionChangeScenario, + scrubPreviewScenarioNames: scenarioResults.map((scenario) => scenario.name), + scrubPreviewScenarioResults: scenarioResults.map((scenario) => ({ + name: scenario.name, + audible: scenario.audible, + firstDragAudible: scenario.firstDragAudible, + startLatencyMs: scenario.startLatencyMs, + stopLatencyMs: scenario.stopLatencyMs, + repeatStability: scenario.repeatStability, + lastPeak: scenario.lastPeak, + routingStatus: scenario.routingStatus, + })), + scrubPreviewStatus: finalStatus ?? undefined, + analysisResult: contour, + }; +} + +async function runRenderRegressionJob(job: PitchRegressionJob): Promise<PitchRegressionResult> { + const normalizedNotes = normalizePitchNotes(job.notes); + const targetShiftRequest = applyTargetShiftSemitones(normalizedNotes, job.targetShiftSemitones); + const requestedNotes = targetShiftRequest.notes; + const renderMode = job.renderMode ?? "single"; + if (!job.projectFixturePath || !job.trackId) { + throw new Error("Render regression job is missing required fixture project metadata."); + } + if (requestedNotes.length === 0) { + throw new Error("Regression job did not contain any editable notes."); + } + if (Math.abs(job.globalFormantSemitones ?? 0) > 0.001 || requestedNotes.some((note) => Math.abs(note.formantShift ?? 0) > 0.001)) { + throw new Error("The current UI+bridge regression harness is pitch-only for now. Formant edits must stay at zero."); + } + + const loadOk = await useDAWStore.getState().loadProject(job.projectFixturePath, { bypassFX: true }); + if (!loadOk) { + throw new Error(`Failed to load fixture project: ${job.projectFixturePath}`); + } + + replaceFixtureClipSource(job); + await useDAWStore.getState().syncClipsWithBackend(); + useDAWStore.getState().openPitchEditor(job.trackId, job.clipId, -1); + + const pitchStore = usePitchEditorStore.getState(); + await pitchStore.analyze(); + const contour = await waitForContourReady(); + + if (!contour) { + throw new Error("Pitch analysis did not produce a contour."); + } + + usePitchEditorStore.setState((state) => ({ + ...state, + contour: job.frames + ? { + ...contour, + frames: job.frames, + } + : contour, + notes: requestedNotes.map((note) => ({ ...note })), + selectedNoteIds: requestedNotes.length > 0 ? [requestedNotes[0].id] : [], + globalFormantCents: Math.round((job.globalFormantSemitones ?? 0) * 100), + })); + + const startedAt = performance.now(); + const completionPromise = waitForPitchCorrectionCompletion(job.clipId); + if (renderMode === "single" || renderMode === "note_hq") { + await usePitchEditorStore.getState().applyCorrection(); + } else { + const requestId = `pitch-regression-${Date.now()}`; + const requestGroupId = `${requestId}-${renderMode}`; + const accepted = (await nativeBridge.applyPitchCorrection( + job.trackId, + job.clipId, + requestedNotes, + job.frames + ? { + times: normalizeArray<number>(job.frames.times), + midi: normalizeArray<number>(job.frames.midi), + confidence: normalizeArray<number>(job.frames.confidence), + rms: normalizeArray<number>(job.frames.rms), + voiced: normalizeArray<boolean>(job.frames.voiced), + } + : contour.frames, + requestId, + job.globalFormantSemitones ?? 0, + job.windowStartSec, + job.windowEndSec, + renderMode, + requestGroupId, + )) as boolean | { outputFile: string; success: boolean } | null; + const acceptedOk = + accepted === true || + (typeof accepted === "object" && accepted !== null && accepted.success === true); + if (!acceptedOk) { + throw new Error(`Pitch correction request was not accepted for renderMode='${renderMode}'.`); + } + } + const completion = await completionPromise; + const elapsedMs = performance.now() - startedAt; + + if (!completion.success || !completion.outputFile) { + const reason = completion.hardFailReason || completion.fallbackReason || "no output file"; + throw new Error(`Pitch correction failed for clip ${job.clipId}: ${reason}`); + } + + await useDAWStore.getState().syncClipsWithBackend(); + + return { + success: true, + jobType: "render", + outputFile: completion.outputFile, + clipId: completion.clipId, + requestId: completion.requestId, + renderMode: completion.renderMode ?? renderMode, + targetShiftSemitones: targetShiftRequest.targetShiftSemitones, + actualRequestedShiftSemitones: targetShiftRequest.actualRequestedShiftSemitones, + requestedShiftErrorCents: targetShiftRequest.requestedShiftErrorCents, + chromaticSnapBypassed: targetShiftRequest.chromaticSnapBypassed, + requestedRendererBranch: completion.requestedRendererBranch, + actualRendererBranch: completion.actualRendererBranch, + pitchOnlyRecoveryPath: completion.pitchOnlyRecoveryPath, + pitchOnlyNeutralFormantUsed: completion.pitchOnlyNeutralFormantUsed, + processingMode: completion.processingMode, + formantCurveUsed: completion.formantCurveUsed, + explicitFormantRequested: completion.explicitFormantRequested, + pitchOnlyFormantSuppressed: completion.pitchOnlyFormantSuppressed, + usedFallback: completion.usedFallback, + fallbackReason: completion.fallbackReason, + hardFailReason: completion.hardFailReason, + pitchRenderStrategy: completion.pitchRenderStrategy, + pitchRenderProductPath: completion.pitchRenderProductPath, + pitchRenderBackendId: completion.pitchRenderBackendId, + pitchRenderBackendVersion: completion.pitchRenderBackendVersion, + pitchRenderBackendFailureCode: completion.pitchRenderBackendFailureCode, + pitchRenderBackendCapabilities: completion.pitchRenderBackendCapabilities, + pitchRenderBackendDiagnostics: completion.pitchRenderBackendDiagnostics, + pitchRenderDirection: completion.pitchRenderDirection, + downshiftFormantGuardUsed: completion.downshiftFormantGuardUsed, + downshiftFormantGuardAlpha: completion.downshiftFormantGuardAlpha, + noteHqEffectiveStartSec: completion.noteHqEffectiveStartSec, + noteHqEffectiveEndSec: completion.noteHqEffectiveEndSec, + noteHqContextStartSec: completion.noteHqContextStartSec, + noteHqContextEndSec: completion.noteHqContextEndSec, + noteHqAudibleCommitStartSec: completion.noteHqAudibleCommitStartSec, + noteHqAudibleCommitEndSec: completion.noteHqAudibleCommitEndSec, + noteHqPreBodyDryProtectedSamples: completion.noteHqPreBodyDryProtectedSamples, + noteHqEntryInsideBodyFadeMs: completion.noteHqEntryInsideBodyFadeMs, + noteHqExitLeadInMs: completion.noteHqExitLeadInMs, + noteHqEntryBridgeStartSec: completion.noteHqEntryBridgeStartSec, + noteHqEntryBridgeEndSec: completion.noteHqEntryBridgeEndSec, + noteHqEntryBridgeWetLagMs: completion.noteHqEntryBridgeWetLagMs, + noteHqEntryBridgeEnvelopeGainDb: completion.noteHqEntryBridgeEnvelopeGainDb, + noteHqEntryBridgeUsed: completion.noteHqEntryBridgeUsed, + noteHqEntryTransientDryPreservedMs: completion.noteHqEntryTransientDryPreservedMs, + pitchOnlyEntrySimpleHandoffUsed: completion.pitchOnlyEntrySimpleHandoffUsed, + pitchOnlyEntrySafeHandoffUsed: completion.pitchOnlyEntrySafeHandoffUsed, + pitchOnlyEntryDryHoldMs: completion.pitchOnlyEntryDryHoldMs, + pitchOnlyEntrySafeBridgeMs: completion.pitchOnlyEntrySafeBridgeMs, + pitchOnlyEntryWetAlignmentMs: completion.pitchOnlyEntryWetAlignmentMs, + pitchOnlyEntryWetGainDb: completion.pitchOnlyEntryWetGainDb, + pitchOnlyEntryWetVsDryRmsDb: completion.pitchOnlyEntryWetVsDryRmsDb, + pitchOnlyEntryEqualPowerBlendUsed: completion.pitchOnlyEntryEqualPowerBlendUsed, + pitchOnlyEntryRmsContinuityUsed: completion.pitchOnlyEntryRmsContinuityUsed, + pitchOnlyEntryRmsContinuityGainDb: completion.pitchOnlyEntryRmsContinuityGainDb, + pitchOnlyEntryRmsContinuityMs: completion.pitchOnlyEntryRmsContinuityMs, + pitchOnlyEntryPhaseSafeUsed: completion.pitchOnlyEntryPhaseSafeUsed, + pitchOnlyEntryWetAlignmentAccepted: completion.pitchOnlyEntryWetAlignmentAccepted, + pitchOnlyEntryFirstCycleCorrelation: completion.pitchOnlyEntryFirstCycleCorrelation, + pitchOnlyEntryZeroCrossOffsetMs: completion.pitchOnlyEntryZeroCrossOffsetMs, + pitchOnlyEntryBridgeGainRampDb: completion.pitchOnlyEntryBridgeGainRampDb, + pitchOnlyDownshiftCoreEnvelopePassUsed: completion.pitchOnlyDownshiftCoreEnvelopePassUsed, + pitchOnlyDownshiftCoreRmsTrimDb: completion.pitchOnlyDownshiftCoreRmsTrimDb, + pitchOnlyDownshiftCoreEnvelopeMaxDb: completion.pitchOnlyDownshiftCoreEnvelopeMaxDb, + pitchOnlyDownshiftCoreEnvelopeFrames: completion.pitchOnlyDownshiftCoreEnvelopeFrames, + pitchOnlyEntryWetLagMs: completion.pitchOnlyEntryWetLagMs, + pitchOnlyEntryBridgeDurationMs: completion.pitchOnlyEntryBridgeDurationMs, + pitchOnlyExitDryRestoreUsed: completion.pitchOnlyExitDryRestoreUsed, + pitchOnlyExitDryRestoreStartSec: completion.pitchOnlyExitDryRestoreStartSec, + pitchOnlyExitDryRestoreEndSec: completion.pitchOnlyExitDryRestoreEndSec, + noteHqEntryBoundaryKind: completion.noteHqEntryBoundaryKind, + noteHqExitBoundaryKind: completion.noteHqExitBoundaryKind, + noteHqEntryBoundaryScore: completion.noteHqEntryBoundaryScore, + noteHqExitBoundaryScore: completion.noteHqExitBoundaryScore, + noteHqRendererEntryBoundaryKind: completion.noteHqRendererEntryBoundaryKind, + noteHqRendererExitBoundaryKind: completion.noteHqRendererExitBoundaryKind, + noteHqEditIslandCount: completion.noteHqEditIslandCount, + noteHqEditedNoteCount: completion.noteHqEditedNoteCount, + noteHqEntryPitchHandoffUsed: completion.noteHqEntryPitchHandoffUsed, + noteHqEntryPitchHandoffStartSec: completion.noteHqEntryPitchHandoffStartSec, + noteHqEntryPitchHandoffEndSec: completion.noteHqEntryPitchHandoffEndSec, + noteHqEntryPitchHandoffPreMs: completion.noteHqEntryPitchHandoffPreMs, + noteHqEntryPitchHandoffBodyMs: completion.noteHqEntryPitchHandoffBodyMs, + noteHqEntryPitchSlopeJumpStPerSec: completion.noteHqEntryPitchSlopeJumpStPerSec, + noteHqEntryPitchAccelerationLimited: completion.noteHqEntryPitchAccelerationLimited, + phraseHqRenderUsed: completion.phraseHqRenderUsed, + phraseHqExpandedToFullClip: completion.phraseHqExpandedToFullClip, + phraseHqStartSec: completion.phraseHqStartSec, + phraseHqEndSec: completion.phraseHqEndSec, + elapsedMs, + outputDurationSec: completion.outputDurationSec, + bridgeUsed: completion.bridgeUsed, + bridgeFallbackUsed: completion.bridgeFallbackUsed, + bridgeStartSec: completion.bridgeStartSec, + bridgeLengthMs: completion.bridgeLengthMs, + bridgeAlignmentLagSamples: completion.bridgeAlignmentLagSamples, + bridgeCorrelationScore: completion.bridgeCorrelationScore, + bridgeGainDeltaDb: completion.bridgeGainDeltaDb, + bodyReplacementUsed: completion.bodyReplacementUsed, + bodyReplacementFallbackUsed: completion.bodyReplacementFallbackUsed, + entryLockStartSec: completion.entryLockStartSec, + entryLockLengthMs: completion.entryLockLengthMs, + exitLockStartSec: completion.exitLockStartSec, + renderedBodyStartSec: completion.renderedBodyStartSec, + renderedBodyEndSec: completion.renderedBodyEndSec, + islandNativeUsed: completion.islandNativeUsed, + islandNativeFallbackUsed: completion.islandNativeFallbackUsed, + islandRenderStartSec: completion.islandRenderStartSec, + islandRenderEndSec: completion.islandRenderEndSec, + transientMaskPeak: completion.transientMaskPeak, + voicedCoreMaskPeak: completion.voicedCoreMaskPeak, + hpssUsed: completion.hpssUsed, + hpssFallbackUsed: completion.hpssFallbackUsed, + harmonicMaskPeak: completion.harmonicMaskPeak, + aperiodicMaskPeak: completion.aperiodicMaskPeak, + spectralEnvelopeCorrectionUsed: completion.spectralEnvelopeCorrectionUsed, + pitchOnlyCoreTimbreCorrectionUsed: completion.pitchOnlyCoreTimbreCorrectionUsed, + pitchOnlyCoreEnvelopeMix: completion.pitchOnlyCoreEnvelopeMix, + pitchOnlyCoreRmsTrimDb: completion.pitchOnlyCoreRmsTrimDb, + pitchOnlyCoreEnvelopeLifter: completion.pitchOnlyCoreEnvelopeLifter, + pitchOnlyEntryTimbreCorrectionUsed: completion.pitchOnlyEntryTimbreCorrectionUsed, + pitchOnlyEntryRmsTrimDb: completion.pitchOnlyEntryRmsTrimDb, + pitchOnlyEntryTiltDb: completion.pitchOnlyEntryTiltDb, + pitchOnlyEntryHandoffUsed: completion.pitchOnlyEntryHandoffUsed, + pitchOnlyExitHandoffUsed: completion.pitchOnlyExitHandoffUsed, + vocalSourceFilterUsed: completion.vocalSourceFilterUsed, + vocalSourceFilterVoicedCoverage: completion.vocalSourceFilterVoicedCoverage, + vocalSourceFilterResidualMix: completion.vocalSourceFilterResidualMix, + vocalSourceFilterFallbackUsed: completion.vocalSourceFilterFallbackUsed, + vocalSourceFilterFallbackReason: completion.vocalSourceFilterFallbackReason, + vocalSourceFilterEntryDryMs: completion.vocalSourceFilterEntryDryMs, + vocalSourceFilterExitDryMs: completion.vocalSourceFilterExitDryMs, + wsolaUsed: completion.wsolaUsed, + wsolaFallbackUsed: completion.wsolaFallbackUsed, + wsolaEntryLagSamples: completion.wsolaEntryLagSamples, + wsolaExitLagSamples: completion.wsolaExitLagSamples, + wsolaCorrelationScore: completion.wsolaCorrelationScore, + phaseLockUsed: completion.phaseLockUsed, + phaseLockFallbackUsed: completion.phaseLockFallbackUsed, + phaseAlignedEntry: completion.phaseAlignedEntry, + phaseAlignedExit: completion.phaseAlignedExit, + phasePeakCount: completion.phasePeakCount, + transitionHqUsed: completion.transitionHqUsed, + transitionHqFallbackUsed: completion.transitionHqFallbackUsed, + transitionStartSec: completion.transitionStartSec, + transitionEndSec: completion.transitionEndSec, + transitionTransientPeak: completion.transitionTransientPeak, + transitionVoicedCorePeak: completion.transitionVoicedCorePeak, + transitionResidualPeak: completion.transitionResidualPeak, + transitionEnvelopeCorrectionUsed: completion.transitionEnvelopeCorrectionUsed, + engineV2Used: completion.engineV2Used, + engineV2FallbackUsed: completion.engineV2FallbackUsed, + engineV2TransitionCount: completion.engineV2TransitionCount, + engineV2TransitionStartSec: completion.engineV2TransitionStartSec, + engineV2TransitionEndSec: completion.engineV2TransitionEndSec, + engineV2HarmonicSupportPeak: completion.engineV2HarmonicSupportPeak, + engineV2ResidualSupportPeak: completion.engineV2ResidualSupportPeak, + engineV2EnvelopeSupportPeak: completion.engineV2EnvelopeSupportPeak, + transientBypassUsed: completion.transientBypassUsed, + residualCarryUsed: completion.residualCarryUsed, + cepstralCutoffUsed: completion.cepstralCutoffUsed, + fftSizeUsed: completion.fftSizeUsed, + hopSizeUsed: completion.hopSizeUsed, + immediateLeftNeighborUsed: completion.immediateLeftNeighborUsed, + immediateRightNeighborUsed: completion.immediateRightNeighborUsed, + leftNeighborSamplesRendered: completion.leftNeighborSamplesRendered, + rightNeighborSamplesRendered: completion.rightNeighborSamplesRendered, + leftNeighborSmoothMs: completion.leftNeighborSmoothMs, + rightNeighborSmoothMs: completion.rightNeighborSmoothMs, + nonImmediateNeighborTouched: completion.nonImmediateNeighborTouched, + entryAlignmentOffsetMs: completion.entryAlignmentOffsetMs, + exitAlignmentOffsetMs: completion.exitAlignmentOffsetMs, + firstVoicedCyclesEntryUsed: completion.firstVoicedCyclesEntryUsed, + firstVoicedCyclesExitUsed: completion.firstVoicedCyclesExitUsed, + v3TransitionPairUsed: completion.v3TransitionPairUsed, + v3ContinuousRenderUsed: completion.v3ContinuousRenderUsed, + v3EntryAnchorMs: completion.v3EntryAnchorMs, + v3ExitAnchorMs: completion.v3ExitAnchorMs, + v3FirstCyclesEntryCount: completion.v3FirstCyclesEntryCount, + v3FirstCyclesExitCount: completion.v3FirstCyclesExitCount, + v3ShellDurationMs: completion.v3ShellDurationMs, + v3BodyDurationMs: completion.v3BodyDurationMs, + v3ResidualMix: completion.v3ResidualMix, + v3FormantMode: completion.v3FormantMode, + v3NeighborLeftOverlapMs: completion.v3NeighborLeftOverlapMs, + v3NeighborRightOverlapMs: completion.v3NeighborRightOverlapMs, + sourceAudioPath: job.sourceAudioPath, + referenceAudioPath: job.referenceAudioPath, + windowStartSec: job.windowStartSec, + windowEndSec: job.windowEndSec, + noteBodyStartSec: job.noteBodyStartSec, + noteBodyEndSec: job.noteBodyEndSec, + entryWindowSec: job.entryWindowSec, + exitWindowSec: job.exitWindowSec, + neighborWindowSec: job.neighborWindowSec, + previewCoverageStartSec: completion.previewCoverageStartSec, + previewCoverageEndSec: completion.previewCoverageEndSec, + candidateCoverageStartSec: completion.candidateCoverageStartSec, + candidateCoverageEndSec: completion.candidateCoverageEndSec, + label: job.label, + }; +} + +async function runExportRegressionJob(job: PitchRegressionJob): Promise<PitchRegressionResult> { + const applyResult = await runRenderRegressionJob({ ...job, renderMode: "note_hq" }); + const store = useDAWStore.getState(); + const track = store.tracks.find((candidate) => candidate.id === job.trackId); + const clip = track?.clips.find((candidate) => candidate.id === job.clipId); + if (!clip) { + throw new Error(`Fixture clip not found after pitch apply: ${job.clipId}`); + } + + await store.syncClipsWithBackend(); + const exportOutputPath = deriveExportOutputPath(job); + const renderEndTime = Math.max(clip.startTime + clip.duration, job.windowEndSec ?? 0); + const exportOk = await nativeBridge.renderProject({ + source: "master", + startTime: 0, + endTime: renderEndTime, + filePath: exportOutputPath, + format: "wav", + sampleRate: 44100, + bitDepth: 24, + channels: 2, + normalize: false, + addTail: false, + tailLength: 0, + }); + if (!exportOk) { + throw new Error(`Export regression render failed: ${exportOutputPath}`); + } + + return { + ...applyResult, + jobType: "export", + outputFile: exportOutputPath, + pitchCorrectionOutputFile: applyResult.outputFile, + exportOutputFile: exportOutputPath, + }; +} + +async function runCleanExportRegressionJob(job: PitchRegressionJob): Promise<PitchRegressionResult> { + if (!job.projectFixturePath || !job.trackId) { + throw new Error("Clean export regression job is missing required fixture project metadata."); + } + + const loadOk = await useDAWStore.getState().loadProject(job.projectFixturePath, { bypassFX: true }); + if (!loadOk) { + throw new Error(`Failed to load fixture project: ${job.projectFixturePath}`); + } + + replaceFixtureClipSource(job); + await useDAWStore.getState().syncClipsWithBackend(); + + const store = useDAWStore.getState(); + const track = store.tracks.find((candidate) => candidate.id === job.trackId); + const clip = track?.clips.find((candidate) => candidate.id === job.clipId); + if (!clip) { + throw new Error(`Fixture clip not found for clean export: ${job.clipId}`); + } + + const exportOutputPath = deriveExportOutputPath(job); + const renderEndTime = Math.max(clip.startTime + clip.duration, job.windowEndSec ?? 0); + const exportOk = await nativeBridge.renderProject({ + source: "master", + startTime: 0, + endTime: renderEndTime, + filePath: exportOutputPath, + format: "wav", + sampleRate: 44100, + bitDepth: 24, + channels: 2, + normalize: false, + addTail: false, + tailLength: 0, + }); + if (!exportOk) { + throw new Error(`Clean export regression render failed: ${exportOutputPath}`); + } + + const targetShift = finiteNumberOrNull(job.targetShiftSemitones) ?? 0; + return { + success: true, + jobType: "clean_export", + outputFile: exportOutputPath, + exportOutputFile: exportOutputPath, + clipId: job.clipId, + renderMode: job.renderMode ?? "note_hq", + targetShiftSemitones: targetShift, + actualRequestedShiftSemitones: 0, + requestedShiftErrorCents: Math.abs(targetShift) * 100, + chromaticSnapBypassed: true, + processingMode: "no-pitch-clean-export", + formantCurveUsed: false, + explicitFormantRequested: false, + pitchOnlyFormantSuppressed: false, + sourceAudioPath: job.sourceAudioPath, + referenceAudioPath: job.referenceAudioPath, + windowStartSec: job.windowStartSec, + windowEndSec: job.windowEndSec, + noteBodyStartSec: job.noteBodyStartSec, + noteBodyEndSec: job.noteBodyEndSec, + entryWindowSec: job.entryWindowSec, + exitWindowSec: job.exitWindowSec, + neighborWindowSec: job.neighborWindowSec, + label: job.label, + }; +} + +async function runAuditionRegressionJob(job: PitchRegressionJob): Promise<PitchRegressionResult> { + if (!job.trackId) { + throw new Error("Audition regression job is missing trackId."); + } + + const notes = normalizePitchNotes(job.notes); + if (notes.length === 0) { + throw new Error("Audition regression job did not contain any editable notes."); + } + + const noteStart = job.noteBodyStartSec ?? notes[0].startTime; + const auditionStart = job.auditionStartSec ?? Math.max(0, noteStart - 1.0); + const auditionDuration = job.auditionDurationSec ?? 4.5; + if (auditionDuration < 4.0) { + throw new Error(`Audition capture duration must be at least 4 seconds; got ${auditionDuration}.`); + } + + const applyResult = await runRenderRegressionJob({ ...job, renderMode: "note_hq" }); + const playbackOutputPath = deriveAuditionPlaybackOutputPath(job); + const exportOutputPath = deriveAuditionExportOutputPath(job); + const routeReportPath = playbackOutputPath.replace(/\.[^.\\/]+$/, "_route.json"); + + await useDAWStore.getState().syncClipsWithBackend(); + const appFinalCapture = await nativeBridge.capturePitchAppFinalContext({ + trackId: job.trackId, + clipId: job.clipId, + startTime: auditionStart, + duration: auditionDuration, + wavPath: playbackOutputPath, + routeJsonPath: routeReportPath, + sampleRate: 44100, + metadata: { + jobLabel: job.label ?? null, + renderMode: "note_hq", + outputFile: applyResult.outputFile ?? null, + targetShiftSemitones: applyResult.targetShiftSemitones ?? job.targetShiftSemitones ?? null, + actualRequestedShiftSemitones: applyResult.actualRequestedShiftSemitones ?? null, + formantCurveUsed: applyResult.formantCurveUsed ?? null, + actualRendererBranch: applyResult.actualRendererBranch ?? null, + }, + }); + const capture = appFinalCapture?.capture; + if (!capture?.success || !capture.filePath) { + throw new Error(`Pitch audition playback capture failed: ${capture?.error ?? "unknown error"}`); + } + + const exportOk = await nativeBridge.renderProject({ + source: "master", + startTime: auditionStart, + endTime: auditionStart + auditionDuration, + filePath: exportOutputPath, + format: "wav", + sampleRate: 44100, + bitDepth: 24, + channels: 2, + normalize: false, + addTail: false, + tailLength: 0, + }); + if (!exportOk) { + throw new Error(`Pitch audition export render failed: ${exportOutputPath}`); + } + + return { + ...applyResult, + jobType: "audition", + outputFile: playbackOutputPath, + auditionPlaybackOutputFile: playbackOutputPath, + auditionExportOutputFile: exportOutputPath, + auditionStartSec: auditionStart, + auditionDurationSec: auditionDuration, + auditionCapture: capture, + appFinalCapture: appFinalCapture ?? undefined, + exportOutputFile: exportOutputPath, + pitchCorrectionOutputFile: applyResult.outputFile, + }; +} + +export async function maybeRunPitchRegressionDriver() { + if (started) { + return; + } + + started = true; + + const job = await nativeBridge.getPitchRegressionJob(); + if (!job) { + return; + } + + try { + const result = job.jobType === "analysis" + ? await runAnalysisRegressionJob(job) + : job.jobType === "scrub" + ? await runScrubRegressionJob(job) + : job.jobType === "export" + ? await runExportRegressionJob(job) + : job.jobType === "clean_export" + ? await runCleanExportRegressionJob(job) + : job.jobType === "audition" + ? await runAuditionRegressionJob(job) + : await runRenderRegressionJob(job); + await reportRegressionResult(result); + } catch (error) { + await reportRegressionResult({ + success: false, + error: error instanceof Error ? error.stack || error.message : String(error), + }); + } +} diff --git a/frontend/src/utils/platform.ts b/frontend/src/utils/platform.ts new file mode 100644 index 0000000..3759a81 --- /dev/null +++ b/frontend/src/utils/platform.ts @@ -0,0 +1,104 @@ +/** + * Platform detection and keyboard shortcut formatting utilities. + * + * Shortcut strings throughout the codebase use Windows-style canonical names: + * "Ctrl+Z", "Ctrl+Shift+Z", "Alt+Enter", "Ctrl+Alt+R" + * + * macOS modifier mapping: + * Windows Ctrl → macOS Cmd (metaKey) + * Windows Alt → macOS Ctrl (ctrlKey) + * Shift stays Shift + * + * This file handles both matching (dispatcher) and display formatting. + */ + +export const isMac: boolean = + typeof navigator !== "undefined" && + (/Mac|iPhone|iPad|iPod/.test(navigator.platform) || + /Mac/.test(navigator.userAgent)); + +/** + * Format a canonical shortcut string for display on the current platform. + * + * Examples on Windows: "Ctrl+Z" → "Ctrl+Z", "Alt+Enter" → "Alt+Enter" + * Examples on macOS: "Ctrl+Z" → "Cmd+Z", "Alt+Enter" → "Ctrl+Enter", + * "Ctrl+Alt+R" → "Cmd+Ctrl+R", "Ctrl+Shift+Z" → "Cmd+Shift+Z" + */ +export function formatShortcut(shortcut: string | undefined): string { + if (!shortcut) return ""; + // Skip descriptive pseudo-shortcuts like "Space (while playing)" + if (shortcut.includes("(")) return shortcut; + + if (!isMac) return shortcut; + + // Parse canonical parts: everything before the last segment is a modifier + const segments = shortcut.split("+"); + const key = segments[segments.length - 1]; + const mods = segments.slice(0, -1); + + const mapped: string[] = []; + for (const mod of mods) { + if (mod === "Ctrl") mapped.push("Cmd"); + else if (mod === "Alt") mapped.push("Ctrl"); + else mapped.push(mod); // Shift stays Shift + } + mapped.push(key); + return mapped.join("+"); +} + +/** + * Given a raw KeyboardEvent, build the canonical shortcut string used in + * actionRegistry ("Ctrl+Z", "Alt+Enter", etc.). + * + * On macOS: + * metaKey (Cmd) → "Ctrl" + * ctrlKey (Ctrl) → "Alt" + * altKey (Option) → not mapped (ignored) + * + * On Windows/Linux: + * ctrlKey | metaKey → "Ctrl" + * altKey → "Alt" + */ +export function keyEventToCanonicalShortcut(e: KeyboardEvent): string { + const parts: string[] = []; + + if (isMac) { + if (e.metaKey) parts.push("Ctrl"); // Cmd → Ctrl + if (e.ctrlKey) parts.push("Alt"); // Ctrl → Alt + } else { + if (e.ctrlKey || e.metaKey) parts.push("Ctrl"); + if (e.altKey) parts.push("Alt"); + } + if (e.shiftKey) parts.push("Shift"); + + let key = e.key; + if (["Control", "Shift", "Alt", "Meta"].includes(key)) return ""; + if (key === " ") key = "Space"; + else if (key === "ArrowLeft") key = "Left"; + else if (key === "ArrowRight") key = "Right"; + else if (key === "ArrowUp") key = "Up"; + else if (key === "ArrowDown") key = "Down"; + else if (key === "Escape") key = "Esc"; + else if (key.length === 1) key = key.toUpperCase(); + + parts.push(key); + return parts.join("+"); +} + +/** + * Returns true if the event represents the platform's primary modifier key. + * On Mac = Cmd (metaKey). On Windows = Ctrl. + * Use this instead of bare `e.ctrlKey` in keyboard handlers so they work on both platforms. + */ +export function isPrimaryModifier(e: KeyboardEvent | MouseEvent): boolean { + return isMac ? e.metaKey : e.ctrlKey; +} + +/** + * Returns true if the event represents the platform's secondary modifier key. + * On Mac = Ctrl (ctrlKey). On Windows = Alt. + * This is the "Alt" modifier in canonical shortcut strings. + */ +export function isSecondaryModifier(e: KeyboardEvent | MouseEvent): boolean { + return isMac ? e.ctrlKey : e.altKey; +} diff --git a/frontend/src/utils/renderPreparation.ts b/frontend/src/utils/renderPreparation.ts new file mode 100644 index 0000000..d213915 --- /dev/null +++ b/frontend/src/utils/renderPreparation.ts @@ -0,0 +1,47 @@ +import { nativeBridge } from "../services/NativeBridge"; +import { usePitchEditorStore } from "../store/pitchEditorStore"; + +const PITCH_RENDER_BUSY_STATES = new Set([ + "queued", + "processing", + "preview_processing", + "final_processing", +]); + +export async function waitForPitchEditorRenderReady(timeoutMs = 120000): Promise<void> { + const startedAt = performance.now(); + while (performance.now() - startedAt < timeoutMs) { + const state = usePitchEditorStore.getState(); + if (!state.clipId || !PITCH_RENDER_BUSY_STATES.has(state.applyState)) { + if (state.applyState === "error") { + throw new Error("Pitch note render failed. Fix the pitch render before exporting."); + } + return; + } + await new Promise((resolve) => window.setTimeout(resolve, 100)); + } + throw new Error("Timed out waiting for the pitch note render to finish before export."); +} + +export async function clearPitchEditorTransientPreviewsForRender(): Promise<void> { + const { clipId } = usePitchEditorStore.getState(); + await nativeBridge.clearPitchPreviewRoutesForCorrectedSources().catch(() => 0); + if (!clipId) return; + await Promise.allSettled([ + nativeBridge.clearAllPitchPreviewRoutes(clipId), + ]); +} + +export async function prepareForManualRender( + syncClipsWithBackend: () => Promise<void>, + label = "manual-render", +): Promise<void> { + console.log(`[render.preflight] ${label}: waiting for pitch renders`); + await waitForPitchEditorRenderReady(); + console.log(`[render.preflight] ${label}: clearing transient preview routes`); + await clearPitchEditorTransientPreviewsForRender(); + console.log(`[render.preflight] ${label}: syncing backend clips`); + await syncClipsWithBackend(); + await nativeBridge.clearPitchPreviewRoutesForCorrectedSources().catch(() => 0); + console.log(`[render.preflight] ${label}: ready`); +} diff --git a/frontend/src/utils/rulerClickSnap.ts b/frontend/src/utils/rulerClickSnap.ts new file mode 100644 index 0000000..e82f53a --- /dev/null +++ b/frontend/src/utils/rulerClickSnap.ts @@ -0,0 +1,44 @@ +import { snapToGrid, type GridSize } from "./snapToGrid"; + +export const RULER_CLICK_SNAP_PX = 10; + +interface TimeSignature { + numerator: number; + denominator: number; +} + +interface GetRulerClickSnapTimeParams { + time: number; + pixelsPerSecond: number; + tempo: number; + timeSignature: TimeSignature; + gridSize: GridSize; + snapEnabled: boolean; + ctrlBypass?: boolean; +} + +export function getRulerClickSnapTime({ + time, + pixelsPerSecond, + tempo, + timeSignature, + gridSize, + snapEnabled, + ctrlBypass = false, +}: GetRulerClickSnapTimeParams): number { + // Beat snap is always preferred for ruler clicks because it is navigation-oriented. + const secondsPerBeat = 60 / tempo; + const beatSnapped = Math.round(time / secondsPerBeat) * secondsPerBeat; + const beatSnapDistPx = Math.abs(beatSnapped - time) * pixelsPerSecond; + if (beatSnapDistPx <= RULER_CLICK_SNAP_PX) { + return beatSnapped; + } + + if (!snapEnabled || ctrlBypass) { + return time; + } + + const gridSnapped = snapToGrid(time, tempo, timeSignature, gridSize); + const gridSnapDistPx = Math.abs(gridSnapped - time) * pixelsPerSecond; + return gridSnapDistPx <= RULER_CLICK_SNAP_PX ? gridSnapped : time; +} diff --git a/frontend/src/utils/trackCreation.ts b/frontend/src/utils/trackCreation.ts index cec0f40..ac6da97 100644 --- a/frontend/src/utils/trackCreation.ts +++ b/frontend/src/utils/trackCreation.ts @@ -1,11 +1,13 @@ import { type TrackType, useDAWStore } from "../store/useDAWStore"; +import { getDefaultWorkflowParams } from "../data/aiWorkflows"; -export type InsertableTrackType = Extract<TrackType, "audio" | "midi" | "instrument">; +export type InsertableTrackType = Extract<TrackType, "audio" | "midi" | "instrument" | "ai">; const DEFAULT_PREFIX: Record<InsertableTrackType, string> = { audio: "Audio", midi: "MIDI", instrument: "Instrument", + ai: "AI", }; function getTrackName(type: InsertableTrackType, prefix?: string) { @@ -27,6 +29,7 @@ export async function createTrackOfType( const trackId = crypto.randomUUID(); const state = useDAWStore.getState(); const isMidiType = type === "midi" || type === "instrument"; + const isAiType = type === "ai"; state.addTrack({ id: trackId, @@ -36,6 +39,37 @@ export async function createTrackOfType( inputChannelCount: isMidiType ? 1 : 2, armed: type === "instrument", monitorEnabled: type === "instrument", + aiWorkflow: isAiType ? "text-to-music" : undefined, + aiWorkflowParams: isAiType + ? getDefaultWorkflowParams("text-to-music") + : undefined, + aiGenerationState: isAiType ? "idle" : undefined, + aiGenerationProgress: isAiType ? 0 : undefined, + aiGenerationError: isAiType ? "" : undefined, + aiGenerationPhase: isAiType ? "" : undefined, + aiGenerationMessage: isAiType ? "" : undefined, + aiGenerationBackend: isAiType ? "" : undefined, + aiGenerationElapsedMs: isAiType ? 0 : undefined, + aiGenerationHeartbeatTs: isAiType ? 0 : undefined, + aiGenerationPhaseProgress: isAiType ? undefined : undefined, + aiGenerationEtaMs: isAiType ? undefined : undefined, + aiGenerationRunMode: isAiType ? undefined : undefined, + aiGenerationRuntimeProfile: isAiType ? "" : undefined, + aiGenerationLmModel: isAiType ? "" : undefined, + aiGenerationStatusNote: isAiType ? "" : undefined, + aiGenerationFailureKind: isAiType ? "" : undefined, + aiGenerationSessionMode: isAiType ? "" : undefined, + aiGenerationWorkerExitCode: isAiType ? 0 : undefined, + aiGenerationLastStdoutLine: isAiType ? "" : undefined, + aiGenerationLastStderrLine: isAiType ? "" : undefined, + aiGenerationAttemptMode: isAiType ? "" : undefined, + aiGenerationAttemptIndex: isAiType ? 0 : undefined, + aiGenerationProtocolVersion: isAiType ? 0 : undefined, + aiGenerationScriptVersion: isAiType ? "" : undefined, + aiGenerationRequestId: isAiType ? "" : undefined, + aiGenerationPriorFailure: isAiType ? "" : undefined, + aiGenerationLastProgressAgeMs: isAiType ? 0 : undefined, + icon: isAiType ? "ai" : undefined, }); state.selectTrack(trackId); diff --git a/packaging/macos/OpenStudio.entitlements b/packaging/macos/OpenStudio.entitlements new file mode 100644 index 0000000..c950b88 --- /dev/null +++ b/packaging/macos/OpenStudio.entitlements @@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" + "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>com.apple.security.device.audio-input</key> + <true/> + <key>com.apple.security.cs.disable-library-validation</key> + <true/> +</dict> +</plist> diff --git a/tests/fixtures/pitch-regression/README.md b/tests/fixtures/pitch-regression/README.md new file mode 100644 index 0000000..5eeff70 --- /dev/null +++ b/tests/fixtures/pitch-regression/README.md @@ -0,0 +1,117 @@ +`pitch_regression_fixture.osproj` is the tracked one-track fixture for the app-path pitch regression harness. + +Stable IDs: +- `trackId`: `pitch-regression-track-1` +- `clipId`: `pitch-regression-clip-1` + +The fixture clip intentionally starts with an empty `filePath`. The regression driver replaces it at runtime with the source WAV from the job file before syncing clips to the backend and opening the pitch editor. + +Example runner flow: + +```powershell +tools\run-pitch-headless-regression.ps1 ` + -SourceAudioPath "d:\test projects\pitchTestOrg.wav" ` + -NotesJsonPath "tests\fixtures\pitch-regression\example_plus4_notes.json" ` + -TargetShiftSemitones 4.00 ` + -WindowStart 0.90 ` + -WindowEnd 1.55 ` + -Label "pitchTest_plus4" +``` + +`example_plus4_notes.json` is only a schema/example starting point. Replace the note timing and pitch values with the real target edit for the clip family you are testing. + +`example_minus4_notes.json` is the matching downward-shift example for the same note window, so the regression harness can compare against `-4 st` references without reusing the `+4 st` note payload by mistake. + +`example_plus4_two_adjacent_word_fragments.json` is the adjacent-fragment regression fixture: both notes share one `wordGroupId` and should render as one note-HQ edit island with no internal bridge or doubled voice. + +`example_pitchTest_plus4_notes.json` and `example_pitchTest_minus4_notes.json` are clip-family-specific fixtures for the later edited note in the `pitchTestOrg` references. Those references do not use the same note timing as `pitchOrg`, so they should not reuse the older `0.90s -> 1.55s` fixture. + +Renderer selection: + +- The tracked deterministic harness allows `default` and `pitch_only_vocal_source_filter_hq`. +- Historical UI/bakeoff helpers are local diagnostics only and are not part of the tracked PR gate. + +Harness notes: + +- The tracked PR gate is deterministic only: exact requested relative shift, output sanity, renderer branch recording, pitch-only formant curve disabled, and corrected-source route state. +- Spectral, null, residual, and double-voice scripts are local diagnostics only. They must not be used as proof that subjective artifact/timbre issues are resolved. +- Subjective audition remains the pass/fail source for start artifacts, doubled voice, robotic tone, naturalness, formant shift, and timbre. +- Debug layer dumps can be enabled with `OPENSTUDIO_VSF_LAYER_DUMP_ENABLE=1`; dumps include dry input, source/filter core, residual/noise layer, wet envelope, optional local hybrid diagnostics, and final output. +- The runner now derives note-body metadata from the first note by default and passes onset/release/neighbor windows into the scorer. +- Analyzer/product segmentation gates: + - pitch-curve corners are `boundaryCandidates` by default, not automatic editable note cuts. + - `destructiveCornerSplitCount` must be `0` unless `OPENSTUDIO_ANALYZER_APPLY_CORNER_SPLITS=1` is explicitly set for research. + - pitch jumps and sustained contour deviations are `pitch_hysteresis_*` boundary candidates by default, not automatic editable note cuts. + - `destructivePitchJumpSplitCount` must be `0` in product/default analyzer runs. + - strong acoustic `hard_word_like` candidates may split default analyzer regions and are reported separately from destructive pitch-corner/pitch-jump failures. + - vibrato-like periodic reversals should be reported as `internal_vibrato`/suppressed diagnostics and must not split a continuous same-breath word. + - short voiced detector dropouts inside a phrase are bridged up to about `80 ms`; hard automatic splits require longer unvoiced gaps or sustained energy-break evidence. + - expected vocal regions should have word-group overlap `>= 0.85`. + - expected vocal regions should not have large overhang from collapsed surrounding words; the analysis gate fails suspicious overhang above `0.65 s`. + - the editor should select and move only explicitly selected notes by default. + - `wordGroupId` is assistive metadata for diagnostics/render grouping, not normal click/drag ownership. +- Adjacent note-HQ edit gates: + - adjacent selected/moved notes should produce one edit island with one entry bridge and one exit bridge. + - internal boundaries inside that island must not report a separate note-HQ bridge. + - double-voice checks should fail on internal-boundary energy lift above `+3 dB`, duplicated onset/repeat spikes, or excess spectral flux. +- `note_hq` comparison semantics: + - `preview_segment` candidates are window-local and must be compared with `--candidate-is-window`. + - `note_hq` candidates are window-local only when the app returns a true segment render. + - phrase/full-clip `note_hq` results are compared as full-clip audio; do not shift the candidate slice to the request window. + - boundary metrics require non-empty pre/post neighbor regions when the note body and neighbor window fit inside the reference duration. +- Pitch-only `note_hq` gates for the portable `pitchOrg +4` and `pitchOrg -4` cases: + - analyzer diagnostics should include note boundary kinds when analysis-derived notes are used: + - `hard_word_like` means previous/next word bodies must remain dry except for a tiny bridge. + - `soft_legato` means a continuous sustain/legato transition may use wider phrase smoothing. + - corner boundaries must be conservative; vibrato-like periodic reversals should not create false note splits. + - body/core pitch error must stay within the configured cents limit. + - harmonic-envelope drift, low/mid/high band deltas, and F1/F2 proxy drift are hard failures, not informational-only spectrogram stats. + - boundary timing, onset derivative/repeat/flux, and pre/post neighbor residual checks must pass together. + - current production note-HQ pitch-only defaults to `pitch_only_vocal_source_filter_hq`; retired renderer branches are not part of the tracked fixture contract. + - upward edits should report direction `upward` and the selected renderer branch. + - downward edits should report direction `downward`; `downshiftFormantGuardUsed=true` is still expected on canonical VSF downshift runs. + - downward primary renders should report `spectralEnvelopeCorrectionUsed=true`, because voiced-core envelope transfer is now part of the production downshift timbre guard. + - production note-HQ pitch-only renders use the native VSF path; historical benchmark engines are not part of this fixture contract. + - the aspirational downshift harmonic-envelope target is `<= 0.35`; the practical hard gate is currently `<= 0.36` so the measured `pitchOrg -4` native directional result at about `0.343` passes while still leaving polish room. + - low/mid/high downshift band deltas target `<= 3 dB`. + - F1/F2 proxy drift target is `<= 120 Hz`; on the canonical downshift case the hard gate uses the note-body proxy because the core F2 proxy is unstable on this fixture. + - edited-note end/next-note handoff is tracked by the exit-to-next-note artifact score, which combines exit-side discontinuity, high-band burst, derivative jump, repeat excess, and flux delta. It intentionally does not gate on raw next-note mel distance, because the reference may pitch-edit the body while the next note remains dry. + - product commit rule: render context may extend before the edited note, and final apply may use a bounded entry bridge, but audio before `noteHqEntryBridgeStartSec` must remain original/dry. + - previous-word/left-shoulder handoff is tracked by a bridge-aware ownership audit, not just the old narrow `preCommitArtifactScore`: + - `preBridgeTailOriginalResidualDb` compares candidate vs original over `[noteHqEntryBridgeStartSec - 80 ms, noteHqEntryBridgeStartSec)` and must be `<= -70 dB` for primary render/apply gates. + - `noteHqEntryBridgeStartSec` must be no earlier than `note.startTime - 24 ms`. + - `candidateActiveDifferenceStartSec` may start at the bridge, but not earlier than `noteHqEntryBridgeStartSec - 5 ms`. + - `preBridgeTailArtifactScore` must be `<= 1.0`. + - entry lag absolute target is `<= 8 ms`, onset artifact target is `<= 1.6`, onset derivative target is `<= 1.6`, and exit-next artifact remains `<= 3.0`. + - the audit uses `noteHqEntryBridgeStartSec` when the result provides it, so body/core analysis windows can be narrower than the true note ownership start without creating false previous-word failures. + - entry contour diagnostics: + - `noteHqEntryPitchHandoffUsed`, start/end, pre/body milliseconds, slope jump, and acceleration-limit status are emitted for note-HQ native renders. + - hard/unknown canonical edits keep pitch-ratio render pre-roll but reach the target by `note.startTime`; they should not use a delayed audible pitch handoff because the `pitchOrg` references behave like step edits. + - explicit continuous/internal transitions (`soft_legato`, `internal_bend`, `internal_vibrato`, or adjacent selected notes inside one edit island) may use a bounded pitch handoff; slope/acceleration gates apply to those cases. + - `entryPitchSlopeJumpStPerSec` and `entryPitchAccelerationSpike` are reported by the scorer, but the hard gate is skipped when no pitch handoff was used. + - the corrected lesson from 2026-04-27: increasing left shoulder ownership can hide an entry issue by moving the stutter backward into the previous word, but forcing dry audio until the exact note start can leave a phase/envelope discontinuity at the edited-note entry. + - dry-neighbor residual checks only inspect the part of the neighbor region outside the effective note-HQ commit range; owned shoulders are scored by pre-commit/onset/exit-next artifacts instead. + - export jobs do not use the source-vs-export pre-body residual as a hard dry-ownership gate because the mixer/export path can change the whole file from time zero; export parity is checked against the note-HQ product. +- Default diagnostics: + - entry window: `80 ms` + - exit window: `80 ms` + - neighbor windows: `120 ms` +- Each run also writes a short `audition` folder beside the summary JSON with: + - `orig_phrase.wav` + - `ref_phrase.wav` + - `cand_phrase.wav` + - `cand_entry.wav` + - `cand_core.wav` + - `cand_exit.wav` + - `cand_exit_next.wav` + - `cand_pre_commit.wav` + - `orig_pre_body_tail.wav` + - `cand_pre_body_tail.wav` + - `diff_pre_body_tail.wav` + - `orig_entry_bridge.wav` + - `ref_entry_bridge.wav` + - `cand_entry_bridge.wav` + - `diff_pre_bridge_tail.wav` + - `cand_first_40ms.wav` + - `cand_pre_neighbor.wav` + - `cand_post_neighbor.wav` diff --git a/tests/fixtures/pitch-regression/example_minus4_notes.json b/tests/fixtures/pitch-regression/example_minus4_notes.json new file mode 100644 index 0000000..ea95ff2 --- /dev/null +++ b/tests/fixtures/pitch-regression/example_minus4_notes.json @@ -0,0 +1,20 @@ +[ + { + "id": "pitch-note-1", + "startTime": 0.9, + "endTime": 1.55, + "effectiveStartTime": 0.86, + "effectiveEndTime": 1.61, + "detectedPitch": 60, + "correctedPitch": 56, + "driftCorrectionAmount": 0, + "vibratoDepth": 1, + "vibratoRate": 0, + "transitionIn": 40, + "transitionOut": 60, + "formantShift": 0, + "gain": 0, + "voiced": true, + "pitchDrift": [] + } +] diff --git a/tests/fixtures/pitch-regression/example_pitchOrg_scrub_multinote.json b/tests/fixtures/pitch-regression/example_pitchOrg_scrub_multinote.json new file mode 100644 index 0000000..244e93c --- /dev/null +++ b/tests/fixtures/pitch-regression/example_pitchOrg_scrub_multinote.json @@ -0,0 +1,38 @@ +[ + { + "id": "note_0", + "startTime": 0.214784577488899, + "endTime": 0.534058928489685, + "effectiveStartTime": 0.214784577488899, + "effectiveEndTime": 0.534058928489685, + "detectedPitch": 67.04881286621094, + "correctedPitch": 67.04881286621094, + "driftCorrectionAmount": 0, + "vibratoDepth": 1, + "vibratoRate": 0, + "transitionIn": 0, + "transitionOut": 0, + "formantShift": 0, + "gain": 0, + "voiced": true, + "pitchDrift": [] + }, + { + "id": "note_1", + "startTime": 0.615328788757324, + "endTime": 1.52671205997467, + "effectiveStartTime": 0.615328788757324, + "effectiveEndTime": 1.120362758636475, + "detectedPitch": 67.56072998046875, + "correctedPitch": 67.56072998046875, + "driftCorrectionAmount": 0, + "vibratoDepth": 1, + "vibratoRate": 0, + "transitionIn": 0, + "transitionOut": 0, + "formantShift": 0, + "gain": 0, + "voiced": true, + "pitchDrift": [] + } +] diff --git a/tests/fixtures/pitch-regression/example_pitchTest_minus4_notes.json b/tests/fixtures/pitch-regression/example_pitchTest_minus4_notes.json new file mode 100644 index 0000000..f719144 --- /dev/null +++ b/tests/fixtures/pitch-regression/example_pitchTest_minus4_notes.json @@ -0,0 +1,20 @@ +[ + { + "id": "pitchtest-minus4-note-1", + "startTime": 1.193, + "endTime": 2.283, + "effectiveStartTime": 1.153, + "effectiveEndTime": 2.333, + "detectedPitch": 67.64092835801603, + "correctedPitch": 63.486820576352436, + "driftCorrectionAmount": 0.0, + "vibratoDepth": 1.0, + "vibratoRate": 0.0, + "transitionIn": 40.0, + "transitionOut": 60.0, + "formantShift": 0.0, + "gain": 0.0, + "voiced": true, + "pitchDrift": [] + } +] diff --git a/tests/fixtures/pitch-regression/example_pitchTest_plus4_notes.json b/tests/fixtures/pitch-regression/example_pitchTest_plus4_notes.json new file mode 100644 index 0000000..d93851f --- /dev/null +++ b/tests/fixtures/pitch-regression/example_pitchTest_plus4_notes.json @@ -0,0 +1,20 @@ +[ + { + "id": "pitchtest-plus4-note-1", + "startTime": 1.193, + "endTime": 2.283, + "effectiveStartTime": 1.153, + "effectiveEndTime": 2.333, + "detectedPitch": 67.64092835801603, + "correctedPitch": 71.21309485364912, + "driftCorrectionAmount": 0.0, + "vibratoDepth": 1.0, + "vibratoRate": 0.0, + "transitionIn": 40.0, + "transitionOut": 60.0, + "formantShift": 0.0, + "gain": 0.0, + "voiced": true, + "pitchDrift": [] + } +] diff --git a/tests/fixtures/pitch-regression/example_pitchTest_scrub_multinote.json b/tests/fixtures/pitch-regression/example_pitchTest_scrub_multinote.json new file mode 100644 index 0000000..c629fd1 --- /dev/null +++ b/tests/fixtures/pitch-regression/example_pitchTest_scrub_multinote.json @@ -0,0 +1,38 @@ +[ + { + "id": "note_0", + "startTime": 0.748843550682068, + "endTime": 1.06811785697937, + "effectiveStartTime": 0.748843550682068, + "effectiveEndTime": 1.06811785697937, + "detectedPitch": 67.03276062011719, + "correctedPitch": 67.03276062011719, + "driftCorrectionAmount": 0, + "vibratoDepth": 1, + "vibratoRate": 0, + "transitionIn": 0, + "transitionOut": 0, + "formantShift": 0, + "gain": 0, + "voiced": true, + "pitchDrift": [] + }, + { + "id": "note_1", + "startTime": 1.149387717247009, + "endTime": 2.060770988464355, + "effectiveStartTime": 1.149387717247009, + "effectiveEndTime": 1.654421806335449, + "detectedPitch": 67.55762481689453, + "correctedPitch": 67.55762481689453, + "driftCorrectionAmount": 0, + "vibratoDepth": 1, + "vibratoRate": 0, + "transitionIn": 0, + "transitionOut": 0, + "formantShift": 0, + "gain": 0, + "voiced": true, + "pitchDrift": [] + } +] diff --git a/tests/fixtures/pitch-regression/example_plus4_notes.json b/tests/fixtures/pitch-regression/example_plus4_notes.json new file mode 100644 index 0000000..fe3bc1e --- /dev/null +++ b/tests/fixtures/pitch-regression/example_plus4_notes.json @@ -0,0 +1,20 @@ +[ + { + "id": "pitch-note-1", + "startTime": 0.9, + "endTime": 1.55, + "effectiveStartTime": 0.86, + "effectiveEndTime": 1.61, + "detectedPitch": 60, + "correctedPitch": 64, + "driftCorrectionAmount": 0, + "vibratoDepth": 1, + "vibratoRate": 0, + "transitionIn": 40, + "transitionOut": 60, + "formantShift": 0, + "gain": 0, + "voiced": true, + "pitchDrift": [] + } +] diff --git a/tests/fixtures/pitch-regression/example_plus4_two_adjacent_word_fragments.json b/tests/fixtures/pitch-regression/example_plus4_two_adjacent_word_fragments.json new file mode 100644 index 0000000..d6707f2 --- /dev/null +++ b/tests/fixtures/pitch-regression/example_plus4_two_adjacent_word_fragments.json @@ -0,0 +1,40 @@ +[ + { + "id": "pitch-note-1a", + "wordGroupId": "word_fixture_1", + "startTime": 0.9, + "endTime": 1.12, + "effectiveStartTime": 0.86, + "effectiveEndTime": 1.12, + "detectedPitch": 60, + "correctedPitch": 64, + "driftCorrectionAmount": 0, + "vibratoDepth": 1, + "vibratoRate": 0, + "transitionIn": 40, + "transitionOut": 0, + "formantShift": 0, + "gain": 0, + "voiced": true, + "pitchDrift": [] + }, + { + "id": "pitch-note-1b", + "wordGroupId": "word_fixture_1", + "startTime": 1.143, + "endTime": 1.55, + "effectiveStartTime": 1.143, + "effectiveEndTime": 1.61, + "detectedPitch": 60, + "correctedPitch": 64, + "driftCorrectionAmount": 0, + "vibratoDepth": 1, + "vibratoRate": 0, + "transitionIn": 0, + "transitionOut": 60, + "formantShift": 0, + "gain": 0, + "voiced": true, + "pitchDrift": [] + } +] diff --git a/tests/fixtures/pitch-regression/pitch_regression_fixture.osproj b/tests/fixtures/pitch-regression/pitch_regression_fixture.osproj new file mode 100644 index 0000000..c4fa89b --- /dev/null +++ b/tests/fixtures/pitch-regression/pitch_regression_fixture.osproj @@ -0,0 +1,136 @@ +{ + "version": "1.1.0", + "savedAt": 0, + "projectName": "Pitch Regression Fixture", + "projectNotes": "Single-track fixture for app-path pitch regression.", + "projectSampleRate": 48000, + "projectBitDepth": 24, + "processingPrecision": "float32", + "tempo": 120, + "timeSignature": { + "numerator": 4, + "denominator": 4 + }, + "metronomeEnabled": false, + "metronomeVolume": 0.5, + "metronomeAccentBeats": [ + true, + false, + false, + false + ], + "projectRange": { + "start": 0, + "end": 10 + }, + "markers": [], + "regions": [], + "tempoMarkers": [], + "masterVolume": 1, + "masterPan": 0, + "isMasterMuted": false, + "masterMono": false, + "masterAutomationLanes": [], + "showMasterAutomation": false, + "masterAutomationEnabled": true, + "suspendedMasterAutomationState": null, + "tracks": [ + { + "id": "pitch-regression-track-1", + "name": "Pitch Regression Track", + "color": "#22c55e", + "type": "audio", + "inputType": "stereo", + "volume": 1, + "volumeDB": 0, + "pan": 0, + "muted": false, + "soloed": false, + "armed": false, + "monitorEnabled": false, + "recordSafe": false, + "inputChannel": null, + "inputStartChannel": 0, + "inputChannelCount": 2, + "inputFxCount": 0, + "trackFxCount": 0, + "fxBypassed": false, + "automationLanes": [], + "showAutomation": false, + "automationEnabled": true, + "suspendedAutomationState": null, + "frozen": false, + "takes": [], + "activeTakeIndex": 0, + "sends": [], + "phaseInverted": false, + "stereoWidth": 100, + "masterSendEnabled": true, + "outputStartChannel": 0, + "outputChannelCount": 2, + "playbackOffsetMs": 0, + "trackChannelCount": 2, + "midiOutputDevice": "", + "waveformZoom": 1, + "spectralView": false, + "clips": [ + { + "id": "pitch-regression-clip-1", + "filePath": "", + "name": "Fixture Clip", + "startTime": 0, + "duration": 6, + "offset": 0, + "color": "#86efac", + "volumeDB": 0, + "fadeIn": 0, + "fadeOut": 0 + } + ], + "midiClips": [] + } + ], + "masterFXPaths": [], + "masterFXStates": [], + "mixerSnapshots": [], + "trackGroups": [], + "clipLauncher": { + "enabled": false, + "scenes": [], + "slots": {} + }, + "renderMetadata": { + "title": "", + "artist": "", + "album": "", + "genre": "", + "year": "", + "description": "", + "isrc": "" + }, + "renderDialogOptions": { + "source": "master", + "bounds": "entire-project", + "directory": "", + "fileName": "render", + "format": "wav", + "sampleRate": 48000, + "bitDepth": 24, + "channels": "stereo", + "tailLength": 0, + "addTail": false, + "normalize": false, + "dither": "none" + }, + "secondaryOutputEnabled": false, + "secondaryOutputFormat": "mp3", + "secondaryOutputBitDepth": 16, + "onlineRender": false, + "addToProjectAfterRender": false, + "projectAuthor": "", + "projectRevisionNotes": [], + "undoHistory": { + "commands": [], + "index": -1 + } +} diff --git a/tests/fixtures/pitch-regression/suites/formant_richer_suite.json b/tests/fixtures/pitch-regression/suites/formant_richer_suite.json new file mode 100644 index 0000000..34db830 --- /dev/null +++ b/tests/fixtures/pitch-regression/suites/formant_richer_suite.json @@ -0,0 +1,100 @@ +{ + "cases": [ + { + "label": "pitchOrg_plus4_formant_body_richer", + "sourceAudioPath": "D:/test projects/pitchOrg.wav", + "referenceAudioPath": "D:/test projects/pitchOrg+4s.wav", + "notesJsonPath": "../example_plus4_notes.json", + "targetShiftSemitones": 4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "windowStartSec": 0.82, + "windowEndSec": 1.66, + "noteBodyStartSec": 1.04, + "noteBodyEndSec": 1.41, + "entryWindowSec": 0.06, + "exitWindowSec": 0.06, + "neighborWindowSec": 0.08 + }, + { + "label": "pitchOrg_minus4_formant_body_richer", + "sourceAudioPath": "D:/test projects/pitchOrg.wav", + "referenceAudioPath": "D:/test projects/pitchOrg-4s.wav", + "notesJsonPath": "../example_minus4_notes.json", + "targetShiftSemitones": -4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "windowStartSec": 0.82, + "windowEndSec": 1.66, + "noteBodyStartSec": 1.04, + "noteBodyEndSec": 1.41, + "entryWindowSec": 0.06, + "exitWindowSec": 0.06, + "neighborWindowSec": 0.08 + }, + { + "label": "pitchTest_plus4_formant_body_richer", + "sourceAudioPath": "D:/test projects/pitchTestOrg.wav", + "referenceAudioPath": "D:/test projects/pitchTestOrg+4s.wav", + "notesJsonPath": "../example_pitchTest_plus4_notes.json", + "targetShiftSemitones": 4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "windowStartSec": 1.12, + "windowEndSec": 2.34, + "noteBodyStartSec": 1.38, + "noteBodyEndSec": 2.10, + "entryWindowSec": 0.06, + "exitWindowSec": 0.06, + "neighborWindowSec": 0.08 + }, + { + "label": "pitchTest_minus4_formant_body_richer", + "sourceAudioPath": "D:/test projects/pitchTestOrg.wav", + "referenceAudioPath": "D:/test projects/pitchTestOrg-4s.wav", + "notesJsonPath": "../example_pitchTest_minus4_notes.json", + "targetShiftSemitones": -4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "windowStartSec": 1.12, + "windowEndSec": 2.34, + "noteBodyStartSec": 1.38, + "noteBodyEndSec": 2.10, + "entryWindowSec": 0.06, + "exitWindowSec": 0.06, + "neighborWindowSec": 0.08 + }, + { + "label": "pitchOrg_plus4_formant_transition_richer", + "sourceAudioPath": "D:/test projects/pitchOrg.wav", + "referenceAudioPath": "D:/test projects/pitchOrg+4s.wav", + "notesJsonPath": "../example_plus4_notes.json", + "targetShiftSemitones": 4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "windowStartSec": 0.75, + "windowEndSec": 1.72, + "noteBodyStartSec": 1.02, + "noteBodyEndSec": 1.43, + "entryWindowSec": 0.08, + "exitWindowSec": 0.08, + "neighborWindowSec": 0.12 + }, + { + "label": "pitchTest_plus4_formant_transition_richer", + "sourceAudioPath": "D:/test projects/pitchTestOrg.wav", + "referenceAudioPath": "D:/test projects/pitchTestOrg+4s.wav", + "notesJsonPath": "../example_pitchTest_plus4_notes.json", + "targetShiftSemitones": 4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "windowStartSec": 1.05, + "windowEndSec": 2.42, + "noteBodyStartSec": 1.33, + "noteBodyEndSec": 2.14, + "entryWindowSec": 0.08, + "exitWindowSec": 0.08, + "neighborWindowSec": 0.12 + } + ] +} diff --git a/tests/fixtures/pitch-regression/suites/formant_smoke_suite.json b/tests/fixtures/pitch-regression/suites/formant_smoke_suite.json new file mode 100644 index 0000000..35ffc1e --- /dev/null +++ b/tests/fixtures/pitch-regression/suites/formant_smoke_suite.json @@ -0,0 +1,28 @@ +{ + "cases": [ + { + "label": "pitchOrg_plus4_formant_smoke", + "sourceAudioPath": "D:/test projects/pitchOrg.wav", + "referenceAudioPath": "D:/test projects/pitchOrg+4s.wav", + "notesJsonPath": "../example_plus4_notes.json", + "targetShiftSemitones": 4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "entryWindowSec": 0.08, + "exitWindowSec": 0.08, + "neighborWindowSec": 0.12 + }, + { + "label": "pitchTest_plus4_formant_smoke", + "sourceAudioPath": "D:/test projects/pitchTestOrg.wav", + "referenceAudioPath": "D:/test projects/pitchTestOrg+4s.wav", + "notesJsonPath": "../example_pitchTest_plus4_notes.json", + "targetShiftSemitones": 4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "entryWindowSec": 0.08, + "exitWindowSec": 0.08, + "neighborWindowSec": 0.12 + } + ] +} diff --git a/tests/fixtures/pitch-regression/suites/transient_richer_suite.json b/tests/fixtures/pitch-regression/suites/transient_richer_suite.json new file mode 100644 index 0000000..4cf8ec7 --- /dev/null +++ b/tests/fixtures/pitch-regression/suites/transient_richer_suite.json @@ -0,0 +1,132 @@ +{ + "cases": [ + { + "label": "pitchOrg_plus4_transient_entry_richer", + "sourceAudioPath": "D:/test projects/pitchOrg.wav", + "referenceAudioPath": "D:/test projects/pitchOrg+4s.wav", + "notesJsonPath": "../example_plus4_notes.json", + "targetShiftSemitones": 4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "windowStartSec": 0.75, + "windowEndSec": 1.72, + "noteBodyStartSec": 1.02, + "noteBodyEndSec": 1.43, + "entryWindowSec": 0.05, + "exitWindowSec": 0.08, + "neighborWindowSec": 0.10 + }, + { + "label": "pitchOrg_plus4_transient_exit_richer", + "sourceAudioPath": "D:/test projects/pitchOrg.wav", + "referenceAudioPath": "D:/test projects/pitchOrg+4s.wav", + "notesJsonPath": "../example_plus4_notes.json", + "targetShiftSemitones": 4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "windowStartSec": 0.75, + "windowEndSec": 1.72, + "noteBodyStartSec": 1.02, + "noteBodyEndSec": 1.43, + "entryWindowSec": 0.08, + "exitWindowSec": 0.05, + "neighborWindowSec": 0.10 + }, + { + "label": "pitchOrg_minus4_transient_entry_richer", + "sourceAudioPath": "D:/test projects/pitchOrg.wav", + "referenceAudioPath": "D:/test projects/pitchOrg-4s.wav", + "notesJsonPath": "../example_minus4_notes.json", + "targetShiftSemitones": -4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "windowStartSec": 0.75, + "windowEndSec": 1.72, + "noteBodyStartSec": 1.02, + "noteBodyEndSec": 1.43, + "entryWindowSec": 0.05, + "exitWindowSec": 0.08, + "neighborWindowSec": 0.10 + }, + { + "label": "pitchOrg_minus4_transient_exit_richer", + "sourceAudioPath": "D:/test projects/pitchOrg.wav", + "referenceAudioPath": "D:/test projects/pitchOrg-4s.wav", + "notesJsonPath": "../example_minus4_notes.json", + "targetShiftSemitones": -4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "windowStartSec": 0.75, + "windowEndSec": 1.72, + "noteBodyStartSec": 1.02, + "noteBodyEndSec": 1.43, + "entryWindowSec": 0.08, + "exitWindowSec": 0.05, + "neighborWindowSec": 0.10 + }, + { + "label": "pitchTest_plus4_transient_entry_richer", + "sourceAudioPath": "D:/test projects/pitchTestOrg.wav", + "referenceAudioPath": "D:/test projects/pitchTestOrg+4s.wav", + "notesJsonPath": "../example_pitchTest_plus4_notes.json", + "targetShiftSemitones": 4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "windowStartSec": 1.05, + "windowEndSec": 2.42, + "noteBodyStartSec": 1.33, + "noteBodyEndSec": 2.14, + "entryWindowSec": 0.05, + "exitWindowSec": 0.08, + "neighborWindowSec": 0.10 + }, + { + "label": "pitchTest_plus4_transient_exit_richer", + "sourceAudioPath": "D:/test projects/pitchTestOrg.wav", + "referenceAudioPath": "D:/test projects/pitchTestOrg+4s.wav", + "notesJsonPath": "../example_pitchTest_plus4_notes.json", + "targetShiftSemitones": 4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "windowStartSec": 1.05, + "windowEndSec": 2.42, + "noteBodyStartSec": 1.33, + "noteBodyEndSec": 2.14, + "entryWindowSec": 0.08, + "exitWindowSec": 0.05, + "neighborWindowSec": 0.10 + }, + { + "label": "pitchTest_minus4_transient_entry_richer", + "sourceAudioPath": "D:/test projects/pitchTestOrg.wav", + "referenceAudioPath": "D:/test projects/pitchTestOrg-4s.wav", + "notesJsonPath": "../example_pitchTest_minus4_notes.json", + "targetShiftSemitones": -4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "windowStartSec": 1.05, + "windowEndSec": 2.42, + "noteBodyStartSec": 1.33, + "noteBodyEndSec": 2.14, + "entryWindowSec": 0.05, + "exitWindowSec": 0.08, + "neighborWindowSec": 0.10 + }, + { + "label": "pitchTest_minus4_transient_exit_richer", + "sourceAudioPath": "D:/test projects/pitchTestOrg.wav", + "referenceAudioPath": "D:/test projects/pitchTestOrg-4s.wav", + "notesJsonPath": "../example_pitchTest_minus4_notes.json", + "targetShiftSemitones": -4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "windowStartSec": 1.05, + "windowEndSec": 2.42, + "noteBodyStartSec": 1.33, + "noteBodyEndSec": 2.14, + "entryWindowSec": 0.08, + "exitWindowSec": 0.05, + "neighborWindowSec": 0.10 + } + ] +} diff --git a/tests/fixtures/pitch-regression/suites/transient_smoke_suite.json b/tests/fixtures/pitch-regression/suites/transient_smoke_suite.json new file mode 100644 index 0000000..af6ab9a --- /dev/null +++ b/tests/fixtures/pitch-regression/suites/transient_smoke_suite.json @@ -0,0 +1,28 @@ +{ + "cases": [ + { + "label": "pitchOrg_plus4_boundary_entry_smoke", + "sourceAudioPath": "D:/test projects/pitchOrg.wav", + "referenceAudioPath": "D:/test projects/pitchOrg+4s.wav", + "notesJsonPath": "../example_plus4_notes.json", + "targetShiftSemitones": 4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "entryWindowSec": 0.08, + "exitWindowSec": 0.08, + "neighborWindowSec": 0.12 + }, + { + "label": "pitchTest_plus4_boundary_entry_smoke", + "sourceAudioPath": "D:/test projects/pitchTestOrg.wav", + "referenceAudioPath": "D:/test projects/pitchTestOrg+4s.wav", + "notesJsonPath": "../example_pitchTest_plus4_notes.json", + "targetShiftSemitones": 4.0, + "renderMode": "note_hq", + "rendererBranch": "pitch_only_vocal_source_filter_hq", + "entryWindowSec": 0.08, + "exitWindowSec": 0.08, + "neighborWindowSec": 0.12 + } + ] +} diff --git a/tests/test_install_ai_tools_runtime_repair.py b/tests/test_install_ai_tools_runtime_repair.py new file mode 100644 index 0000000..5e4342b --- /dev/null +++ b/tests/test_install_ai_tools_runtime_repair.py @@ -0,0 +1,192 @@ +import sys +import tempfile +import unittest +from pathlib import Path +from subprocess import CompletedProcess +from unittest.mock import Mock, patch + + +REPO_ROOT = Path(__file__).resolve().parents[1] +TOOLS_DIR = REPO_ROOT / "tools" +if str(TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(TOOLS_DIR)) + +import install_ai_tools as installer # noqa: E402 + + +class FailCalled(Exception): + def __init__(self, message: str, **kwargs) -> None: + super().__init__(message) + self.message = message + self.kwargs = kwargs + + +class RepairWindowsReusedRuntimeTests(unittest.TestCase): + def make_runtime_layout(self, temp_dir: str) -> tuple[Path, Path, Path, Path]: + runtime_root = Path(temp_dir) / "stem-runtime" + runtime_python = runtime_root / "Scripts" / "python.exe" + site_packages = runtime_root / "Lib" / "site-packages" + bootstrap_python = Path(temp_dir) / "python311.exe" + runtime_python.parent.mkdir(parents=True, exist_ok=True) + site_packages.mkdir(parents=True, exist_ok=True) + runtime_python.touch() + bootstrap_python.touch() + return runtime_root, runtime_python, site_packages, bootstrap_python + + def fail_stub(self, message: str, **kwargs): + raise FailCalled(message, **kwargs) + + def test_healthy_reused_runtime_skips_repair(self): + with tempfile.TemporaryDirectory() as temp_dir: + runtime_root, runtime_python, _site_packages, bootstrap_python = self.make_runtime_layout(temp_dir) + with ( + patch.object(installer.platform, "system", return_value="Windows"), + patch.object(installer, "probe_windows_reused_runtime_health", return_value=True), + patch.object(installer, "run_step_process") as run_step_process, + patch.object(installer, "terminate_windows_runtime_lock_holders") as terminate_holders, + patch.object(installer, "emit", Mock()), + patch.object(installer, "log_event", Mock()), + ): + returned_python, returned_version, reused = installer.repair_windows_reused_runtime( + runtime_python, + runtime_root, + bootstrap_python, + (3, 11, 9), + (3, 11, 9), + install_source="externalPython", + requires_external_python=True, + python_detected=True, + build_runtime_mode="unbundled-dev", + ) + + self.assertEqual(returned_python, runtime_python) + self.assertEqual(returned_version, (3, 11, 9)) + self.assertTrue(reused) + run_step_process.assert_not_called() + terminate_holders.assert_not_called() + + def test_locked_runtime_file_triggers_single_rebuild(self): + with tempfile.TemporaryDirectory() as temp_dir: + runtime_root, runtime_python, site_packages, bootstrap_python = self.make_runtime_layout(temp_dir) + locked_file = site_packages / "81d243bd2c585b0f4821__mypyc.cp311-win_amd64.pyd" + rebuild_python = runtime_root / "Scripts" / "python.exe" + repair_result = CompletedProcess( + args=["pip", "install"], + returncode=1, + stdout="", + stderr=( + "ERROR: Could not install packages due to an OSError: [WinError 5] " + f"Access is denied: '{locked_file}'\nCheck the permissions.\n" + ), + ) + + with ( + patch.object(installer.platform, "system", return_value="Windows"), + patch.object(installer, "probe_windows_reused_runtime_health", return_value=False), + patch.object(installer, "run_step_process", return_value=repair_result) as run_step_process, + patch.object(installer, "remove_tree_with_retries", return_value=[]), + patch.object(installer, "run_step") as run_step, + patch.object(installer, "resolve_runtime_python", return_value=rebuild_python), + patch.object(installer, "read_python_version_info", return_value=(3, 11, 9)), + patch.object(installer, "emit", Mock()), + patch.object(installer, "log_event", Mock()), + ): + returned_python, returned_version, reused = installer.repair_windows_reused_runtime( + runtime_python, + runtime_root, + bootstrap_python, + (3, 11, 9), + (3, 11, 9), + install_source="externalPython", + requires_external_python=True, + python_detected=True, + build_runtime_mode="unbundled-dev", + ) + + self.assertEqual(returned_python, rebuild_python) + self.assertEqual(returned_version, (3, 11, 9)) + self.assertFalse(reused) + run_step_process.assert_called_once() + run_step.assert_called_once() + self.assertEqual(run_step.call_args.kwargs["error_code"], "runtime_locked_rebuild_failed") + + def test_non_lock_failure_does_not_rebuild(self): + with tempfile.TemporaryDirectory() as temp_dir: + runtime_root, runtime_python, _site_packages, bootstrap_python = self.make_runtime_layout(temp_dir) + repair_result = CompletedProcess( + args=["pip", "install"], + returncode=1, + stdout="", + stderr="ERROR: Could not find a version that satisfies the requirement broken-package\n", + ) + + with ( + patch.object(installer.platform, "system", return_value="Windows"), + patch.object(installer, "probe_windows_reused_runtime_health", return_value=False), + patch.object(installer, "run_step_process", return_value=repair_result), + patch.object(installer, "remove_tree_with_retries") as remove_tree, + patch.object(installer, "run_step") as run_step, + patch.object(installer, "fail", side_effect=self.fail_stub), + patch.object(installer, "emit", Mock()), + patch.object(installer, "log_event", Mock()), + ): + with self.assertRaises(FailCalled) as failure: + installer.repair_windows_reused_runtime( + runtime_python, + runtime_root, + bootstrap_python, + (3, 11, 9), + (3, 11, 9), + install_source="externalPython", + requires_external_python=True, + python_detected=True, + build_runtime_mode="unbundled-dev", + ) + + self.assertEqual(failure.exception.kwargs["error_code"], "dependency_bootstrap_failed") + remove_tree.assert_not_called() + run_step.assert_not_called() + + def test_locked_runtime_remove_failure_reports_specific_error(self): + with tempfile.TemporaryDirectory() as temp_dir: + runtime_root, runtime_python, site_packages, bootstrap_python = self.make_runtime_layout(temp_dir) + locked_file = site_packages / "81d243bd2c585b0f4821__mypyc.cp311-win_amd64.pyd" + repair_result = CompletedProcess( + args=["pip", "install"], + returncode=1, + stdout="", + stderr=( + "ERROR: Could not install packages due to an OSError: [WinError 5] " + f"Access is denied: '{locked_file}'\nCheck the permissions.\n" + ), + ) + + with ( + patch.object(installer.platform, "system", return_value="Windows"), + patch.object(installer, "probe_windows_reused_runtime_health", return_value=False), + patch.object(installer, "run_step_process", return_value=repair_result), + patch.object(installer, "remove_tree_with_retries", side_effect=OSError("still locked")), + patch.object(installer, "run_step") as run_step, + patch.object(installer, "fail", side_effect=self.fail_stub), + patch.object(installer, "emit", Mock()), + patch.object(installer, "log_event", Mock()), + ): + with self.assertRaises(FailCalled) as failure: + installer.repair_windows_reused_runtime( + runtime_python, + runtime_root, + bootstrap_python, + (3, 11, 9), + (3, 11, 9), + install_source="externalPython", + requires_external_python=True, + python_detected=True, + build_runtime_mode="unbundled-dev", + ) + + self.assertEqual(failure.exception.kwargs["error_code"], "runtime_rebuild_remove_failed") + run_step.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_openstudio_ace15_config_detection.py b/tests/test_openstudio_ace15_config_detection.py new file mode 100644 index 0000000..29aa398 --- /dev/null +++ b/tests/test_openstudio_ace15_config_detection.py @@ -0,0 +1,473 @@ +import argparse +import io +import json +import os +import sys +import tempfile +import types +import unittest +from pathlib import Path +from unittest import mock + +import torch + + +VENDOR_RUNTIME_ROOT = Path(__file__).resolve().parents[1] / "tools" / "openstudio_ace_backend" / "vendor_runtime" +if str(VENDOR_RUNTIME_ROOT) not in sys.path: + sys.path.insert(0, str(VENDOR_RUNTIME_ROOT)) +TOOLS_ROOT = Path(__file__).resolve().parents[1] / "tools" +if str(TOOLS_ROOT) not in sys.path: + sys.path.insert(0, str(TOOLS_ROOT)) + +from comfy.model_detection import detect_unet_config +from comfy.ops import disable_weight_init +from comfy.ldm.ace.ace_step15 import AceStepConditionGenerationModel +from comfy_extras.nodes_audio import vae_decode_audio +from generate_music import ( + build_native_split_request, + classify_generation_failure_kind, + is_out_of_memory_error, + normalize_generation_params, + resolve_one_shot_params_json, +) +from install_ai_tools import normalize_installer_command +from comfy_extras.nodes_ace import EmptyAceStep15LatentAudio +from openstudio_ace_runner import build_openstudio_ace_prompt, choose_existing, run_ace_split_request + + +class DetectAceStep15ConfigTests(unittest.TestCase): + def test_native_split_request_matches_expected_comfy_shape(self): + raw_params = { + "prompt": "prompt text", + "lyrics": "lyrics text", + "seed": 0, + "bpm": 170, + "duration": 191, + "timesignature": "3/4", + "language": "en", + "keyscale": "C# minor", + "generate_audio_codes": True, + "cfg_scale": 2, + "guidance_scale": 1, + "inferenceSteps": 8, + "shift": 3, + "temperature": 0.85, + "top_p": 0.9, + "top_k": 0, + "min_p": 0, + } + + normalized = build_native_split_request(normalize_generation_params(raw_params)) + + self.assertEqual(normalized["timesignature"], "3") + self.assertEqual(normalized["sampler_name"], "euler") + self.assertEqual(normalized["scheduler"], "simple") + self.assertEqual(normalized["denoise"], 1.0) + self.assertEqual(normalized["clip_type"], "ace") + self.assertEqual(normalized["model_mode"], "default") + self.assertEqual(normalized["decode_mode"], "full") + + def test_detects_split_encoder_and_decoder_dimensions(self): + state_dict = { + "encoder.lyric_encoder.layers.0.input_layernorm.weight": torch.empty(2048), + "decoder.proj_in.1.weight": torch.empty(2560, 192, 2), + "decoder.proj_out.1.weight": torch.empty(2560, 64, 2), + "decoder.layers.0.self_attn.q_norm.weight": torch.empty(128), + "decoder.layers.0.self_attn.q_proj.weight": torch.empty(4096, 2560), + "decoder.layers.0.self_attn.k_proj.weight": torch.empty(1024, 2560), + "decoder.layers.0.mlp.gate_proj.weight": torch.empty(9728, 2560), + "decoder.layers.1.self_attn.q_proj.weight": torch.empty(4096, 2560), + "encoder.text_projector.weight": torch.empty(2048, 1024), + "encoder.timbre_encoder.embed_tokens.weight": torch.empty(2048, 64), + "encoder.lyric_encoder.layers.0.self_attn.q_proj.weight": torch.empty(2048, 2048), + "encoder.lyric_encoder.layers.0.self_attn.k_proj.weight": torch.empty(1024, 2048), + "encoder.lyric_encoder.layers.0.mlp.gate_proj.weight": torch.empty(6144, 2048), + "encoder.lyric_encoder.layers.1.self_attn.q_proj.weight": torch.empty(2048, 2048), + "encoder.timbre_encoder.layers.0.self_attn.q_proj.weight": torch.empty(2048, 2048), + "encoder.timbre_encoder.layers.1.self_attn.q_proj.weight": torch.empty(2048, 2048), + "tokenizer.attention_pooler.layers.0.self_attn.q_proj.weight": torch.empty(2048, 2048), + "tokenizer.attention_pooler.layers.1.self_attn.q_proj.weight": torch.empty(2048, 2048), + "tokenizer.quantizer.project_in.weight": torch.empty(6, 2048), + "detokenizer.special_tokens": torch.empty(1, 5, 2048), + } + + config = detect_unet_config(state_dict, "") + + self.assertEqual(config["audio_model"], "ace1.5") + self.assertEqual(config["encoder_hidden_size"], 2048) + self.assertEqual(config["decoder_hidden_size"], 2560) + self.assertEqual(config["encoder_intermediate_size"], 6144) + self.assertEqual(config["decoder_intermediate_size"], 9728) + + def test_split_model_uses_2048_to_2560_condition_projection(self): + model = AceStepConditionGenerationModel( + encoder_hidden_size=2048, + decoder_hidden_size=2560, + encoder_num_heads=16, + encoder_num_kv_heads=8, + decoder_num_heads=32, + decoder_num_kv_heads=8, + encoder_intermediate_size=6144, + decoder_intermediate_size=9728, + operations=disable_weight_init, + ) + + self.assertEqual(tuple(model.decoder.condition_embedder.weight.shape), (2560, 2048)) + + def test_out_of_memory_errors_classify_as_native_decode_failures(self): + error_text = ( + "ACE-Step full audio decode ran out of GPU memory while decoding audio from the VAE." + ) + + self.assertTrue(is_out_of_memory_error(error_text)) + self.assertEqual( + classify_generation_failure_kind(error_text, generation_mode="lm_first"), + "native_decode_failure", + ) + + def test_audio_decode_normalization_avoids_inplace_updates(self): + class DummyVae: + audio_sample_rate = 44100 + + def decode(self, samples): + return torch.ones((1, 64, 32), dtype=torch.float32) + + decoded = vae_decode_audio(DummyVae(), {"samples": torch.zeros((1, 1, 1))}) + + self.assertEqual(tuple(decoded["waveform"].shape), (1, 32, 64)) + self.assertEqual(decoded["sample_rate"], 44100) + + def test_tiled_audio_decode_uses_requested_tile_size_for_1d_audio(self): + class DummyVae: + audio_sample_rate = 44100 + + def __init__(self): + self.kwargs = None + + def decode_tiled(self, samples, **kwargs): + self.kwargs = kwargs + return torch.ones((1, 64, 32), dtype=torch.float32) + + vae = DummyVae() + decoded = vae_decode_audio( + vae, + {"samples": torch.zeros((1, 1, 1))}, + tile=512, + overlap=64, + ) + + self.assertEqual(decoded["sample_rate"], 44100) + self.assertEqual(vae.kwargs, {"tile_x": 512, "tile_y": 512, "overlap": 64}) + + def test_ace15_latent_uses_comfy_intermediate_dtype(self): + with mock.patch("comfy.model_management.intermediate_device", return_value="cpu"): + with mock.patch("comfy.model_management.intermediate_dtype", return_value=torch.float16): + latent = EmptyAceStep15LatentAudio.execute(1.0, 1)[0] + + self.assertEqual(latent["samples"].dtype, torch.float16) + self.assertEqual(tuple(latent["samples"].shape), (1, 64, 25)) + + def test_installer_disables_pip_http_cache_for_installs(self): + command = normalize_installer_command( + ["python", "-m", "pip", "install", "--upgrade", "torch"] + ) + + self.assertEqual( + command, + ["python", "-m", "pip", "install", "--no-cache-dir", "--upgrade", "torch"], + ) + + def test_one_shot_params_loader_accepts_inline_file_stdin_and_bom_file(self): + payload = json.dumps({"prompt": "hello"}) + bom_payload = "\ufeff" + payload + with tempfile.TemporaryDirectory() as temp_dir: + params_path = Path(temp_dir) / "params.json" + bom_path = Path(temp_dir) / "params-bom.json" + params_path.write_text(payload, encoding="utf-8") + bom_path.write_text(bom_payload, encoding="utf-8") + + inline_args = argparse.Namespace(params=payload, params_file="", params_stdin=False) + file_args = argparse.Namespace(params="", params_file=str(params_path), params_stdin=False) + bom_args = argparse.Namespace(params="", params_file=str(bom_path), params_stdin=False) + stdin_args = argparse.Namespace(params="", params_file="", params_stdin=True) + + self.assertEqual(resolve_one_shot_params_json(inline_args), payload) + self.assertEqual(resolve_one_shot_params_json(file_args), payload) + self.assertEqual(resolve_one_shot_params_json(bom_args), payload) + with mock.patch("sys.stdin", io.StringIO(payload)): + self.assertEqual(resolve_one_shot_params_json(stdin_args), payload) + + def test_one_shot_params_loader_rejects_ambiguous_sources(self): + args = argparse.Namespace(params="{}", params_file="x.json", params_stdin=False) + with self.assertRaises(SystemExit): + resolve_one_shot_params_json(args) + + def test_openstudio_ace_prompt_maps_request_to_known_graph_values(self): + request = { + "prompt": "prompt", + "lyrics": "lyrics", + "seed": -1, + "bpm": 170, + "duration": 191.0, + "timesignature": "3/4", + "language": "en", + "keyscale": "C# minor", + "generate_audio_codes": True, + "cfg_scale": 2.0, + "guidance_scale": 1.0, + "inferenceSteps": 8, + "shift": 3.0, + "temperature": 0.85, + "top_p": 0.9, + "top_k": 0, + "min_p": 0.0, + "sampler_name": "euler", + "scheduler": "simple", + "denoise": 1.0, + } + + prompt = build_openstudio_ace_prompt( + request=request, + unet_name="acestep_v1.5_xl_turbo_bf16.safetensors", + clip_name1="qwen_0.6b_ace15.safetensors", + clip_name2="qwen_4b_ace15.safetensors", + vae_name="ace_1.5_vae.safetensors", + ) + + self.assertEqual(prompt["104"]["class_type"], "UNETLoader") + self.assertEqual(prompt["105"]["inputs"]["type"], "ace") + self.assertEqual(prompt["94"]["class_type"], "TextEncodeAceStepAudio1.5") + self.assertEqual(prompt["94"]["inputs"]["seed"], 0) + self.assertEqual(prompt["94"]["inputs"]["duration"], 191.0) + self.assertEqual(prompt["94"]["inputs"]["timesignature"], "3") + self.assertEqual(prompt["47"]["class_type"], "ConditioningZeroOut") + self.assertEqual(prompt["3"]["inputs"]["negative"], ["47", 0]) + self.assertEqual(prompt["3"]["inputs"]["sampler_name"], "euler") + self.assertEqual(prompt["3"]["inputs"]["scheduler"], "simple") + self.assertEqual(prompt["3"]["inputs"]["cfg"], 1.0) + self.assertEqual(prompt["18"]["class_type"], "VAEDecodeAudio") + + def test_missing_ace_asset_error_points_to_ai_setup(self): + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaises(FileNotFoundError) as exc_info: + choose_existing(Path(temp_dir), ["vae/ace_1.5_vae.safetensors"]) + + self.assertIn("Run OpenStudio AI setup", str(exc_info.exception)) + + def test_runner_executes_openstudio_ace_graph_through_executor(self): + with tempfile.TemporaryDirectory() as temp_dir: + temp_root = Path(temp_dir) + backend_root = temp_root / "backend" + backend_root.mkdir(parents=True, exist_ok=True) + output_path = temp_root / "out.wav" + for relative_path in ( + "diffusion_models/acestep_v1.5_xl_turbo_bf16.safetensors", + "text_encoders/qwen_0.6b_ace15.safetensors", + "text_encoders/qwen_4b_ace15.safetensors", + "vae/ace_1.5_vae.safetensors", + ): + asset_path = temp_root / relative_path + asset_path.parent.mkdir(parents=True, exist_ok=True) + asset_path.write_bytes(b"stub") + + comfy_mod = types.ModuleType("comfy") + comfy_utils_mod = types.ModuleType("comfy.utils") + comfy_utils_mod.set_progress_bar_global_hook = lambda hook: None + comfy_mod.utils = comfy_utils_mod + + nodes_mod = types.ModuleType("nodes") + nodes_mod.NODE_CLASS_MAPPINGS = {} + init_calls = [] + + async def fake_init_extra_nodes(**kwargs): + init_calls.append(kwargs) + return [] + + nodes_mod.init_extra_nodes = fake_init_extra_nodes + + class FakeCacheEntry: + def __init__(self, outputs): + self.outputs = outputs + + class FakeOutputs: + def __init__(self): + self.entries = {} + + async def get(self, node_id): + return self.entries.get(node_id) + + class FakeCaches: + def __init__(self): + self.outputs = FakeOutputs() + + class FakePromptExecutor: + captured_prompt = None + + def __init__(self, server, **kwargs): + self.server = server + self.cache_args = kwargs.get("cache_args") + self.success = True + self.caches = FakeCaches() + + def execute(self, prompt, prompt_id, extra_data=None, execute_outputs=None): + type(self).captured_prompt = prompt + self.caches.outputs.entries["18"] = FakeCacheEntry( + [[{"waveform": torch.full((1, 2, 32), 0.1, dtype=torch.float32), "sample_rate": 48000}]] + ) + self.caches.outputs.entries["3"] = FakeCacheEntry( + [[{"samples": torch.full((1, 64, 8), 0.25, dtype=torch.float32)}]] + ) + + execution_mod = types.ModuleType("execution") + execution_mod.PromptExecutor = FakePromptExecutor + nodes_model_adv_mod = types.ModuleType("comfy_extras.nodes_model_advanced") + nodes_model_adv_mod.NODE_CLASS_MAPPINGS = {"ModelSamplingAuraFlow": object} + + fake_modules = { + "comfy": comfy_mod, + "comfy.utils": comfy_utils_mod, + "nodes": nodes_mod, + "execution": execution_mod, + "comfy_extras.nodes_model_advanced": nodes_model_adv_mod, + } + + request = { + "requestId": "test-request", + "prompt": "prompt", + "lyrics": "lyrics", + "seed": 0, + "bpm": 120, + "duration": 10.0, + "timesignature": "4", + "language": "en", + "keyscale": "C major", + "generate_audio_codes": True, + "cfg_scale": 2.0, + "guidance_scale": 1.0, + "inferenceSteps": 1, + "shift": 3.0, + "temperature": 0.85, + "top_p": 0.9, + "top_k": 0, + "min_p": 0.0, + "sampler_name": "euler", + "scheduler": "simple", + "denoise": 1.0, + "clip_type": "ace", + "model_mode": "default", + } + + with mock.patch.dict(sys.modules, fake_modules, clear=False): + with mock.patch.dict(os.environ, {}, clear=True): + with mock.patch("openstudio_ace_runner.get_backend_root", return_value=backend_root): + with mock.patch("openstudio_ace_runner.get_runtime_workspace", return_value=temp_root / "workspace"): + with mock.patch("openstudio_ace_runner.configure_vendor_paths", return_value=None): + run_ace_split_request( + checkpoint_root=temp_root, + request=request, + output_path=output_path, + ) + self.assertEqual(os.environ["HF_HUB_OFFLINE"], "1") + self.assertEqual(os.environ["TRANSFORMERS_OFFLINE"], "1") + self.assertEqual(os.environ["HF_DATASETS_OFFLINE"], "1") + + self.assertTrue(output_path.exists()) + self.assertEqual(init_calls, [{"init_custom_nodes": False, "init_api_nodes": False}]) + self.assertIn("ModelSamplingAuraFlow", nodes_mod.NODE_CLASS_MAPPINGS) + self.assertEqual(FakePromptExecutor.captured_prompt["18"]["class_type"], "VAEDecodeAudio") + + def test_runner_executor_failure_does_not_write_output(self): + with tempfile.TemporaryDirectory() as temp_dir: + temp_root = Path(temp_dir) + backend_root = temp_root / "backend" + backend_root.mkdir(parents=True, exist_ok=True) + output_path = temp_root / "out.wav" + for relative_path in ( + "diffusion_models/acestep_v1.5_xl_turbo_bf16.safetensors", + "text_encoders/qwen_0.6b_ace15.safetensors", + "text_encoders/qwen_4b_ace15.safetensors", + "vae/ace_1.5_vae.safetensors", + ): + asset_path = temp_root / relative_path + asset_path.parent.mkdir(parents=True, exist_ok=True) + asset_path.write_bytes(b"stub") + + comfy_mod = types.ModuleType("comfy") + comfy_utils_mod = types.ModuleType("comfy.utils") + comfy_utils_mod.set_progress_bar_global_hook = lambda hook: None + comfy_mod.utils = comfy_utils_mod + + nodes_mod = types.ModuleType("nodes") + nodes_mod.NODE_CLASS_MAPPINGS = {} + + async def fake_init_extra_nodes(**kwargs): + return [] + + nodes_mod.init_extra_nodes = fake_init_extra_nodes + + class FakePromptExecutor: + def __init__(self, server, **kwargs): + self.server = server + self.cache_args = kwargs.get("cache_args") + self.success = False + self.caches = types.SimpleNamespace(outputs=None) + + def execute(self, prompt, prompt_id, extra_data=None, execute_outputs=None): + return None + + execution_mod = types.ModuleType("execution") + execution_mod.PromptExecutor = FakePromptExecutor + nodes_model_adv_mod = types.ModuleType("comfy_extras.nodes_model_advanced") + nodes_model_adv_mod.NODE_CLASS_MAPPINGS = {"ModelSamplingAuraFlow": object} + + fake_modules = { + "comfy": comfy_mod, + "comfy.utils": comfy_utils_mod, + "nodes": nodes_mod, + "execution": execution_mod, + "comfy_extras.nodes_model_advanced": nodes_model_adv_mod, + } + + request = { + "requestId": "test-request", + "prompt": "prompt", + "lyrics": "lyrics", + "seed": 0, + "bpm": 120, + "duration": 10.0, + "timesignature": "4", + "language": "en", + "keyscale": "C major", + "generate_audio_codes": True, + "cfg_scale": 2.0, + "guidance_scale": 1.0, + "inferenceSteps": 1, + "shift": 3.0, + "temperature": 0.85, + "top_p": 0.9, + "top_k": 0, + "min_p": 0.0, + "sampler_name": "euler", + "scheduler": "simple", + "denoise": 1.0, + "clip_type": "ace", + "model_mode": "default", + } + + with mock.patch.dict(sys.modules, fake_modules, clear=False): + with mock.patch("openstudio_ace_runner.get_backend_root", return_value=backend_root): + with mock.patch("openstudio_ace_runner.get_runtime_workspace", return_value=temp_root / "workspace"): + with mock.patch("openstudio_ace_runner.configure_vendor_paths", return_value=None): + with self.assertRaises(RuntimeError) as exc_info: + run_ace_split_request( + checkpoint_root=temp_root, + request=request, + output_path=output_path, + ) + + self.assertIn("graph execution failed", str(exc_info.exception)) + self.assertFalse(output_path.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/OpenStudio.desktop b/tools/OpenStudio.desktop new file mode 100644 index 0000000..19ac71a --- /dev/null +++ b/tools/OpenStudio.desktop @@ -0,0 +1,11 @@ +[Desktop Entry] +Name=OpenStudio +GenericName=Digital Audio Workstation +Comment=Professional audio recording and editing +Exec=OpenStudio %f +Icon=OpenStudio +Type=Application +Categories=AudioVideo;Audio;Music; +MimeType=application/x-openstudio-project; +StartupWMClass=OpenStudio +Keywords=DAW;audio;recording;mixing;music;studio; diff --git a/tools/ai-runtime-install-plan-linux-cuda.json b/tools/ai-runtime-install-plan-linux-cuda.json new file mode 100644 index 0000000..3879c80 --- /dev/null +++ b/tools/ai-runtime-install-plan-linux-cuda.json @@ -0,0 +1,63 @@ +{ + "steps": [ + { + "type": "pip_install", + "description": "Installing GPU stem separation packages", + "stepLabel": "Installing GPU stem separation packages", + "packages": ["audio-separator[gpu]==0.41.1"] + }, + { "type": "pip_uninstall", "packages": ["torch", "torchvision", "torchaudio"] }, + { + "type": "pip_install", + "description": "Installing CUDA-capable PyTorch wheels", + "stepLabel": "Installing CUDA PyTorch wheels", + "indexUrl": "https://download.pytorch.org/whl/cu121", + "packages": ["torch==2.5.1", "torchvision==0.20.1", "torchaudio==2.5.1"] + }, + { + "type": "pip_install", + "description": "Installing OpenStudio ACE runtime dependencies", + "stepLabel": "Installing OpenStudio ACE runtime dependencies", + "packages": [ + "torchsde", + "numpy>=1.25.0", + "einops", + "transformers>=4.51.0,<4.58.0", + "tokenizers>=0.13.3", + "sentencepiece", + "safetensors>=0.4.2", + "diffusers==0.35.2", + "accelerate>=1.12.0", + "vector-quantize-pytorch>=1.27.15", + "huggingface_hub>=0.34,<1.0", + "loguru>=0.7.3", + "xxhash>=3.5.0", + "aiohttp>=3.11.8", + "yarl>=1.18.0", + "pyyaml", + "Pillow", + "scipy", + "tqdm", + "psutil", + "filelock", + "av>=14.2.0", + "comfy-aimdo>=0.2.12", + "requests", + "simpleeval>=1.0.0", + "blake3", + "pydantic~=2.0", + "pydantic-settings~=2.0" + ], + "errorCode": "music_generation_runtime_install_failed", + "nonFatalForStemSeparation": true + }, + { + "type": "pip_install", + "description": "Installing ACE-Step source bridge", + "stepLabel": "Installing ACE-Step source bridge", + "packages": ["git+https://github.com/ace-step/ACE-Step.git@1bee4c9f5b43e30995f8d4d33b3919197ce1bd68"], + "errorCode": "music_generation_runtime_install_failed", + "nonFatalForStemSeparation": true + } + ] +} diff --git a/tools/ai-runtime-install-plan-linux-rocm.json b/tools/ai-runtime-install-plan-linux-rocm.json new file mode 100644 index 0000000..25f9d17 --- /dev/null +++ b/tools/ai-runtime-install-plan-linux-rocm.json @@ -0,0 +1,63 @@ +{ + "steps": [ + { + "type": "pip_install", + "description": "Installing CPU stem separation packages", + "stepLabel": "Installing CPU stem separation packages", + "packages": ["audio-separator[cpu]==0.41.1"] + }, + { "type": "pip_uninstall", "packages": ["torch", "torchvision", "torchaudio"] }, + { + "type": "pip_install", + "description": "Installing ROCm PyTorch wheels", + "stepLabel": "Installing ROCm PyTorch wheels", + "indexUrl": "https://download.pytorch.org/whl/rocm6.2", + "packages": ["torch==2.5.1", "torchvision==0.20.1", "torchaudio==2.5.1"] + }, + { + "type": "pip_install", + "description": "Installing OpenStudio ACE runtime dependencies", + "stepLabel": "Installing OpenStudio ACE runtime dependencies", + "packages": [ + "torchsde", + "numpy>=1.25.0", + "einops", + "transformers>=4.51.0,<4.58.0", + "tokenizers>=0.13.3", + "sentencepiece", + "safetensors>=0.4.2", + "diffusers==0.35.2", + "accelerate>=1.12.0", + "vector-quantize-pytorch>=1.27.15", + "huggingface_hub>=0.34,<1.0", + "loguru>=0.7.3", + "xxhash>=3.5.0", + "aiohttp>=3.11.8", + "yarl>=1.18.0", + "pyyaml", + "Pillow", + "scipy", + "tqdm", + "psutil", + "filelock", + "av>=14.2.0", + "comfy-aimdo>=0.2.12", + "requests", + "simpleeval>=1.0.0", + "blake3", + "pydantic~=2.0", + "pydantic-settings~=2.0" + ], + "errorCode": "music_generation_runtime_install_failed", + "nonFatalForStemSeparation": true + }, + { + "type": "pip_install", + "description": "Installing ACE-Step source bridge", + "stepLabel": "Installing ACE-Step source bridge", + "packages": ["git+https://github.com/ace-step/ACE-Step.git@1bee4c9f5b43e30995f8d4d33b3919197ce1bd68"], + "errorCode": "music_generation_runtime_install_failed", + "nonFatalForStemSeparation": true + } + ] +} diff --git a/tools/ai-runtime-install-plan-windows-cuda.json b/tools/ai-runtime-install-plan-windows-cuda.json index 252fd73..bb4c58d 100644 --- a/tools/ai-runtime-install-plan-windows-cuda.json +++ b/tools/ai-runtime-install-plan-windows-cuda.json @@ -1,11 +1,11 @@ { "schemaVersion": 1, - "id": "windows-cuda-cu121-v1", + "id": "windows-cuda-cu128-v2", "backend": "cuda", "platform": "windows", "architecture": "x64", - "pythonFamily": "3.10", - "packageSource": "official-upstream", + "pythonFamily": "3.11", + "packageSource": "repo-pinned-windows-acceleration-manifest", "steps": [ { "type": "pip_install", @@ -31,12 +31,64 @@ "stepLabel": "Downloading CUDA PyTorch wheels", "downloadHint": "This is the largest Windows AI download and can take several minutes depending on your connection.", "isLargeDownload": true, - "indexUrl": "https://download.pytorch.org/whl/cu121", + "indexUrl": "https://download.pytorch.org/whl/cu128", "packages": [ - "torch==2.5.1", - "torchvision==0.20.1", - "torchaudio==2.5.1" + "torch==2.8.0", + "torchvision==0.23.0", + "torchaudio==2.8.0" ] + }, + { + "type": "pip_install", + "description": "Installing Triton for the ACE-Step accelerated LM runtime", + "stepLabel": "Installing Triton for music generation", + "downloadHint": "OpenStudio is installing the Windows Triton runtime used by ACE-Step acceleration.", + "isLargeDownload": true, + "packages": [ + "triton-windows==3.6.0.post26" + ], + "errorCode": "music_acceleration_setup_failed", + "nonFatalForStemSeparation": true + }, + { + "type": "install_pinned_wheel", + "description": "Installing Flash Attention for the ACE-Step accelerated LM runtime", + "stepLabel": "Installing Flash Attention for music generation", + "downloadHint": "OpenStudio is installing the pinned Flash Attention wheel for the Windows ACE-Step stack.", + "isLargeDownload": true, + "url": "https://github.com/sdbds/flash-attention-for-windows/releases/download/2.8.3/flash_attn-2.8.3%2Bcu128torch2.8.0cxx11abiFALSEfullbackward-cp311-cp311-win_amd64.whl", + "fileName": "flash_attn-2.8.3+cu128torch2.8.0cxx11abiFALSEfullbackward-cp311-cp311-win_amd64.whl", + "sha256": "9850d5010e62dde2bf1ec415bd267da23c996f05939501e316fd2e0a7c729076", + "errorCode": "music_acceleration_setup_failed", + "nonFatalForStemSeparation": true + }, + { + "type": "pip_install", + "description": "Installing ACE-Step 1.5 runtime dependencies", + "stepLabel": "Installing ACE-Step 1.5 runtime dependencies", + "packages": [ + "transformers>=4.51.0,<4.58.0", + "diffusers==0.35.2", + "accelerate>=1.12.0", + "vector-quantize-pytorch>=1.27.15", + "huggingface_hub>=0.34,<1.0", + "loguru>=0.7.3", + "xxhash>=3.5.0" + ], + "errorCode": "music_generation_runtime_install_failed", + "nonFatalForStemSeparation": true + }, + { + "type": "hf_snapshot_pip_install", + "description": "Preparing the ACE-Step 1.5 runtime source", + "stepLabel": "Installing the ACE-Step 1.5 runtime bridge", + "downloadHint": "OpenStudio is downloading the pinned ACE-Step 1.5 runtime bridge from Hugging Face.", + "isLargeDownload": true, + "repoId": "ACE-Step/Ace-Step-v1.5", + "repoType": "space", + "localDirName": "ace-step-v1.5-source", + "errorCode": "music_generation_runtime_install_failed", + "nonFatalForStemSeparation": true } ] } diff --git a/tools/ai-runtime-install-plan-windows-directml.json b/tools/ai-runtime-install-plan-windows-directml.json index dfdb7ef..5427e56 100644 --- a/tools/ai-runtime-install-plan-windows-directml.json +++ b/tools/ai-runtime-install-plan-windows-directml.json @@ -1,11 +1,11 @@ { "schemaVersion": 1, - "id": "windows-directml-v1", + "id": "windows-directml-v2", "backend": "directml", "platform": "windows", "architecture": "x64", - "pythonFamily": "3.10", - "packageSource": "official-upstream", + "pythonFamily": "3.11", + "packageSource": "repo-pinned-ace-step-v15", "steps": [ { "type": "pip_install", @@ -16,6 +16,34 @@ "packages": [ "audio-separator[dml]==0.41.1" ] + }, + { + "type": "pip_install", + "description": "Installing ACE-Step 1.5 runtime dependencies", + "stepLabel": "Installing ACE-Step 1.5 runtime dependencies", + "packages": [ + "transformers>=4.51.0,<4.58.0", + "diffusers==0.35.2", + "accelerate>=1.12.0", + "vector-quantize-pytorch>=1.27.15", + "huggingface_hub>=0.34,<1.0", + "loguru>=0.7.3", + "xxhash>=3.5.0" + ], + "errorCode": "music_generation_runtime_install_failed", + "nonFatalForStemSeparation": true + }, + { + "type": "hf_snapshot_pip_install", + "description": "Preparing the ACE-Step 1.5 runtime source", + "stepLabel": "Installing the ACE-Step 1.5 runtime bridge", + "downloadHint": "OpenStudio is downloading the pinned ACE-Step 1.5 runtime bridge from Hugging Face.", + "isLargeDownload": true, + "repoId": "ACE-Step/Ace-Step-v1.5", + "repoType": "space", + "localDirName": "ace-step-v1.5-source", + "errorCode": "music_generation_runtime_install_failed", + "nonFatalForStemSeparation": true } ] } diff --git a/tools/ai-runtime-requirements-linux-cuda.txt b/tools/ai-runtime-requirements-linux-cuda.txt new file mode 100644 index 0000000..d5920ba --- /dev/null +++ b/tools/ai-runtime-requirements-linux-cuda.txt @@ -0,0 +1,30 @@ +audio-separator[gpu]==0.41.1 +torchsde +numpy>=1.25.0 +einops +transformers>=4.51.0,<4.58.0 +tokenizers>=0.13.3 +sentencepiece +safetensors>=0.4.2 +diffusers==0.35.2 +accelerate>=1.12.0 +vector-quantize-pytorch>=1.27.15 +huggingface_hub>=0.34,<1.0 +loguru>=0.7.3 +xxhash>=3.5.0 +aiohttp>=3.11.8 +yarl>=1.18.0 +pyyaml +Pillow +scipy +tqdm +psutil +filelock +av>=14.2.0 +comfy-aimdo>=0.2.12 +requests +simpleeval>=1.0.0 +blake3 +pydantic~=2.0 +pydantic-settings~=2.0 +git+https://github.com/ace-step/ACE-Step.git@1bee4c9f5b43e30995f8d4d33b3919197ce1bd68 diff --git a/tools/ai-runtime-requirements-linux-rocm.txt b/tools/ai-runtime-requirements-linux-rocm.txt new file mode 100644 index 0000000..067cc23 --- /dev/null +++ b/tools/ai-runtime-requirements-linux-rocm.txt @@ -0,0 +1,30 @@ +audio-separator[cpu]==0.41.1 +torchsde +numpy>=1.25.0 +einops +transformers>=4.51.0,<4.58.0 +tokenizers>=0.13.3 +sentencepiece +safetensors>=0.4.2 +diffusers==0.35.2 +accelerate>=1.12.0 +vector-quantize-pytorch>=1.27.15 +huggingface_hub>=0.34,<1.0 +loguru>=0.7.3 +xxhash>=3.5.0 +aiohttp>=3.11.8 +yarl>=1.18.0 +pyyaml +Pillow +scipy +tqdm +psutil +filelock +av>=14.2.0 +comfy-aimdo>=0.2.12 +requests +simpleeval>=1.0.0 +blake3 +pydantic~=2.0 +pydantic-settings~=2.0 +git+https://github.com/ace-step/ACE-Step.git@1bee4c9f5b43e30995f8d4d33b3919197ce1bd68 diff --git a/tools/ai-runtime-requirements-linux.txt b/tools/ai-runtime-requirements-linux.txt new file mode 100644 index 0000000..e4f4a8e --- /dev/null +++ b/tools/ai-runtime-requirements-linux.txt @@ -0,0 +1,33 @@ +audio-separator[cpu]==0.41.1 +torch +torchvision +torchaudio +torchsde +numpy>=1.25.0 +einops +transformers>=4.51.0,<4.58.0 +tokenizers>=0.13.3 +sentencepiece +safetensors>=0.4.2 +diffusers==0.35.2 +accelerate>=1.12.0 +vector-quantize-pytorch>=1.27.15 +huggingface_hub>=0.34,<1.0 +loguru>=0.7.3 +xxhash>=3.5.0 +aiohttp>=3.11.8 +yarl>=1.18.0 +pyyaml +Pillow +scipy +tqdm +psutil +filelock +av>=14.2.0 +comfy-aimdo>=0.2.12 +requests +simpleeval>=1.0.0 +blake3 +pydantic~=2.0 +pydantic-settings~=2.0 +git+https://github.com/ace-step/ACE-Step.git@1bee4c9f5b43e30995f8d4d33b3919197ce1bd68 diff --git a/tools/ai-runtime-requirements-macos.txt b/tools/ai-runtime-requirements-macos.txt index 8cbd273..e4f4a8e 100644 --- a/tools/ai-runtime-requirements-macos.txt +++ b/tools/ai-runtime-requirements-macos.txt @@ -1 +1,33 @@ audio-separator[cpu]==0.41.1 +torch +torchvision +torchaudio +torchsde +numpy>=1.25.0 +einops +transformers>=4.51.0,<4.58.0 +tokenizers>=0.13.3 +sentencepiece +safetensors>=0.4.2 +diffusers==0.35.2 +accelerate>=1.12.0 +vector-quantize-pytorch>=1.27.15 +huggingface_hub>=0.34,<1.0 +loguru>=0.7.3 +xxhash>=3.5.0 +aiohttp>=3.11.8 +yarl>=1.18.0 +pyyaml +Pillow +scipy +tqdm +psutil +filelock +av>=14.2.0 +comfy-aimdo>=0.2.12 +requests +simpleeval>=1.0.0 +blake3 +pydantic~=2.0 +pydantic-settings~=2.0 +git+https://github.com/ace-step/ACE-Step.git@1bee4c9f5b43e30995f8d4d33b3919197ce1bd68 diff --git a/tools/ai-runtime-requirements-windows-cuda.txt b/tools/ai-runtime-requirements-windows-cuda.txt index 1247356..f44d149 100644 --- a/tools/ai-runtime-requirements-windows-cuda.txt +++ b/tools/ai-runtime-requirements-windows-cuda.txt @@ -1,2 +1,8 @@ audio-separator==0.41.1 onnxruntime-gpu>=1.17 +transformers>=4.51.0,<4.58.0 +diffusers==0.35.2 +accelerate>=1.12.0 +vector-quantize-pytorch>=1.27.15 +huggingface_hub>=0.34,<1.0 +loguru>=0.7.3 diff --git a/tools/ai-runtime-requirements-windows-directml.txt b/tools/ai-runtime-requirements-windows-directml.txt index e07c357..fe37d40 100644 --- a/tools/ai-runtime-requirements-windows-directml.txt +++ b/tools/ai-runtime-requirements-windows-directml.txt @@ -1 +1,7 @@ audio-separator[dml]==0.41.1 +transformers>=4.51.0,<4.58.0 +diffusers==0.35.2 +accelerate>=1.12.0 +vector-quantize-pytorch>=1.27.15 +huggingface_hub>=0.34,<1.0 +loguru>=0.7.3 diff --git a/tools/ai_runtime_probe.py b/tools/ai_runtime_probe.py index a77dee4..d9469e5 100644 --- a/tools/ai_runtime_probe.py +++ b/tools/ai_runtime_probe.py @@ -9,15 +9,148 @@ from __future__ import annotations import argparse +import importlib.util import json import os import platform +import subprocess import sys import time from importlib import metadata from pathlib import Path from typing import Any +DEFAULT_MUSIC_GEN_MODEL = "acestep-v15-xl-turbo" +DEFAULT_MUSIC_GEN_MODEL_REPO = "Comfy-Org/ace_step_1.5_ComfyUI_files" +DEFAULT_MUSIC_GEN_SHARED_REPO = "Comfy-Org/ace_step_1.5_ComfyUI_files" +REQUIRED_MUSIC_GEN_PYTHON = (3, 11) +REQUIRED_MUSIC_GEN_NATIVE_FILES: tuple[dict[str, str], ...] = ( + { + "id": "diffusion_model", + "label": "ACE-Step XL Turbo diffusion model", + "relativePath": "diffusion_models/acestep_v1.5_xl_turbo_bf16.safetensors", + "sourceRelativePaths": ( + "diffusion_models/acestep_v1.5_xl_turbo_bf16.safetensors", + "unet/acestep_v1.5_xl_turbo_bf16.safetensors", + ), + }, + { + "id": "vae", + "label": "ACE-Step 1.5 VAE", + "relativePath": "vae/ace_1.5_vae.safetensors", + "sourceRelativePaths": ("vae/ace_1.5_vae.safetensors",), + }, + { + "id": "text_encoder_0_6b", + "label": "ACE-Step Qwen 0.6B text encoder", + "relativePath": "text_encoders/qwen_0.6b_ace15.safetensors", + "sourceRelativePaths": ("text_encoders/qwen_0.6b_ace15.safetensors",), + }, + { + "id": "text_encoder_4b", + "label": "ACE-Step Qwen 4B text encoder", + "relativePath": "text_encoders/qwen_4b_ace15.safetensors", + "sourceRelativePaths": ("text_encoders/qwen_4b_ace15.safetensors",), + }, +) +MUSIC_RUNTIME_PROFILE_SPECS: dict[str, dict[str, Any]] = { + "native-xl-turbo": { + "label": "OpenStudio ACE Split", + "runtimeProfileName": "openstudio-ace-split", + "lmModel": "qwen_4b_ace15.safetensors", + "requiredAssets": tuple(spec["relativePath"] for spec in REQUIRED_MUSIC_GEN_NATIVE_FILES), + }, +} +WINDOWS_ACCELERATION_MANIFEST_PATH = Path(__file__).with_name( + "windows-ai-acceleration-manifest.json" +) +_WINDOWS_ACCELERATION_MANIFEST_CACHE: dict[str, Any] | None = None + + +def load_windows_acceleration_manifest() -> dict[str, Any]: + global _WINDOWS_ACCELERATION_MANIFEST_CACHE + if _WINDOWS_ACCELERATION_MANIFEST_CACHE is None: + _WINDOWS_ACCELERATION_MANIFEST_CACHE = json.loads( + WINDOWS_ACCELERATION_MANIFEST_PATH.read_text(encoding="utf-8") + ) + return _WINDOWS_ACCELERATION_MANIFEST_CACHE + + +def get_windows_acceleration_target() -> dict[str, Any]: + return load_windows_acceleration_manifest().get("target", {}) + + +def get_windows_cuda_pytorch_index_url() -> str: + return str( + get_windows_acceleration_target() + .get("pytorch", {}) + .get("indexUrl", "https://download.pytorch.org/whl/cu128") + ) + + +def get_windows_cuda_pytorch_packages() -> tuple[str, ...]: + pytorch = get_windows_acceleration_target().get("pytorch", {}) + return tuple( + str(package).strip() + for package in pytorch.get("packages", []) + if str(package).strip() + ) + + +def get_windows_triton_package_spec() -> str: + return str( + get_windows_acceleration_target() + .get("tritonWindows", {}) + .get("package", "triton-windows") + ).strip() + + +def get_windows_flash_attn_asset() -> dict[str, Any]: + return dict(get_windows_acceleration_target().get("flashAttn", {})) + + +def _strip_local_version(version: str | None) -> str: + if not version: + return "" + return str(version).split("+", 1)[0].strip() + + +def _expected_torch_packages_by_name() -> dict[str, str]: + expected: dict[str, str] = {} + for package in get_windows_cuda_pytorch_packages(): + name, separator, version = str(package).partition("==") + if separator and name and version: + expected[name.strip()] = version.strip() + return expected + + +def _get_windows_expected_stack_versions() -> dict[str, str]: + target = get_windows_acceleration_target() + triton_spec = get_windows_triton_package_spec() + triton_name, _separator, triton_version = triton_spec.partition("==") + flash_attn = get_windows_flash_attn_asset() + expected = _expected_torch_packages_by_name() + expected["triton-windows"] = triton_version.strip() + expected["flash-attn"] = str(flash_attn.get("version", "")).strip() + expected["cuda"] = str(target.get("cuda", "")).strip() + return expected + + +def _set_music_generation_status( + report: dict[str, Any], + *, + ready: bool, + message: str = "", + error_code: str = "", +) -> None: + report["musicGenerationReady"] = ready + report["musicGenerationStatusMessage"] = message + report["musicGenerationFailureCode"] = error_code + + +def _is_music_generation_python_compatible() -> bool: + return sys.version_info[:2] == REQUIRED_MUSIC_GEN_PYTHON + def _get_dist_version(package_name: str) -> str | None: try: @@ -30,6 +163,161 @@ def _has_distribution(package_name: str) -> bool: return _get_dist_version(package_name) is not None +def _has_module(module_name: str) -> bool: + try: + return importlib.util.find_spec(module_name) is not None + except Exception: + return False + + +def _try_import_module(module_name: str) -> tuple[bool, str]: + try: + __import__(module_name) + except Exception as exc: + return False, f"{type(exc).__name__}: {exc}" + return True, "" + + +def _has_windows_nvidia_hardware() -> bool: + if platform.system() != "Windows": + return False + try: + result = subprocess.run( + ["nvidia-smi", "-L"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=5, + ) + except (FileNotFoundError, OSError, subprocess.SubprocessError): + return False + + return result.returncode == 0 and bool((result.stdout or "").strip()) + + +def _ensure_optional_music_acceleration_paths() -> None: + try: + import acestep + + nano_vllm_root = ( + Path(acestep.__file__).resolve().parent / "third_parts" / "nano-vllm" + ) + if nano_vllm_root.exists(): + nano_vllm_root_str = str(nano_vllm_root) + if nano_vllm_root_str not in sys.path: + sys.path.insert(0, nano_vllm_root_str) + except Exception: + pass + + +def _probe_music_generation_acceleration( + *, + compute_backend: str, + report: dict[str, Any], +) -> tuple[bool, str]: + if compute_backend != "cuda": + if report.get("platform") == "windows" and _has_windows_nvidia_hardware(): + report["backendDecisionTrace"].append( + "ACE-Step CUDA acceleration unavailable on a Windows NVIDIA machine" + ) + return ( + False, + "ACE-Step CUDA acceleration is required on this Windows NVIDIA machine, " + "but the managed runtime is not exposing CUDA to PyTorch yet.", + ) + return True, "" + + _ensure_optional_music_acceleration_paths() + missing: list[str] = [] + mismatch_details: list[str] = [] + import_failures: list[str] = [] + + for module_name, friendly_name in ( + ("nanovllm", "nano-vllm"), + ("triton", "triton"), + ("flash_attn", "flash-attn"), + ): + import_ok, import_error = _try_import_module(module_name) + if not import_ok: + if not _has_module(module_name): + missing.append(friendly_name) + else: + import_failures.append(f"{friendly_name} ({import_error})") + + if report.get("platform") == "windows": + expected_versions = _get_windows_expected_stack_versions() + installed_versions = { + "torch": report.get("torchVersion"), + "torchvision": _get_dist_version("torchvision"), + "torchaudio": _get_dist_version("torchaudio"), + "triton-windows": _get_dist_version("triton-windows"), + "flash-attn": _get_dist_version("flash-attn"), + } + + for package_name in ("torch", "torchvision", "torchaudio"): + expected_version = expected_versions.get(package_name, "") + installed_version = str(installed_versions.get(package_name) or "") + if not installed_version: + missing.append(package_name) + continue + + if ( + _strip_local_version(installed_version) != expected_version + or expected_versions.get("cuda", "") not in installed_version + ): + mismatch_details.append( + f"{package_name}={installed_version} (expected {expected_version}+{expected_versions.get('cuda', '')})" + ) + + for package_name in ("triton-windows", "flash-attn"): + expected_version = expected_versions.get(package_name, "") + installed_version = str(installed_versions.get(package_name) or "") + if not installed_version: + if package_name not in missing: + missing.append(package_name) + continue + if _strip_local_version(installed_version) != expected_version: + mismatch_details.append( + f"{package_name}={installed_version} (expected {expected_version})" + ) + + if not missing and not mismatch_details and not import_failures: + report["backendDecisionTrace"].append( + "ACE-Step accelerated LM runtime detected with the pinned Windows CUDA stack" + ) + return True, "ACE-Step accelerated LM runtime is installed." + + detail_parts: list[str] = [] + if missing: + detail_parts.append("missing " + ", ".join(missing)) + if mismatch_details: + detail_parts.append("mismatched " + "; ".join(mismatch_details)) + if import_failures: + detail_parts.append("import failures " + "; ".join(import_failures)) + + report["backendDecisionTrace"].append( + "ACE-Step CUDA acceleration incomplete: " + " | ".join(detail_parts) + ) + return ( + False, + "ACE-Step music generation is installed, but the pinned Windows CUDA acceleration " + "stack is incomplete: " + " | ".join(detail_parts) + ".", + ) + + +def _can_import_music_generation_bridge() -> tuple[bool, str]: + try: + import acestep.handler # noqa: F401 + import acestep.inference # noqa: F401 + import acestep.llm_inference # noqa: F401 + import av # noqa: F401 + import torchsde # noqa: F401 + except Exception as exc: + return False, f"{type(exc).__name__}: {exc}" + return True, "" + + def _normalize_arch(machine: str) -> str: value = machine.lower() if value in {"amd64", "x86_64"}: @@ -39,13 +327,170 @@ def _normalize_arch(machine: str) -> str: return value +def resolve_music_gen_checkpoint_root(checkpoint_root: str = "") -> Path: + if checkpoint_root.strip(): + return Path(checkpoint_root).expanduser().resolve() + return (Path.home() / ".cache" / "ace-step" / "checkpoints").resolve() + + +def get_openstudio_native_backend_root() -> Path: + return Path(__file__).resolve().parent / "openstudio_ace_backend" / "vendor_runtime" + + +def get_openstudio_native_backend_status() -> tuple[bool, str, list[str]]: + backend_root = get_openstudio_native_backend_root() + required_paths = ( + backend_root / "nodes.py", + backend_root / "folder_paths.py", + backend_root / "comfy" / "sd.py", + backend_root / "comfy_extras" / "nodes_ace.py", + ) + missing = [str(path) for path in required_paths if not path.exists()] + return not missing, str(backend_root), missing + + +def get_candidate_comfy_model_roots() -> list[Path]: + candidates = [ + Path(os.environ.get("COMFYUI_MODEL_DIR", "")).expanduser(), + Path(os.environ.get("COMFYUI_ROOT", "")).expanduser() / "models", + ] + home = Path.home() + candidates.extend( + [ + home / "Documents" / "ComfyUI" / "models", + home / "Documents" / "Codes" / "ComfyUI" / "models", + home / "ComfyUI" / "models", + ] + ) + + unique_candidates: list[Path] = [] + seen: set[str] = set() + for candidate in candidates: + candidate_str = str(candidate).strip() + if not candidate_str or candidate_str == ".": + continue + try: + resolved = candidate.resolve() + except OSError: + resolved = candidate + key = str(resolved).lower() + if key in seen: + continue + seen.add(key) + unique_candidates.append(resolved) + return unique_candidates + + +def find_local_comfy_native_assets() -> tuple[dict[str, Path], list[str]]: + found: dict[str, Path] = {} + searched_roots: list[str] = [] + + for model_root in get_candidate_comfy_model_roots(): + searched_roots.append(str(model_root)) + if not model_root.is_dir(): + continue + + for spec in REQUIRED_MUSIC_GEN_NATIVE_FILES: + if spec["id"] in found: + continue + for source_relative in spec.get("sourceRelativePaths", (spec["relativePath"],)): + candidate = model_root / Path(source_relative) + if candidate.exists() and candidate.is_file(): + found[spec["id"]] = candidate.resolve() + break + + return found, searched_roots + + +def get_music_generation_required_paths( + checkpoint_root: str = "", + model_name: str = DEFAULT_MUSIC_GEN_MODEL, +) -> dict[str, Any]: + root = resolve_music_gen_checkpoint_root(checkpoint_root) + required_paths = [root / Path(spec["relativePath"]) for spec in REQUIRED_MUSIC_GEN_NATIVE_FILES] + missing_paths = [str(path) for path in required_paths if not path.exists()] + + return { + "checkpointRoot": str(root), + "modelId": model_name, + "modelRepoId": DEFAULT_MUSIC_GEN_MODEL_REPO, + "sharedRepoId": DEFAULT_MUSIC_GEN_SHARED_REPO, + "mainModelPath": str(required_paths[0]) if required_paths else "", + "sharedPaths": [str(path) for path in required_paths[1:]], + "requiredPaths": [str(path) for path in required_paths], + "missingPaths": missing_paths, + "layoutValid": not missing_paths, + "requiredAssets": [ + { + "id": spec["id"], + "label": spec["label"], + "relativePath": spec["relativePath"], + "path": str(root / Path(spec["relativePath"])), + } + for spec in REQUIRED_MUSIC_GEN_NATIVE_FILES + ], + } + + +def get_music_runtime_profiles(checkpoint_root: str = "") -> dict[str, Any]: + root = resolve_music_gen_checkpoint_root(checkpoint_root) + installed_assets = ( + { + str(path.relative_to(root)).replace("\\", "/") + for path in root.rglob("*") + if path.is_file() + } + if root.exists() + else set() + ) + + profiles: dict[str, dict[str, Any]] = {} + available_profiles: list[str] = [] + unavailable_profiles: list[dict[str, Any]] = [] + for profile_id, spec in MUSIC_RUNTIME_PROFILE_SPECS.items(): + missing_assets = [asset for asset in spec["requiredAssets"] if asset not in installed_assets] + profile = { + "id": profile_id, + "label": spec["label"], + "runtimeProfileName": spec["runtimeProfileName"], + "lmModel": spec["lmModel"], + "requiredAssets": list(spec["requiredAssets"]), + "missingAssets": missing_assets, + "available": not missing_assets, + } + profiles[profile_id] = profile + if profile["available"]: + available_profiles.append(profile_id) + else: + unavailable_profiles.append(profile) + + default_profile = ( + "native-xl-turbo" + if profiles.get("native-xl-turbo", {}).get("available") + else next(iter(available_profiles), "") + ) + return { + "defaultProfile": default_profile, + "profiles": profiles, + "availableProfiles": available_profiles, + "unavailableProfiles": unavailable_profiles, + "warmSessionCapable": True, + } + + def probe_runtime_capabilities( *, models_dir: str = "", model_name: str = "", acceleration_mode: str = "auto", + music_checkpoint_root: str = "", + music_model_id: str = DEFAULT_MUSIC_GEN_MODEL, ) -> dict[str, Any]: started_at = time.perf_counter() + music_layout = get_music_generation_required_paths( + checkpoint_root=music_checkpoint_root, + model_name=music_model_id, + ) report: dict[str, Any] = { "schemaVersion": 1, "baseRuntimeReady": False, @@ -57,13 +502,38 @@ def probe_runtime_capabilities( "runtimeExecutable": sys.executable, "accelerationMode": acceleration_mode, "audioSeparatorVersion": None, + "aceStepVersion": None, "torchVersion": None, + "torchvisionVersion": None, + "torchaudioVersion": None, + "tritonWindowsVersion": None, + "flashAttnVersion": None, "onnxRuntimePackages": {}, "onnxProviders": [], "packagedBackends": [], "supportedBackends": ["cpu"], "selectedBackend": "cpu", "modelInstalled": False, + "musicGenerationReady": False, + "musicGenerationLayoutValid": bool(music_layout["layoutValid"]), + "musicGenerationStatusMessage": "", + "musicGenerationFailureCode": "", + "musicGenerationPerformanceReady": False, + "musicGenerationPerformanceStatusMessage": "", + "musicGenerationComputeBackend": "cpu", + "musicGenerationModelId": music_layout["modelId"], + "musicGenerationModelRepoId": music_layout["modelRepoId"], + "musicGenerationSharedRepoId": music_layout["sharedRepoId"], + "musicGenerationCheckpointRoot": music_layout["checkpointRoot"], + "musicGenerationBackendRoot": "", + "musicGenerationMainModelPath": music_layout["mainModelPath"], + "musicGenerationRequiredPaths": music_layout["requiredPaths"], + "musicGenerationMissingPaths": music_layout["missingPaths"], + "musicGenerationRuntimeProfiles": {}, + "musicGenerationAvailableProfiles": [], + "musicGenerationUnavailableProfiles": [], + "musicGenerationDefaultProfile": "", + "musicGenerationWarmSessionCapable": True, "modelVersion": model_name or "", "fallbackReason": "", "errorCode": "", @@ -74,6 +544,15 @@ def probe_runtime_capabilities( if models_dir and model_name: report["modelInstalled"] = (Path(models_dir) / model_name).exists() + runtime_profiles = get_music_runtime_profiles(music_checkpoint_root) + report["musicGenerationRuntimeProfiles"] = runtime_profiles["profiles"] + report["musicGenerationAvailableProfiles"] = runtime_profiles["availableProfiles"] + report["musicGenerationUnavailableProfiles"] = runtime_profiles["unavailableProfiles"] + report["musicGenerationDefaultProfile"] = runtime_profiles["defaultProfile"] + report["musicGenerationWarmSessionCapable"] = runtime_profiles["warmSessionCapable"] + backend_runtime_ok, backend_root, backend_missing = get_openstudio_native_backend_status() + report["musicGenerationBackendRoot"] = backend_root + report["baseRuntimeReady"] = True try: @@ -89,7 +568,12 @@ def probe_runtime_capabilities( report["runtimeReady"] = True report["audioSeparatorVersion"] = _get_dist_version("audio-separator") + report["aceStepVersion"] = _get_dist_version("ace-step") report["torchVersion"] = getattr(torch, "__version__", None) + report["torchvisionVersion"] = _get_dist_version("torchvision") + report["torchaudioVersion"] = _get_dist_version("torchaudio") + report["tritonWindowsVersion"] = _get_dist_version("triton-windows") + report["flashAttnVersion"] = _get_dist_version("flash-attn") report["onnxRuntimePackages"] = { "onnxruntime": _get_dist_version("onnxruntime"), "onnxruntime-gpu": _get_dist_version("onnxruntime-gpu"), @@ -105,6 +589,111 @@ def probe_runtime_capabilities( report["backendDecisionTrace"].append("onnxruntime.get_available_providers failed") report["onnxProviders"] = ort_providers + if not music_layout["layoutValid"]: + _set_music_generation_status( + report, + ready=False, + message="Pinned ACE-Step native split-model files are still missing.", + error_code="missing_checkpoint_files", + ) + report["backendDecisionTrace"].append( + "music generation checkpoint layout missing: " + + ", ".join(music_layout["missingPaths"]) + ) + elif not _is_music_generation_python_compatible(): + _set_music_generation_status( + report, + ready=False, + message=( + "Pinned ACE-Step music generation currently requires Python " + f"{REQUIRED_MUSIC_GEN_PYTHON[0]}.{REQUIRED_MUSIC_GEN_PYTHON[1]}.x, " + f"but this managed runtime is using Python {report['pythonVersion']}." + ), + error_code="music_generation_python_incompatible", + ) + report["backendDecisionTrace"].append( + "music generation python incompatible: " + f"expected {REQUIRED_MUSIC_GEN_PYTHON[0]}.{REQUIRED_MUSIC_GEN_PYTHON[1]}.x, " + f"got {report['pythonVersion']}" + ) + elif report["aceStepVersion"] is None: + _set_music_generation_status( + report, + ready=False, + message="ACE-Step is not installed in the managed AI runtime yet.", + error_code="missing_ace_step_runtime", + ) + report["backendDecisionTrace"].append("ace-step package not installed") + elif not backend_runtime_ok: + _set_music_generation_status( + report, + ready=False, + message=( + "ACE-Step 1.5 assets are installed, but the packaged OpenStudio split backend " + "is missing runtime files: " + ", ".join(backend_missing) + ), + error_code="missing_openstudio_native_backend", + ) + report["backendDecisionTrace"].append( + "openstudio native backend missing: " + ", ".join(backend_missing) + ) + elif str(report["aceStepVersion"]).startswith("1."): + try: + import torchaudio # noqa: F401 + except Exception as exc: + _set_music_generation_status( + report, + ready=False, + message=( + "ACE-Step 1.5 dependencies are installed, but this runtime cannot " + f"load torchaudio yet: {exc}" + ), + error_code="torchaudio_runtime_incompatible", + ) + report["backendDecisionTrace"].append(f"torchaudio import failed: {exc}") + else: + bridge_import_ok, bridge_import_error = _can_import_music_generation_bridge() + if bridge_import_ok: + report["musicGenerationComputeBackend"] = "cuda" if torch.cuda.is_available() else "cpu" + acceleration_ready, acceleration_message = _probe_music_generation_acceleration( + compute_backend=report["musicGenerationComputeBackend"], + report=report, + ) + report["musicGenerationPerformanceReady"] = acceleration_ready + report["musicGenerationPerformanceStatusMessage"] = acceleration_message + _set_music_generation_status( + report, + ready=True, + message="OpenStudio ACE split backend is ready.", + ) + else: + _set_music_generation_status( + report, + ready=False, + message=( + "ACE-Step 1.5 is installed, but the packaged OpenStudio split backend " + f"cannot import its ACE-Step bridge in this environment: {bridge_import_error}" + ), + error_code="broken_v15_runtime_bridge", + ) + report["backendDecisionTrace"].append( + "openstudio split backend ace-step import failed: " + bridge_import_error + ) + else: + _set_music_generation_status( + report, + ready=False, + message=( + "The managed runtime still has the legacy ACE-Step " + f"{report['aceStepVersion']} package, but the pinned model requires the " + "ACE-Step 1.5 split backend." + ), + error_code="legacy_ace_step_runtime", + ) + report["backendDecisionTrace"].append( + f"legacy ace-step runtime detected: {report['aceStepVersion']}" + ) + packaged_backends: list[str] = [] if ( report["platform"] == "windows" @@ -124,6 +713,24 @@ def probe_runtime_capabilities( packaged_backends.append("directml") report["backendDecisionTrace"].append("packaged directml runtime detected") + if ( + report["platform"] == "linux" + and not str(report["torchVersion"] or "").endswith("+cpu") + and _has_distribution("onnxruntime-gpu") + and "CUDAExecutionProvider" in ort_providers + ): + packaged_backends.append("cuda") + report["backendDecisionTrace"].append("packaged cuda runtime detected (linux)") + + if report["platform"] == "linux": + # ROCm: torch built against ROCm exposes torch.version.hip + try: + if hasattr(torch, "version") and getattr(torch.version, "hip", None) is not None: + packaged_backends.append("rocm") + report["backendDecisionTrace"].append("packaged rocm runtime detected (linux)") + except Exception: + pass + mps_available = bool( hasattr(torch.backends, "mps") and torch.backends.mps.is_available() @@ -181,6 +788,25 @@ def probe_runtime_capabilities( if not supported_backends: fallback_reason = "no Apple Silicon acceleration backend could be configured" report["errorCode"] = "probe_backend_unavailable" + elif report["platform"] == "linux": + if "cuda" in report["packagedBackends"] and torch.cuda.is_available(): + supported_backends.append("cuda") + report["backendDecisionTrace"].append("cuda backend available on linux machine") + if "rocm" in report["packagedBackends"]: + try: + # ROCm appears as CUDA to torch.cuda on linux; double-check via hip version + rocm_available = bool( + torch.cuda.is_available() + and getattr(torch.version, "hip", None) is not None + ) + if rocm_available: + supported_backends.append("rocm") + report["backendDecisionTrace"].append("rocm backend available on linux machine") + except Exception: + pass + if not supported_backends: + fallback_reason = "no GPU backend could be configured on this Linux machine" + report["errorCode"] = "probe_backend_unavailable" else: fallback_reason = "no accelerated backend is supported for this platform" report["errorCode"] = "probe_backend_unavailable" @@ -204,6 +830,13 @@ def probe_runtime_capabilities( report["selectedBackend"] = "mps" else: report["selectedBackend"] = "cpu" + elif report["platform"] == "linux": + if "cuda" in report["supportedBackends"]: + report["selectedBackend"] = "cuda" + elif "rocm" in report["supportedBackends"]: + report["selectedBackend"] = "rocm" + else: + report["selectedBackend"] = "cpu" else: report["selectedBackend"] = "cpu" @@ -218,6 +851,16 @@ def main() -> int: parser = argparse.ArgumentParser(description="Probe OpenStudio AI runtime capabilities") parser.add_argument("--models-dir", default="", help="Optional models directory") parser.add_argument("--model", default="", help="Optional model filename to check") + parser.add_argument( + "--music-gen-checkpoint-root", + default="", + help="Pinned ACE-Step checkpoint root to validate", + ) + parser.add_argument( + "--music-gen-model", + default=DEFAULT_MUSIC_GEN_MODEL, + help="Pinned ACE-Step model id to validate", + ) parser.add_argument( "--acceleration-mode", choices=["auto", "cpu-only"], @@ -230,6 +873,8 @@ def main() -> int: models_dir=args.models_dir, model_name=args.model, acceleration_mode=args.acceleration_mode, + music_checkpoint_root=args.music_gen_checkpoint_root, + music_model_id=args.music_gen_model, ) print(json.dumps(report), flush=True) return 0 if report.get("baseRuntimeReady") else 1 diff --git a/tools/analyze_repitch.ps1 b/tools/analyze_repitch.ps1 deleted file mode 100644 index 45a2e41..0000000 --- a/tools/analyze_repitch.ps1 +++ /dev/null @@ -1,23 +0,0 @@ -$path = 'C:\Program Files\Common Files\VST3\RePitch.vst3\Contents\x86_64-win\RePitch.vst3' -$data = [System.IO.File]::ReadAllBytes($path) -Write-Host ('Size: ' + $data.Length) - -# Extract ASCII strings >= 6 chars -$text = [System.Text.Encoding]::ASCII.GetString($data) -$pattern = [regex]'[ -~]{6,}' -$m_list = $pattern.Matches($text) - -$keywords = @('pitch','fft','partial','formant','phase','vocoder','rubber','sinusoid','harmonic','autocorr','lpc','spectrum','transient','residual','stft','overlap','additive','hop size','hann','window size','analysis','resynth','synthesis') - -$found = @{} -foreach ($m in $m_list) { - $v = $m.Value.ToLower() - foreach ($k in $keywords) { - if ($v.Contains($k)) { - $found[$m.Value] = 1 - break - } - } -} - -$found.Keys | Sort-Object diff --git a/tools/analyze_repitch2.ps1 b/tools/analyze_repitch2.ps1 deleted file mode 100644 index d5a2a0b..0000000 --- a/tools/analyze_repitch2.ps1 +++ /dev/null @@ -1,21 +0,0 @@ -$path = 'C:\Program Files\Common Files\VST3\RePitch.vst3\Contents\x86_64-win\RePitch.vst3' -$data = [System.IO.File]::ReadAllBytes($path) - -# Extract longer ASCII strings >= 12 chars that look like real text (not binary noise) -$text = [System.Text.Encoding]::ASCII.GetString($data) -$pattern = [regex]'[A-Za-z][A-Za-z0-9_ .,\-/\\:()]{11,}' -$m_list = $pattern.Matches($text) - -$found = @{} -foreach ($m in $m_list) { - $found[$m.Value] = 1 -} - -# Also look for error/log messages -$pattern2 = [regex]'[A-Za-z][A-Za-z0-9_ .!?,\-/\\:()]{20,}' -$m_list2 = $pattern2.Matches($text) -foreach ($m in $m_list2) { - $found[$m.Value] = 1 -} - -$found.Keys | Sort-Object | Select-Object -First 500 diff --git a/tools/analyze_repitch3.ps1 b/tools/analyze_repitch3.ps1 deleted file mode 100644 index b2e4786..0000000 --- a/tools/analyze_repitch3.ps1 +++ /dev/null @@ -1,20 +0,0 @@ -$path = 'C:\Program Files\Common Files\VST3\RePitch.vst3\Contents\x86_64-win\RePitch.vst3' -$data = [System.IO.File]::ReadAllBytes($path) -$text = [System.Text.Encoding]::ASCII.GetString($data) - -# Only extract strings with mostly printable ASCII (letters, numbers, common symbols) -# Filter out binary noise by requiring high proportion of letters -$pattern = [regex]'[A-Za-z][A-Za-z0-9 _.,\-/:()!?+*=<>]{8,}' -$m_list = $pattern.Matches($text) - -$results = @{} -foreach ($m in $m_list) { - $val = $m.Value - # Check that at least 60% are letters/digits/space - $letters = ($val.ToCharArray() | Where-Object { $_ -match '[A-Za-z0-9 ]' }).Count - if ($letters / $val.Length -ge 0.7) { - $results[$val] = 1 - } -} - -$results.Keys | Sort-Object diff --git a/tools/analyze_repitch4.ps1 b/tools/analyze_repitch4.ps1 deleted file mode 100644 index ce24b79..0000000 --- a/tools/analyze_repitch4.ps1 +++ /dev/null @@ -1,39 +0,0 @@ -$path = 'C:\Program Files\Common Files\VST3\RePitch.vst3\Contents\x86_64-win\RePitch.vst3' -$data = [System.IO.File]::ReadAllBytes($path) -$text = [System.Text.Encoding]::ASCII.GetString($data) - -# Search for specific algorithm-related substrings -$searches = @( - 'sample rate', 'samplerate', 'sample_rate', - 'fft size', 'fftsize', 'fft_size', - 'window size', 'windowsize', - 'hop size', 'hopsize', 'hop_size', - 'overlap', 'phase vocoder', 'phase_vocoder', - 'sinusoidal', 'partial track', 'harmonic', - 'formant', 'lpc order', 'autocorr', - 'pitch shift', 'pitchshift', 'pitch_shift', - 'resynthes', 'residual', - 'onset', 'transient', - 'vibrato', 'portamento', 'glide', - 'midi note', 'note detect', - 'repitch', 'RePitch', - 'Synchro', 'synchro', - 'algorithm', 'process', - 'version', 'copyright', - 'interpolat', 'resampl', - 'magnitude', 'spectrum', - 'autocorrelation', 'yin', 'crepe', 'pyin', - 'nnls', 'nmf', 'ica', - 'sms', 'loris', 'spear' -) - -foreach ($s in $searches) { - $idx = $text.ToLower().IndexOf($s.ToLower()) - if ($idx -ge 0) { - $start = [Math]::Max(0, $idx - 20) - $len = [Math]::Min(200, $text.Length - $start) - Write-Host "=== Found: '$s' ===" - Write-Host $text.Substring($start, $len) - Write-Host "" - } -} diff --git a/tools/analyze_wav_tails.py b/tools/analyze_wav_tails.py deleted file mode 100644 index 3428aa6..0000000 --- a/tools/analyze_wav_tails.py +++ /dev/null @@ -1,205 +0,0 @@ -""" -Analyze WAV files: duration, sample rate, channels, bit depth, peak/RMS, -and pitch (F0) in the last 2 seconds using YIN-style autocorrelation. -""" -import sys -import io -sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace') -import numpy as np -import soundfile as sf -from scipy.signal import correlate - -FILES = { - "original": "D:/test projects/untitled3.wav", - "repitch": "D:/test projects/untitled3-001.wav", - "studio13": "D:/test projects/Untitled Project26.wav", -} - -# ────────────────────────────────────────────────────────────────────────────── -# YIN-style autocorrelation F0 estimator (no librosa needed) -# ────────────────────────────────────────────────────────────────────────────── -def yin_frame(frame, sr, fmin=60, fmax=800): - """ - Compute F0 of a single mono frame using YIN difference function. - Returns F0 in Hz, or 0 if unvoiced/unreliable. - """ - N = len(frame) - tau_min = int(sr / fmax) - tau_max = min(int(sr / fmin), N // 2) - if tau_max <= tau_min: - return 0.0 - - # Squared difference function - def sdf(tau): - diff = frame[:N - tau] - frame[tau:] - return np.dot(diff, diff) - - # Cumulative mean normalised difference (CMND) - d = np.array([sdf(tau) for tau in range(tau_max + 1)], dtype=np.float64) - d[0] = 1.0 - cmnd = np.empty_like(d) - cmnd[0] = 1.0 - running = 0.0 - for tau in range(1, tau_max + 1): - running += d[tau] - cmnd[tau] = d[tau] * tau / running if running > 0 else 1.0 - - # Find first dip below threshold 0.10 - threshold = 0.10 - tau_est = 0 - for tau in range(tau_min, tau_max): - if cmnd[tau] < threshold: - # Parabolic interpolation - if 0 < tau < len(cmnd) - 1: - denom = cmnd[tau-1] - 2*cmnd[tau] + cmnd[tau+1] - if denom != 0: - tau_interp = tau + 0.5 * (cmnd[tau-1] - cmnd[tau+1]) / denom - return sr / tau_interp - return sr / tau - # Fallback: global minimum in range - idx = np.argmin(cmnd[tau_min:tau_max]) + tau_min - if cmnd[idx] < 0.3: - return sr / idx if idx > 0 else 0.0 - return 0.0 - - -def midi_note(f0_hz): - if f0_hz <= 0: - return "—" - midi = 69 + 12 * np.log2(f0_hz / 440.0) - note_names = ["A","A#","B","C","C#","D","D#","E","F","F#","G","G#"] - n = int(round(midi)) % 12 - octave = (int(round(midi)) // 12) - 1 - cents = (midi - round(midi)) * 100 - sign = "+" if cents >= 0 else "" - return f"{note_names[n]}{octave} ({sign}{cents:.1f}¢) [{f0_hz:.2f} Hz]" - - -def analyze_file(label, path): - print(f"\n{'='*60}") - print(f" {label}") - print(f" {path}") - print(f"{'='*60}") - - try: - info = sf.info(path) - except Exception as e: - print(f" ERROR reading info: {e}") - return - - duration = info.frames / info.samplerate - print(f" Duration : {duration:.4f} s ({info.frames} frames)") - print(f" Sample rate : {info.samplerate} Hz") - print(f" Channels : {info.channels}") - print(f" Subtype : {info.subtype} (bit depth)") - - # Load full file for peak/RMS - data, sr = sf.read(path, dtype='float32', always_2d=True) - mono = data.mean(axis=1) - - peak = float(np.max(np.abs(data))) - peak_db = 20 * np.log10(peak + 1e-12) - rms = float(np.sqrt(np.mean(mono**2))) - rms_db = 20 * np.log10(rms + 1e-12) - print(f" Peak amp : {peak:.6f} ({peak_db:.2f} dBFS)") - print(f" RMS (mono) : {rms:.6f} ({rms_db:.2f} dBFS)") - - # ── TAIL ANALYSIS ────────────────────────────────────────────────────── - tail_sec = 2.0 - tail_samples = int(tail_sec * sr) - tail = mono[-tail_samples:] if len(mono) >= tail_samples else mono - tail_dur = len(tail) / sr - - tail_peak = float(np.max(np.abs(tail))) - tail_rms = float(np.sqrt(np.mean(tail**2))) - tail_peak_db = 20 * np.log10(tail_peak + 1e-12) - tail_rms_db = 20 * np.log10(tail_rms + 1e-12) - print(f"\n -- Last {tail_dur:.2f}s tail --") - print(f" Tail peak : {tail_peak:.6f} ({tail_peak_db:.2f} dBFS)") - print(f" Tail RMS : {tail_rms:.6f} ({tail_rms_db:.2f} dBFS)") - - # ── PITCH TRACKING IN TAIL ──────────────────────────────────────────── - frame_len = int(0.04 * sr) # 40 ms frames - hop_len = int(0.01 * sr) # 10 ms hop - f0_values = [] - positions = [] - start = 0 - while start + frame_len <= len(tail): - frame = tail[start : start + frame_len].copy().astype(np.float64) - # Window - frame *= np.hanning(len(frame)) - f0 = yin_frame(frame, sr) - t_offset = (start + frame_len // 2) / sr # time from tail start - t_abs = duration - tail_dur + t_offset # absolute time in file - if f0 > 0: - f0_values.append(f0) - positions.append(t_abs) - start += hop_len - - print(f"\n Pitch track ({len(f0_values)} voiced frames out of " - f"{(len(tail) - frame_len) // hop_len + 1} total):") - if f0_values: - f0_arr = np.array(f0_values) - print(f" F0 median : {midi_note(float(np.median(f0_arr)))}") - print(f" F0 mean : {np.mean(f0_arr):.2f} Hz") - print(f" F0 std : {np.std(f0_arr):.2f} Hz") - print(f" F0 min/max : {np.min(f0_arr):.2f} / {np.max(f0_arr):.2f} Hz") - - # Print per-second summary for the tail - print(f"\n Per-0.5s F0 summary:") - seg_size = int(0.5 * sr / hop_len) - for seg_i in range(0, len(f0_values), max(1, seg_size)): - seg = f0_arr[seg_i : seg_i + seg_size] - t_start_abs = duration - tail_dur + (seg_i * hop_len) / sr - if len(seg) > 0: - print(f" t={t_start_abs:.2f}s median={np.median(seg):.2f} Hz " - f"({midi_note(float(np.median(seg)))})") - else: - print(" No voiced frames detected in tail (silence or below threshold)") - - return {"duration": duration, "sr": sr, "channels": info.channels, - "peak_db": peak_db, "rms_db": rms_db, - "tail_peak_db": tail_peak_db, "tail_rms_db": tail_rms_db, - "f0_values": f0_values} - - -results = {} -for label, path in FILES.items(): - results[label] = analyze_file(label, path) - - -# ── COMPARISON SUMMARY ──────────────────────────────────────────────────────── -print(f"\n\n{'='*60}") -print(" COMPARISON SUMMARY") -print(f"{'='*60}") - -# Duration diff -labels = list(FILES.keys()) -for l in labels: - r = results.get(l) - if r: - print(f" {l:12s}: {r['duration']:.4f}s peak={r['peak_db']:.1f}dBFS " - f"tail_peak={r['tail_peak_db']:.1f}dBFS tail_rms={r['tail_rms_db']:.1f}dBFS") - -# F0 comparison -print("\n Median tail F0 comparison:") -medians = {} -for l in labels: - r = results.get(l) - if r and r.get("f0_values"): - m = float(np.median(r["f0_values"])) - medians[l] = m - print(f" {l:12s}: {midi_note(m)}") - else: - print(f" {l:12s}: no voiced frames") - -if len(medians) >= 2: - print("\n Pitch differences (cents) relative to original:") - orig_m = medians.get("original", 0) - for l in ["repitch", "studio13"]: - if l in medians and orig_m > 0: - cents_diff = 1200 * np.log2(medians[l] / orig_m) - print(f" {l:12s} vs original: {cents_diff:+.1f} cents") - if "repitch" in medians and "studio13" in medians: - cents_diff = 1200 * np.log2(medians["studio13"] / medians["repitch"]) - print(f" studio13 vs repitch: {cents_diff:+.1f} cents") diff --git a/tools/formant_reference_compare.py b/tools/formant_reference_compare.py deleted file mode 100644 index 0e89bbf..0000000 --- a/tools/formant_reference_compare.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python3 -"""Compare Studio13 formant renders against RePitch reference clips. - -Usage: - python tools/formant_reference_compare.py ^ - --original "d:\\test projects\\org.wav" ^ - --plus-target "d:\\test projects\\org+200.wav" ^ - --minus-target "d:\\test projects\\org-200.wav" ^ - [--candidate-plus "..."] [--candidate-minus "..."] - -The script measures: - - best-fit simple spectral-envelope warp in semitones - - residual envelope distance after that best fit - - coarse low/body/presence/air band drift - - optional candidate-vs-target error if Studio13 renders are provided -""" - -from __future__ import annotations - -import argparse -import math -import wave -from dataclasses import dataclass -from pathlib import Path - -import numpy as np - - -@dataclass -class AudioData: - sample_rate: int - samples: np.ndarray - - -def load_wav(path: Path) -> AudioData: - with wave.open(str(path), "rb") as wf: - sample_rate = wf.getframerate() - channels = wf.getnchannels() - width = wf.getsampwidth() - frames = wf.getnframes() - raw = wf.readframes(frames) - - if width == 2: - data = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0 - elif width == 4: - data = np.frombuffer(raw, dtype="<i4").astype(np.float32) / 2147483648.0 - else: - raise ValueError(f"Unsupported sample width: {width}") - - if channels > 1: - data = data.reshape(-1, channels).mean(axis=1) - return AudioData(sample_rate=sample_rate, samples=data) - - -def stft_mag(samples: np.ndarray, fft_size: int = 2048, hop: int = 512) -> np.ndarray: - if samples.size < fft_size: - samples = np.pad(samples, (0, fft_size - samples.size)) - window = np.hanning(fft_size).astype(np.float32) - frames = [] - for start in range(0, max(1, samples.size - fft_size + 1), hop): - frame = samples[start:start + fft_size] - if frame.size < fft_size: - frame = np.pad(frame, (0, fft_size - frame.size)) - frames.append(np.abs(np.fft.rfft(frame * window))) - return np.stack(frames, axis=0) - - -def smooth_envelope(mag: np.ndarray, smooth_bins: int = 24) -> np.ndarray: - kernel = np.ones(smooth_bins, dtype=np.float32) / smooth_bins - padded = np.pad(mag, ((0, 0), (smooth_bins, smooth_bins)), mode="edge") - out = np.empty_like(mag) - for i in range(mag.shape[0]): - out[i] = np.convolve(padded[i], kernel, mode="same")[smooth_bins:-smooth_bins] - return np.maximum(out, 1e-7) - - -def voiced_mask(samples: np.ndarray, fft_size: int = 2048, hop: int = 512) -> np.ndarray: - frames = [] - for start in range(0, max(1, samples.size - fft_size + 1), hop): - frame = samples[start:start + fft_size] - if frame.size < fft_size: - frame = np.pad(frame, (0, fft_size - frame.size)) - rms = math.sqrt(float(np.mean(frame * frame)) + 1e-12) - zcr = float(np.mean(np.abs(np.diff(np.signbit(frame)).astype(np.float32)))) - frames.append(rms > 0.01 and zcr < 0.25) - mask = np.array(frames, dtype=bool) - if not np.any(mask): - mask[:] = True - return mask - - -def warp_env(env: np.ndarray, semitones: float) -> np.ndarray: - ratio = 2.0 ** (semitones / 12.0) - bins = np.arange(env.shape[1], dtype=np.float32) - src = bins / ratio - src = np.clip(src, 0, env.shape[1] - 1) - lo = np.floor(src).astype(np.int32) - hi = np.clip(lo + 1, 0, env.shape[1] - 1) - frac = src - lo - warped = env[:, lo] * (1.0 - frac) + env[:, hi] * frac - return np.maximum(warped, 1e-7) - - -def env_error(a: np.ndarray, b: np.ndarray, mask: np.ndarray) -> float: - la = np.log(a[mask]) - lb = np.log(b[mask]) - return float(np.sqrt(np.mean((la - lb) ** 2))) - - -def best_simple_warp(reference_env: np.ndarray, target_env: np.ndarray, mask: np.ndarray) -> tuple[float, float]: - best_st = 0.0 - best_err = float("inf") - for semitones in np.linspace(-3.0, 3.0, 241): - warped = warp_env(reference_env, float(semitones)) - err = env_error(warped, target_env, mask) - if err < best_err: - best_err = err - best_st = float(semitones) - return best_st, best_err - - -def band_energy_delta(env_a: np.ndarray, env_b: np.ndarray, sr: int) -> dict[str, float]: - freqs = np.linspace(0, sr / 2, env_a.shape[1], dtype=np.float32) - bands = { - "sub_body": (80, 350), - "body": (350, 1200), - "presence": (1200, 4000), - "air": (4000, 9000), - } - out: dict[str, float] = {} - for name, (lo, hi) in bands.items(): - idx = np.where((freqs >= lo) & (freqs < hi))[0] - if idx.size == 0: - out[name] = 0.0 - continue - ea = float(np.mean(np.log(env_a[:, idx] + 1e-7))) - eb = float(np.mean(np.log(env_b[:, idx] + 1e-7))) - out[name] = eb - ea - return out - - -def analyze_pair(original: AudioData, target: AudioData, label: str) -> dict[str, float]: - if original.sample_rate != target.sample_rate: - raise ValueError(f"{label}: sample rate mismatch") - min_len = min(original.samples.size, target.samples.size) - src = original.samples[:min_len] - dst = target.samples[:min_len] - src_mag = stft_mag(src) - dst_mag = stft_mag(dst) - src_env = smooth_envelope(src_mag) - dst_env = smooth_envelope(dst_mag) - mask = voiced_mask(src) - best_st, best_err = best_simple_warp(src_env, dst_env, mask) - warped = warp_env(src_env, best_st) - src_err = env_error(src_env, dst_env, mask) - residual_ratio = best_err / max(src_err, 1e-9) - bands = band_energy_delta(src_env[mask], dst_env[mask], original.sample_rate) - return { - "best_st": best_st, - "best_cents": best_st * 100.0, - "best_err": best_err, - "src_err": src_err, - "residual_ratio": residual_ratio, - **{f"band_{k}": v for k, v in bands.items()}, - } - - -def compare_candidate(original: AudioData, candidate: AudioData, target: AudioData, label: str) -> dict[str, float]: - min_len = min(original.samples.size, candidate.samples.size, target.samples.size) - src_env = smooth_envelope(stft_mag(original.samples[:min_len])) - cand_env = smooth_envelope(stft_mag(candidate.samples[:min_len])) - target_env = smooth_envelope(stft_mag(target.samples[:min_len])) - mask = voiced_mask(original.samples[:min_len]) - return { - f"{label}_candidate_to_target_err": env_error(cand_env, target_env, mask), - f"{label}_candidate_vs_original_err": env_error(cand_env, src_env, mask), - } - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--original", required=True) - parser.add_argument("--plus-target", required=True) - parser.add_argument("--minus-target", required=True) - parser.add_argument("--candidate-plus") - parser.add_argument("--candidate-minus") - args = parser.parse_args() - - original = load_wav(Path(args.original)) - plus_target = load_wav(Path(args.plus_target)) - minus_target = load_wav(Path(args.minus_target)) - - plus = analyze_pair(original, plus_target, "+200") - minus = analyze_pair(original, minus_target, "-200") - - print("RePitch reference fit") - for name, metrics in (("+200", plus), ("-200", minus)): - print(name) - for key, value in metrics.items(): - print(f" {key}: {value:.6f}") - - if args.candidate_plus: - candidate_plus = load_wav(Path(args.candidate_plus)) - for key, value in compare_candidate(original, candidate_plus, plus_target, "plus").items(): - print(f"{key}: {value:.6f}") - - if args.candidate_minus: - candidate_minus = load_wav(Path(args.candidate_minus)) - for key, value in compare_candidate(original, candidate_minus, minus_target, "minus").items(): - print(f"{key}: {value:.6f}") - - -if __name__ == "__main__": - main() diff --git a/tools/generate-release-metadata.ps1 b/tools/generate-release-metadata.ps1 index 99ba9f8..ae97744 100644 --- a/tools/generate-release-metadata.ps1 +++ b/tools/generate-release-metadata.ps1 @@ -36,6 +36,12 @@ param( [Parameter(Mandatory = $false)] [string]$MacAssetUrl = "", + [Parameter(Mandatory = $false)] + [string]$LinuxAssetPath = "", + + [Parameter(Mandatory = $false)] + [string]$LinuxAssetUrl = "", + [Parameter(Mandatory = $false)] [string]$WindowsAiRuntimeAssetPath = "", @@ -84,6 +90,24 @@ param( [Parameter(Mandatory = $false)] [string]$MacX64AiRuntimeAssetUrl = "", + [Parameter(Mandatory = $false)] + [string]$LinuxAiRuntimeAssetPath = "", + + [Parameter(Mandatory = $false)] + [string]$LinuxAiRuntimeAssetUrl = "", + + [Parameter(Mandatory = $false)] + [string]$LinuxX64AiRuntimeAssetPath = "", + + [Parameter(Mandatory = $false)] + [string]$LinuxX64AiRuntimeAssetUrl = "", + + [Parameter(Mandatory = $false)] + [string]$LinuxArm64AiRuntimeAssetPath = "", + + [Parameter(Mandatory = $false)] + [string]$LinuxArm64AiRuntimeAssetUrl = "", + [Parameter(Mandatory = $false)] [string]$AiRuntimeVersion = "", @@ -223,6 +247,7 @@ function Get-AppcastMimeType { if ($FileName -like "*.dmg") { return "application/x-apple-diskimage" } if ($FileName -like "*.exe") { return "application/vnd.microsoft.portable-executable" } + if ($FileName -like "*.AppImage") { return "application/x-iso9660-appimage" } return "application/octet-stream" } @@ -265,6 +290,7 @@ if (-not [string]::IsNullOrWhiteSpace($MacMinimumSystemVersion)) { $windows = Get-AssetMetadata -AssetPath $WindowsAssetPath -AssetUrl $WindowsAssetUrl -AdditionalProperties $windowsAdditional $macos = Get-AssetMetadata -AssetPath $MacAssetPath -AssetUrl $MacAssetUrl -AdditionalProperties $macAdditional +$linux = Get-AssetMetadata -AssetPath $LinuxAssetPath -AssetUrl $LinuxAssetUrl $windowsAiRuntime = Get-AssetMetadata -AssetPath $WindowsAiRuntimeAssetPath -AssetUrl $WindowsAiRuntimeAssetUrl $windowsBaseAiRuntime = Get-AssetMetadata -AssetPath $WindowsBaseAiRuntimeAssetPath -AssetUrl $WindowsBaseAiRuntimeAssetUrl $windowsDirectmlAiRuntime = Get-AssetMetadata -AssetPath $WindowsDirectmlAiRuntimeAssetPath -AssetUrl $WindowsDirectmlAiRuntimeAssetUrl @@ -272,6 +298,9 @@ $windowsCudaAiRuntime = Get-AssetMetadata -AssetPath $WindowsCudaAiRuntimeAssetP $macosAiRuntime = Get-AssetMetadata -AssetPath $MacAiRuntimeAssetPath -AssetUrl $MacAiRuntimeAssetUrl $macosArm64AiRuntime = Get-AssetMetadata -AssetPath $MacArm64AiRuntimeAssetPath -AssetUrl $MacArm64AiRuntimeAssetUrl $macosX64AiRuntime = Get-AssetMetadata -AssetPath $MacX64AiRuntimeAssetPath -AssetUrl $MacX64AiRuntimeAssetUrl +$linuxAiRuntime = Get-AssetMetadata -AssetPath $LinuxAiRuntimeAssetPath -AssetUrl $LinuxAiRuntimeAssetUrl +$linuxX64AiRuntime = Get-AssetMetadata -AssetPath $LinuxX64AiRuntimeAssetPath -AssetUrl $LinuxX64AiRuntimeAssetUrl +$linuxArm64AiRuntime = Get-AssetMetadata -AssetPath $LinuxArm64AiRuntimeAssetPath -AssetUrl $LinuxArm64AiRuntimeAssetUrl $windowsCudaInstallPlan = Load-OptionalJsonFile -PathValue $WindowsCudaInstallPlanPath $windowsDirectmlInstallPlan = Load-OptionalJsonFile -PathValue $WindowsDirectmlInstallPlanPath @@ -295,10 +324,12 @@ if (-not [string]::IsNullOrWhiteSpace($FullReleaseNotesUrl)) { if ($windows) { $manifest.platforms.windows = $windows } if ($macos) { $manifest.platforms.macos = $macos } +if ($linux) { $manifest.platforms.linux = $linux } $checksums = @() if ($windows) { $checksums += "{0} {1}" -f $windows.sha256, $windows.fileName } if ($macos) { $checksums += "{0} {1}" -f $macos.sha256, $macos.fileName } +if ($linux) { $checksums += "{0} {1}" -f $linux.sha256, $linux.fileName } if ($windowsAiRuntime) { $checksums += "{0} {1}" -f $windowsAiRuntime.sha256, $windowsAiRuntime.fileName } if ($windowsBaseAiRuntime) { $checksums += "{0} {1}" -f $windowsBaseAiRuntime.sha256, $windowsBaseAiRuntime.fileName } if ($windowsDirectmlAiRuntime) { $checksums += "{0} {1}" -f $windowsDirectmlAiRuntime.sha256, $windowsDirectmlAiRuntime.fileName } @@ -306,12 +337,15 @@ if ($windowsCudaAiRuntime) { $checksums += "{0} {1}" -f $windowsCudaAiRuntime.s if ($macosAiRuntime) { $checksums += "{0} {1}" -f $macosAiRuntime.sha256, $macosAiRuntime.fileName } if ($macosArm64AiRuntime) { $checksums += "{0} {1}" -f $macosArm64AiRuntime.sha256, $macosArm64AiRuntime.fileName } if ($macosX64AiRuntime) { $checksums += "{0} {1}" -f $macosX64AiRuntime.sha256, $macosX64AiRuntime.fileName } +if ($linuxAiRuntime) { $checksums += "{0} {1}" -f $linuxAiRuntime.sha256, $linuxAiRuntime.fileName } +if ($linuxX64AiRuntime) { $checksums += "{0} {1}" -f $linuxX64AiRuntime.sha256, $linuxX64AiRuntime.fileName } +if ($linuxArm64AiRuntime) { $checksums += "{0} {1}" -f $linuxArm64AiRuntime.sha256, $linuxArm64AiRuntime.fileName } Set-Content -Path (Join-Path $resolvedOutputDir "OpenStudio-checksums.txt") -Value ($checksums -join [Environment]::NewLine) Set-Content -Path (Join-Path $releaseDir "latest.json") -Value ($manifest | ConvertTo-Json -Depth 12) Set-Content -Path (Join-Path $channelReleaseDir "latest.json") -Value ($manifest | ConvertTo-Json -Depth 12) -if (($windowsAiRuntime -or $windowsBaseAiRuntime -or $windowsDirectmlAiRuntime -or $windowsCudaAiRuntime -or $macosAiRuntime -or $macosArm64AiRuntime -or $macosX64AiRuntime) -and -not [string]::IsNullOrWhiteSpace($AiRuntimeVersion)) { +if (($windowsAiRuntime -or $windowsBaseAiRuntime -or $windowsDirectmlAiRuntime -or $windowsCudaAiRuntime -or $macosAiRuntime -or $macosArm64AiRuntime -or $macosX64AiRuntime -or $linuxAiRuntime -or $linuxX64AiRuntime -or $linuxArm64AiRuntime) -and -not [string]::IsNullOrWhiteSpace($AiRuntimeVersion)) { $aiRuntimeManifest = [ordered]@{ schemaVersion = [Math]::Max($SchemaVersion, 4) channel = $Channel @@ -381,6 +415,15 @@ if (($windowsAiRuntime -or $windowsBaseAiRuntime -or $windowsDirectmlAiRuntime - $aiRuntimeManifest.platforms.macos = $macosAiRuntime } + if ($linuxX64AiRuntime -or $linuxArm64AiRuntime) { + $aiRuntimeManifest.platforms.linux = [ordered]@{} + if ($linuxX64AiRuntime) { $aiRuntimeManifest.platforms.linux.x64 = $linuxX64AiRuntime } + if ($linuxArm64AiRuntime) { $aiRuntimeManifest.platforms.linux.arm64 = $linuxArm64AiRuntime } + } + elseif ($linuxAiRuntime) { + $aiRuntimeManifest.platforms.linux = $linuxAiRuntime + } + Set-Content -Path (Join-Path $aiRuntimeDir "latest.json") -Value ($aiRuntimeManifest | ConvertTo-Json -Depth 12) Set-Content -Path (Join-Path $channelAiRuntimeDir "latest.json") -Value ($aiRuntimeManifest | ConvertTo-Json -Depth 12) } @@ -449,4 +492,36 @@ $(if (-not [string]::IsNullOrWhiteSpace($FullReleaseNotesUrl)) { " <sparkle } } +if ($linux) { + $linuxPubDate = [DateTime]::Parse($PublishedAt).ToUniversalTime().ToString("r") + $linuxAppcast = @" +<?xml version="1.0" encoding="utf-8"?> +<rss version="2.0" xmlns:sparkle="http://www.andymatuschak.org/xml-namespaces/sparkle" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:openstudio="https://openstudio.org.in/xmlns/appcast"> + <channel> + <title>OpenStudio Linux $($Channel.Substring(0,1).ToUpper() + $Channel.Substring(1)) + $ReleasePageUrl + $($Channel.Substring(0,1).ToUpper() + $Channel.Substring(1)) OpenStudio releases for Linux. + + OpenStudio $Version + $linuxPubDate + + +$(if (-not [string]::IsNullOrWhiteSpace($FullReleaseNotesUrl)) { " $FullReleaseNotesUrl" }) + + + +"@ + Set-Content -Path (Join-Path $appcastDir "linux-$Channel.xml") -Value $linuxAppcast + if ($Channel -eq "stable") { + Set-Content -Path (Join-Path $appcastDir "linux-stable.xml") -Value $linuxAppcast + } +} + Write-Host "Release metadata written to $resolvedOutputDir" diff --git a/tools/generate_music.py b/tools/generate_music.py new file mode 100644 index 0000000..aa1f112 --- /dev/null +++ b/tools/generate_music.py @@ -0,0 +1,2764 @@ +#!/usr/bin/env python3 +""" +OpenStudio AI music generation helper. + +This script supports two modes: +1. One-shot CLI generation for direct invocation. +2. A persistent localhost worker used by the native app so ACE-Step models stay warm. +""" + +from __future__ import annotations + +import argparse +import ctypes +import functools +import hashlib +import importlib.util +import json +import os +import shutil +import socket +import struct +import subprocess +import sys +import threading +import time +import traceback +import uuid +from importlib import metadata +from pathlib import Path +from typing import Any + +from ai_runtime_probe import ( + DEFAULT_MUSIC_GEN_MODEL, + REQUIRED_MUSIC_GEN_NATIVE_FILES, + find_local_comfy_native_assets, + get_windows_cuda_pytorch_packages, + get_windows_flash_attn_asset, + get_windows_triton_package_spec, + get_music_generation_required_paths, + resolve_music_gen_checkpoint_root, +) +DEFAULT_INFER_STEP = 8 +DEFAULT_LM_MODEL = "qwen_4b_ace15.safetensors" +DEFAULT_LM_SELECTION = "auto" +DEFAULT_RUNTIME_PROFILE = "native-xl-turbo" +HEARTBEAT_INTERVAL_SEC = 5.0 +ORIGINAL_STDOUT = sys.stdout +ORIGINAL_STDERR = sys.stderr +WORKER_PROTOCOL_VERSION = 2 +MAX_FRAMED_PAYLOAD_BYTES = 8 * 1024 * 1024 +SCRIPT_PATH = Path(__file__).resolve() +SCRIPT_VERSION = hashlib.md5(SCRIPT_PATH.read_bytes()).hexdigest()[:16] +DECODE_STALL_THRESHOLD_COLD_SEC = 240.0 +GENERATION_MODE_LM_FIRST = "lm_first" +GENERATION_MODE_DIT_MANUAL = "dit_manual" +GENERATION_MODE_OPTIONS = { + GENERATION_MODE_LM_FIRST, + GENERATION_MODE_DIT_MANUAL, +} +LM_SHAPE_MISMATCH_MARKERS = ( + "error generating from formatted prompt", + "size of tensor a", + "must match the size of tensor b", +) + +ACTIVE_TRACE_LOCK = threading.Lock() +ACTIVE_TRACE_SESSION: "AITraceSession | None" = None +ACTIVE_PROGRESS_REPORTER: "ProgressReporter | None" = None +LM_DIAGNOSTICS_INSTALLED = False + +RUNTIME_PROFILE_SPECS: dict[str, dict[str, Any]] = { + "native-xl-turbo": { + "label": "OpenStudio ACE Split", + "runtimeProfileName": "openstudio-ace-split", + "lmModel": DEFAULT_LM_MODEL, + "requiredAssets": tuple(spec["relativePath"] for spec in REQUIRED_MUSIC_GEN_NATIVE_FILES), + "fallbackProfiles": (), + "notes": ( + "Targets the packaged OpenStudio ACE-Step 1.5 split graph.", + ), + }, +} + +LM_MODEL_OPTIONS = { + DEFAULT_LM_SELECTION, + "qwen_1.7b_ace15.safetensors", + "qwen_4b_ace15.safetensors", +} +DEFAULT_MUSICAL_METADATA = { + "bpm": 120, + "duration": 30.0, + "timesignature": "4/4", + "keyscale": "C major", +} +ACE15_MANAGED_LM_SHARED_FILES = ( + "tokenizer.json", + "tokenizer_config.json", + "special_tokens_map.json", + "vocab.json", + "merges.txt", + "added_tokens.json", + "chat_template.jinja", +) +ACE15_MANAGED_4B_CONFIG_OVERRIDES = { + "hidden_size": 2560, + "intermediate_size": 9728, + "num_hidden_layers": 36, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "max_position_embeddings": 40960, + "max_window_layers": 36, + "vocab_size": 217204, + "architectures": ["Qwen3Model"], + "model_type": "qwen3", +} + +PHASE_PROGRESS_BOUNDS: dict[str, tuple[float, float]] = { + "validating_request": (0.04, 0.08), + "loading_text_encoders": (0.08, 0.2), + "encoding_conditioning": (0.2, 0.42), + "loading_diffusion_model": (0.42, 0.5), + "sampling": (0.5, 0.9), + "decoding_audio": (0.9, 0.96), + "writing_output": (0.96, 0.99), + "done": (1.0, 1.0), + "error": (0.0, 0.0), +} + + +def get_openstudio_log_root() -> Path: + override = os.environ.get("OPENSTUDIO_AI_TRACE_ROOT", "").strip() + if override: + return Path(override).expanduser().resolve() + + if sys.platform == "win32": + local_app_data = os.environ.get("LOCALAPPDATA", "").strip() + if local_app_data: + return Path(local_app_data).expanduser().resolve() / "OpenStudio" / "logs" + if sys.platform == "darwin": + return Path.home() / "Library" / "Application Support" / "OpenStudio" / "logs" + + xdg_state_home = os.environ.get("XDG_STATE_HOME", "").strip() + if xdg_state_home: + return Path(xdg_state_home).expanduser().resolve() / "OpenStudio" / "logs" + return Path.home() / ".local" / "state" / "OpenStudio" / "logs" + + +def get_ai_trace_root() -> Path: + root = get_openstudio_log_root() / "ai" / "music-generation" + root.mkdir(parents=True, exist_ok=True) + return root + + +def resolve_openstudio_native_backend() -> dict[str, str]: + runner_path = (SCRIPT_PATH.parent / "openstudio_ace_runner.py").resolve() + backend_root = (SCRIPT_PATH.parent / "openstudio_ace_backend").resolve() + vendor_root = backend_root / "vendor_runtime" + required_paths = ( + runner_path, + backend_root / "__init__.py", + vendor_root / "nodes.py", + vendor_root / "folder_paths.py", + vendor_root / "comfy" / "sd.py", + vendor_root / "comfy_extras" / "nodes_ace.py", + ) + missing = [str(path) for path in required_paths if not path.exists()] + if missing: + raise GenerationFailure( + "The packaged OpenStudio ACE split backend is incomplete: " + ", ".join(missing), + progress=0.04, + failureKind="native_asset_missing", + ) + return { + "pythonExe": sys.executable, + "runnerPath": str(runner_path), + "backendRoot": str(backend_root), + "runtimeKind": "openstudio_ace_executor", + } + + +def resolve_native_split_backend() -> dict[str, str]: + return resolve_openstudio_native_backend() + + +def utc_timestamp() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + f".{int((time.time() % 1) * 1000):03d}Z" + + +def trace_safe_request_id(request_id: str) -> str: + normalized = normalize_text(request_id) + if not normalized: + return "unknown-request" + return "".join(char if char.isalnum() or char in {"-", "_"} else "_" for char in normalized)[:96] + + +def summarize_value(value: Any, *, depth: int = 0) -> Any: + if depth > 3: + return f"<{type(value).__name__}>" + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, Path): + return str(value) + if isinstance(value, dict): + items = list(value.items())[:20] + return { + str(key): summarize_value(item, depth=depth + 1) + for key, item in items + } + if isinstance(value, (list, tuple)): + items = list(value)[:20] + return [summarize_value(item, depth=depth + 1) for item in items] + if hasattr(value, "shape") and hasattr(value, "dtype"): + summary = { + "type": type(value).__name__, + "shape": list(getattr(value, "shape", [])), + "dtype": str(getattr(value, "dtype", "")), + } + device = getattr(value, "device", None) + if device is not None: + summary["device"] = str(device) + numel = getattr(value, "numel", None) + if callable(numel): + try: + summary["numel"] = int(numel()) + except Exception: + pass + return summary + if isinstance(value, BaseException): + return {"type": type(value).__name__, "message": str(value)} + if hasattr(value, "__dict__") and depth < 2: + try: + return { + "__class__": type(value).__name__, + **{ + str(key): summarize_value(item, depth=depth + 1) + for key, item in list(vars(value).items())[:20] + }, + } + except Exception: + pass + return repr(value) + + +def summarize_traceback_frames(tb: Any) -> list[dict[str, Any]]: + frames: list[dict[str, Any]] = [] + cursor = tb + while cursor is not None: + frame = cursor.tb_frame + code = frame.f_code + filename = str(code.co_filename) + if "acestep" in filename.lower() or filename == str(SCRIPT_PATH): + frames.append( + { + "file": filename, + "line": int(cursor.tb_lineno), + "function": code.co_name, + "locals": { + str(key): summarize_value(value) + for key, value in list(frame.f_locals.items())[:30] + }, + } + ) + cursor = cursor.tb_next + return frames[-8:] + + +def count_prompt_tokens(handler: Any, formatted_prompt: str) -> int | None: + tokenizer = getattr(handler, "tokenizer", None) + if tokenizer is None or not formatted_prompt: + return None + try: + encoded = tokenizer(formatted_prompt, return_tensors="pt") + input_ids = encoded.get("input_ids") if isinstance(encoded, dict) else None + if input_ids is None: + return None + shape = getattr(input_ids, "shape", None) + if shape is not None and len(shape) >= 2: + return int(shape[-1]) + except Exception: + return None + return None + + +class AITraceSession: + def __init__( + self, + *, + request_id: str, + workflow: str, + session_mode: str, + model_id: str, + ) -> None: + self.request_id = normalize_text(request_id) or str(uuid.uuid4()) + self.workflow = normalize_text(workflow) or "text-to-music" + self.session_mode = normalize_text(session_mode) or "persistent" + self.model_id = normalize_text(model_id) or DEFAULT_MUSIC_GEN_MODEL + self.root = get_ai_trace_root() + self.root.mkdir(parents=True, exist_ok=True) + self.started_at = time.time() + timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime(self.started_at)) + basename = f"{timestamp}_{trace_safe_request_id(self.request_id)}" + self.jsonl_path = (self.root / f"{basename}.jsonl").resolve() + self.summary_path = (self.root / f"{basename}.txt").resolve() + self.latest_failure_path = (self.root / "latest_failure.txt").resolve() + self._lock = threading.Lock() + self._metadata: dict[str, Any] = { + "requestId": self.request_id, + "workflow": self.workflow, + "sessionMode": self.session_mode, + "musicGenerationModelId": self.model_id, + "pid": os.getpid(), + "scriptPath": str(SCRIPT_PATH), + "scriptVersion": SCRIPT_VERSION, + "protocolVersion": WORKER_PROTOCOL_VERSION, + "argv": list(sys.argv), + } + self._last_payload: dict[str, Any] = {} + self._terminal_payload: dict[str, Any] = {} + self.log_event("trace_started", mirror=True, **self._metadata) + + def trace_path(self) -> str: + return str(self.jsonl_path) + + def set_metadata(self, **metadata: Any) -> None: + with self._lock: + self._metadata.update({key: summarize_value(value) for key, value in metadata.items() if value is not None}) + + def log_event(self, event: str, *, mirror: bool = False, **payload: Any) -> None: + entry = { + "ts": utc_timestamp(), + "event": event, + **self._metadata, + **{key: summarize_value(value) for key, value in payload.items()}, + } + line = json.dumps(entry, ensure_ascii=False) + try: + with self._lock: + with self.jsonl_path.open("a", encoding="utf-8") as handle: + handle.write(line + "\n") + except Exception as exc: + ORIGINAL_STDERR.write(f"[OpenStudio AI][trace-error] failed to write trace event {event}: {exc}\n") + ORIGINAL_STDERR.flush() + if mirror: + ORIGINAL_STDERR.write(f"[OpenStudio AI][trace] {event}: {line}\n") + ORIGINAL_STDERR.flush() + + def record_payload(self, payload: dict[str, Any]) -> None: + normalized = {str(key): summarize_value(value) for key, value in payload.items()} + with self._lock: + self._last_payload = normalized + state = str(normalized.get("state", "")) + if state in {"done", "error", "cancelled"}: + self._terminal_payload = normalized + self.log_event("progress_payload", payload=normalized) + + def _build_summary_text(self) -> str: + payload = self._terminal_payload or self._last_payload + lines = [ + f"Request ID: {self.request_id}", + f"Workflow: {self.workflow}", + f"Session mode: {self.session_mode}", + f"Model: {self.model_id}", + f"Trace JSONL: {self.jsonl_path}", + f"Summary TXT: {self.summary_path}", + "", + f"Final state: {payload.get('state', '')}", + f"Failure kind: {payload.get('failureKind', '')}", + f"Failure detail: {payload.get('failureDetail', '')}", + f"LM backend: {payload.get('lmBackend', '')}", + f"LM stage: {payload.get('lmStage', '')}", + f"Runtime profile: {payload.get('runtimeProfile', '')}", + f"LM model: {payload.get('lmModel', '')}", + f"Trace path: {payload.get('tracePath', self.trace_path())}", + f"Last stderr: {payload.get('lastStderrLine', '')}", + f"Last stdout: {payload.get('lastStdoutLine', '')}", + "", + "Metadata:", + json.dumps(self._metadata, indent=2, ensure_ascii=False), + "", + "Terminal payload:", + json.dumps(payload, indent=2, ensure_ascii=False), + ] + return "\n".join(lines).strip() + "\n" + + def finalize(self) -> None: + summary_text = self._build_summary_text() + payload = self._terminal_payload or self._last_payload + try: + self.summary_path.write_text(summary_text, encoding="utf-8") + if str(payload.get("state", "")) == "error": + self.latest_failure_path.write_text(summary_text, encoding="utf-8") + except Exception as exc: + ORIGINAL_STDERR.write(f"[OpenStudio AI][trace-error] failed to finalize trace summary: {exc}\n") + ORIGINAL_STDERR.flush() + self.log_event( + "trace_finalized", + mirror=True, + tracePath=self.trace_path(), + summaryPath=str(self.summary_path), + terminalState=payload.get("state"), + failureKind=payload.get("failureKind"), + ) + + +def set_active_diagnostics_context( + trace_session: AITraceSession | None, + reporter: "ProgressReporter | None", +) -> None: + global ACTIVE_TRACE_SESSION, ACTIVE_PROGRESS_REPORTER + with ACTIVE_TRACE_LOCK: + ACTIVE_TRACE_SESSION = trace_session + ACTIVE_PROGRESS_REPORTER = reporter + + +def get_active_trace_session() -> AITraceSession | None: + with ACTIVE_TRACE_LOCK: + return ACTIVE_TRACE_SESSION + + +def get_active_progress_reporter() -> "ProgressReporter | None": + with ACTIVE_TRACE_LOCK: + return ACTIVE_PROGRESS_REPORTER + + +def emit_payload(payload: dict[str, Any]) -> None: + normalized = dict(payload) + normalized["progress"] = round(float(normalized.get("progress", 0.0)), 4) + ORIGINAL_STDOUT.write(json.dumps(normalized) + "\n") + ORIGINAL_STDOUT.flush() + trace_session = get_active_trace_session() + if trace_session is not None: + trace_session.record_payload(normalized) + + +def emit(state: str, progress: float, **kwargs: Any) -> None: + payload = {"state": state, "progress": progress} + payload.update(kwargs) + emit_payload(payload) + + +class GenerationFailure(RuntimeError): + def __init__(self, message: str, *, progress: float = 0.0, **payload: Any) -> None: + super().__init__(message) + self.message = message + self.progress = progress + self.payload = payload + + +class StructuredStderrMirror: + def __init__(self, downstream: Any) -> None: + self._downstream = downstream + self._buffer = "" + self._lock = threading.Lock() + + def write(self, data: str) -> int: + written = self._downstream.write(data) + self._downstream.flush() + if not data: + return written + + lines_to_emit: list[str] = [] + with self._lock: + self._buffer += data + while True: + newline_pos = self._buffer.find("\n") + carriage_pos = self._buffer.find("\r") + split_pos_candidates = [pos for pos in (newline_pos, carriage_pos) if pos >= 0] + if not split_pos_candidates: + break + split_pos = min(split_pos_candidates) + line = self._buffer[:split_pos].strip() + self._buffer = self._buffer[split_pos + 1 :] + if line: + lines_to_emit.append(line) + + for line in lines_to_emit: + emit_payload( + { + "event": "stderr", + "phase": "stderr", + "message": line, + "line": line, + "pid": os.getpid(), + } + ) + + return written + + def flush(self) -> None: + self._downstream.flush() + + def isatty(self) -> bool: + return bool(getattr(self._downstream, "isatty", lambda: False)()) + + +def install_stream_mirrors() -> None: + sys.stderr = StructuredStderrMirror(ORIGINAL_STDERR) + + +def normalize_text(value: Any) -> str: + return str(value or "").strip() + + +def normalize_int(value: Any, default: int) -> int: + try: + if value is None or value == "": + return default + return int(float(value)) + except (TypeError, ValueError): + return default + + +def normalize_float(value: Any, default: float) -> float: + try: + if value is None or value == "": + return default + return float(value) + except (TypeError, ValueError): + return default + + +def normalize_bool(value: Any, default: bool) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in {"true", "1", "yes", "on"}: + return True + if lowered in {"false", "0", "no", "off"}: + return False + if isinstance(value, (int, float)): + return bool(value) + return default + + +def normalize_timesignature(value: Any) -> str: + raw = normalize_text(value) + if not raw: + return "4/4" + if "/" in raw: + return raw + try: + beats = int(float(raw)) + except ValueError: + return raw + return f"{beats}/4" + + +def normalize_generation_mode(raw_params: dict[str, Any]) -> str: + direct_value = normalize_text(raw_params.get("generationMode")).lower() + if direct_value in GENERATION_MODE_OPTIONS: + return direct_value + + if "generate_audio_codes" in raw_params: + return ( + GENERATION_MODE_LM_FIRST + if normalize_bool(raw_params.get("generate_audio_codes"), True) + else GENERATION_MODE_DIT_MANUAL + ) + + return GENERATION_MODE_LM_FIRST + + +def normalize_infer_method(value: Any) -> str: + normalized = normalize_text(value).lower() + if normalized in {"ode", "sde"}: + return normalized + return "ode" + + +def normalize_time_signature_for_generation(value: str) -> str: + normalized = normalize_text(value) + if "/" in normalized: + return normalized.split("/", 1)[0].strip() + return normalized + + +def has_explicit_musical_metadata(params: dict[str, Any]) -> bool: + return ( + normalize_int(params.get("bpm"), DEFAULT_MUSICAL_METADATA["bpm"]) + != DEFAULT_MUSICAL_METADATA["bpm"] + or normalize_float( + params.get("duration"), + DEFAULT_MUSICAL_METADATA["duration"], + ) + != DEFAULT_MUSICAL_METADATA["duration"] + or normalize_timesignature( + params.get("timesignature"), + ) + != DEFAULT_MUSICAL_METADATA["timesignature"] + or (normalize_text(params.get("keyscale")) or DEFAULT_MUSICAL_METADATA["keyscale"]) + != DEFAULT_MUSICAL_METADATA["keyscale"] + ) + + +def normalize_generation_params(raw_params: dict[str, Any]) -> dict[str, Any]: + generation_mode = normalize_generation_mode(raw_params) + generate_audio_codes = generation_mode == GENERATION_MODE_LM_FIRST + normalized = { + "prompt": normalize_text(raw_params.get("prompt")), + "lyrics": normalize_text(raw_params.get("lyrics")), + "seed": normalize_int(raw_params.get("seed"), -1), + "bpm": normalize_int(raw_params.get("bpm"), DEFAULT_MUSICAL_METADATA["bpm"]), + "duration": normalize_float( + raw_params.get("duration"), + DEFAULT_MUSICAL_METADATA["duration"], + ), + "timesignature": normalize_timesignature(raw_params.get("timesignature")), + "language": normalize_text(raw_params.get("language")) or "en", + "keyscale": normalize_text(raw_params.get("keyscale")) + or DEFAULT_MUSICAL_METADATA["keyscale"], + "generate_audio_codes": generate_audio_codes, + "cfg_scale": normalize_float(raw_params.get("cfg_scale"), 2.0), + "guidance_scale": normalize_float(raw_params.get("guidance_scale"), 1.0), + "shift": normalize_float(raw_params.get("shift"), 3.0), + "temperature": normalize_float(raw_params.get("temperature"), 0.85), + "top_p": normalize_float(raw_params.get("top_p"), 0.9), + "top_k": normalize_int(raw_params.get("top_k"), 0), + "min_p": normalize_float(raw_params.get("min_p"), 0.0), + "inferMethod": normalize_infer_method(raw_params.get("inferMethod")), + "debugForceLmShapeMismatch": normalize_bool( + raw_params.get("debugForceLmShapeMismatch"), + False, + ), + "debugDecodeStallSeconds": max( + 0.0, + normalize_float(raw_params.get("debugDecodeStallSeconds"), 0.0), + ), + "runtimeProfile": DEFAULT_RUNTIME_PROFILE, + "lmModel": DEFAULT_LM_MODEL, + "inferenceSteps": normalize_int( + raw_params.get("inferenceSteps"), DEFAULT_INFER_STEP + ), + } + return normalized + + +def get_installed_checkpoint_assets(checkpoint_root: Path) -> set[str]: + if not checkpoint_root.exists(): + return set() + return { + str(path.relative_to(checkpoint_root)).replace("\\", "/") + for path in checkpoint_root.rglob("*") + if path.is_file() + } + + +def build_runtime_profile_catalog(checkpoint_root: Path) -> dict[str, Any]: + installed_assets = get_installed_checkpoint_assets(checkpoint_root) + profiles: dict[str, dict[str, Any]] = {} + available_profile_ids: list[str] = [] + unavailable_profiles: list[dict[str, Any]] = [] + + for profile_id, spec in RUNTIME_PROFILE_SPECS.items(): + missing_assets = [ + asset for asset in spec.get("requiredAssets", ()) if asset not in installed_assets + ] + profile = { + "id": profile_id, + "label": spec["label"], + "runtimeProfileName": spec["runtimeProfileName"], + "lmModel": spec["lmModel"], + "requiredAssets": list(spec.get("requiredAssets", ())), + "missingAssets": missing_assets, + "available": not missing_assets, + "notes": list(spec.get("notes", ())), + } + profiles[profile_id] = profile + if profile["available"]: + available_profile_ids.append(profile_id) + else: + unavailable_profiles.append(profile) + + default_profile = ( + DEFAULT_RUNTIME_PROFILE + if profiles.get(DEFAULT_RUNTIME_PROFILE, {}).get("available") + else next(iter(available_profile_ids), "") + ) + return { + "defaultProfile": default_profile, + "profiles": profiles, + "availableProfiles": available_profile_ids, + "unavailableProfiles": unavailable_profiles, + "warmSessionCapable": True, + } + + +def resolve_runtime_selection( + *, + requested_profile: str, + requested_lm_model: str, + checkpoint_root: Path, +) -> dict[str, Any]: + catalog = build_runtime_profile_catalog(checkpoint_root) + profiles = catalog["profiles"] + status_notes: list[str] = [] + + selected_profile = ( + requested_profile if requested_profile in profiles else DEFAULT_RUNTIME_PROFILE + ) + selected_profile_info = profiles.get(selected_profile) + + if selected_profile_info is None: + selected_profile = catalog["defaultProfile"] or DEFAULT_RUNTIME_PROFILE + selected_profile_info = profiles.get(selected_profile) + + if selected_profile_info is None: + raise GenerationFailure( + "No ACE-Step runtime profile definitions are available in this bridge.", + progress=0.04, + ) + + if not selected_profile_info["available"]: + raise GenerationFailure( + "Native XL Turbo split-model assets are missing for the primary ACE-Step path.", + progress=0.04, + runtimeProfile=selected_profile, + availableProfiles=catalog["availableProfiles"], + unavailableProfiles=catalog["unavailableProfiles"], + ) + + return { + "requestedProfile": requested_profile, + "selectedProfile": selected_profile, + "selectedProfileLabel": selected_profile_info["label"], + "runtimeProfileName": selected_profile_info["runtimeProfileName"], + "requestedLmModel": DEFAULT_LM_MODEL, + "selectedLmModel": DEFAULT_LM_MODEL, + "catalog": catalog, + "statusNotes": status_notes, + } + + +def validate_checkpoint_layout(checkpoint_root: Path, model_id: str) -> dict[str, Any]: + hydrate_native_split_assets(checkpoint_root) + layout = get_music_generation_required_paths( + checkpoint_root=str(checkpoint_root), + model_name=model_id, + ) + if not layout["layoutValid"]: + raise GenerationFailure( + "Pinned ACE-Step checkpoint layout is incomplete.", + progress=0.05, + musicGenerationModelId=model_id, + musicGenerationCheckpointRoot=str(checkpoint_root), + musicGenerationMissingPaths=layout["missingPaths"], + ) + if normalize_bool(os.environ.get("OPENSTUDIO_USE_LEGACY_ACE_WRAPPER"), False): + ensure_hidden_legacy_lm_bridge(checkpoint_root) + return layout + + +def hardlink_or_copy_file(source: Path, destination: Path) -> str: + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists(): + destination.unlink() + try: + os.link(source, destination) + return "hardlink" + except OSError: + shutil.copy2(source, destination) + return "copy" + + +def hydrate_native_split_assets(checkpoint_root: Path) -> list[str]: + found_sources, _searched_dirs = find_local_comfy_native_assets() + imported_assets: list[str] = [] + + for spec in REQUIRED_MUSIC_GEN_NATIVE_FILES: + destination = checkpoint_root / Path(spec["relativePath"]) + if destination.exists() and destination.is_file(): + continue + + source_file = found_sources.get(spec["id"]) + if source_file is None: + continue + + hardlink_or_copy_file(source_file, destination) + imported_assets.append(spec["relativePath"]) + + if imported_assets: + trace_session = get_active_trace_session() + if trace_session is not None: + trace_session.log_event( + "native_assets_hydrated", + mirror=True, + importedAssets=imported_assets, + checkpointRoot=str(checkpoint_root), + ) + return imported_assets + + +def ensure_hidden_legacy_lm_bridge(checkpoint_root: Path) -> None: + target_dir = checkpoint_root / DEFAULT_LM_MODEL + target_model = target_dir / "model.safetensors" + template_dir = checkpoint_root / "acestep-5Hz-lm-1.7B" + template_config = template_dir / "config.json" + source_qwen_4b = checkpoint_root / "text_encoders" / "qwen_4b_ace15.safetensors" + if not template_config.exists() or not source_qwen_4b.exists(): + return + + config = json.loads(template_config.read_text(encoding="utf-8")) + config.update(ACE15_MANAGED_4B_CONFIG_OVERRIDES) + config["layer_types"] = ["full_attention"] * int(config["num_hidden_layers"]) + target_dir.mkdir(parents=True, exist_ok=True) + + for shared_name in ACE15_MANAGED_LM_SHARED_FILES: + source_file = template_dir / shared_name + if source_file.exists() and source_file.is_file(): + shutil.copy2(source_file, target_dir / shared_name) + + (target_dir / "config.json").write_text( + json.dumps(config, indent=2, ensure_ascii=True) + "\n", + encoding="utf-8", + ) + link_mode = "existing" + if not target_model.exists(): + link_mode = hardlink_or_copy_file(source_qwen_4b, target_model) + (target_dir / "openstudio-source.json").write_text( + json.dumps( + { + "source": "nativeSplitAssets", + "sourceFile": str(source_qwen_4b), + "generatedAt": utc_timestamp(), + "linkMode": link_mode, + "targetName": DEFAULT_LM_MODEL, + }, + indent=2, + ensure_ascii=True, + ) + + "\n", + encoding="utf-8", + ) + + +def lower_process_priority_for_cpu() -> None: + if sys.platform != "win32": + return + try: + BELOW_NORMAL_PRIORITY_CLASS = 0x00004000 + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + handle = kernel32.GetCurrentProcess() + kernel32.SetPriorityClass(handle, BELOW_NORMAL_PRIORITY_CLASS) + except Exception: + pass + + +def ensure_optional_acceleration_paths() -> None: + try: + import acestep + + nano_vllm_root = ( + Path(acestep.__file__).resolve().parent / "third_parts" / "nano-vllm" + ) + if nano_vllm_root.exists(): + nano_vllm_root_str = str(nano_vllm_root) + if nano_vllm_root_str not in sys.path: + sys.path.insert(0, nano_vllm_root_str) + except Exception: + pass + + +def has_optional_module(name: str) -> bool: + try: + return importlib.util.find_spec(name) is not None + except Exception: + return False + + +def get_distribution_version(package_name: str) -> str | None: + try: + return metadata.version(package_name) + except metadata.PackageNotFoundError: + return None + + +def strip_local_version(version: str | None) -> str: + if not version: + return "" + return str(version).split("+", 1)[0].strip() + + +def resolve_lm_backend(device: str) -> tuple[str, str, bool]: + ensure_optional_acceleration_paths() + + if device != "cuda": + return "pt", f"PyTorch fallback on {device.upper()}", False + + missing: list[str] = [] + import_failures: list[str] = [] + for module_name, friendly_name in ( + ("nanovllm", "nano-vllm"), + ("triton", "triton"), + ("flash_attn", "flash-attn"), + ): + if not has_optional_module(module_name): + missing.append(friendly_name) + continue + try: + __import__(module_name) + except Exception as exc: + import_failures.append(f"{friendly_name} ({type(exc).__name__}: {exc})") + + expected_torch_versions: dict[str, str] = {} + for package in get_windows_cuda_pytorch_packages(): + name, separator, version = str(package).partition("==") + if separator and name and version: + expected_torch_versions[name.strip()] = version.strip() + + triton_spec = get_windows_triton_package_spec() + _triton_name, _separator, triton_version = triton_spec.partition("==") + flash_attn_asset = get_windows_flash_attn_asset() + + version_mismatches: list[str] = [] + try: + import torch + + torch_version = getattr(torch, "__version__", "") + except Exception as exc: + missing.append(f"torch ({type(exc).__name__}: {exc})") + torch_version = "" + + installed_versions = { + "torch": torch_version, + "torchvision": get_distribution_version("torchvision") or "", + "torchaudio": get_distribution_version("torchaudio") or "", + "triton-windows": get_distribution_version("triton-windows") or "", + "flash-attn": get_distribution_version("flash-attn") or "", + } + + for package_name, expected_version in expected_torch_versions.items(): + installed_version = installed_versions.get(package_name, "") + if not installed_version: + if package_name not in missing: + missing.append(package_name) + continue + if strip_local_version(installed_version) != expected_version or "cu128" not in installed_version: + version_mismatches.append( + f"{package_name}={installed_version} (expected {expected_version}+cu128)" + ) + + installed_triton_version = installed_versions["triton-windows"] + if not installed_triton_version: + if "triton-windows" not in missing and "triton" not in missing: + missing.append("triton-windows") + elif strip_local_version(installed_triton_version) != triton_version.strip(): + version_mismatches.append( + f"triton-windows={installed_triton_version} (expected {triton_version.strip()})" + ) + + installed_flash_version = installed_versions["flash-attn"] + expected_flash_version = str(flash_attn_asset.get("version", "")).strip() + if not installed_flash_version: + if "flash-attn" not in missing: + missing.append("flash-attn") + elif strip_local_version(installed_flash_version) != expected_flash_version: + version_mismatches.append( + f"flash-attn={installed_flash_version} (expected {expected_flash_version})" + ) + + if not missing and not import_failures and not version_mismatches: + return "vllm", "nano-vllm acceleration available", True + + details: list[str] = [] + if missing: + details.append("missing " + ", ".join(missing)) + if import_failures: + details.append("import failures " + "; ".join(import_failures)) + if version_mismatches: + details.append("mismatched " + "; ".join(version_mismatches)) + return "pt", "PyTorch fallback (" + " | ".join(details) + ")", False + + +def classify_phase(description: str) -> str: + lowered = description.strip().lower() + if "phase 1" in lowered or "phase 2" in lowered: + return "encoding_conditioning" + if "preparing inputs" in lowered: + return "loading_diffusion_model" + if "generating music" in lowered: + return "sampling" + if "decoding audio" in lowered: + return "decoding_audio" + if "preparing audio data" in lowered: + return "decoding_audio" + return "sampling" + + +def is_lm_shape_mismatch(error_text: str) -> bool: + lowered = normalize_text(error_text).lower() + if not lowered: + return False + return all(marker in lowered for marker in LM_SHAPE_MISMATCH_MARKERS) + + +def is_out_of_memory_error(error_text: str) -> bool: + lowered = normalize_text(error_text).lower() + if not lowered: + return False + return any( + marker in lowered + for marker in ( + "out of memory", + "cuda error: out of memory", + "cuda out of memory", + "cublas_status_alloc_failed", + "ran out of gpu memory", + ) + ) + + +def classify_generation_failure_kind( + error_text: str, + *, + generation_mode: str, +) -> str: + if generation_mode == GENERATION_MODE_LM_FIRST and is_lm_shape_mismatch(error_text): + return "native_conditioning_failure" + lowered = normalize_text(error_text).lower() + if is_out_of_memory_error(lowered): + if any(marker in lowered for marker in ("decode", "decoding", "vae", "audio", "finalizing")): + return "native_decode_failure" + return "native_sampling_failure" + if "missing" in lowered and any( + marker in lowered for marker in ("text encoder", "diffusion", "vae", "asset", "checkpoint") + ): + return "native_asset_missing" + if "formatted prompt" in lowered or "conditioning" in lowered: + return "native_conditioning_failure" + if "decode" in lowered: + return "native_decode_failure" + return "native_sampling_failure" + + +def build_native_split_request(params: dict[str, Any]) -> dict[str, Any]: + request_timesignature = normalize_timesignature(params.get("timesignature")) + if "/" in request_timesignature: + request_timesignature = request_timesignature.split("/", 1)[0].strip() or "4" + request = { + "prompt": normalize_text(params.get("prompt")), + "lyrics": normalize_text(params.get("lyrics")), + "seed": normalize_int(params.get("seed"), -1), + "bpm": normalize_int(params.get("bpm"), DEFAULT_MUSICAL_METADATA["bpm"]), + "duration": normalize_float(params.get("duration"), DEFAULT_MUSICAL_METADATA["duration"]), + "timesignature": request_timesignature, + "language": normalize_text(params.get("language")) or "en", + "keyscale": normalize_text(params.get("keyscale")) or DEFAULT_MUSICAL_METADATA["keyscale"], + "generate_audio_codes": normalize_bool(params.get("generate_audio_codes"), True), + "cfg_scale": normalize_float(params.get("cfg_scale"), 2.0), + "guidance_scale": normalize_float(params.get("guidance_scale"), 1.0), + "inferenceSteps": normalize_int(params.get("inferenceSteps"), DEFAULT_INFER_STEP), + "shift": normalize_float(params.get("shift"), 3.0), + "temperature": normalize_float(params.get("temperature"), 0.85), + "top_p": normalize_float(params.get("top_p"), 0.9), + "top_k": normalize_int(params.get("top_k"), 0), + "min_p": normalize_float(params.get("min_p"), 0.0), + "sampler_name": "euler", + "scheduler": "simple", + "denoise": 1.0, + "clip_type": "ace", + "model_mode": "default", + "decode_mode": "full", + } + return request + + +def log_native_split_request_payload( + *, + request_id: str, + workflow: str, + session_mode: str, + normalized_params: dict[str, Any], + runtime_selection: dict[str, Any], + split_request: dict[str, Any], + split_runtime: dict[str, str], +) -> None: + diagnostic_payload = { + "requestId": request_id, + "workflow": workflow, + "sessionMode": session_mode, + "normalizedParams": normalized_params, + "runtimeSelection": { + "selectedProfile": runtime_selection["selectedProfile"], + "selectedLmModel": runtime_selection["selectedLmModel"], + "runtimeProfileName": runtime_selection["runtimeProfileName"], + "statusNotes": runtime_selection["statusNotes"], + }, + "backendFamily": "openstudio_ace_split", + "nativeSplitRequest": split_request, + "backendRuntime": split_runtime, + } + trace_session = get_active_trace_session() + if trace_session is not None: + trace_session.set_metadata( + normalizedParams=normalized_params, + runtimeSelection=diagnostic_payload["runtimeSelection"], + nativeSplitRequest=split_request, + backendRuntime=split_runtime, + backendFamily="openstudio_ace_split", + ) + trace_session.log_event( + "native_split_request", + mirror=True, + normalizedParams=normalized_params, + runtimeSelection=diagnostic_payload["runtimeSelection"], + nativeSplitRequest=split_request, + backendRuntime=split_runtime, + ) + ORIGINAL_STDERR.write( + "[OpenStudio AI] native_split_request " + + json.dumps(diagnostic_payload, ensure_ascii=False) + + "\n" + ) + ORIGINAL_STDERR.flush() + + +def update_reporter_from_openstudio_ace_event( + reporter: "ProgressReporter", + event: dict[str, Any], + *, + backend_label: str, + selection: dict[str, Any], + status_note: str, +) -> None: + kind = normalize_text(event.get("kind")).lower() + phase = normalize_text(event.get("phase")) or "sampling" + progress = normalize_float(event.get("progress"), 0.0) + message = normalize_text(event.get("message")) or phase.replace("_", " ").title() + details = event.get("details") if isinstance(event.get("details"), dict) else None + + if kind == "phase": + reporter.update( + "loading" if phase != "done" else "done", + progress, + phase=phase, + message=message, + lmBackend=backend_label, + phaseProgress=0.0, + statusNote=status_note, + backendFamily="openstudio_ace_split", + runtimeProfile=selection["selectedProfile"], + lmModel=selection["selectedLmModel"], + failureDetail=normalize_text(event.get("errorType")) or None, + loadedAssets=details, + ) + return + + if kind == "progress": + fraction = normalize_float(event.get("fraction"), 0.0) + reporter.update( + "loading", + progress, + phase=phase, + message=message, + lmBackend=backend_label, + phaseProgress=fraction, + statusNote=status_note, + backendFamily="openstudio_ace_split", + runtimeProfile=selection["selectedProfile"], + lmModel=selection["selectedLmModel"], + ) + return + + if kind == "result": + reporter.update( + "writing", + 0.99, + phase="writing_output", + message="Finalizing OpenStudio ACE output...", + lmBackend=backend_label, + statusNote=status_note, + backendFamily="openstudio_ace_split", + runtimeProfile=selection["selectedProfile"], + lmModel=selection["selectedLmModel"], + loadedAssets=event.get("assets"), + ) + + +def run_native_split_generation( + *, + reporter: "ProgressReporter", + trace_session: AITraceSession, + request_id: str, + workflow: str, + session_mode: str, + normalized_params: dict[str, Any], + selection: dict[str, Any], + checkpoint_root: Path, + output_path: Path, +) -> str: + split_runtime = resolve_native_split_backend() + split_request = build_native_split_request(normalized_params) + runtime_kind = split_runtime.get("runtimeKind", "openstudio_ace_executor") + status_note = ( + "OpenStudio's native ACE graph is using explicit BPM, duration, time signature, key, " + + ("and LM audio-code generation." if split_request["generate_audio_codes"] else "with LM audio-code generation disabled.") + ) + log_native_split_request_payload( + request_id=request_id, + workflow=workflow, + session_mode=session_mode, + normalized_params=normalized_params, + runtime_selection=selection, + split_request=split_request, + split_runtime=split_runtime, + ) + command = [ + split_runtime["pythonExe"], + split_runtime["runnerPath"], + ] + command.extend(["--checkpoint-root", str(checkpoint_root)]) + command.extend( + [ + "--request-json", + json.dumps(split_request, ensure_ascii=False), + "--output", + str(output_path), + ] + ) + + trace_session.log_event( + "native_split_spawn", + mirror=True, + command=command, + runtimeKind=runtime_kind, + ) + + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + bufsize=1, + ) + + final_output = "" + last_stdout_line = "" + last_stderr_line = "" + + def pump_stderr() -> None: + nonlocal last_stderr_line + if process.stderr is None: + return + for line in process.stderr: + cleaned = line.rstrip() + if cleaned: + last_stderr_line = cleaned + trace_session.log_event("native_split_stderr", line=cleaned) + + stderr_thread = threading.Thread(target=pump_stderr, name="OpenStudioAcesplitStderr", daemon=True) + stderr_thread.start() + + try: + if process.stdout is None: + raise GenerationFailure( + "OpenStudio ACE split runner did not expose stdout.", + progress=0.1, + failureKind="native_conditioning_failure", + ) + for line in process.stdout: + cleaned = line.rstrip() + if not cleaned: + continue + last_stdout_line = cleaned + try: + event = json.loads(cleaned) + except json.JSONDecodeError: + trace_session.log_event("native_split_stdout", line=cleaned) + continue + + trace_session.log_event("native_split_event", payload=event) + kind = normalize_text(event.get("kind")).lower() + if kind == "error": + error_message = normalize_text(event.get("message")) or "OpenStudio ACE split generation failed." + error_phase = normalize_text(event.get("phase")).lower() + reporter.set_failure_detail( + error_message or normalize_text(event.get("errorType")) + ) + raise GenerationFailure( + error_message, + progress=normalize_float(event.get("progress"), 0.24), + failureKind=( + "native_decode_failure" + if error_phase == "decoding_audio" + else classify_generation_failure_kind( + error_message, + generation_mode=( + GENERATION_MODE_LM_FIRST + if split_request["generate_audio_codes"] + else GENERATION_MODE_DIT_MANUAL + ), + ) + ), + ) + if kind == "result": + final_output = normalize_text(event.get("outputPath")) + update_reporter_from_openstudio_ace_event( + reporter, + event, + backend_label="openstudio-ace-split", + selection=selection, + status_note=status_note, + ) + + if reporter.should_abort(): + process.kill() + failure_detail = normalize_text(getattr(reporter, "_failure_detail", "")) + if is_out_of_memory_error(failure_detail): + raise GenerationFailure( + "ACE-Step ran out of GPU memory while finalizing audio.", + progress=normalize_float(event.get("progress"), 0.95), + failureKind="native_decode_failure", + failureDetail=failure_detail, + ) + raise GenerationFailure( + "ACE-Step decode stalled while finalizing audio.", + progress=normalize_float(event.get("progress"), 0.95), + failureKind="decode_stalled", + failureDetail=failure_detail or None, + ) + finally: + if process.stdout is not None: + process.stdout.close() + process.wait(timeout=None) + stderr_thread.join(timeout=2.0) + if last_stdout_line: + reporter.set_last_stdout_line(last_stdout_line) + if last_stderr_line: + reporter.set_last_stderr_line(last_stderr_line) + trace_session.set_metadata( + nativeRunnerExitCode=process.returncode, + lastStdoutLine=last_stdout_line, + lastStderrLine=last_stderr_line, + ) + + if process.returncode != 0: + runner_error = last_stderr_line or last_stdout_line or "unknown error" + raise GenerationFailure( + "OpenStudio ACE split runner exited with " + f"code {process.returncode}: {runner_error}", + progress=0.24, + failureKind=classify_generation_failure_kind( + runner_error, + generation_mode=( + GENERATION_MODE_LM_FIRST + if split_request["generate_audio_codes"] + else GENERATION_MODE_DIT_MANUAL + ), + ), + ) + if not final_output: + final_output = str(output_path) + return final_output + + +def resolve_one_shot_params_json(args: argparse.Namespace) -> str: + params_sources = [ + bool(getattr(args, "params", "")), + bool(getattr(args, "params_file", "")), + bool(getattr(args, "params_stdin", False)), + ] + if sum(1 for source in params_sources if source) != 1: + raise SystemExit( + "Exactly one of --params, --params-file, or --params-stdin is required unless --worker is used." + ) + if getattr(args, "params", ""): + return str(args.params) + if getattr(args, "params_file", ""): + return Path(args.params_file).expanduser().resolve().read_text(encoding="utf-8-sig") + return sys.stdin.read() + + +def build_generation_param_payload( + params: dict[str, Any], + *, + generation_mode: str, +) -> dict[str, Any]: + use_lm_audio_codes = bool(params.get("generate_audio_codes", True)) + lyrics = normalize_text(params.get("lyrics")) + instrumental = not lyrics + + return { + "task_type": "text2music", + "caption": normalize_text(params.get("prompt")), + "lyrics": lyrics, + "instrumental": instrumental, + "vocal_language": normalize_text(params.get("language")) or "en", + "bpm": params["bpm"], + "keyscale": normalize_text(params.get("keyscale")), + "timesignature": normalize_time_signature_for_generation(params.get("timesignature", "")), + "duration": params["duration"], + "inference_steps": max(4, params["inferenceSteps"]), + "seed": params["seed"], + "guidance_scale": params["guidance_scale"], + "shift": params["shift"], + "infer_method": normalize_infer_method(params.get("inferMethod")), + "thinking": use_lm_audio_codes, + "lm_temperature": params["temperature"], + "lm_cfg_scale": params["cfg_scale"], + "lm_top_k": params["top_k"], + "lm_top_p": params["top_p"], + "use_cot_metas": False, + "use_cot_caption": use_lm_audio_codes, + "use_cot_language": use_lm_audio_codes, + "use_constrained_decoding": use_lm_audio_codes, + } + + +def build_generation_params(params: dict[str, Any], *, generation_mode: str): + from acestep.inference import GenerationParams + + return GenerationParams(**build_generation_param_payload(params, generation_mode=generation_mode)) + + +def log_normalized_request_payload( + *, + request_id: str, + workflow: str, + session_mode: str, + normalized_params: dict[str, Any], + runtime_selection: dict[str, Any], + generation_param_payload: dict[str, Any], +) -> None: + diagnostic_payload = { + "requestId": request_id, + "workflow": workflow, + "sessionMode": session_mode, + "normalizedParams": normalized_params, + "runtimeSelection": { + "selectedProfile": runtime_selection["selectedProfile"], + "selectedLmModel": runtime_selection["selectedLmModel"], + "runtimeProfileName": runtime_selection["runtimeProfileName"], + "statusNotes": runtime_selection["statusNotes"], + }, + "generationParams": generation_param_payload, + } + trace_session = get_active_trace_session() + if trace_session is not None: + trace_session.set_metadata( + normalizedParams=normalized_params, + runtimeSelection=diagnostic_payload["runtimeSelection"], + generationParams=generation_param_payload, + ) + trace_session.log_event( + "normalized_request", + mirror=True, + normalizedParams=normalized_params, + runtimeSelection=diagnostic_payload["runtimeSelection"], + generationParams=generation_param_payload, + ) + ORIGINAL_STDERR.write( + "[OpenStudio AI] normalized_request " + + json.dumps(diagnostic_payload, ensure_ascii=False) + + "\n" + ) + ORIGINAL_STDERR.flush() + + +def build_method_call_details(handler: Any, method_name: str, args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]: + details: dict[str, Any] = {} + if method_name == "initialize": + details.update( + { + "checkpointDir": kwargs.get("checkpoint_dir", args[0] if len(args) > 0 else None), + "lmModelPath": kwargs.get("lm_model_path", args[1] if len(args) > 1 else None), + "backend": kwargs.get("backend", args[2] if len(args) > 2 else None), + "device": kwargs.get("device", args[3] if len(args) > 3 else None), + "offloadToCpu": kwargs.get("offload_to_cpu", args[4] if len(args) > 4 else None), + "dtype": kwargs.get("dtype", args[5] if len(args) > 5 else None), + } + ) + elif method_name in {"build_formatted_prompt", "build_formatted_prompt_with_cot"}: + caption = kwargs.get("caption", args[0] if len(args) > 0 else "") + lyrics = kwargs.get("lyrics", args[1] if len(args) > 1 else "") + details.update( + { + "captionLength": len(normalize_text(caption)), + "lyricsLength": len(normalize_text(lyrics)), + "generationPhase": kwargs.get("generation_phase"), + } + ) + if method_name == "build_formatted_prompt_with_cot": + cot_text = kwargs.get("cot_text", args[2] if len(args) > 2 else "") + details["cotLength"] = len(normalize_text(cot_text)) + elif method_name == "generate_from_formatted_prompt": + formatted_prompt = kwargs.get("formatted_prompt", args[0] if len(args) > 0 else "") + details.update( + { + "formattedPromptLength": len(normalize_text(formatted_prompt)), + "formattedPromptTokenCount": count_prompt_tokens(handler, normalize_text(formatted_prompt)), + "useConstrainedDecoding": kwargs.get("use_constrained_decoding", True), + "stopAtReasoning": kwargs.get("stop_at_reasoning", False), + "cfg": summarize_value(kwargs.get("cfg", args[1] if len(args) > 1 else None)), + } + ) + elif method_name in {"_run_pt_single", "_run_pt", "_run_vllm"}: + formatted_prompt = kwargs.get("formatted_prompt") + if formatted_prompt is None and args: + formatted_prompt = args[0] + prompt_value = normalize_text(formatted_prompt if isinstance(formatted_prompt, str) else "") + details.update( + { + "backend": "vllm" if method_name == "_run_vllm" else "pt", + "formattedPromptLength": len(prompt_value), + "formattedPromptTokenCount": count_prompt_tokens(handler, prompt_value), + "temperature": kwargs.get("temperature", args[1] if len(args) > 1 else None), + "cfgScale": kwargs.get("cfg_scale", args[2] if len(args) > 2 else None), + "topK": kwargs.get("top_k", args[4] if len(args) > 4 else None), + "topP": kwargs.get("top_p", args[5] if len(args) > 5 else None), + "targetDuration": kwargs.get("target_duration"), + "userMetadata": summarize_value(kwargs.get("user_metadata")), + "generationPhase": kwargs.get("generation_phase"), + "captionLength": len(normalize_text(kwargs.get("caption", ""))), + "lyricsLength": len(normalize_text(kwargs.get("lyrics", ""))), + "cotLength": len(normalize_text(kwargs.get("cot_text", ""))), + } + ) + return {key: value for key, value in details.items() if value is not None} + + +def summarize_method_result(handler: Any, method_name: str, result: Any) -> dict[str, Any]: + if method_name == "initialize" and isinstance(result, tuple): + return { + "status": summarize_value(result[0] if len(result) > 0 else None), + "ok": summarize_value(result[1] if len(result) > 1 else None), + } + if method_name in {"build_formatted_prompt", "build_formatted_prompt_with_cot"}: + text = normalize_text(result) + return { + "formattedPromptLength": len(text), + "formattedPromptTokenCount": count_prompt_tokens(handler, text), + } + if method_name == "generate_from_formatted_prompt" and isinstance(result, tuple): + cot_text = normalize_text(result[0] if len(result) > 0 else "") + codes = normalize_text(result[1] if len(result) > 1 else "") + return { + "cotLength": len(cot_text), + "codesLength": len(codes), + } + return {"result": summarize_value(result)} + + +def install_lm_diagnostics_instrumentation() -> None: + global LM_DIAGNOSTICS_INSTALLED + if LM_DIAGNOSTICS_INSTALLED: + return + + import acestep.llm_inference as lli + + handler_cls = lli.LLMHandler + methods_to_wrap = ( + ("initialize", "initialize_lm"), + ("build_formatted_prompt", "build_formatted_prompt"), + ("build_formatted_prompt_with_cot", "build_formatted_prompt_with_cot"), + ("generate_from_formatted_prompt", "generate_from_formatted_prompt"), + ("_run_pt", "run_pt"), + ("_run_pt_single", "run_pt_single"), + ("_run_vllm", "run_vllm"), + ) + + for method_name, lm_stage in methods_to_wrap: + original = getattr(handler_cls, method_name, None) + if original is None: + trace_session = get_active_trace_session() + if trace_session is not None: + trace_session.log_event( + "lm_symbol_missing", + mirror=True, + method=method_name, + lmStage=lm_stage, + ) + continue + if getattr(original, "_openstudio_ai_instrumented", False): + continue + + @functools.wraps(original) + def wrapped(self: Any, *args: Any, __original=original, __method_name=method_name, __lm_stage=lm_stage, **kwargs: Any) -> Any: + trace_session = get_active_trace_session() + reporter = get_active_progress_reporter() + call_details = build_method_call_details(self, __method_name, args, kwargs) + if reporter is not None: + reporter.set_lm_stage(__lm_stage) + if trace_session is not None: + trace_session.log_event( + "lm_stage_enter", + method=__method_name, + lmStage=__lm_stage, + details=call_details, + ) + try: + result = __original(self, *args, **kwargs) + except Exception as exc: + failure_detail = f"{type(exc).__name__}: {exc}" + if reporter is not None: + reporter.set_lm_stage(__lm_stage) + reporter.set_failure_detail(failure_detail) + if trace_session is not None: + trace_session.log_event( + "lm_stage_exception", + mirror=True, + method=__method_name, + lmStage=__lm_stage, + details=call_details, + failureDetail=failure_detail, + exceptionType=type(exc).__name__, + traceback="".join(traceback.format_exception(type(exc), exc, exc.__traceback__)), + tracebackFrames=summarize_traceback_frames(exc.__traceback__), + ) + raise + if trace_session is not None: + trace_session.log_event( + "lm_stage_exit", + method=__method_name, + lmStage=__lm_stage, + details=summarize_method_result(self, __method_name, result), + ) + return result + + setattr(wrapped, "_openstudio_ai_instrumented", True) + setattr(handler_cls, method_name, wrapped) + + LM_DIAGNOSTICS_INSTALLED = True + + +class ProgressReporter: + def __init__( + self, + *, + model_id: str, + checkpoint_root: Path, + backend: str = "unknown", + run_mode: str = "cold", + session_mode: str = "persistent", + runtime_profile: str = DEFAULT_RUNTIME_PROFILE, + lm_model: str = DEFAULT_LM_MODEL, + request_id: str = "", + trace_path: str = "", + ) -> None: + self.model_id = model_id + self.checkpoint_root = checkpoint_root + self.backend = backend + self.run_mode = run_mode + self.session_mode = session_mode + self.runtime_profile = runtime_profile + self.lm_model = lm_model + self.request_id = normalize_text(request_id) or str(uuid.uuid4()) + self.started_at = time.monotonic() + self._lock = threading.Lock() + self._stop_heartbeat = threading.Event() + self._heartbeat_thread: threading.Thread | None = None + self._last_progress_emit_at = self.started_at + self._phase_entered_at = self.started_at + self._decode_stall_reported = False + self._abort_due_to_stall = False + self._attempt_mode = "lm_dit" + self._attempt_index = 1 + self._prior_failure = "" + self._debug_decode_stall_seconds = 0.0 + self._debug_decode_stall_used = False + self.trace_path = normalize_text(trace_path) + self._failure_detail = "" + self._lm_stage = "" + self._lm_backend = "" + self._payload: dict[str, Any] = { + "state": "idle", + "progress": 0.0, + "phase": "idle", + "message": "", + "backend": backend, + "musicGenerationModelId": model_id, + "musicGenerationCheckpointRoot": str(checkpoint_root), + "runMode": run_mode, + "sessionMode": session_mode, + "runtimeProfile": runtime_profile, + "lmModel": lm_model, + "protocolVersion": WORKER_PROTOCOL_VERSION, + "scriptVersion": SCRIPT_VERSION, + "requestId": self.request_id, + "attemptMode": self._attempt_mode, + "attemptIndex": self._attempt_index, + "tracePath": self.trace_path or None, + } + + def _timestamp_ms(self) -> int: + return int(time.time() * 1000) + + def _elapsed_ms(self) -> int: + return int((time.monotonic() - self.started_at) * 1000) + + def _last_progress_age_ms_locked(self) -> int: + return int((time.monotonic() - self._last_progress_emit_at) * 1000) + + def _decode_stall_threshold_sec_locked(self) -> float: + return DECODE_STALL_THRESHOLD_COLD_SEC + + def _emit_locked(self, *, heartbeat: bool = False) -> None: + payload = dict(self._payload) + payload["elapsedMs"] = self._elapsed_ms() + payload["heartbeatTs"] = self._timestamp_ms() + payload["lastProgressAgeMs"] = self._last_progress_age_ms_locked() + progress = float(payload.get("progress", 0.0)) + if progress > 0.02: + payload["etaMs"] = max( + 0, + int((payload["elapsedMs"] / progress) - payload["elapsedMs"]), + ) + if heartbeat: + payload["heartbeat"] = True + emit_payload(payload) + + def start_heartbeat(self) -> None: + if self._heartbeat_thread is not None: + return + + def heartbeat_loop() -> None: + while not self._stop_heartbeat.wait(HEARTBEAT_INTERVAL_SEC): + stall_payload: dict[str, Any] | None = None + with self._lock: + state = str(self._payload.get("state", "idle")) + if state in {"idle", "done", "error", "cancelled"}: + continue + phase = str(self._payload.get("phase", "")) + if ( + phase == "decoding_audio" + and not self._decode_stall_reported + and self._last_progress_age_ms_locked() + >= int(self._decode_stall_threshold_sec_locked() * 1000) + ): + self._decode_stall_reported = True + self._abort_due_to_stall = True + prior_failure = normalize_text(self._prior_failure) + failure_detail = normalize_text(self._failure_detail) + failure_kind = "decode_stalled" + phase_name = "decode_stalled" + message = "ACE-Step decode stalled while finalizing audio." + if is_out_of_memory_error(failure_detail): + failure_kind = "native_decode_failure" + phase_name = "error" + message = "ACE-Step ran out of GPU memory while finalizing audio." + if prior_failure: + message += f" Prior failure: {prior_failure}" + self._payload.update( + { + "state": "error", + "progress": float(self._payload.get("progress", 0.0)), + "phase": phase_name, + "message": message, + "error": message, + "failureKind": failure_kind, + "priorFailure": prior_failure or None, + "phaseElapsedMs": int( + (time.monotonic() - self._phase_entered_at) * 1000 + ), + } + ) + stall_payload = dict(self._payload) + stall_payload["elapsedMs"] = self._elapsed_ms() + stall_payload["heartbeatTs"] = self._timestamp_ms() + stall_payload["lastProgressAgeMs"] = self._last_progress_age_ms_locked() + stall_payload["heartbeat"] = True + else: + self._emit_locked(heartbeat=True) + if stall_payload is not None: + emit_payload(stall_payload) + + self._heartbeat_thread = threading.Thread( + target=heartbeat_loop, + name="OpenStudioAIMusicHeartbeat", + daemon=True, + ) + self._heartbeat_thread.start() + + def stop_heartbeat(self) -> None: + self._stop_heartbeat.set() + if self._heartbeat_thread is not None: + self._heartbeat_thread.join(timeout=2.0) + self._heartbeat_thread = None + + def update( + self, + state: str, + progress: float, + *, + phase: str, + message: str, + **extra: Any, + ) -> None: + with self._lock: + phase_changed = phase != str(self._payload.get("phase", "")) + if phase_changed: + self._phase_entered_at = time.monotonic() + self._decode_stall_reported = False + payload = { + "state": state, + "progress": progress, + "phase": phase, + "message": message, + "backend": self.backend, + "runMode": self.run_mode, + "sessionMode": self.session_mode, + "runtimeProfile": self.runtime_profile, + "lmModel": self.lm_model, + "protocolVersion": WORKER_PROTOCOL_VERSION, + "scriptVersion": SCRIPT_VERSION, + "requestId": self.request_id, + "attemptMode": self._attempt_mode, + "attemptIndex": self._attempt_index, + "priorFailure": self._prior_failure or None, + "tracePath": self.trace_path or None, + "failureDetail": self._failure_detail or None, + "lmBackend": self._lm_backend or None, + "lmStage": self._lm_stage or None, + **extra, + } + payload = {key: value for key, value in payload.items() if value is not None} + self._payload.update(payload) + self._last_progress_emit_at = time.monotonic() + self._emit_locked() + + def set_backend(self, backend: str) -> None: + with self._lock: + self.backend = backend + self._payload["backend"] = backend + + def set_lm_backend(self, lm_backend: str) -> None: + with self._lock: + self._lm_backend = normalize_text(lm_backend) + if self._lm_backend: + self._payload["lmBackend"] = self._lm_backend + else: + self._payload.pop("lmBackend", None) + + def set_lm_stage(self, lm_stage: str) -> None: + with self._lock: + self._lm_stage = normalize_text(lm_stage) + if self._lm_stage: + self._payload["lmStage"] = self._lm_stage + else: + self._payload.pop("lmStage", None) + + def set_failure_detail(self, failure_detail: str) -> None: + with self._lock: + self._failure_detail = normalize_text(failure_detail) + if self._failure_detail: + self._payload["failureDetail"] = self._failure_detail + else: + self._payload.pop("failureDetail", None) + + def set_last_stdout_line(self, line: str) -> None: + with self._lock: + cleaned = normalize_text(line) + if cleaned: + self._payload["lastStdoutLine"] = cleaned + else: + self._payload.pop("lastStdoutLine", None) + + def set_last_stderr_line(self, line: str) -> None: + with self._lock: + cleaned = normalize_text(line) + if cleaned: + self._payload["lastStderrLine"] = cleaned + else: + self._payload.pop("lastStderrLine", None) + + def set_request_context( + self, + *, + attempt_mode: str, + attempt_index: int, + prior_failure: str = "", + debug_decode_stall_seconds: float = 0.0, + ) -> None: + with self._lock: + self._attempt_mode = attempt_mode + self._attempt_index = max(1, int(attempt_index)) + self._prior_failure = normalize_text(prior_failure) + self._debug_decode_stall_seconds = max(0.0, float(debug_decode_stall_seconds)) + self._debug_decode_stall_used = False + self._payload["attemptMode"] = self._attempt_mode + self._payload["attemptIndex"] = self._attempt_index + if self._prior_failure: + self._payload["priorFailure"] = self._prior_failure + else: + self._payload.pop("priorFailure", None) + + def set_runtime_context( + self, + *, + run_mode: str, + session_mode: str, + runtime_profile: str, + lm_model: str, + ) -> None: + with self._lock: + self.run_mode = run_mode + self.session_mode = session_mode + self.runtime_profile = runtime_profile + self.lm_model = lm_model + self._payload["runMode"] = run_mode + self._payload["sessionMode"] = session_mode + self._payload["runtimeProfile"] = runtime_profile + self._payload["lmModel"] = lm_model + + def _map_stage_progress( + self, phase: str, raw_progress: float + ) -> tuple[float, float | None]: + start, end = PHASE_PROGRESS_BOUNDS.get(phase, (0.0, 1.0)) + phase_progress: float | None = None + + if phase == "lm_phase_1" and 0.0 <= raw_progress <= 0.5: + phase_progress = min(1.0, max(0.0, raw_progress / 0.5)) + elif phase == "lm_phase_2" and raw_progress >= 0.5: + phase_progress = min(1.0, max(0.0, (raw_progress - 0.5) / 0.5)) + elif phase == "dit_generation" and 0.0 <= raw_progress <= 1.0: + phase_progress = min(1.0, max(0.0, raw_progress)) + elif phase == "decoding_audio" and 0.0 <= raw_progress <= 1.0: + phase_progress = min(1.0, max(0.0, raw_progress)) + elif phase == "preparing_audio" and 0.0 <= raw_progress <= 1.0: + phase_progress = min(1.0, max(0.0, raw_progress)) + + if phase_progress is None: + return start, None + return start + ((end - start) * phase_progress), phase_progress + + def ace_progress(self, value: float, desc: str | None = None, **_: Any) -> None: + description = normalize_text(desc) or "Generating music..." + phase = classify_phase(description) + should_debug_stall = False + if ( + phase == "decoding_audio" + and self._debug_decode_stall_seconds > 0 + and not self._debug_decode_stall_used + ): + self._debug_decode_stall_used = True + should_debug_stall = True + mapped_progress, phase_progress = self._map_stage_progress(phase, float(value)) + self.update( + "generating", + mapped_progress, + phase=phase, + message=description, + phaseProgress=phase_progress, + ) + if should_debug_stall: + time.sleep(self._debug_decode_stall_seconds) + + def fail(self, message: str, *, progress: float = 0.0, **extra: Any) -> None: + self.update( + "error", + progress, + phase="error", + message=message, + error=message, + **extra, + ) + + def done(self, output_file: str) -> None: + self.update( + "done", + 1.0, + phase="done", + message="Music generation completed.", + outputFile=output_file, + ) + + def should_abort(self) -> bool: + with self._lock: + return self._abort_due_to_stall + + +class WorkerSession: + def __init__(self, checkpoint_root: Path, model_id: str) -> None: + self.checkpoint_root = checkpoint_root + self.model_id = model_id + self.runtime_handlers: dict[str, Any] | None = None + self.runtime_signature: tuple[str, str] | None = None + self.backend = "unknown" + self._busy = False + + def _ensure_runtime_handlers( + self, + reporter: ProgressReporter, + *, + lm_model: str, + ) -> dict[str, Any]: + if self.runtime_handlers is not None and self.runtime_signature == ( + self.model_id, + lm_model, + ): + reporter.set_backend(self.backend) + return self.runtime_handlers + + reporter.update( + "loading", + 0.08, + phase="loading_runtime", + message="Loading the legacy ACE-Step runtime bridge...", + ) + try: + from acestep.handler import AceStepHandler + from acestep.llm_inference import LLMHandler + install_lm_diagnostics_instrumentation() + except Exception as exc: # pragma: no cover - runtime dependent + raise GenerationFailure( + f"ACE-Step is not available in this runtime: {exc}", + progress=0.08, + ) from exc + + checkpoint_storage_root = self.checkpoint_root.parent + dit_handler = AceStepHandler( + persistent_storage_path=str(checkpoint_storage_root) + ) + llm_handler = LLMHandler( + persistent_storage_path=str(checkpoint_storage_root) + ) + + device = "cuda" + try: + import torch + + if not torch.cuda.is_available(): + device = "cpu" + except Exception: + device = "cpu" + + self.backend = device + reporter.set_backend(device) + if device == "cpu": + lower_process_priority_for_cpu() + + reporter.update( + "loading", + 0.12, + phase="initializing_model", + message=f"Initializing ACE-Step on {device.upper()}...", + ) + lm_backend, lm_backend_reason, lm_acceleration_ready = resolve_lm_backend(device) + reporter.set_lm_backend(lm_backend) + if sys.platform == "win32" and device == "cuda" and not lm_acceleration_ready: + raise GenerationFailure( + "OpenStudio requires the pinned Windows ACE-Step acceleration stack before music generation can start. " + + lm_backend_reason, + progress=0.12, + lmBackend=lm_backend, + lmBackendReason=lm_backend_reason, + ) + init_status, init_ok = dit_handler.initialize_service( + project_root=str(self.checkpoint_root), + config_path=self.model_id, + device=device, + use_flash_attention=device == "cuda" and lm_acceleration_ready, + compile_model=False, + offload_to_cpu=device == "cpu", + offload_dit_to_cpu=False, + ) + if not init_ok: + raise GenerationFailure( + f"ACE-Step model initialization failed: {init_status}", + progress=0.12, + ) + + reporter.update( + "loading", + 0.18, + phase="initializing_lm", + message="Initializing ACE-Step language model...", + ) + lm_status, lm_ok = llm_handler.initialize( + checkpoint_dir=str(self.checkpoint_root), + lm_model_path=lm_model, + backend=lm_backend, + device=device, + offload_to_cpu=device == "cpu", + dtype=dit_handler.dtype, + ) + if not lm_ok: + raise GenerationFailure( + f"ACE-Step LM initialization failed: {lm_status}", + progress=0.18, + ) + actual_lm_backend = normalize_text(getattr(llm_handler, "llm_backend", "")) or lm_backend + if actual_lm_backend != lm_backend: + lm_backend_reason = ( + lm_backend_reason + + f"; ACE-Step runtime fell back to {actual_lm_backend}" + ) + reporter.set_lm_backend(actual_lm_backend) + + self.runtime_handlers = { + "dit_handler": dit_handler, + "llm_handler": llm_handler, + "device": device, + "lm_backend": actual_lm_backend, + "lm_backend_reason": lm_backend_reason, + "lm_model": lm_model, + } + self.runtime_signature = (self.model_id, lm_model) + return self.runtime_handlers + + def _run_generation( + self, + *, + workflow: str, + raw_params_json: str, + output_path: Path, + session_mode: str, + request_id: str, + ) -> str: + if workflow == "continuation": + raise GenerationFailure( + "Continuation workflow is not yet supported by this ACE-Step bridge.", + progress=0.1, + ) + + try: + raw_params = json.loads(raw_params_json) + except json.JSONDecodeError as exc: + raise GenerationFailure(f"Invalid params JSON: {exc}") from exc + + output_path.parent.mkdir(parents=True, exist_ok=True) + layout = validate_checkpoint_layout(self.checkpoint_root, self.model_id) + selection = resolve_runtime_selection( + requested_profile=normalize_text(raw_params.get("runtimeProfile")) + or DEFAULT_RUNTIME_PROFILE, + requested_lm_model=normalize_text(raw_params.get("lmModel")) + or DEFAULT_LM_SELECTION, + checkpoint_root=self.checkpoint_root, + ) + use_legacy_wrapper = normalize_bool( + os.environ.get("OPENSTUDIO_USE_LEGACY_ACE_WRAPPER"), + False, + ) + run_mode = ( + "warm" + if use_legacy_wrapper + and self.runtime_handlers is not None + and self.runtime_signature == (self.model_id, selection["selectedLmModel"]) + else "cold" + ) + trace_session = AITraceSession( + request_id=request_id, + workflow=workflow, + session_mode=session_mode, + model_id=self.model_id, + ) + trace_session.set_metadata( + rawParams=raw_params, + selectedProfile=selection["selectedProfile"], + selectedLmModel=selection["selectedLmModel"], + runtimeProfileName=selection["runtimeProfileName"], + nativeRequiredAssets=layout.get("requiredAssets", []), + outputPath=str(output_path), + checkpointRoot=str(self.checkpoint_root), + runMode=run_mode, + workerPid=os.getpid(), + ) + + reporter = ProgressReporter( + model_id=self.model_id, + checkpoint_root=self.checkpoint_root, + backend=self.backend, + run_mode=run_mode, + session_mode=session_mode, + runtime_profile=selection["selectedProfile"], + lm_model=selection["selectedLmModel"], + request_id=request_id, + trace_path=trace_session.trace_path(), + ) + set_active_diagnostics_context(trace_session, reporter) + reporter.start_heartbeat() + + try: + reporter.update( + "loading", + 0.05, + phase="validating_request", + message=( + "Validating the ACE-Step request " + f"({selection['selectedProfileLabel']} / {selection['selectedLmModel']})..." + ), + runMode=run_mode, + runtimeProfile=selection["selectedProfile"], + lmModel=selection["selectedLmModel"], + statusNote=" ".join(selection["statusNotes"]).strip() or None, + ) + params = normalize_generation_params(raw_params) + reporter.set_runtime_context( + run_mode=run_mode, + session_mode=session_mode, + runtime_profile=selection["selectedProfile"], + lm_model=selection["selectedLmModel"], + ) + trace_session.set_metadata( + backend=self.backend, + tracePath=trace_session.trace_path(), + ) + generation_mode = ( + GENERATION_MODE_LM_FIRST + if bool(params.get("generate_audio_codes", True)) + else GENERATION_MODE_DIT_MANUAL + ) + use_lm_audio_codes = generation_mode == GENERATION_MODE_LM_FIRST + status_notes = list(selection["statusNotes"]) + status_notes.append( + "OpenStudio's native split graph is using TextEncodeAceStepAudio1.5 -> AuraFlow -> KSampler -> VAEDecodeAudio." + ) + status_notes.append( + "Manual workflow mode is active; explicit BPM, duration, time signature, language, and key are passed directly to the split graph encoder." + ) + if use_lm_audio_codes: + status_notes.append( + "LM audio-code generation is enabled to match the native split graph workflow." + ) + else: + status_notes.append( + "LM audio-code generation is disabled for manual direct conditioning." + ) + attempt_mode = ( + "legacy_ace_wrapper" if use_legacy_wrapper else "native_split_graph" + ) + reporter.set_request_context( + attempt_mode=attempt_mode, + attempt_index=1, + debug_decode_stall_seconds=params["debugDecodeStallSeconds"], + ) + reporter.set_lm_stage("encoding_conditioning") + trace_session.log_event( + "request_contract", + mirror=True, + rawParams=raw_params, + normalizedParams=params, + backendFamily=("legacy_ace_wrapper" if use_legacy_wrapper else "openstudio_ace_split"), + workflowMode="manual", + generateAudioCodes=use_lm_audio_codes, + ) + + reporter.update( + "loading", + 0.08, + phase="loading_text_encoders", + message=( + "Launching the OpenStudio ACE split runner " + f"({selection['selectedProfileLabel']} / {selection['selectedLmModel']})..." + ), + lmBackend=("legacy-ace-wrapper" if use_legacy_wrapper else "openstudio-ace-split"), + phaseProgress=0.0, + statusNote=" ".join(status_notes).strip() or None, + ) + if not use_legacy_wrapper: + reporter.set_lm_backend("openstudio-ace-split") + trace_session.set_metadata( + lmBackend="openstudio-ace-split", + backendFamily="openstudio_ace_split", + ) + final_output = run_native_split_generation( + reporter=reporter, + trace_session=trace_session, + request_id=request_id, + workflow=workflow, + session_mode=session_mode, + normalized_params=params, + selection=selection, + checkpoint_root=self.checkpoint_root, + output_path=output_path, + ) + final_output_path = Path(final_output).expanduser().resolve() + if final_output_path != output_path.resolve(): + reporter.update( + "writing", + 0.96, + phase="writing_output", + message="Copying the generated audio into OpenStudio's output path...", + ) + try: + shutil.copyfile(final_output_path, output_path) + except Exception as exc: # pragma: no cover - file-system dependent + raise GenerationFailure( + f"Failed to save generated audio: {exc}", + progress=0.97, + ) from exc + + reporter.done(str(output_path)) + return str(output_path) + + trace_session.log_event( + "legacy_wrapper_requested", + mirror=True, + note="OPENSTUDIO_USE_LEGACY_ACE_WRAPPER is enabled; routing to the old ACE-Step wrapper path.", + ) + runtime_handlers = self._ensure_runtime_handlers( + reporter, + lm_model=selection["selectedLmModel"], + ) + reporter.set_lm_backend(runtime_handlers.get("lm_backend", "")) + trace_session.set_metadata( + lmBackend=runtime_handlers.get("lm_backend"), + lmBackendReason=runtime_handlers.get("lm_backend_reason"), + backendFamily="legacy_ace_wrapper", + ) + try: + from acestep.inference import ( + GenerationConfig, + generate_music, + ) + except Exception as exc: # pragma: no cover - runtime dependent + raise GenerationFailure( + f"ACE-Step inference API is not available in this runtime: {exc}", + progress=0.2, + ) from exc + + generation_config = GenerationConfig( + batch_size=1, + allow_lm_batch=False, + use_random_seed=params["seed"] < 0, + seeds=None if params["seed"] < 0 else [params["seed"]], + audio_format="wav", + ) + generation_param_payload = build_generation_param_payload( + params, + generation_mode=generation_mode, + ) + generation_params = build_generation_params( + params, + generation_mode=generation_mode, + ) + log_normalized_request_payload( + request_id=request_id, + workflow=workflow, + session_mode=session_mode, + normalized_params=params, + runtime_selection=selection, + generation_param_payload=generation_param_payload, + ) + trace_session.log_event( + "legacy_request_contract", + mirror=True, + generationParams=generation_param_payload, + ) + reporter.update( + "loading", + 0.2, + phase="encoding_conditioning", + message=( + "Submitting the request to the legacy ACE-Step wrapper " + f"({runtime_handlers.get('lm_backend', 'pt').upper()} LM / {selection['selectedLmModel']})..." + ), + lmBackend=runtime_handlers.get("lm_backend"), + lmBackendReason=runtime_handlers.get("lm_backend_reason"), + phaseProgress=0.0, + statusNote=" ".join(status_notes).strip() or None, + ) + + generation_error: str | None = None + result = None + if params["debugForceLmShapeMismatch"] and use_lm_audio_codes: + generation_error = ( + "❌ Error generating from formatted prompt: " + "The size of tensor a (151669) must match the size of tensor b " + "(217204) at non-singleton dimension 1" + ) + else: + try: + trace_session.log_event( + "generate_music_enter", + generationMode=generation_mode, + lmBackend=runtime_handlers.get("lm_backend"), + lmModel=selection["selectedLmModel"], + generateAudioCodes=bool(params.get("generate_audio_codes", True)), + explicitMusicalMetadata=has_explicit_musical_metadata(params), + ) + result = generate_music( + runtime_handlers["dit_handler"], + runtime_handlers["llm_handler"], + generation_params, + generation_config, + save_dir=str(output_path.parent), + progress=reporter.ace_progress, + ) + trace_session.log_event( + "generate_music_exit", + result=summarize_value(result), + ) + except Exception as exc: # pragma: no cover - runtime/model dependent + reporter.set_failure_detail(f"{type(exc).__name__}: {exc}") + trace_session.log_event( + "generate_music_exception", + mirror=True, + exceptionType=type(exc).__name__, + failureDetail=f"{type(exc).__name__}: {exc}", + traceback="".join(traceback.format_exception(type(exc), exc, exc.__traceback__)), + tracebackFrames=summarize_traceback_frames(exc.__traceback__), + ) + generation_error = str(exc) + + if generation_error is None and ( + result is None or not result.success or not result.audios + ): + generation_error = ( + result.error + if result is not None + else "ACE-Step generation did not return any audio outputs." + ) or "ACE-Step generation did not return any audio outputs." + + if generation_error is None and reporter.should_abort(): + raise GenerationFailure( + "ACE-Step decode stalled while finalizing audio.", + progress=0.95, + failureKind="decode_stalled", + priorFailure=reporter._prior_failure or None, + ) + + if generation_error is not None: + reporter.set_failure_detail(generation_error) + raise GenerationFailure( + f"ACE-Step generation failed: {generation_error}", + progress=0.2, + failureKind=classify_generation_failure_kind( + generation_error, + generation_mode=generation_mode, + ), + ) + + if not result.success or not result.audios: + raise GenerationFailure( + result.error + or "ACE-Step generation did not return any audio outputs.", + progress=0.2, + ) + + output_candidate = normalize_text(result.audios[0].get("path")) + if not output_candidate: + raise GenerationFailure( + "ACE-Step generation finished without an output file path.", + progress=0.92, + ) + + reporter.update( + "writing", + 0.96, + phase="writing_output", + message="Writing the generated audio file...", + ) + try: + shutil.copyfile(output_candidate, output_path) + except Exception as exc: # pragma: no cover - file-system dependent + raise GenerationFailure( + f"Failed to save generated audio: {exc}", + progress=0.97, + ) from exc + + reporter.done(str(output_path)) + return str(output_path) + except GenerationFailure as failure: + reporter.set_failure_detail( + normalize_text( + failure.payload.get("failureDetail") + or failure.payload.get("error") + or failure.message + ) + ) + reporter.fail( + failure.message, + progress=failure.progress, + **failure.payload, + ) + return "" + finally: + reporter.stop_heartbeat() + trace_session.finalize() + set_active_diagnostics_context(None, None) + + def generate( + self, + *, + workflow: str, + raw_params_json: str, + output_path: Path, + session_mode: str, + request_id: str, + ) -> bool: + if self._busy: + emit( + "error", + 0.0, + phase="busy", + message="Another generation is already active in the worker.", + error="Another generation is already active in the worker.", + requestId=request_id, + protocolVersion=WORKER_PROTOCOL_VERSION, + scriptVersion=SCRIPT_VERSION, + ) + return False + + self._busy = True + try: + return bool( + self._run_generation( + workflow=workflow, + raw_params_json=raw_params_json, + output_path=output_path, + session_mode=session_mode, + request_id=request_id, + ) + ) + finally: + self._busy = False + + +def recv_exact(connection: socket.socket, byte_count: int) -> bytes: + chunks: list[bytes] = [] + bytes_remaining = byte_count + while bytes_remaining > 0: + data = connection.recv(bytes_remaining) + if not data: + raise ConnectionError("Socket closed before the framed payload was fully received.") + chunks.append(data) + bytes_remaining -= len(data) + return b"".join(chunks) + + +def recv_framed_json(connection: socket.socket) -> dict[str, Any]: + header = recv_exact(connection, 4) + payload_length = struct.unpack(">I", header)[0] + if payload_length <= 0 or payload_length > MAX_FRAMED_PAYLOAD_BYTES: + raise ValueError(f"Invalid framed payload length: {payload_length}") + payload_bytes = recv_exact(connection, payload_length) + decoded = payload_bytes.decode("utf-8", errors="replace") + parsed = json.loads(decoded) + if not isinstance(parsed, dict): + raise ValueError("Framed payload must decode to a JSON object.") + parsed["_framedPayloadLength"] = payload_length + return parsed + + +def send_framed_json(connection: socket.socket, payload: dict[str, Any]) -> int: + payload_bytes = json.dumps(payload, ensure_ascii=False).encode("utf-8") + if len(payload_bytes) > MAX_FRAMED_PAYLOAD_BYTES: + raise ValueError( + f"Framed payload exceeds the maximum allowed size: {len(payload_bytes)} bytes" + ) + connection.sendall(struct.pack(">I", len(payload_bytes)) + payload_bytes) + return len(payload_bytes) + + +def run_worker_server(checkpoint_root: Path, model_id: str) -> None: + session = WorkerSession(checkpoint_root=checkpoint_root, model_id=model_id) + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + server.listen(1) + port = server.getsockname()[1] + + emit( + "idle", + 0.0, + event="ready", + phase="worker_ready", + message="ACE-Step worker is ready.", + port=port, + backend=session.backend, + sessionMode="persistent", + protocolVersion=WORKER_PROTOCOL_VERSION, + scriptVersion=SCRIPT_VERSION, + pid=os.getpid(), + scriptPath=str(SCRIPT_PATH), + musicGenerationModelId=model_id, + musicGenerationCheckpointRoot=str(checkpoint_root), + availableProfiles=build_runtime_profile_catalog(checkpoint_root)["availableProfiles"], + warmSessionCapable=True, + ) + + try: + while True: + connection, _address = server.accept() + with connection: + try: + request = recv_framed_json(connection) + except Exception as exc: + send_framed_json( + connection, + { + "accepted": False, + "error": f"Invalid worker request frame: {exc}", + "failureKind": "worker_protocol", + "protocolVersion": WORKER_PROTOCOL_VERSION, + "scriptVersion": SCRIPT_VERSION, + }, + ) + continue + + command = normalize_text(request.get("command")) + request_id = normalize_text(request.get("requestId")) or str(uuid.uuid4()) + request_protocol_version = normalize_int( + request.get("protocolVersion"), WORKER_PROTOCOL_VERSION + ) + request_script_version = normalize_text(request.get("scriptVersion")) + if command == "shutdown": + send_framed_json( + connection, + { + "accepted": True, + "requestId": request_id, + "protocolVersion": WORKER_PROTOCOL_VERSION, + "scriptVersion": SCRIPT_VERSION, + "pid": os.getpid(), + }, + ) + break + + if request_protocol_version != WORKER_PROTOCOL_VERSION: + send_framed_json( + connection, + { + "accepted": False, + "requestId": request_id, + "error": ( + "Worker protocol mismatch: " + f"app={request_protocol_version}, worker={WORKER_PROTOCOL_VERSION}" + ), + "failureKind": "worker_protocol", + "protocolVersion": WORKER_PROTOCOL_VERSION, + "scriptVersion": SCRIPT_VERSION, + "pid": os.getpid(), + }, + ) + continue + + if request_script_version and request_script_version != SCRIPT_VERSION: + send_framed_json( + connection, + { + "accepted": False, + "requestId": request_id, + "error": ( + "Worker script version mismatch: " + f"app={request_script_version}, worker={SCRIPT_VERSION}" + ), + "failureKind": "worker_protocol", + "protocolVersion": WORKER_PROTOCOL_VERSION, + "scriptVersion": SCRIPT_VERSION, + "pid": os.getpid(), + }, + ) + continue + + if command != "generate": + send_framed_json( + connection, + { + "accepted": False, + "requestId": request_id, + "error": f"Unsupported worker command: {command}", + "failureKind": "worker_protocol", + "protocolVersion": WORKER_PROTOCOL_VERSION, + "scriptVersion": SCRIPT_VERSION, + "pid": os.getpid(), + }, + ) + continue + + workflow = normalize_text(request.get("workflow")) or "text-to-music" + raw_params_json = request.get("params") + output = normalize_text(request.get("output")) + if not isinstance(raw_params_json, str) or not output: + send_framed_json( + connection, + { + "accepted": False, + "error": "Generate requests require params JSON and output path.", + "requestId": request_id, + "failureKind": "worker_protocol", + "protocolVersion": WORKER_PROTOCOL_VERSION, + "scriptVersion": SCRIPT_VERSION, + "pid": os.getpid(), + }, + ) + continue + + framed_payload_length = int(request.get("_framedPayloadLength", 0)) + send_framed_json( + connection, + { + "accepted": True, + "requestId": request_id, + "protocolVersion": WORKER_PROTOCOL_VERSION, + "scriptVersion": SCRIPT_VERSION, + "pid": os.getpid(), + "framedPayloadLength": framed_payload_length, + }, + ) + session.generate( + workflow=workflow, + raw_params_json=raw_params_json, + output_path=Path(output).expanduser().resolve(), + session_mode="persistent", + request_id=request_id, + ) + finally: + server.close() + + +def run_one_shot(args: argparse.Namespace) -> int: + if not args.workflow or not args.output: + raise SystemExit( + "--workflow and --output are required unless --worker is used." + ) + raw_params_json = resolve_one_shot_params_json(args) + + session = WorkerSession( + checkpoint_root=resolve_music_gen_checkpoint_root(args.checkpoint_root), + model_id=args.music_gen_model, + ) + success = session.generate( + workflow=args.workflow, + raw_params_json=raw_params_json, + output_path=Path(args.output).expanduser().resolve(), + session_mode=normalize_text(getattr(args, "session_mode", "")) or "oneshot", + request_id=normalize_text(getattr(args, "request_id", "")) or str(uuid.uuid4()), + ) + return 0 if success else 1 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate music with ACE-Step") + parser.add_argument("--worker", action="store_true", help="Run as a persistent worker") + parser.add_argument("--workflow", help="Workflow id to execute") + parser.add_argument("--params", help="Workflow parameters JSON") + parser.add_argument("--params-file", help="Path to workflow parameters JSON") + parser.add_argument("--params-stdin", action="store_true", help="Read workflow parameters JSON from stdin") + parser.add_argument("--output", help="Output WAV file path") + parser.add_argument("--request-id", help="Request identifier for diagnostics") + parser.add_argument( + "--checkpoint-root", + required=True, + help="Pinned ACE-Step checkpoint root", + ) + parser.add_argument( + "--music-gen-model", + default=DEFAULT_MUSIC_GEN_MODEL, + help="Pinned ACE-Step model id", + ) + parser.add_argument( + "--session-mode", + default="oneshot", + help="Session mode label to include in progress payloads", + ) + return parser.parse_args() + + +def main() -> None: + install_stream_mirrors() + args = parse_args() + checkpoint_root = resolve_music_gen_checkpoint_root(args.checkpoint_root) + if args.worker: + run_worker_server(checkpoint_root=checkpoint_root, model_id=args.music_gen_model) + return + + raise SystemExit(run_one_shot(args)) + + +if __name__ == "__main__": + main() diff --git a/tools/install_ai_tools.py b/tools/install_ai_tools.py index 5524c59..20422c2 100644 --- a/tools/install_ai_tools.py +++ b/tools/install_ai_tools.py @@ -16,20 +16,117 @@ import argparse from collections import deque from datetime import datetime, timezone +import hashlib import json +import os import platform +import re +import socket import shutil import subprocess import sys +import threading import time +from urllib.error import HTTPError, URLError +from urllib.parse import urlparse +from urllib.request import Request, urlopen from pathlib import Path from typing import Any -from ai_runtime_probe import probe_runtime_capabilities +from ai_runtime_probe import ( + DEFAULT_MUSIC_GEN_MODEL, + DEFAULT_MUSIC_GEN_MODEL_REPO, + DEFAULT_MUSIC_GEN_SHARED_REPO, + REQUIRED_MUSIC_GEN_NATIVE_FILES, + find_local_comfy_native_assets, + get_windows_cuda_pytorch_index_url, + get_windows_cuda_pytorch_packages, + get_windows_flash_attn_asset, + get_windows_triton_package_spec, + get_music_generation_required_paths, + get_music_runtime_profiles, + probe_runtime_capabilities, + resolve_music_gen_checkpoint_root, +) DEFAULT_MODEL_NAME = "BS-Roformer-SW.ckpt" FALLBACK_MIN_PYTHON = (3, 10) -FALLBACK_MAX_PYTHON_EXCLUSIVE = (3, 14) +FALLBACK_MAX_PYTHON_EXCLUSIVE = (3, 13) +MUSIC_GEN_REQUIRED_PYTHON = (3, 11) +MUSIC_GEN_SPACE_REPO = "ACE-Step/Ace-Step-v1.5" +MUSIC_GEN_SPACE_REPO_TYPE = "space" +MUSIC_GEN_SOURCE_DIRNAME = "ace-step-v1.5-source" +MUSIC_GEN_HUB_CACHE_DIRNAME = ".hub-cache" +MODEL_DOWNLOAD_MANIFEST_URL = "https://raw.githubusercontent.com/TRvlvr/application_data/main/filelists/download_checks.json" +UVR_MODEL_REPO_URL_PREFIX = "https://github.com/TRvlvr/model_repo/releases/download/all_public_uvr_models" +UVR_MODEL_CONFIG_URL_PREFIX = f"{UVR_MODEL_REPO_URL_PREFIX}/mdx_model_data/mdx_c_configs" +AUDIO_SEPARATOR_MODEL_REPO_URL_PREFIX = "https://github.com/nomadkaraoke/python-audio-separator/releases/download/model-configs" +AUDIO_SEPARATOR_HF_REPO_URL_PREFIX = "https://huggingface.co/lainlives/audio-separator-models/resolve/main" +DOWNLOAD_CHUNK_SIZE = 1024 * 512 +DOWNLOAD_TIMEOUT_SECONDS = 60 +DOWNLOAD_HEARTBEAT_SECONDS = 5.0 +MODEL_DOWNLOAD_PROGRESS_START = 0.82 +MODEL_DOWNLOAD_PROGRESS_END = 0.95 +ACE15_COMFY_TEXT_ENCODER_FILENAMES: dict[str, str] = { + "acestep-5Hz-lm-0.6B": "qwen_0.6b_ace15.safetensors", + "acestep-5Hz-lm-1.7B": "qwen_1.7b_ace15.safetensors", + "acestep-5Hz-lm-4B": "qwen_4b_ace15.safetensors", +} +ACE15_MANAGED_LM_CONFIG_OVERRIDES: dict[str, dict[str, Any]] = { + "acestep-5Hz-lm-0.6B": { + "hidden_size": 1024, + "intermediate_size": 3072, + "num_hidden_layers": 28, + "num_attention_heads": 16, + "num_key_value_heads": 8, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "vocab_size": 151669, + "architectures": ["Qwen3Model"], + "model_type": "qwen3", + }, + "acestep-5Hz-lm-1.7B": { + "hidden_size": 2048, + "intermediate_size": 6144, + "num_hidden_layers": 28, + "num_attention_heads": 16, + "num_key_value_heads": 8, + "max_position_embeddings": 40960, + "max_window_layers": 28, + "vocab_size": 217204, + "architectures": ["Qwen3Model"], + "model_type": "qwen3", + }, + "acestep-5Hz-lm-4B": { + "hidden_size": 2560, + "intermediate_size": 9728, + "num_hidden_layers": 36, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "max_position_embeddings": 40960, + "max_window_layers": 36, + "vocab_size": 217204, + "architectures": ["Qwen3Model"], + "model_type": "qwen3", + }, +} +ACE15_MANAGED_LM_SHARED_FILES = ( + "tokenizer.json", + "tokenizer_config.json", + "special_tokens_map.json", + "vocab.json", + "merges.txt", + "added_tokens.json", + "chat_template.jinja", +) +WINDOWS_REUSED_RUNTIME_REPAIR_PACKAGES = ( + "audio-separator==0.41.1", + "requests>=2", + "certifi>=2023.5.7", + "charset_normalizer<4,>=2", + "idna<4,>=2.5", + "urllib3<3,>=1.26", +) LOG_PATH: Path | None = None SESSION_ID = "" @@ -46,6 +143,13 @@ def __init__(self, message: str, *, error_code: str, progress: float) -> None: self.progress = progress +class ModelDownloadError(Exception): + def __init__(self, message: str, *, error_code: str) -> None: + super().__init__(message) + self.message = message + self.error_code = error_code + + def utc_now_iso() -> str: return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") @@ -85,7 +189,20 @@ def emit(state: str, progress: float, **kwargs) -> None: payload.setdefault("elapsedMs", int((time.monotonic() - START_TIME_MONOTONIC) * 1000)) if LOG_PATH is not None and "detailLogPath" not in payload: payload["detailLogPath"] = str(LOG_PATH) - print(json.dumps(payload), flush=True) + serialized = json.dumps(payload) + write_log(serialized) + # When OpenStudio launches this installer it tails the install log directly + # and does not drain the child process stdout pipe. Continuing to print every + # structured progress update would eventually block the installer once the + # pipe buffer fills, which is exactly what can happen during large model + # downloads. Keep stdout for standalone/debug runs that do not provide a + # log path, and allow forcing stdout explicitly for troubleshooting. + should_print_stdout = LOG_PATH is None or os.environ.get("OPENSTUDIO_INSTALLER_FORCE_STDOUT") == "1" + if should_print_stdout: + try: + print(serialized, flush=True) + except (BrokenPipeError, OSError): + pass def fail( @@ -127,57 +244,537 @@ def resolve_runtime_python(runtime_root: Path) -> Path: raise AssertionError("unreachable") -def get_requirement_specifiers() -> list[str]: +def _detect_linux_gpu_backend() -> str: + """Detect available GPU backend on Linux. Returns 'cuda', 'rocm', or 'cpu'.""" + # NVIDIA: nvidia-smi exits 0 when a GPU is present and the driver is loaded + try: + result = subprocess.run( + ["nvidia-smi"], capture_output=True, timeout=5 + ) + if result.returncode == 0: + return "cuda" + except (FileNotFoundError, subprocess.TimeoutExpired): + pass + # AMD ROCm: rocm-smi exits 0 when ROCm-compatible hardware is present + try: + result = subprocess.run( + ["rocm-smi"], capture_output=True, timeout=5 + ) + if result.returncode == 0: + return "rocm" + except (FileNotFoundError, subprocess.TimeoutExpired): + pass + return "cpu" + + +def read_python_version_info(python_executable: Path) -> tuple[int, int, int] | None: + try: + result = subprocess.run( + [ + str(python_executable), + "-c", + "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}.{sys.version_info[2]}')", + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return None + + if result.returncode != 0: + return None + + version_text = result.stdout.strip() + parts = version_text.split(".") + if len(parts) != 3: + return None + + try: + return (int(parts[0]), int(parts[1]), int(parts[2])) + except ValueError: + return None + + +def format_python_version(version: tuple[int, int, int] | None) -> str: + if version is None: + return "unknown" + return ".".join(str(part) for part in version) + + +def is_music_generation_python_compatible(version: tuple[int, int, int] | None) -> bool: + return version is not None and version[:2] == MUSIC_GEN_REQUIRED_PYTHON + + +def find_windows_python_311() -> Path | None: + if platform.system() != "Windows": + return None + + py_launcher = shutil.which("py") + if not py_launcher: + return None + + try: + result = subprocess.run( + [ + py_launcher, + "-3.11", + "-c", + "import sys; print(sys.executable)", + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return None + + if result.returncode != 0: + return None + + candidate = Path(result.stdout.strip()).expanduser() + if candidate.exists(): + return candidate.resolve() + return None + + +def is_windows_nvidia_machine() -> bool: + if platform.system() != "Windows": + return False + + try: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=10, + ) + except (FileNotFoundError, subprocess.TimeoutExpired, OSError, subprocess.SubprocessError): + return False + + return result.returncode == 0 and bool(result.stdout.strip()) + + +def terminate_windows_runtime_lock_holders(runtime_root: Path) -> list[str]: + if platform.system() != "Windows": + return [] + + resolved_runtime_root = safe_resolve(runtime_root) + runtime_python = safe_resolve(runtime_root / "Scripts" / "python.exe") + runtime_root_text = str(resolved_runtime_root).replace("\\", "\\\\").replace("'", "''") + runtime_python_text = str(runtime_python).replace("\\", "\\\\").replace("'", "''") + powershell_script = f""" +$runtimeRoot = [System.IO.Path]::GetFullPath('{runtime_root_text}') +$runtimePython = [System.IO.Path]::GetFullPath('{runtime_python_text}') +Get-CimInstance Win32_Process | ForEach-Object {{ + $exe = $_.ExecutablePath + $cmd = $_.CommandLine + $matchesRuntime = $false + + if ($exe) {{ + try {{ + $resolvedExe = [System.IO.Path]::GetFullPath($exe) + if ($resolvedExe -ieq $runtimePython -or $resolvedExe.StartsWith($runtimeRoot, [System.StringComparison]::OrdinalIgnoreCase)) {{ + $matchesRuntime = $true + }} + }} catch {{ + }} + }} + + if (-not $matchesRuntime -and $cmd) {{ + if ( + $cmd.IndexOf('generate_music.py --worker', [System.StringComparison]::OrdinalIgnoreCase) -ge 0 + -or $cmd.IndexOf($runtimeRoot, [System.StringComparison]::OrdinalIgnoreCase) -ge 0 + ) {{ + $matchesRuntime = $true + }} + }} + + if ($matchesRuntime) {{ + [PSCustomObject]@{{ + ProcessId = $_.ProcessId + Name = $_.Name + }} + }} +}} | ConvertTo-Json -Compress +""" + + try: + result = subprocess.run( + ["powershell", "-NoProfile", "-Command", powershell_script], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=15, + ) + except (OSError, subprocess.SubprocessError): + return [] + + if result.returncode != 0: + return [] + + stdout = (result.stdout or "").strip() + if not stdout: + return [] + + try: + parsed = json.loads(stdout) + except json.JSONDecodeError: + return [] + + if isinstance(parsed, dict): + process_rows = [parsed] + elif isinstance(parsed, list): + process_rows = [row for row in parsed if isinstance(row, dict)] + else: + return [] + + killed: list[str] = [] + for row in process_rows: + pid = row.get("ProcessId") + name = str(row.get("Name", "")).strip() or "process" + try: + pid_int = int(pid) + except (TypeError, ValueError): + continue + + try: + subprocess.run( + ["taskkill", "/PID", str(pid_int), "/F", "/T"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=10, + check=False, + ) + killed.append(f"{pid_int}:{name}") + except (OSError, subprocess.SubprocessError): + continue + + return killed + + +def _handle_remove_readonly(func: Any, path: str, exc_info: Any) -> None: + try: + os.chmod(path, 0o700) + except OSError: + pass + func(path) + + +def remove_tree_with_retries(target: Path, *, retries: int = 3, retry_delay_seconds: float = 1.0) -> list[str]: + terminated_processes: list[str] = [] + if not target.exists(): + return terminated_processes + if platform.system() == "Windows": - return ["audio-separator[dml]==0.41.1", "onnxruntime-gpu>=1.17"] - return ["audio-separator[cpu]==0.41.1"] + terminated_processes.extend(terminate_windows_runtime_lock_holders(target)) + if terminated_processes: + time.sleep(retry_delay_seconds) + last_error: OSError | None = None + for attempt in range(1, retries + 1): + try: + shutil.rmtree(target, onerror=_handle_remove_readonly) + return terminated_processes + except OSError as exc: + last_error = exc + if platform.system() == "Windows": + terminated_processes.extend(terminate_windows_runtime_lock_holders(target)) + if attempt < retries: + time.sleep(retry_delay_seconds) + continue + raise + + if last_error is not None: + raise last_error + return terminated_processes + + +def get_candidate_comfy_text_encoder_dirs() -> list[Path]: + candidates: list[Path] = [] + + for env_name in ( + "OPENSTUDIO_ACESTEP_TEXT_ENCODER_DIR", + "COMFYUI_TEXT_ENCODER_DIR", + ): + raw = os.environ.get(env_name, "").strip() + if raw: + candidates.append(Path(raw).expanduser()) + + home = Path.home() + candidates.extend( + [ + home / "Documents" / "ComfyUI" / "models" / "text_encoders", + home / "Documents" / "Codes" / "ComfyUI" / "models" / "text_encoders", + home / "ComfyUI" / "models" / "text_encoders", + ] + ) -def log_subprocess_output(result: subprocess.CompletedProcess[str], description: str) -> None: - write_log(f"$ {description}") - write_log(f"[exitCode] {result.returncode}") - if result.stdout: - write_log("[stdout]") - write_log(result.stdout) - if result.stderr: - write_log("[stderr]") - write_log(result.stderr) + unique_candidates: list[Path] = [] + seen: set[str] = set() + for candidate in candidates: + try: + resolved = candidate.expanduser().resolve() + except OSError: + resolved = candidate.expanduser() + key = str(resolved).lower() + if key in seen: + continue + seen.add(key) + unique_candidates.append(resolved) + return unique_candidates + + +def find_local_comfy_text_encoder_assets() -> tuple[dict[str, Path], list[str]]: + found: dict[str, Path] = {} + searched_dirs: list[str] = [] + + for candidate_dir in get_candidate_comfy_text_encoder_dirs(): + searched_dirs.append(str(candidate_dir)) + if not candidate_dir.is_dir(): + continue + for target_name, source_filename in ACE15_COMFY_TEXT_ENCODER_FILENAMES.items(): + if target_name in found: + continue + candidate = candidate_dir / source_filename + if candidate.exists() and candidate.is_file(): + found[target_name] = candidate.resolve() + return found, searched_dirs -def safe_resolve(path: Path) -> Path: + +def hardlink_or_copy_file(source: Path, destination: Path) -> str: + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists(): + destination.unlink() try: - return path.resolve() + os.link(source, destination) + return "hardlink" except OSError: - return path + shutil.copy2(source, destination) + return "copy" -def run_step( - command: list[str], +def synthesize_managed_ace15_lm_checkpoint( + *, + checkpoint_root: Path, + target_name: str, + source_safetensors: Path, + template_checkpoint_dir: Path, +) -> Path: + if target_name not in ACE15_MANAGED_LM_CONFIG_OVERRIDES: + raise ValueError(f"Unsupported ACE15 managed LM target: {target_name}") + + target_dir = checkpoint_root / target_name + target_dir.mkdir(parents=True, exist_ok=True) + + template_config_path = template_checkpoint_dir / "config.json" + if not template_config_path.exists(): + raise FileNotFoundError(f"Template config is missing: {template_config_path}") + + config = json.loads(template_config_path.read_text(encoding="utf-8")) + config.update(ACE15_MANAGED_LM_CONFIG_OVERRIDES[target_name]) + if "num_hidden_layers" in config: + config["layer_types"] = ["full_attention"] * int(config["num_hidden_layers"]) + + for shared_name in ACE15_MANAGED_LM_SHARED_FILES: + source_file = template_checkpoint_dir / shared_name + if source_file.exists() and source_file.is_file(): + shutil.copy2(source_file, target_dir / shared_name) + + (target_dir / "config.json").write_text( + json.dumps(config, indent=2, ensure_ascii=True) + "\n", + encoding="utf-8", + ) + + link_mode = hardlink_or_copy_file(source_safetensors, target_dir / "model.safetensors") + metadata = { + "source": "localModelImport", + "sourceFile": str(source_safetensors), + "linkMode": link_mode, + "generatedAt": utc_now_iso(), + "targetName": target_name, + } + (target_dir / "openstudio-source.json").write_text( + json.dumps(metadata, indent=2, ensure_ascii=True) + "\n", + encoding="utf-8", + ) + return target_dir + + +def hydrate_comfy_runtime_profiles(checkpoint_root: Path) -> dict[str, Any]: + template_dir = checkpoint_root / "acestep-5Hz-lm-1.7B" + if not template_dir.is_dir(): + return { + "importedTargets": [], + "searchedDirs": [], + "foundSources": {}, + "missingTargets": list(ACE15_COMFY_TEXT_ENCODER_FILENAMES.keys()), + "error": f"Template checkpoint is missing: {template_dir}", + } + + found_sources, searched_dirs = find_local_comfy_text_encoder_assets() + imported_targets: list[str] = [] + missing_targets: list[str] = [] + + for target_name in ACE15_COMFY_TEXT_ENCODER_FILENAMES: + target_dir = checkpoint_root / target_name + if (target_dir / "model.safetensors").exists(): + continue + + source_safetensors = found_sources.get(target_name) + if source_safetensors is None: + missing_targets.append(target_name) + continue + + synthesize_managed_ace15_lm_checkpoint( + checkpoint_root=checkpoint_root, + target_name=target_name, + source_safetensors=source_safetensors, + template_checkpoint_dir=template_dir, + ) + imported_targets.append(target_name) + + return { + "importedTargets": imported_targets, + "searchedDirs": searched_dirs, + "foundSources": {key: str(value) for key, value in found_sources.items()}, + "missingTargets": missing_targets, + "error": "", + } + + +def hydrate_native_split_assets(checkpoint_root: Path) -> dict[str, Any]: + found_sources, searched_dirs = find_local_comfy_native_assets() + imported_assets: list[str] = [] + missing_assets: list[str] = [] + + for spec in REQUIRED_MUSIC_GEN_NATIVE_FILES: + destination = checkpoint_root / Path(spec["relativePath"]) + if destination.exists() and destination.is_file(): + continue + + source_file = found_sources.get(spec["id"]) + if source_file is None: + missing_assets.append(spec["relativePath"]) + continue + + hardlink_or_copy_file(source_file, destination) + imported_assets.append(spec["relativePath"]) + + return { + "importedAssets": imported_assets, + "searchedDirs": searched_dirs, + "foundSources": {key: str(value) for key, value in found_sources.items()}, + "missingAssets": missing_assets, + } + + +def get_requirement_specifiers(*, python_version: tuple[int, int, int] | None) -> list[str]: + if platform.system() == "Windows": + windows_audio_separator_variant = ( + "audio-separator[gpu]==0.41.1" + if is_windows_nvidia_machine() + else "audio-separator[dml]==0.41.1" + ) + return [ + windows_audio_separator_variant, + "hf_xet>=1.4.2", + "huggingface_hub>=0.34,<1.0", + ] + if platform.system() == "Linux": + backend = _detect_linux_gpu_backend() + if backend == "cuda": + return [ + "audio-separator[gpu]==0.41.1", + "hf_xet>=1.4.2", + "huggingface_hub>=0.34,<1.0", + ] + # ROCm and CPU both start from the CPU base; the backend plan swaps torch wheels + return [ + "audio-separator[cpu]==0.41.1", + "hf_xet>=1.4.2", + "huggingface_hub>=0.34,<1.0", + ] + return [ + "audio-separator[cpu]==0.41.1", + "hf_xet>=1.4.2", + "huggingface_hub>=0.34,<1.0", + ] + + +def get_music_generation_runtime_requirements( + *, + python_version: tuple[int, int, int] | None, +) -> list[str]: + if not is_music_generation_python_compatible(python_version): + return [] + + return [ + "transformers>=4.51.0,<4.58.0", + "diffusers==0.35.2", + "accelerate>=1.12.0", + "vector-quantize-pytorch>=1.27.15", + "torchsde>=0.2.6", + "av>=12.0.0", + "huggingface_hub>=0.34,<1.0", + "loguru>=0.7.3", + "xxhash>=3.5.0", + ] + + +def compute_file_sha256(file_path: Path) -> str: + digest = hashlib.sha256() + with file_path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest().lower() + + +def download_url_to_path(url: str, destination: Path) -> None: + request = Request(url, headers={"User-Agent": "OpenStudio-AI-Installer/1.0"}) + destination.parent.mkdir(parents=True, exist_ok=True) + with urlopen(request, timeout=DOWNLOAD_TIMEOUT_SECONDS) as response, destination.open( + "wb" + ) as output: + while True: + chunk = response.read(DOWNLOAD_CHUNK_SIZE) + if not chunk: + break + output.write(chunk) + + +def install_pinned_wheel( + runtime_python: Path, + runtime_root: Path, *, + wheel_url: str, + wheel_sha256: str, + wheel_filename: str, + description: str, state: str, progress: float, - description: str, install_source: str, requires_external_python: bool, - error_code: str, python_detected: bool, build_runtime_mode: str, - cwd: Path | None = None, - raise_on_error: bool = False, step_label: str | None = None, step_index: int = 0, step_count: int = 0, download_hint: str = "", is_large_download: bool = False, + raise_on_error: bool = False, + error_code: str = "dependency_bootstrap_failed", ) -> None: - log_event( - "installer", - state, - "step_start", - description=description, - command=command, - buildRuntimeMode=build_runtime_mode, - ) emit( state, progress, @@ -192,46 +789,1151 @@ def run_step( pythonDetected=python_detected, buildRuntimeMode=build_runtime_mode, ) - result = subprocess.run( - command, - cwd=str(cwd) if cwd else None, + + cache_dir = runtime_root / ".openstudio-pinned-wheels" + wheel_path = cache_dir / wheel_filename + expected_sha256 = wheel_sha256.removeprefix("sha256:").strip().lower() + + try: + should_download = True + if wheel_path.exists(): + should_download = compute_file_sha256(wheel_path) != expected_sha256 + if should_download: + wheel_path.unlink() + + if should_download: + download_url_to_path(wheel_url, wheel_path) + + actual_sha256 = compute_file_sha256(wheel_path) + if actual_sha256 != expected_sha256: + raise InstallerStepError( + f"{description} failed. Downloaded wheel checksum mismatch.", + error_code=error_code, + progress=progress, + ) + except InstallerStepError: + raise + except Exception as exc: + if raise_on_error: + raise InstallerStepError( + f"{description} failed: {type(exc).__name__}: {exc}", + error_code=error_code, + progress=progress, + ) from exc + fail( + f"{description} failed: {type(exc).__name__}: {exc}", + progress=progress, + error_code=error_code, + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + + run_step( + [ + str(runtime_python), + "-m", + "pip", + "install", + "--no-deps", + str(wheel_path), + ], + state=state, + progress=progress, + description=description, + install_source=install_source, + requires_external_python=requires_external_python, + error_code=error_code, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + cwd=runtime_root, + raise_on_error=raise_on_error, + step_label=step_label, + step_index=step_index, + step_count=step_count, + download_hint=download_hint, + is_large_download=is_large_download, + ) + + +def apply_windows_cuda_pytorch_overlay( + runtime_python: Path, + runtime_root: Path, + *, + install_source: str, + requires_external_python: bool, + python_detected: bool, + build_runtime_mode: str, +) -> None: + run_step( + [ + str(runtime_python), + "-m", + "pip", + "uninstall", + "-y", + "torch", + "torchvision", + "torchaudio", + ], + state="installing", + progress=0.575, + description="Removing CPU PyTorch wheels before enabling CUDA music generation", + install_source=install_source, + requires_external_python=requires_external_python, + error_code="dependency_bootstrap_failed", + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + cwd=runtime_root, + ) + + stream_step( + [ + str(runtime_python), + "-m", + "pip", + "install", + "--verbose", + "--index-url", + get_windows_cuda_pytorch_index_url(), + *get_windows_cuda_pytorch_packages(), + ], + state="installing", + progress=0.59, + description="Installing CUDA PyTorch wheels for ACE-Step music generation", + install_source=install_source, + requires_external_python=requires_external_python, + error_code="dependency_bootstrap_failed", + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + cwd=runtime_root, + step_label="Installing CUDA PyTorch wheels for ACE-Step music generation", + download_hint="This is the largest Windows AI runtime step. OpenStudio is downloading and installing the pinned CUDA PyTorch wheels, which can stay quiet for several minutes.", + is_large_download=True, + ) + + +def install_windows_music_acceleration_stack( + runtime_python: Path, + runtime_root: Path, + *, + state: str, + install_source: str, + requires_external_python: bool, + python_detected: bool, + build_runtime_mode: str, + step_index_offset: int = 0, + step_count: int = 0, + raise_on_error: bool = False, + error_code: str = "music_acceleration_setup_failed", +) -> None: + triton_package = get_windows_triton_package_spec() + flash_attn_asset = get_windows_flash_attn_asset() + + run_step( + [ + str(runtime_python), + "-m", + "pip", + "install", + "--upgrade", + triton_package, + ], + state=state, + progress=0.605, + description="Installing Triton for the ACE-Step accelerated LM runtime", + install_source=install_source, + requires_external_python=requires_external_python, + error_code=error_code, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + cwd=runtime_root, + raise_on_error=raise_on_error, + step_label="Installing Triton for music generation", + step_index=step_index_offset + 1, + step_count=step_count, + download_hint="OpenStudio is installing the Windows Triton runtime used by ACE-Step acceleration.", + is_large_download=True, + ) + + install_pinned_wheel( + runtime_python, + runtime_root, + wheel_url=str(flash_attn_asset.get("url", "")).strip(), + wheel_sha256=str(flash_attn_asset.get("sha256", "")).strip(), + wheel_filename=str( + flash_attn_asset.get("fileName") + or flash_attn_asset.get("filename") + or "flash_attn.whl" + ).strip(), + description="Installing Flash Attention for the ACE-Step accelerated LM runtime", + state=state, + progress=0.615, + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + step_label="Installing Flash Attention for music generation", + step_index=step_index_offset + 2, + step_count=step_count, + download_hint="OpenStudio is installing the pinned Flash Attention wheel for the Windows ACE-Step stack.", + is_large_download=True, + raise_on_error=raise_on_error, + error_code=error_code, + ) + + +def download_music_generation_source( + runtime_python: Path, + runtime_root: Path, + *, + install_source: str, + requires_external_python: bool, + python_detected: bool, + build_runtime_mode: str, + raise_on_error: bool = False, +) -> Path: + source_root = runtime_root / MUSIC_GEN_SOURCE_DIRNAME + run_step( + [ + str(runtime_python), + "-c", + ( + "from pathlib import Path\n" + "from huggingface_hub import snapshot_download\n" + f"source_root = Path(r'{source_root}')\n" + "source_root.mkdir(parents=True, exist_ok=True)\n" + "snapshot_download(\n" + f" repo_id=r'{MUSIC_GEN_SPACE_REPO}',\n" + f" repo_type=r'{MUSIC_GEN_SPACE_REPO_TYPE}',\n" + " local_dir=str(source_root),\n" + ")\n" + "print('ok')\n" + ), + ], + state="installing", + progress=0.52, + description="Preparing the ACE-Step 1.5 runtime source", + install_source=install_source, + requires_external_python=requires_external_python, + error_code="dependency_bootstrap_failed", + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + cwd=runtime_root, + raise_on_error=raise_on_error, + ) + return source_root + + +def log_subprocess_output(result: subprocess.CompletedProcess[str], description: str) -> None: + write_log(f"$ {description}") + write_log(f"[exitCode] {result.returncode}") + if result.stdout: + write_log("[stdout]") + write_log(result.stdout) + if result.stderr: + write_log("[stderr]") + write_log(result.stderr) + + +def safe_resolve(path: Path) -> Path: + try: + return path.resolve() + except OSError: + return path + + +def is_path_within(path: Path, root: Path) -> bool: + resolved_path = safe_resolve(path) + resolved_root = safe_resolve(root) + try: + resolved_path.relative_to(resolved_root) + return True + except ValueError: + return False + + +def extract_windows_access_denied_path(output: str) -> Path | None: + match = re.search(r"Access is denied:\s*['\"]([^'\"]+)['\"]", output, re.IGNORECASE) + if match is None: + return None + return Path(match.group(1)) + + +def read_json_file(path: Path) -> dict[str, Any]: + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def fetch_json(url: str, *, timeout: int = DOWNLOAD_TIMEOUT_SECONDS) -> dict[str, Any]: + request = Request(url, headers={"User-Agent": "OpenStudio-AI-Installer/1.0"}) + with urlopen(request, timeout=timeout) as response: + payload = response.read() + return json.loads(payload.decode("utf-8")) + + +def resolve_audio_separator_models_json(runtime_root: Path) -> Path: + candidates = [ + runtime_root / "Lib" / "site-packages" / "audio_separator" / "models.json", + runtime_root / "lib" / "site-packages" / "audio_separator" / "models.json", + runtime_root / "python" / "Lib" / "site-packages" / "audio_separator" / "models.json", + runtime_root / "python" / "lib" / "site-packages" / "audio_separator" / "models.json", + ] + for candidate in candidates: + if candidate.exists(): + return candidate + + for candidate in runtime_root.rglob("models.json"): + if candidate.parent.name == "audio_separator": + return candidate + + raise ModelDownloadError( + f"OpenStudio could not find audio_separator/models.json inside {runtime_root}.", + error_code="model_manifest_missing", + ) + + +def append_activity_line(recent_lines: deque[str], line: str) -> list[str]: + recent_lines.append(line) + write_log(line) + return list(recent_lines) + + +def emit_model_download_status( + *, + progress: float, + message: str, + install_source: str, + requires_external_python: bool, + python_detected: bool, + build_runtime_mode: str, + step_label: str, + activity_lines: list[str], + bytes_downloaded: int = 0, + bytes_total: int = 0, + status_warning: str = "", + status_warning_code: str = "", +) -> None: + payload: dict[str, Any] = { + "message": message, + "stepLabel": step_label, + "stepIndex": 1, + "stepCount": 1, + "downloadHint": "The stem model download can take a while on slower connections.", + "isLargeDownload": True, + "activityLines": activity_lines, + "installSource": install_source, + "requiresExternalPython": requires_external_python, + "pythonDetected": python_detected, + "buildRuntimeMode": build_runtime_mode, + } + if bytes_downloaded > 0: + payload["bytesDownloaded"] = bytes_downloaded + if bytes_total > 0: + payload["bytesTotal"] = bytes_total + if status_warning: + payload["statusWarning"] = status_warning + if status_warning_code: + payload["statusWarningCode"] = status_warning_code + emit("downloading_model", progress, **payload) + + +def load_model_manifests(runtime_root: Path, recent_lines: deque[str]) -> tuple[dict[str, Any] | None, dict[str, Any]]: + append_activity_line(recent_lines, "Resolving model manifests...") + bundled_manifest_path = resolve_audio_separator_models_json(runtime_root) + bundled_manifest = read_json_file(bundled_manifest_path) + log_event( + "installer", + "downloading_model", + "bundled_model_manifest_loaded", + manifestPath=str(bundled_manifest_path), + ) + append_activity_line(recent_lines, f"Loaded bundled model list from {bundled_manifest_path}") + + try: + uvr_manifest = fetch_json(MODEL_DOWNLOAD_MANIFEST_URL) + log_event( + "installer", + "downloading_model", + "uvr_model_manifest_loaded", + manifestUrl=MODEL_DOWNLOAD_MANIFEST_URL, + ) + append_activity_line(recent_lines, "Loaded UVR model manifest") + except Exception as exc: + uvr_manifest = None + log_event( + "installer", + "downloading_model", + "uvr_model_manifest_failed", + manifestUrl=MODEL_DOWNLOAD_MANIFEST_URL, + errorCode="model_manifest_fetch_failed", + exceptionType=type(exc).__name__, + exceptionMessage=str(exc), + ) + append_activity_line(recent_lines, "Could not load the UVR model manifest; continuing with bundled model metadata") + + return uvr_manifest, bundled_manifest + + +def build_simple_model_entry(model_name: str, model_type: str, files: list[str], source: str, friendly_name: str) -> dict[str, Any]: + ordered_files = list(dict.fromkeys(files)) + if model_name in ordered_files: + ordered_files = [model_name, *[entry for entry in ordered_files if entry != model_name]] + return { + "source": source, + "modelType": model_type, + "filename": model_name, + "friendlyName": friendly_name, + "downloadFiles": ordered_files, + } + + +def find_model_in_manifest(manifest: dict[str, Any], model_name: str, *, source: str) -> dict[str, Any] | None: + for key, models in manifest.items(): + if not key.endswith("_download_list") or not isinstance(models, dict): + continue + + model_type = key.removesuffix("_download_list").upper() + for friendly_name, value in models.items(): + if isinstance(value, str): + if value == model_name: + return build_simple_model_entry(model_name, model_type, [value], source, friendly_name) + continue + + if not isinstance(value, dict): + continue + + if model_type == "DEMUCS": + yaml_name = next((entry for entry in value.keys() if isinstance(entry, str) and entry.endswith(".yaml")), "") + download_files = list(value.values()) + if yaml_name and (yaml_name == model_name or model_name in download_files): + return build_simple_model_entry(model_name, model_type, download_files, source, friendly_name) + continue + + matching_model_key = next((entry for entry in value.keys() if entry == model_name), None) + matching_yaml_value = next((entry for entry in value.values() if entry == model_name), None) + if matching_model_key is None and matching_yaml_value is None: + continue + + model_file = matching_model_key or next(iter(value.keys())) + sidecar_file = value[model_file] + return build_simple_model_entry(model_name, model_type, [model_file, sidecar_file], source, friendly_name) + + return None + + +def build_download_urls(*, source: str, model_type: str, filename: str) -> list[str]: + if filename.startswith("http://") or filename.startswith("https://"): + return [filename] + + if source == "audio_separator": + return [ + f"{AUDIO_SEPARATOR_HF_REPO_URL_PREFIX}/{filename}", + f"{AUDIO_SEPARATOR_MODEL_REPO_URL_PREFIX}/{filename}", + ] + + if model_type == "MDX23C" and filename.endswith(".yaml"): + return [ + f"{UVR_MODEL_CONFIG_URL_PREFIX}/{filename}", + f"{AUDIO_SEPARATOR_MODEL_REPO_URL_PREFIX}/{filename}", + ] + + return [ + f"{UVR_MODEL_REPO_URL_PREFIX}/{filename}", + f"{AUDIO_SEPARATOR_MODEL_REPO_URL_PREFIX}/{filename}", + ] + + +def resolve_model_download_plan(runtime_root: Path, model_name: str, recent_lines: deque[str]) -> list[dict[str, Any]]: + uvr_manifest, bundled_manifest = load_model_manifests(runtime_root, recent_lines) + + candidates: list[dict[str, Any]] = [] + + bundled_match = find_model_in_manifest(bundled_manifest, model_name, source="audio_separator") + if bundled_match is not None: + candidates.append(bundled_match) + + if uvr_manifest is not None: + uvr_match = find_model_in_manifest(uvr_manifest, model_name, source="uvr") + if uvr_match is not None: + candidates.append(uvr_match) + + if not candidates: + raise ModelDownloadError( + f"OpenStudio could not resolve download information for the stem model {model_name}.", + error_code="model_manifest_missing", + ) + + selected = candidates[0] + log_event( + "installer", + "downloading_model", + "model_resolution_succeeded", + modelName=model_name, + source=selected["source"], + modelType=selected["modelType"], + friendlyName=selected["friendlyName"], + downloadFiles=selected["downloadFiles"], + ) + append_activity_line( + recent_lines, + f"Resolved {model_name} via {selected['source']} model metadata ({selected['friendlyName']})", + ) + + targets: list[dict[str, Any]] = [] + for entry in selected["downloadFiles"]: + filename = urlparse(entry).path.rsplit("/", 1)[-1] if entry.startswith("http") else entry + targets.append( + { + "filename": filename, + "urls": build_download_urls( + source=selected["source"], + model_type=selected["modelType"], + filename=entry, + ), + } + ) + + return targets + + +def _calculate_download_progress(file_index: int, file_count: int, file_progress: float) -> float: + normalized = (file_index + max(0.0, min(file_progress, 1.0))) / max(file_count, 1) + return MODEL_DOWNLOAD_PROGRESS_START + (MODEL_DOWNLOAD_PROGRESS_END - MODEL_DOWNLOAD_PROGRESS_START) * normalized + + +def download_file_with_retries( + *, + urls: list[str], + target_path: Path, + file_label: str, + file_index: int, + file_count: int, + recent_lines: deque[str], + install_source: str, + requires_external_python: bool, + python_detected: bool, + build_runtime_mode: str, + completed_bytes: int, +) -> int: + temp_path = target_path.with_suffix(target_path.suffix + ".part") + if temp_path.exists(): + temp_path.unlink() + + activity_line = f"Preparing to download {file_label}" + activity_lines = append_activity_line(recent_lines, activity_line) + emit_model_download_status( + progress=_calculate_download_progress(file_index, file_count, 0.0), + message=f"Connecting to the host for {file_label}", + step_label=f"Connecting to the host for {file_label}", + activity_lines=activity_lines, + bytes_downloaded=completed_bytes, + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + + for url_index, url in enumerate(urls): + append_activity_line(recent_lines, f"Attempting {file_label} from {url}") + log_event( + "installer", + "downloading_model", + "model_file_download_started", + fileName=target_path.name, + fileLabel=file_label, + url=url, + attempt=url_index + 1, + ) + + heartbeat_stop = threading.Event() + heartbeat_payload: dict[str, Any] = { + "progress": _calculate_download_progress(file_index, file_count, 0.0), + "message": f"Connecting to the host for {file_label}", + "stepLabel": f"Connecting to the host for {file_label}", + "activityLines": list(recent_lines), + "bytesDownloaded": completed_bytes, + "bytesTotal": 0, + } + heartbeat_lock = threading.Lock() + + def heartbeat_loop() -> None: + while not heartbeat_stop.wait(DOWNLOAD_HEARTBEAT_SECONDS): + with heartbeat_lock: + payload = dict(heartbeat_payload) + emit_model_download_status( + progress=payload["progress"], + message=payload["message"], + step_label=payload["stepLabel"], + activity_lines=payload["activityLines"], + bytes_downloaded=payload.get("bytesDownloaded", 0), + bytes_total=payload.get("bytesTotal", 0), + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + + heartbeat_thread = threading.Thread(target=heartbeat_loop, daemon=True) + heartbeat_thread.start() + + downloaded = 0 + total_size = 0 + try: + request = Request(url, headers={"User-Agent": "OpenStudio-AI-Installer/1.0"}) + with urlopen(request, timeout=DOWNLOAD_TIMEOUT_SECONDS) as response: + status_code = getattr(response, "status", response.getcode()) + if status_code != 200: + raise HTTPError(url, status_code, "Unexpected status code", response.headers, None) + + content_length_header = response.headers.get("Content-Length", "0") + total_size = int(content_length_header or "0") + with heartbeat_lock: + heartbeat_payload["message"] = f"Downloading {file_label}" + heartbeat_payload["stepLabel"] = f"Downloading {file_label}" + heartbeat_payload["bytesDownloaded"] = completed_bytes + heartbeat_payload["bytesTotal"] = completed_bytes + total_size if total_size > 0 else 0 + + with temp_path.open("wb") as handle: + while True: + chunk = response.read(DOWNLOAD_CHUNK_SIZE) + if not chunk: + break + handle.write(chunk) + downloaded += len(chunk) + + file_progress = downloaded / total_size if total_size > 0 else 0.0 + aggregate_downloaded = completed_bytes + downloaded + aggregate_total = completed_bytes + total_size if total_size > 0 else 0 + activity_lines = list(recent_lines) + progress = _calculate_download_progress(file_index, file_count, file_progress) + + with heartbeat_lock: + heartbeat_payload["progress"] = progress + heartbeat_payload["message"] = f"Downloading {file_label}" + heartbeat_payload["stepLabel"] = f"Downloading {file_label}" + heartbeat_payload["activityLines"] = activity_lines + heartbeat_payload["bytesDownloaded"] = aggregate_downloaded + heartbeat_payload["bytesTotal"] = aggregate_total + + emit_model_download_status( + progress=progress, + message=f"Downloading {file_label}", + step_label=f"Downloading {file_label}", + activity_lines=activity_lines, + bytes_downloaded=aggregate_downloaded, + bytes_total=aggregate_total, + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + + temp_path.replace(target_path) + final_size = target_path.stat().st_size + completed_total = completed_bytes + final_size + append_activity_line(recent_lines, f"Finished downloading {target_path.name}") + log_event( + "installer", + "downloading_model", + "model_file_download_succeeded", + fileName=target_path.name, + fileLabel=file_label, + url=url, + bytesDownloaded=final_size, + ) + emit_model_download_status( + progress=_calculate_download_progress(file_index + 1, file_count, 0.0), + message=f"Downloaded {file_label}", + step_label=f"Downloaded {file_label}", + activity_lines=list(recent_lines), + bytes_downloaded=completed_total, + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + return final_size + except HTTPError as exc: + append_activity_line(recent_lines, f"{target_path.name} was not available at {url} ({exc.code})") + log_event( + "installer", + "downloading_model", + "model_file_download_http_error", + fileName=target_path.name, + url=url, + httpStatus=exc.code, + ) + if temp_path.exists(): + temp_path.unlink() + if url_index == len(urls) - 1: + raise ModelDownloadError( + f"OpenStudio could not find {target_path.name} at the published model URLs.", + error_code="model_url_unavailable", + ) from exc + except (TimeoutError, socket.timeout) as exc: + if temp_path.exists(): + temp_path.unlink() + error_code = "model_download_timeout" if downloaded == 0 else "model_transfer_interrupted" + message = ( + f"OpenStudio timed out before {target_path.name} started downloading." + if downloaded == 0 + else f"OpenStudio lost connection while downloading {target_path.name}." + ) + append_activity_line(recent_lines, message) + raise ModelDownloadError(message, error_code=error_code) from exc + except (URLError, OSError) as exc: + if temp_path.exists(): + temp_path.unlink() + error_code = "model_download_timeout" if downloaded == 0 else "model_transfer_interrupted" + message = ( + f"OpenStudio could not start downloading {target_path.name}: {exc}." + if downloaded == 0 + else f"OpenStudio lost connection while downloading {target_path.name}: {exc}." + ) + append_activity_line(recent_lines, message) + if url_index == len(urls) - 1: + raise ModelDownloadError(message, error_code=error_code) from exc + finally: + heartbeat_stop.set() + heartbeat_thread.join(timeout=1) + + raise ModelDownloadError( + f"OpenStudio could not download {target_path.name}.", + error_code="model_download_failed", + ) + + +def run_step( + command: list[str], + *, + state: str, + progress: float, + description: str, + install_source: str, + requires_external_python: bool, + error_code: str, + python_detected: bool, + build_runtime_mode: str, + cwd: Path | None = None, + raise_on_error: bool = False, + step_label: str | None = None, + step_index: int = 0, + step_count: int = 0, + download_hint: str = "", + is_large_download: bool = False, +) -> None: + result = run_step_process( + command, + state=state, + progress=progress, + description=description, + install_source=install_source, + requires_external_python=requires_external_python, + error_code=error_code, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + cwd=cwd, + step_label=step_label, + step_index=step_index, + step_count=step_count, + download_hint=download_hint, + is_large_download=is_large_download, + ) + if result.returncode != 0: + if raise_on_error: + raise InstallerStepError( + f"{description} failed. See the install log for details.", + error_code=error_code, + progress=progress, + ) + fail( + f"{description} failed. See the install log for details.", + progress=progress, + error_code=error_code, + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + + +def normalize_installer_command(command: list[str]) -> list[str]: + normalized = list(command) + if ( + len(normalized) >= 4 + and normalized[1:4] == ["-m", "pip", "install"] + and "--no-cache-dir" not in normalized + ): + normalized.insert(4, "--no-cache-dir") + return normalized + + +def run_step_process( + command: list[str], + *, + state: str, + progress: float, + description: str, + install_source: str, + requires_external_python: bool, + error_code: str, + python_detected: bool, + build_runtime_mode: str, + cwd: Path | None = None, + raise_on_error: bool = False, + step_label: str | None = None, + step_index: int = 0, + step_count: int = 0, + download_hint: str = "", + is_large_download: bool = False, +) -> subprocess.CompletedProcess[str]: + command = normalize_installer_command(command) + log_event( + "installer", + state, + "step_start", + description=description, + command=command, + buildRuntimeMode=build_runtime_mode, + ) + emit( + state, + progress, + message=description, + stepLabel=step_label or description, + stepIndex=step_index, + stepCount=step_count, + downloadHint=download_hint, + isLargeDownload=is_large_download, + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + result = subprocess.run( + command, + cwd=str(cwd) if cwd else None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + ) + log_subprocess_output(result, " ".join(command)) + if result.returncode != 0: + log_event( + "installer", + state, + "step_failed", + description=description, + exitCode=result.returncode, + errorCode=error_code, + ) + else: + log_event( + "installer", + state, + "step_succeeded", + description=description, + exitCode=result.returncode, + ) + return result + + +def probe_windows_reused_runtime_health(runtime_python: Path) -> bool: + if platform.system() != "Windows": + return False + + health_check = ( + "from importlib import import_module, metadata\n" + "required = ['audio_separator', 'requests', 'certifi', 'charset_normalizer', 'idna', 'urllib3']\n" + "for module_name in required:\n" + " import_module(module_name)\n" + "version = metadata.version('audio-separator')\n" + "if version != '0.41.1':\n" + " raise SystemExit(f'unexpected audio-separator version: {version}')\n" + "print('ok')\n" + ) + result = subprocess.run( + [str(runtime_python), "-c", health_check], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + ) + log_subprocess_output(result, f"{runtime_python} -c ") + if result.returncode == 0: + log_event( + "installer", + "installing", + "reused_runtime_health_check_succeeded", + runtimePython=str(runtime_python), + ) + return True + + log_event( + "installer", + "installing", + "reused_runtime_health_check_failed", + runtimePython=str(runtime_python), + exitCode=result.returncode, + ) + return False + + +def classify_windows_runtime_lock_error( + result: subprocess.CompletedProcess[str], + runtime_root: Path, +) -> tuple[bool, str]: + if platform.system() != "Windows": + return False, "" + + output = "\n".join(part for part in (result.stderr, result.stdout) if part).strip() + if not output: + return False, "" + if "[WinError 5]" not in output and "Access is denied" not in output: + return False, "" + + locked_path = extract_windows_access_denied_path(output) + if locked_path is None: + return False, "" + + site_packages = runtime_root / "Lib" / "site-packages" + if not is_path_within(locked_path, site_packages): + return False, str(safe_resolve(locked_path)) + + return True, str(safe_resolve(locked_path)) + + +def repair_windows_reused_runtime( + runtime_python: Path, + runtime_root: Path, + bootstrap_python: Path, + runtime_python_version: tuple[int, int, int] | None, + bootstrap_python_version: tuple[int, int, int] | None, + *, + install_source: str, + requires_external_python: bool, + python_detected: bool, + build_runtime_mode: str, +) -> tuple[Path, tuple[int, int, int] | None, bool]: + if platform.system() != "Windows": + run_step( + [ + str(runtime_python), + "-m", + "pip", + "install", + "--ignore-installed", + "--no-deps", + *WINDOWS_REUSED_RUNTIME_REPAIR_PACKAGES, + ], + state="installing", + progress=0.3, + description="Repairing the AI tools environment", + install_source=install_source, + requires_external_python=requires_external_python, + error_code="dependency_bootstrap_failed", + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + return runtime_python, runtime_python_version, True + + if probe_windows_reused_runtime_health(runtime_python): + log_event( + "installer", + "installing", + "reused_runtime_repair_skipped", + runtimePython=str(runtime_python), + runtimePythonVersion=format_python_version(runtime_python_version), + ) + emit( + "installing", + 0.3, + message="Reused AI tools environment looks healthy", + stepLabel="Reused AI tools environment looks healthy", + activityLines=[ + f"Existing runtime uses Python {format_python_version(runtime_python_version)}.", + "audio-separator and the networking packages are already importable.", + ], + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + return runtime_python, runtime_python_version, True + + terminated_before_repair = terminate_windows_runtime_lock_holders(runtime_root) + if terminated_before_repair: + log_event( + "installer", + "installing", + "runtime_lock_holders_terminated_before_repair", + runtimeRoot=str(runtime_root), + terminatedProcesses=terminated_before_repair, + ) + + repair_result = run_step_process( + [ + str(runtime_python), + "-m", + "pip", + "install", + "--ignore-installed", + "--no-deps", + *WINDOWS_REUSED_RUNTIME_REPAIR_PACKAGES, + ], + state="installing", + progress=0.3, + description="Repairing the AI tools environment", + install_source=install_source, + requires_external_python=requires_external_python, + error_code="dependency_bootstrap_failed", + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + if repair_result.returncode == 0: + return runtime_python, runtime_python_version, True + + is_locked_runtime, locked_path = classify_windows_runtime_lock_error(repair_result, runtime_root) + if not is_locked_runtime: + fail( + "Repairing the AI tools environment failed. See the install log for details.", + progress=0.3, + error_code="dependency_bootstrap_failed", + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + + log_event( + "installer", + "installing", + "runtime_locked_file_detected", + runtimeRoot=str(runtime_root), + lockedPath=locked_path, + exitCode=repair_result.returncode, + ) + emit( + "installing", + 0.31, + message="A locked AI runtime file was detected. Rebuilding the AI tools environment.", + stepLabel="Rebuilding the AI tools environment on Windows", + activityLines=[ + "Windows denied access while pip was replacing a managed runtime file.", + f"Locked file: {locked_path}", + f"OpenStudio will rebuild only {runtime_root.name} and keep downloaded models/checkpoints.", + ], + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + + try: + terminated_runtime_processes = remove_tree_with_retries(runtime_root) + except OSError as exc: + fail( + f"OpenStudio could not rebuild the managed runtime because '{runtime_root}' could not be removed: {exc}", + state="installing", + progress=0.31, + error_code="runtime_rebuild_remove_failed", + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + + if terminated_runtime_processes: + log_event( + "installer", + "installing", + "runtime_lock_holders_terminated", + runtimeRoot=str(runtime_root), + terminatedProcesses=terminated_runtime_processes, + ) + + log_event( + "installer", + "creating_venv", + "runtime_locked_rebuild_started", + runtimeRoot=str(runtime_root), + bootstrapPython=str(bootstrap_python), + bootstrapPythonVersion=format_python_version(bootstrap_python_version), + lockedPath=locked_path, + ) + run_step( + [str(bootstrap_python), "-m", "venv", str(runtime_root)], + state="creating_venv", + progress=0.32, + description="Rebuilding the AI tools environment on Windows", + install_source=install_source, + requires_external_python=requires_external_python, + error_code="runtime_locked_rebuild_failed", + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + runtime_python = resolve_runtime_python(runtime_root) + runtime_python_version = read_python_version_info(runtime_python) or bootstrap_python_version + log_event( + "installer", + "creating_venv", + "runtime_locked_rebuild_succeeded", + runtimeRoot=str(runtime_root), + runtimePython=str(runtime_python), + runtimePythonVersion=format_python_version(runtime_python_version), + ) + return runtime_python, runtime_python_version, False + + +def ensure_runtime_pip( + runtime_python: Path, + *, + progress: float, + install_source: str, + requires_external_python: bool, + python_detected: bool, + build_runtime_mode: str, +) -> None: + probe = subprocess.run( + [str(runtime_python), "-m", "pip", "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8", errors="replace", ) - log_subprocess_output(result, " ".join(command)) - if result.returncode != 0: + log_subprocess_output(probe, f"{runtime_python} -m pip --version") + if probe.returncode == 0: log_event( "installer", - state, - "step_failed", - description=description, - exitCode=result.returncode, - errorCode=error_code, - ) - if raise_on_error: - raise InstallerStepError( - f"{description} failed. See the install log for details.", - error_code=error_code, - progress=progress, - ) - fail( - f"{description} failed. See the install log for details.", - progress=progress, - error_code=error_code, - installSource=install_source, - requiresExternalPython=requires_external_python, - pythonDetected=python_detected, - buildRuntimeMode=build_runtime_mode, + "installing", + "runtime_pip_available", + runtimePython=str(runtime_python), ) + return + log_event( "installer", - state, - "step_succeeded", - description=description, - exitCode=result.returncode, + "installing", + "runtime_pip_missing", + runtimePython=str(runtime_python), + exitCode=probe.returncode, + ) + run_step( + [str(runtime_python), "-m", "ensurepip", "--upgrade"], + state="installing", + progress=progress, + description="Repairing Python packaging support in the AI tools environment", + install_source=install_source, + requires_external_python=requires_external_python, + error_code="dependency_bootstrap_failed", + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, ) @@ -254,6 +1956,7 @@ def stream_step( download_hint: str = "", is_large_download: bool = False, ) -> None: + command = normalize_installer_command(command) log_event( "installer", state, @@ -354,13 +2057,22 @@ def verify_runtime( runtime_root: Path, *, require_audio_separator: bool, + require_music_generation: bool = False, install_source: str, requires_external_python: bool, python_detected: bool, build_runtime_mode: str, raise_on_error: bool = False, ) -> None: - if (runtime_root / "pyvenv.cfg").exists(): + # Downloaded OpenStudio runtimes must be relocatable, so they should not retain + # virtualenv markers or depend on external metadata that a plain dev venv lacks. + # The external-Python fallback intentionally creates a venv in runtime_root, and + # that layout is valid for the unbundled dev path. + requires_relocatable_runtime = ( + install_source == "downloadedRuntime" and not requires_external_python + ) + + if requires_relocatable_runtime and (runtime_root / "pyvenv.cfg").exists(): if raise_on_error: raise InstallerStepError( f"The extracted AI runtime at {runtime_root} still looks like a virtual environment and is not relocatable.", @@ -377,7 +2089,7 @@ def verify_runtime( buildRuntimeMode=build_runtime_mode, ) - if not (runtime_root / ".openstudio-ai-runtime.json").exists(): + if requires_relocatable_runtime and not (runtime_root / ".openstudio-ai-runtime.json").exists(): if raise_on_error: raise InstallerStepError( f"The extracted AI runtime at {runtime_root} is missing OpenStudio runtime metadata.", @@ -432,6 +2144,8 @@ def verify_runtime( import audio_separator.separator # noqa: F401 else: import pip # noqa: F401 + if require_music_generation: + import acestep # noqa: F401 except Exception as exc: write_log(f"[in-process verification error] {type(exc).__name__}: {exc}") log_event( @@ -460,7 +2174,14 @@ def verify_runtime( buildRuntimeMode=build_runtime_mode, ) - write_log("[in-process verification] " + ("import audio_separator.separator succeeded" if require_audio_separator else "pip import succeeded")) + write_log( + "[in-process verification] " + + ( + "runtime imports succeeded" + if require_audio_separator or require_music_generation + else "pip import succeeded" + ) + ) log_event( "installer", "verifying_base_runtime" if not require_audio_separator else "verifying_runtime", @@ -471,7 +2192,19 @@ def verify_runtime( return run_step( - [str(runtime_python), "-c", "import audio_separator.separator; print('ok')"] if require_audio_separator else [str(runtime_python), "-m", "pip", "--version"], + ( + [ + str(runtime_python), + "-c", + "import audio_separator.separator; import acestep; print('ok')", + ] + if require_audio_separator and require_music_generation + else [str(runtime_python), "-c", "import audio_separator.separator; print('ok')"] + if require_audio_separator + else [str(runtime_python), "-c", "import acestep; print('ok')"] + if require_music_generation + else [str(runtime_python), "-m", "pip", "--version"] + ), state="verifying_runtime" if require_audio_separator else "verifying_base_runtime", progress=0.65, description="Verifying the AI tools runtime" if require_audio_separator else "Verifying the AI tools base runtime", @@ -565,51 +2298,191 @@ def apply_backend_install_plan( step_label = str(step.get("stepLabel", description)).strip() or description download_hint = str(step.get("downloadHint", "")).strip() is_large_download = bool(step.get("isLargeDownload", False)) - if step_type == "pip_install": - command: list[str] = [str(runtime_python), "-m", "pip", "install", "--upgrade"] - index_url = str(step.get("indexUrl", "")).strip() - if index_url: - command += ["--index-url", index_url] - extra_index_urls = step.get("extraIndexUrls", []) - if isinstance(extra_index_urls, list): - for extra_index_url in extra_index_urls: - if str(extra_index_url).strip(): - command += ["--extra-index-url", str(extra_index_url).strip()] - packages = step.get("packages", []) - if not isinstance(packages, list) or not packages: - fail( - f"Backend install plan step {index} is missing packages.", - progress=0.74, - error_code="backend_install_plan_invalid", - installSource=install_source, - requiresExternalPython=requires_external_python, - pythonDetected=python_detected, - buildRuntimeMode=build_runtime_mode, + step_progress = min(0.74 + (0.12 * index / max(len(steps), 1)), 0.86) + step_error_code = str(step.get("errorCode", "backend_install_failed")).strip() or "backend_install_failed" + non_fatal_for_stem_separation = bool(step.get("nonFatalForStemSeparation", False)) + try: + if step_type == "pip_install": + command: list[str] = [str(runtime_python), "-m", "pip", "install", "--upgrade"] + index_url = str(step.get("indexUrl", "")).strip() + if index_url: + command += ["--index-url", index_url] + extra_index_urls = step.get("extraIndexUrls", []) + if isinstance(extra_index_urls, list): + for extra_index_url in extra_index_urls: + if str(extra_index_url).strip(): + command += ["--extra-index-url", str(extra_index_url).strip()] + packages = step.get("packages", []) + if not isinstance(packages, list) or not packages: + fail( + f"Backend install plan step {index} is missing packages.", + progress=0.74, + error_code="backend_install_plan_invalid", + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + command += [str(package) for package in packages] + stream_step( + command, + state="installing_backend", + progress=step_progress, + description=description, + install_source=install_source, + requires_external_python=requires_external_python, + error_code=step_error_code, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + cwd=runtime_root, + raise_on_error=True, + step_label=step_label, + step_index=index, + step_count=len(steps), + download_hint=download_hint, + is_large_download=is_large_download, ) - command += [str(package) for package in packages] - stream_step( - command, - state="installing_backend", - progress=min(0.74 + (0.12 * index / max(len(steps), 1)), 0.86), - description=description, - install_source=install_source, - requires_external_python=requires_external_python, - error_code="backend_install_failed", - python_detected=python_detected, - build_runtime_mode=build_runtime_mode, - cwd=runtime_root, - raise_on_error=True, - step_label=step_label, - step_index=index, - step_count=len(steps), - download_hint=download_hint, - is_large_download=is_large_download, - ) - elif step_type == "pip_uninstall": - packages = step.get("packages", []) - if not isinstance(packages, list) or not packages: + elif step_type == "pip_uninstall": + packages = step.get("packages", []) + if not isinstance(packages, list) or not packages: + fail( + f"Backend install plan step {index} is missing packages.", + progress=0.74, + error_code="backend_install_plan_invalid", + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + run_step( + [str(runtime_python), "-m", "pip", "uninstall", "-y", *[str(package) for package in packages]], + state="installing_backend", + progress=step_progress, + description=description, + install_source=install_source, + requires_external_python=requires_external_python, + error_code=step_error_code, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + cwd=runtime_root, + raise_on_error=True, + step_label=step_label, + step_index=index, + step_count=len(steps), + download_hint=download_hint, + is_large_download=is_large_download, + ) + elif step_type == "install_pinned_wheel": + wheel_url = str(step.get("url", "")).strip() + wheel_sha256 = str(step.get("sha256", "")).strip() + wheel_filename = str(step.get("fileName", "")).strip() + if not wheel_url or not wheel_sha256 or not wheel_filename: + fail( + f"Backend install plan step {index} is missing a pinned wheel URL, file name, or checksum.", + progress=0.74, + error_code="backend_install_plan_invalid", + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + install_pinned_wheel( + runtime_python, + runtime_root, + wheel_url=wheel_url, + wheel_sha256=wheel_sha256, + wheel_filename=wheel_filename, + description=description, + state="installing_backend", + progress=step_progress, + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + step_label=step_label, + step_index=index, + step_count=len(steps), + download_hint=download_hint, + is_large_download=is_large_download, + raise_on_error=True, + error_code=step_error_code, + ) + elif step_type == "hf_snapshot_pip_install": + repo_id = str(step.get("repoId", "")).strip() + repo_type = str(step.get("repoType", "model")).strip() or "model" + local_dir_name = str(step.get("localDirName", "")).strip() + if not repo_id or not local_dir_name: + fail( + f"Backend install plan step {index} is missing a Hugging Face repo id or localDirName.", + progress=0.74, + error_code="backend_install_plan_invalid", + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + source_root = runtime_root / local_dir_name + run_step( + [ + str(runtime_python), + "-c", + ( + "from pathlib import Path\n" + "from huggingface_hub import snapshot_download\n" + f"source_root = Path(r'{source_root}')\n" + "source_root.mkdir(parents=True, exist_ok=True)\n" + "snapshot_download(\n" + f" repo_id=r'{repo_id}',\n" + f" repo_type=r'{repo_type}',\n" + " local_dir=str(source_root),\n" + ")\n" + "print('ok')\n" + ), + ], + state="installing_backend", + progress=step_progress, + description=description, + install_source=install_source, + requires_external_python=requires_external_python, + error_code=step_error_code, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + cwd=runtime_root, + raise_on_error=True, + step_label=step_label, + step_index=index, + step_count=len(steps), + download_hint=download_hint, + is_large_download=is_large_download, + ) + run_step( + [ + str(runtime_python), + "-m", + "pip", + "install", + "--no-deps", + str(source_root), + ], + state="installing_backend", + progress=step_progress, + description=f"Installing {repo_id} into the managed runtime", + install_source=install_source, + requires_external_python=requires_external_python, + error_code=step_error_code, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + cwd=runtime_root, + raise_on_error=True, + step_label=step_label, + step_index=index, + step_count=len(steps), + download_hint=download_hint, + is_large_download=is_large_download, + ) + else: fail( - f"Backend install plan step {index} is missing packages.", + f"Unsupported backend install step type '{step_type}'.", progress=0.74, error_code="backend_install_plan_invalid", installSource=install_source, @@ -617,29 +2490,35 @@ def apply_backend_install_plan( pythonDetected=python_detected, buildRuntimeMode=build_runtime_mode, ) - run_step( - [str(runtime_python), "-m", "pip", "uninstall", "-y", *[str(package) for package in packages]], - state="installing_backend", - progress=min(0.74 + (0.12 * index / max(len(steps), 1)), 0.86), - description=description, - install_source=install_source, - requires_external_python=requires_external_python, - error_code="backend_install_failed", - python_detected=python_detected, - build_runtime_mode=build_runtime_mode, - cwd=runtime_root, - raise_on_error=True, - step_label=step_label, - step_index=index, - step_count=len(steps), - download_hint=download_hint, - is_large_download=is_large_download, + except InstallerStepError as exc: + if not non_fatal_for_stem_separation: + raise + + log_event( + "installer", + "installing_backend", + "backend_install_step_degraded", + backendRequested=backend_requested, + installPlanId=plan_id, + stepIndex=index, + stepType=step_type, + errorCode=exc.error_code, + errorMessage=exc.message, ) - else: - fail( - f"Unsupported backend install step type '{step_type}'.", - progress=0.74, - error_code="backend_install_plan_invalid", + emit( + "installing_backend", + step_progress, + message=description, + stepLabel=step_label, + stepIndex=index, + stepCount=len(steps), + downloadHint=download_hint, + isLargeDownload=is_large_download, + activityLines=[ + f"{description} could not be completed.", + exc.message, + "OpenStudio will keep going and mark music generation as incomplete if acceleration is still unavailable.", + ], installSource=install_source, requiresExternalPython=requires_external_python, pythonDetected=python_detected, @@ -657,11 +2536,14 @@ def apply_backend_install_plan( def probe_runtime( + runtime_python: Path, runtime_root: Path, models_dir: Path, model_name: str, *, acceleration_mode: str, + music_checkpoint_root: Path, + music_model_id: str, backend_requested: str, install_source: str, requires_external_python: bool, @@ -694,11 +2576,78 @@ def probe_runtime( except Exception: runtime_version = "" - report = probe_runtime_capabilities( - models_dir=str(models_dir), - model_name=model_name, - acceleration_mode=acceleration_mode, - ) + resolved_runtime_python = safe_resolve(runtime_python) + resolved_current_python = safe_resolve(Path(sys.executable)) + + if resolved_runtime_python == resolved_current_python: + report = probe_runtime_capabilities( + models_dir=str(models_dir), + model_name=model_name, + acceleration_mode=acceleration_mode, + music_checkpoint_root=str(music_checkpoint_root), + music_model_id=music_model_id, + ) + else: + probe_script = Path(__file__).with_name("ai_runtime_probe.py") + probe_command = [ + str(runtime_python), + str(probe_script), + "--models-dir", + str(models_dir), + "--model", + model_name, + "--music-gen-checkpoint-root", + str(music_checkpoint_root), + "--music-gen-model", + music_model_id, + "--acceleration-mode", + acceleration_mode, + ] + result = subprocess.run( + probe_command, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + cwd=runtime_root, + ) + if result.stdout: + for line in result.stdout.splitlines(): + write_log(line) + if result.stderr: + for line in result.stderr.splitlines(): + write_log(line) + + report_line = "" + for line in reversed(result.stdout.splitlines() if result.stdout else []): + candidate = line.strip() + if candidate.startswith("{") and candidate.endswith("}"): + report_line = candidate + break + + if report_line: + try: + report = json.loads(report_line) + except json.JSONDecodeError: + report = { + "runtimeReady": False, + "fallbackReason": "runtime probe returned invalid JSON", + "errorCode": "probe_output_invalid", + "supportedBackends": ["cpu"], + "selectedBackend": "cpu", + "backendDecisionTrace": ["runtime probe returned invalid JSON"], + "probeDurationMs": 0, + } + else: + report = { + "runtimeReady": False, + "fallbackReason": "runtime probe produced no JSON output", + "errorCode": "probe_output_missing", + "supportedBackends": ["cpu"], + "selectedBackend": "cpu", + "backendDecisionTrace": ["runtime probe produced no JSON output"], + "probeDurationMs": 0, + } report["runtimeVersion"] = runtime_version report["modelVersion"] = model_name @@ -749,43 +2698,372 @@ def probe_runtime( def resolve_fallback_backend_install_plan(runtime_root: Path, backend_requested: str) -> tuple[str, dict[str, Any]] | None: - if platform.system() != "Windows": - return None - normalized_backend = backend_requested.strip().lower() - if normalized_backend != "cuda": - return None - fallback_plan_path = runtime_root / "openstudio-ai-backend-plans" / "ai-runtime-install-plan-windows-directml.json" - if not fallback_plan_path.exists(): - return None + if platform.system() == "Windows": + # CUDA failed → try DirectML as fallback + if normalized_backend != "cuda": + return None + fallback_plan_path = ( + runtime_root / "openstudio-ai-backend-plans" / "ai-runtime-install-plan-windows-directml.json" + ) + if not fallback_plan_path.exists(): + return None + return ("directml", load_backend_install_plan(fallback_plan_path)) + + if platform.system() == "Linux": + # cuda/rocm failed → no further fallback (CPU base is already installed) + if normalized_backend not in ("cuda", "rocm"): + return None + plan_name = f"ai-runtime-install-plan-linux-{normalized_backend}.json" + fallback_plan_path = runtime_root / "openstudio-ai-backend-plans" / plan_name + if not fallback_plan_path.exists(): + return None + return (normalized_backend, load_backend_install_plan(fallback_plan_path)) + + return None + + +def bootstrap_runtime(runtime_root: Path, bootstrap_python: Path) -> Path: + install_source = "externalPython" + requires_external_python = True + python_detected = True + build_runtime_mode = "unbundled-dev" + reused_existing_runtime = False + runtime_python_version: tuple[int, int, int] | None = None + bootstrap_python_version = read_python_version_info(bootstrap_python) + force_runtime_rebuild = str(os.environ.get("OPENSTUDIO_FORCE_RUNTIME_REBUILD", "")).strip().lower() in { + "1", + "true", + "yes", + "on", + } + + if runtime_root.exists(): + existing_runtime_python = resolve_runtime_python(runtime_root) + if existing_runtime_python.exists(): + if platform.system() == "Windows": + existing_runtime_version = read_python_version_info(existing_runtime_python) + rebuild_windows_runtime = force_runtime_rebuild or ( + bootstrap_python_version is not None + and existing_runtime_version is not None + and existing_runtime_version[:2] != bootstrap_python_version[:2] + ) - return ("directml", load_backend_install_plan(fallback_plan_path)) + if rebuild_windows_runtime: + print("Rebuilding the AI tools environment on Windows") + log_event( + "installer", + "creating_venv", + "venv_rebuild_windows_reset", + runtimeRoot=str(runtime_root), + runtimePython=str(existing_runtime_python), + runtimePythonVersion=format_python_version(existing_runtime_version), + bootstrapPython=str(bootstrap_python), + bootstrapPythonVersion=format_python_version(bootstrap_python_version), + forceRuntimeRebuild=force_runtime_rebuild, + ) + emit( + "creating_venv", + 0.18, + message="Rebuilding the AI tools environment on Windows", + stepLabel="Rebuilding the AI tools environment on Windows", + activityLines=[ + "OpenStudio is rebuilding the managed runtime on Windows.", + f"Selected interpreter uses Python {format_python_version(bootstrap_python_version)}.", + ], + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + try: + terminated_runtime_processes = remove_tree_with_retries(runtime_root) + except OSError as exc: + fail( + f"OpenStudio could not rebuild the managed runtime because '{runtime_root}' is still locked or could not be removed: {exc}", + state="creating_venv", + progress=0.18, + error_code="runtime_rebuild_remove_failed", + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + if terminated_runtime_processes: + log_event( + "installer", + "creating_venv", + "runtime_lock_holders_terminated", + runtimeRoot=str(runtime_root), + terminatedProcesses=terminated_runtime_processes, + ) + run_step( + [str(bootstrap_python), "-m", "venv", str(runtime_root)], + state="creating_venv", + progress=0.2, + description="Creating the AI tools environment", + install_source=install_source, + requires_external_python=requires_external_python, + error_code="dependency_bootstrap_failed", + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + runtime_python = resolve_runtime_python(runtime_root) + runtime_python_version = read_python_version_info(runtime_python) + else: + reused_existing_runtime = True + print("Reusing the existing AI tools environment on Windows") + log_event( + "installer", + "creating_venv", + "venv_reused_windows", + runtimeRoot=str(runtime_root), + runtimePython=str(existing_runtime_python), + runtimePythonVersion=format_python_version(existing_runtime_version), + bootstrapPython=str(bootstrap_python), + bootstrapPythonVersion=format_python_version(bootstrap_python_version), + ) + emit( + "creating_venv", + 0.2, + message="Reusing the AI tools environment", + stepLabel="Reusing the AI tools environment", + activityLines=[ + f"Existing runtime uses Python {format_python_version(existing_runtime_version)}.", + f"Selected interpreter uses Python {format_python_version(bootstrap_python_version)}.", + ], + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + runtime_python = existing_runtime_python + runtime_python_version = existing_runtime_version + elif ( + bootstrap_python_version is not None + and (existing_runtime_version := read_python_version_info(existing_runtime_python)) is not None + and existing_runtime_version[:2] != bootstrap_python_version[:2] + ): + print( + "Rebuilding the AI tools environment to match the selected Python " + f"{format_python_version(bootstrap_python_version)} runtime" + ) + log_event( + "installer", + "creating_venv", + "venv_rebuild_required", + runtimeRoot=str(runtime_root), + runtimePython=str(existing_runtime_python), + runtimePythonVersion=format_python_version(existing_runtime_version), + bootstrapPython=str(bootstrap_python), + bootstrapPythonVersion=format_python_version(bootstrap_python_version), + ) + emit( + "creating_venv", + 0.18, + message="Rebuilding the AI tools environment for the selected Python version", + stepLabel="Rebuilding the AI tools environment for the selected Python version", + activityLines=[ + f"Existing runtime uses Python {format_python_version(existing_runtime_version)}.", + f"Selected interpreter uses Python {format_python_version(bootstrap_python_version)}.", + ], + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + try: + remove_tree_with_retries(runtime_root) + except OSError as exc: + fail( + f"OpenStudio could not rebuild the managed runtime because '{runtime_root}' could not be removed: {exc}", + state="creating_venv", + progress=0.18, + error_code="runtime_rebuild_remove_failed", + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + run_step( + [str(bootstrap_python), "-m", "venv", str(runtime_root)], + state="creating_venv", + progress=0.2, + description="Creating the AI tools environment", + install_source=install_source, + requires_external_python=requires_external_python, + error_code="dependency_bootstrap_failed", + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + runtime_python = resolve_runtime_python(runtime_root) + runtime_python_version = read_python_version_info(runtime_python) + else: + reused_existing_runtime = True + print("Reusing the existing AI tools environment") + log_event( + "installer", + "creating_venv", + "venv_reused", + runtimeRoot=str(runtime_root), + runtimePython=str(existing_runtime_python), + ) + emit( + "creating_venv", + 0.2, + message="Reusing the AI tools environment", + stepLabel="Reusing the AI tools environment", + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + runtime_python = existing_runtime_python + runtime_python_version = read_python_version_info(runtime_python) + else: + try: + remove_tree_with_retries(runtime_root) + except OSError as exc: + fail( + f"OpenStudio could not rebuild the managed runtime because '{runtime_root}' could not be removed: {exc}", + state="creating_venv", + progress=0.18, + error_code="runtime_rebuild_remove_failed", + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + run_step( + [str(bootstrap_python), "-m", "venv", str(runtime_root)], + state="creating_venv", + progress=0.2, + description="Creating the AI tools environment", + install_source=install_source, + requires_external_python=requires_external_python, + error_code="dependency_bootstrap_failed", + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + runtime_python = resolve_runtime_python(runtime_root) + runtime_python_version = read_python_version_info(runtime_python) + else: + run_step( + [str(bootstrap_python), "-m", "venv", str(runtime_root)], + state="creating_venv", + progress=0.2, + description="Creating the AI tools environment", + install_source=install_source, + requires_external_python=requires_external_python, + error_code="dependency_bootstrap_failed", + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + runtime_python = resolve_runtime_python(runtime_root) + runtime_python_version = read_python_version_info(runtime_python) + if reused_existing_runtime: + ensure_runtime_pip( + runtime_python, + progress=0.27, + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + site_packages = runtime_python.parent.parent / "Lib" / "site-packages" + broken_entries: list[Path] = [] + for candidate_name in ("acestep", "ace_step-0.2.0.dist-info"): + candidate = site_packages / candidate_name + if candidate.is_dir(): + try: + is_empty = not any(candidate.iterdir()) + except OSError: + is_empty = False + if is_empty: + broken_entries.append(candidate) + + if broken_entries: + for broken_entry in broken_entries: + shutil.rmtree(broken_entry, ignore_errors=True) + repaired_names = ", ".join(path.name for path in broken_entries) + print(f"Removed broken ACE-Step package entries: {repaired_names}") + log_event( + "installer", + "installing", + "broken_ace_step_package_removed", + removedEntries=[str(path) for path in broken_entries], + ) + emit( + "installing", + 0.28, + message="Repairing the ACE-Step package installation", + stepLabel="Repairing the ACE-Step package installation", + activityLines=[f"Removed broken ACE-Step package entries: {repaired_names}"], + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) -def bootstrap_runtime(runtime_root: Path, bootstrap_python: Path) -> Path: - install_source = "externalPython" - requires_external_python = True - python_detected = True - build_runtime_mode = "unbundled-dev" + if not is_music_generation_python_compatible(runtime_python_version): + incompatible_entries: list[Path] = [] + for candidate in site_packages.glob("ace_step*.dist-info"): + if candidate.is_dir(): + incompatible_entries.append(candidate) + legacy_package_dir = site_packages / "acestep" + if legacy_package_dir.is_dir(): + incompatible_entries.append(legacy_package_dir) + + if incompatible_entries: + for incompatible_entry in incompatible_entries: + shutil.rmtree(incompatible_entry, ignore_errors=True) + removed_names = ", ".join(path.name for path in incompatible_entries) + print(f"Removed incompatible ACE-Step package entries: {removed_names}") + log_event( + "installer", + "installing", + "incompatible_ace_step_package_removed", + removedEntries=[str(path) for path in incompatible_entries], + runtimePythonVersion=format_python_version(runtime_python_version), + ) + emit( + "installing", + 0.29, + message="Removing incompatible ACE-Step runtime entries", + stepLabel="Removing incompatible ACE-Step runtime entries", + activityLines=[ + f"Removed incompatible ACE-Step package entries: {removed_names}", + f"Music generation requires Python {MUSIC_GEN_REQUIRED_PYTHON[0]}.{MUSIC_GEN_REQUIRED_PYTHON[1]}.x; current runtime is Python {format_python_version(runtime_python_version)}.", + ], + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) - if runtime_root.exists(): - shutil.rmtree(runtime_root) + runtime_python, runtime_python_version, reused_existing_runtime = repair_windows_reused_runtime( + runtime_python, + runtime_root, + bootstrap_python, + runtime_python_version, + bootstrap_python_version, + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) - run_step( - [str(bootstrap_python), "-m", "venv", str(runtime_root)], - state="creating_venv", - progress=0.2, - description="Creating the AI tools environment", + ensure_runtime_pip( + runtime_python, + progress=0.34, install_source=install_source, requires_external_python=requires_external_python, - error_code="dependency_bootstrap_failed", python_detected=python_detected, build_runtime_mode=build_runtime_mode, ) - runtime_python = resolve_runtime_python(runtime_root) - run_step( [str(runtime_python), "-m", "pip", "install", "--upgrade", "pip", "setuptools", "wheel"], state="installing", @@ -799,10 +3077,16 @@ def bootstrap_runtime(runtime_root: Path, bootstrap_python: Path) -> Path: ) run_step( - [str(runtime_python), "-m", "pip", "install", *get_requirement_specifiers()], + [ + str(runtime_python), + "-m", + "pip", + "install", + *get_requirement_specifiers(python_version=runtime_python_version), + ], state="installing", progress=0.55, - description="Installing stem separation packages", + description="Installing AI audio packages", install_source=install_source, requires_external_python=requires_external_python, error_code="dependency_bootstrap_failed", @@ -810,15 +3094,148 @@ def bootstrap_runtime(runtime_root: Path, bootstrap_python: Path) -> Path: build_runtime_mode=build_runtime_mode, ) + if ( + platform.system() == "Windows" + and is_music_generation_python_compatible(runtime_python_version) + and is_windows_nvidia_machine() + ): + log_event( + "installer", + "installing", + "windows_cuda_music_overlay_selected", + runtimePython=str(runtime_python), + runtimePythonVersion=format_python_version(runtime_python_version), + pytorchIndexUrl=get_windows_cuda_pytorch_index_url(), + ) + emit( + "installing", + 0.57, + message="Preparing CUDA acceleration for ACE-Step music generation", + stepLabel="Preparing CUDA acceleration for ACE-Step music generation", + activityLines=[ + "NVIDIA hardware detected on this Windows machine.", + "Installing CUDA-capable PyTorch wheels for the ACE-Step runtime.", + ], + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + music_generation_setup_error: InstallerStepError | None = None + music_generation_requirements = get_music_generation_runtime_requirements( + python_version=runtime_python_version + ) + if music_generation_requirements: + try: + if ( + platform.system() == "Windows" + and is_windows_nvidia_machine() + and is_music_generation_python_compatible(runtime_python_version) + ): + apply_windows_cuda_pytorch_overlay( + runtime_python, + runtime_root, + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + install_windows_music_acceleration_stack( + runtime_python, + runtime_root, + state="installing", + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + raise_on_error=True, + error_code="music_acceleration_setup_failed", + ) + + run_step( + [ + str(runtime_python), + "-m", + "pip", + "install", + *music_generation_requirements, + ], + state="installing", + progress=0.6, + description="Installing ACE-Step 1.5 runtime dependencies", + install_source=install_source, + requires_external_python=requires_external_python, + error_code="music_generation_runtime_install_failed", + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + raise_on_error=True, + ) + + ace_step_source_root = download_music_generation_source( + runtime_python, + runtime_root, + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + raise_on_error=True, + ) + + run_step( + [ + str(runtime_python), + "-m", + "pip", + "install", + "--no-deps", + str(ace_step_source_root), + ], + state="installing", + progress=0.62, + description="Installing the ACE-Step 1.5 split backend", + install_source=install_source, + requires_external_python=requires_external_python, + error_code="music_generation_runtime_install_failed", + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + raise_on_error=True, + ) + except InstallerStepError as exc: + music_generation_setup_error = exc + log_event( + "installer", + "installing", + "music_generation_setup_degraded", + errorCode=exc.error_code, + errorMessage=exc.message, + ) + emit( + "installing", + 0.63, + message="Stem separation packages are ready, but music generation acceleration could not be fully installed", + stepLabel="Continuing with stem separation only", + activityLines=[ + "OpenStudio will finish stem separation setup and then report music generation as incomplete.", + exc.message, + ], + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + ) + return runtime_python def format_supported_python_range() -> str: - return "Python 3.10 through 3.13" + if platform.system() == "Windows": + return "Python 3.11" + return "Python 3.10 through 3.12" def download_model( runtime_python: Path, + runtime_root: Path, models_dir: Path, model_name: str, *, @@ -848,56 +3265,66 @@ def download_model( pythonDetected=python_detected, buildRuntimeMode=build_runtime_mode, ) - if safe_resolve(runtime_python) == safe_resolve(Path(sys.executable)): - try: - from audio_separator.separator import Separator - - write_log("[in-process model download] starting") - Separator( - output_dir=str(models_dir), - model_file_dir=str(models_dir), - use_directml=(platform.system() == "Windows" and backend_requested == "directml"), - ).load_model(model_filename=model_name) - write_log("[in-process model download] completed") - except Exception as exc: - write_log(f"[in-process model download error] {type(exc).__name__}: {exc}") - log_event( - "installer", - "downloading_model", - "model_download_failed", - errorCode="model_download_failed", - exceptionType=type(exc).__name__, - exceptionMessage=str(exc), + recent_lines: deque[str] = deque(maxlen=12) + completed_bytes = 0 + + try: + targets = resolve_model_download_plan(runtime_root, model_name, recent_lines) + for file_index, target in enumerate(targets): + target_path = models_dir / target["filename"] + file_label = ( + "the stem separation checkpoint" + if target_path.suffix.lower() in {".ckpt", ".onnx", ".pth", ".th"} + else "the model configuration" ) - fail( - f"Downloading the stem separation model failed: {type(exc).__name__}: {exc}", - progress=0.92, - error_code="model_download_failed", - installSource=install_source, - requiresExternalPython=requires_external_python, - pythonDetected=python_detected, - buildRuntimeMode=build_runtime_mode, + + if target_path.exists() and target_path.is_file(): + existing_size = target_path.stat().st_size + completed_bytes += existing_size + append_activity_line(recent_lines, f"Using cached {target_path.name}") + emit_model_download_status( + progress=_calculate_download_progress(file_index + 1, len(targets), 0.0), + message=f"Using cached {file_label}", + step_label=f"Using cached {file_label}", + activity_lines=list(recent_lines), + bytes_downloaded=completed_bytes, + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + continue + + completed_bytes += download_file_with_retries( + urls=target["urls"], + target_path=target_path, + file_label=file_label, + file_index=file_index, + file_count=len(targets), + recent_lines=recent_lines, + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + completed_bytes=completed_bytes, ) - else: - model_bootstrap = ( - "from audio_separator.separator import Separator; " - f"Separator(output_dir=r'{models_dir}', model_file_dir=r'{models_dir}', use_directml={platform.system() == 'Windows' and backend_requested == 'directml'}).load_model(model_filename=r'{model_name}')" + except ModelDownloadError as exc: + log_event( + "installer", + "downloading_model", + "model_download_failed", + errorCode=exc.error_code, + exceptionType=type(exc).__name__, + exceptionMessage=exc.message, ) - stream_step( - [str(runtime_python), "-c", model_bootstrap], - state="downloading_model", + fail( + exc.message, progress=0.92, - description="Downloading the stem separation model", - install_source=install_source, - requires_external_python=requires_external_python, - error_code="model_download_failed", - python_detected=python_detected, - build_runtime_mode=build_runtime_mode, - step_label="Downloading the stem separation model", - step_index=1, - step_count=1, - download_hint="The stem model download can take a while on slower connections.", - is_large_download=True, + error_code=exc.error_code, + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, ) model_path = models_dir / model_name @@ -928,6 +3355,294 @@ def download_model( return model_path +def download_music_gen_model( + runtime_python: Path, + music_gen_model: str, + music_gen_checkpoint_root: Path, + *, + install_source: str, + requires_external_python: bool, + python_detected: bool, + build_runtime_mode: str, +) -> None: + checkpoint_root = resolve_music_gen_checkpoint_root(str(music_gen_checkpoint_root)) + layout = get_music_generation_required_paths( + checkpoint_root=str(checkpoint_root), + model_name=music_gen_model, + ) + runtime_profiles = get_music_runtime_profiles(str(checkpoint_root)) + native_profile = runtime_profiles["profiles"].get("native-xl-turbo", {}) + missing_profile_assets = list(native_profile.get("missingAssets", [])) + log_event( + "installer", + "downloading_model", + "music_generation_prepare_start", + musicGenModel=music_gen_model, + musicGenCheckpointRoot=str(checkpoint_root), + musicGenModelRepo=DEFAULT_MUSIC_GEN_MODEL_REPO, + musicGenSharedRepo=DEFAULT_MUSIC_GEN_SHARED_REPO, + musicGenerationAvailableProfiles=runtime_profiles.get("availableProfiles", []), + musicGenerationMissingProfileAssets=missing_profile_assets, + ) + + checkpoint_root.mkdir(parents=True, exist_ok=True) + + if layout["layoutValid"] and not missing_profile_assets: + emit( + "downloading_model", + 0.97, + message="Pinned ACE-Step music generation model and runtime profiles are already installed", + stepLabel="Pinned ACE-Step music generation model and runtime profiles are already installed", + stepIndex=1, + stepCount=1, + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + musicGenerationModelId=music_gen_model, + musicGenerationCheckpointRoot=str(checkpoint_root), + musicGenerationLayoutValid=True, + musicGenerationAvailableProfiles=runtime_profiles.get("availableProfiles", []), + musicGenerationDefaultProfile=runtime_profiles.get("defaultProfile", ""), + ) + log_event( + "installer", + "downloading_model", + "music_generation_prepare_skipped_existing", + musicGenModel=music_gen_model, + musicGenCheckpointRoot=str(checkpoint_root), + ) + return + + hub_cache_dir = checkpoint_root.parent / MUSIC_GEN_HUB_CACHE_DIRNAME + hub_cache_dir.mkdir(parents=True, exist_ok=True) + + shared_repo_download_lines = [ + "snapshot_download(", + f" repo_id=r'{DEFAULT_MUSIC_GEN_SHARED_REPO}',", + " local_dir=str(checkpoint_root),", + " cache_dir=str(hub_cache_dir),", + " local_dir_use_symlinks=False,", + ] + if missing_profile_assets: + patterns = [ + f"{asset}/*" for asset in missing_profile_assets + ] + [ + f"{asset}/**" for asset in missing_profile_assets + ] + shared_repo_download_lines.append(f" allow_patterns={patterns!r},") + shared_repo_download_lines.append(")") + + bootstrap_lines = [ + "from pathlib import Path", + "from huggingface_hub import snapshot_download", + f"checkpoint_root = Path(r'{checkpoint_root}')", + f"hub_cache_dir = Path(r'{hub_cache_dir}')", + "checkpoint_root.mkdir(parents=True, exist_ok=True)", + "hub_cache_dir.mkdir(parents=True, exist_ok=True)", + *shared_repo_download_lines, + ] + use_legacy_wrapper = str(os.environ.get("OPENSTUDIO_USE_LEGACY_ACE_WRAPPER", "")).strip().lower() in { + "1", + "true", + "yes", + "on", + } + if use_legacy_wrapper and not (checkpoint_root / music_gen_model).exists(): + bootstrap_lines.extend( + [ + "snapshot_download(", + f" repo_id=r'{DEFAULT_MUSIC_GEN_MODEL_REPO}',", + f" local_dir=str(checkpoint_root / r'{music_gen_model}'),", + " cache_dir=str(hub_cache_dir),", + " local_dir_use_symlinks=False,", + ")", + ] + ) + bootstrap_lines.append("print('ok')") + bootstrap = "\n".join(bootstrap_lines) + + stream_step( + [str(runtime_python), "-c", bootstrap], + state="downloading_model", + progress=0.97, + description=( + "Downloading the pinned ACE-Step XL Turbo model" + if not missing_profile_assets + else "Downloading the pinned ACE-Step fast runtime profile assets" + ), + install_source=install_source, + requires_external_python=requires_external_python, + error_code="music_model_download_failed", + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + step_label=( + "Downloading the pinned ACE-Step XL Turbo model" + if not missing_profile_assets + else "Downloading the pinned ACE-Step Native XL Turbo profile assets" + ), + step_index=1, + step_count=1, + download_hint=( + "Downloading pinned ACE-Step repositories into the configured checkpoint root." + if not missing_profile_assets + else "Downloading the missing ACE-Step assets required for the Native XL Turbo profile." + ), + is_large_download=True, + ) + + layout = get_music_generation_required_paths( + checkpoint_root=str(checkpoint_root), + model_name=music_gen_model, + ) + runtime_profiles = get_music_runtime_profiles(str(checkpoint_root)) + if not layout["layoutValid"]: + fail( + "OpenStudio could not verify the pinned ACE-Step checkpoint layout after download.", + progress=0.97, + error_code="music_model_download_failed", + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + musicGenerationModelId=music_gen_model, + musicGenerationCheckpointRoot=str(checkpoint_root), + ) + imported_profile_targets: list[str] = [] + native_asset_import = { + "importedAssets": [], + "searchedDirs": [], + "foundSources": {}, + "missingAssets": [], + } + comfy_profile_import = { + "importedTargets": [], + "searchedDirs": [], + "foundSources": {}, + "missingTargets": [], + "error": "", + } + if "native-xl-turbo" not in runtime_profiles.get("availableProfiles", []): + native_asset_import = hydrate_native_split_assets(checkpoint_root) + imported_native_assets = list(native_asset_import.get("importedAssets", [])) + if imported_native_assets: + emit( + "downloading_model", + 0.972, + message="Importing local ACE-Step split-model assets", + stepLabel="Importing local ACE-Step split-model assets", + stepIndex=1, + stepCount=1, + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + musicGenerationModelId=music_gen_model, + musicGenerationCheckpointRoot=str(checkpoint_root), + musicGenerationImportedNativeAssets=imported_native_assets, + ) + log_event( + "installer", + "downloading_model", + "music_generation_imported_native_assets", + musicGenModel=music_gen_model, + musicGenCheckpointRoot=str(checkpoint_root), + importedAssets=imported_native_assets, + searchedDirs=native_asset_import.get("searchedDirs", []), + foundSources=native_asset_import.get("foundSources", {}), + ) + + runtime_profiles = get_music_runtime_profiles(str(checkpoint_root)) + + if use_legacy_wrapper and not (checkpoint_root / "acestep-5Hz-lm-4B" / "model.safetensors").exists(): + comfy_profile_import = hydrate_comfy_runtime_profiles(checkpoint_root) + imported_profile_targets = list(comfy_profile_import.get("importedTargets", [])) + if imported_profile_targets: + emit( + "downloading_model", + 0.975, + message="Importing local legacy ACE runtime profile assets", + stepLabel="Importing local legacy ACE runtime profile assets", + stepIndex=1, + stepCount=1, + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + musicGenerationModelId=music_gen_model, + musicGenerationCheckpointRoot=str(checkpoint_root), + musicGenerationImportedProfiles=imported_profile_targets, + ) + log_event( + "installer", + "downloading_model", + "music_generation_imported_local_profiles", + musicGenModel=music_gen_model, + musicGenCheckpointRoot=str(checkpoint_root), + importedTargets=imported_profile_targets, + searchedDirs=comfy_profile_import.get("searchedDirs", []), + foundSources=comfy_profile_import.get("foundSources", {}), + ) + elif comfy_profile_import.get("error"): + log_event( + "installer", + "downloading_model", + "music_generation_local_profile_import_skipped", + musicGenModel=music_gen_model, + musicGenCheckpointRoot=str(checkpoint_root), + error=comfy_profile_import.get("error", ""), + searchedDirs=comfy_profile_import.get("searchedDirs", []), + ) + + runtime_profiles = get_music_runtime_profiles(str(checkpoint_root)) + + if "native-xl-turbo" not in runtime_profiles.get("availableProfiles", []): + native_profile = runtime_profiles["profiles"].get("native-xl-turbo", {}) + missing_profile_assets = list(native_profile.get("missingAssets", [])) + searched_dirs = list( + dict.fromkeys( + list(native_asset_import.get("searchedDirs", [])) + + list(comfy_profile_import.get("searchedDirs", [])) + ) + ) + fail( + "OpenStudio installed the ACE-Step split backend, but could not verify the Native XL Turbo split-model assets afterward. " + + ( + "Still missing: " + ", ".join(missing_profile_assets) + ". " + if missing_profile_assets + else "" + ) + + ( + "Looked for local ACE-Step model files in: " + ", ".join(searched_dirs) + "." + if searched_dirs + else "No local ACE-Step model directories were configured or discovered." + ), + progress=0.97, + error_code="music_profile_assets_incomplete", + installSource=install_source, + requiresExternalPython=requires_external_python, + pythonDetected=python_detected, + buildRuntimeMode=build_runtime_mode, + musicGenerationModelId=music_gen_model, + musicGenerationCheckpointRoot=str(checkpoint_root), + musicGenerationAvailableProfiles=runtime_profiles.get("availableProfiles", []), + musicGenerationDefaultProfile=runtime_profiles.get("defaultProfile", ""), + musicGenerationMissingProfileAssets=missing_profile_assets, + musicGenerationLocalProfileImportSearchedDirs=searched_dirs, + musicGenerationLocalProfileImportFoundSources=found_sources, + ) + + log_event( + "installer", + "downloading_model", + "music_generation_prepare_succeeded", + musicGenModel=music_gen_model, + musicGenCheckpointRoot=str(checkpoint_root), + musicGenerationAvailableProfiles=runtime_profiles.get("availableProfiles", []), + importedTargets=imported_profile_targets, + ) + + def main() -> None: global LOG_PATH, SESSION_ID, RUNTIME_CANDIDATE, FALLBACK_ATTEMPTED global START_TIME_MONOTONIC @@ -936,6 +3651,8 @@ def main() -> None: parser.add_argument("--runtime-root", required=True, help="Directory for the prepared AI runtime") parser.add_argument("--models-dir", required=True, help="Directory where stem-separation models should be stored") parser.add_argument("--model", default=DEFAULT_MODEL_NAME, help="Stem-separation model filename") + parser.add_argument("--music-gen-model", default=DEFAULT_MUSIC_GEN_MODEL, help="Music-generation model identifier") + parser.add_argument("--music-gen-checkpoint-root", default="", help="Pinned ACE-Step checkpoint root") parser.add_argument("--bootstrap-with", help="Python executable to use for dev fallback bootstrapping") parser.add_argument("--verify-existing-runtime", action="store_true", help="Verify the already-prepared OpenStudio runtime and download the model") parser.add_argument("--log-path", help="Detailed installer log file path") @@ -948,6 +3665,7 @@ def main() -> None: runtime_root = Path(args.runtime_root).expanduser().resolve() models_dir = Path(args.models_dir).expanduser().resolve() + music_gen_checkpoint_root = resolve_music_gen_checkpoint_root(args.music_gen_checkpoint_root) bootstrap_python = Path(args.bootstrap_with).expanduser().resolve() if args.bootstrap_with else None LOG_PATH = Path(args.log_path).expanduser().resolve() if args.log_path else None SESSION_ID = args.session_id @@ -963,6 +3681,8 @@ def main() -> None: write_log(f"sys.executable={sys.executable}") write_log(f"runtime_root={runtime_root}") write_log(f"models_dir={models_dir}") + write_log(f"music_gen_checkpoint_root={music_gen_checkpoint_root}") + write_log(f"music_gen_model={args.music_gen_model}") if SESSION_ID: write_log(f"sessionId={SESSION_ID}") if RUNTIME_CANDIDATE: @@ -979,6 +3699,8 @@ def main() -> None: pythonExecutable=sys.executable, runtimeRoot=str(runtime_root), modelsDir=str(models_dir), + musicGenCheckpointRoot=str(music_gen_checkpoint_root), + musicGenModel=args.music_gen_model, verifyExistingRuntime=bool(args.verify_existing_runtime), backendRequested=args.backend_requested, backendInstallPlan=args.backend_install_plan, @@ -1003,6 +3725,7 @@ def main() -> None: runtime_python, runtime_root, require_audio_separator=backend_install_plan is None, + require_music_generation=False, install_source=install_source, requires_external_python=requires_external_python, python_detected=python_detected, @@ -1025,6 +3748,7 @@ def main() -> None: runtime_python, runtime_root, require_audio_separator=True, + require_music_generation=False, install_source=install_source, requires_external_python=requires_external_python, python_detected=python_detected, @@ -1032,10 +3756,13 @@ def main() -> None: raise_on_error=True, ) probe_runtime( + runtime_python, runtime_root, models_dir, args.model, acceleration_mode="auto", + music_checkpoint_root=music_gen_checkpoint_root, + music_model_id=args.music_gen_model, backend_requested=effective_backend_requested, install_source=install_source, requires_external_python=requires_external_python, @@ -1098,6 +3825,7 @@ def main() -> None: runtime_python, runtime_root, require_audio_separator=True, + require_music_generation=False, install_source=install_source, requires_external_python=requires_external_python, python_detected=python_detected, @@ -1105,10 +3833,13 @@ def main() -> None: raise_on_error=True, ) probe_runtime( + runtime_python, runtime_root, models_dir, args.model, acceleration_mode="auto", + music_checkpoint_root=music_gen_checkpoint_root, + music_model_id=args.music_gen_model, backend_requested=fallback_backend_requested, install_source=install_source, requires_external_python=requires_external_python, @@ -1157,7 +3888,14 @@ def main() -> None: buildRuntimeMode=build_runtime_mode, ) - if sys.version_info < FALLBACK_MIN_PYTHON or sys.version_info >= FALLBACK_MAX_PYTHON_EXCLUSIVE: + bootstrap_python_version = read_python_version_info(bootstrap_python) + if ( + bootstrap_python_version is not None + and ( + bootstrap_python_version[:2] < FALLBACK_MIN_PYTHON + or bootstrap_python_version[:2] >= FALLBACK_MAX_PYTHON_EXCLUSIVE + ) + ): fail( f"This dev fallback only supports {format_supported_python_range()}. Reinstall a proper release build or use a supported Python version.", state="pythonMissing", @@ -1168,10 +3906,23 @@ def main() -> None: buildRuntimeMode=build_runtime_mode, ) + preferred_music_gen_python = find_windows_python_311() + if preferred_music_gen_python is not None and safe_resolve(preferred_music_gen_python) != safe_resolve(bootstrap_python): + write_log(f"musicGenPreferredPython={preferred_music_gen_python}") + log_event( + "installer", + "checking", + "music_generation_python_selected", + selectedPython=str(preferred_music_gen_python), + previousPython=str(bootstrap_python), + ) + bootstrap_python = preferred_music_gen_python + bootstrap_python_version = read_python_version_info(bootstrap_python) + emit( "checking", 0.05, - message=f"Using Python {platform.python_version()}", + message=f"Using Python {format_python_version(bootstrap_python_version)}", installSource=install_source, requiresExternalPython=requires_external_python, pythonDetected=True, @@ -1182,6 +3933,7 @@ def main() -> None: runtime_python, runtime_root, require_audio_separator=True, + require_music_generation=False, install_source=install_source, requires_external_python=requires_external_python, python_detected=True, @@ -1190,6 +3942,7 @@ def main() -> None: model_path = download_model( runtime_python, + runtime_root, models_dir, args.model, backend_requested=effective_backend_requested, @@ -1199,11 +3952,35 @@ def main() -> None: build_runtime_mode=build_runtime_mode, ) + download_music_gen_model( + runtime_python, + args.music_gen_model, + music_gen_checkpoint_root, + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + + verify_runtime( + runtime_python, + runtime_root, + require_audio_separator=True, + require_music_generation=False, + install_source=install_source, + requires_external_python=requires_external_python, + python_detected=python_detected, + build_runtime_mode=build_runtime_mode, + ) + runtime_probe = probe_runtime( + runtime_python, runtime_root, models_dir, args.model, acceleration_mode="auto", + music_checkpoint_root=music_gen_checkpoint_root, + music_model_id=args.music_gen_model, backend_requested=effective_backend_requested, install_source=install_source, requires_external_python=requires_external_python, @@ -1211,16 +3988,42 @@ def main() -> None: build_runtime_mode=build_runtime_mode, ) + music_generation_ready = bool(runtime_probe.get("musicGenerationReady", False)) + music_generation_layout_valid = bool(runtime_probe.get("musicGenerationLayoutValid", False)) + music_generation_status_message = runtime_probe.get("musicGenerationStatusMessage", "") + music_generation_performance_ready = bool(runtime_probe.get("musicGenerationPerformanceReady", True)) + music_generation_performance_message = runtime_probe.get("musicGenerationPerformanceStatusMessage", "") + ready_message = ( + "AI tools are ready." + if music_generation_ready and music_generation_layout_valid and music_generation_performance_ready + else ( + "AI tools are installed, but music generation acceleration is incomplete." + if music_generation_ready and music_generation_layout_valid + else "Stem separation is ready, but music generation still needs the OpenStudio ACE split backend." + ) + ) + emit( "ready", 1.0, - message="AI tools are ready.", + message=ready_message, runtimeRoot=str(runtime_root), modelsDir=str(models_dir), modelPath=str(model_path), runtimeInstalled=True, modelInstalled=True, available=True, + musicGenerationReady=music_generation_ready, + musicGenerationLayoutValid=music_generation_layout_valid, + musicGenerationStatusMessage=music_generation_status_message, + musicGenerationFailureCode=runtime_probe.get("musicGenerationFailureCode", ""), + musicGenerationPerformanceReady=music_generation_performance_ready, + musicGenerationPerformanceStatusMessage=music_generation_performance_message, + musicGenerationModelId=runtime_probe.get("musicGenerationModelId", args.music_gen_model), + musicGenerationModelRepoId=runtime_probe.get("musicGenerationModelRepoId", DEFAULT_MUSIC_GEN_MODEL_REPO), + musicGenerationSharedRepoId=runtime_probe.get("musicGenerationSharedRepoId", DEFAULT_MUSIC_GEN_SHARED_REPO), + musicGenerationCheckpointRoot=runtime_probe.get("musicGenerationCheckpointRoot", str(music_gen_checkpoint_root)), + aceStepVersion=runtime_probe.get("aceStepVersion"), installSource=install_source, requiresExternalPython=requires_external_python, pythonDetected=python_detected, @@ -1232,6 +4035,8 @@ def main() -> None: modelVersion=runtime_probe.get("modelVersion", args.model), verificationMode="in-process" if safe_resolve(runtime_python) == safe_resolve(Path(sys.executable)) else "subprocess", restartRequired=bool(runtime_probe.get("restartRequired", False)), + statusWarning=music_generation_performance_message if music_generation_ready and music_generation_layout_valid and not music_generation_performance_ready else "", + statusWarningCode="music_generation_acceleration_incomplete" if music_generation_ready and music_generation_layout_valid and not music_generation_performance_ready else "", ) log_event( "installer", diff --git a/tools/mime/application-x-openstudio-project.xml b/tools/mime/application-x-openstudio-project.xml new file mode 100644 index 0000000..4913669 --- /dev/null +++ b/tools/mime/application-x-openstudio-project.xml @@ -0,0 +1,10 @@ + + + + OpenStudio Project + OpenStudio Project + + + + + diff --git a/tools/openstudio_ace_backend/__init__.py b/tools/openstudio_ace_backend/__init__.py new file mode 100644 index 0000000..58bf63e --- /dev/null +++ b/tools/openstudio_ace_backend/__init__.py @@ -0,0 +1,6 @@ +""" +OpenStudio-owned ACE split backend package. + +This directory vendors the minimal upstream runtime needed to execute the +packaged ACE-Step split graph without relying on an external app install. +""" diff --git a/tools/openstudio_ace_runner.py b/tools/openstudio_ace_runner.py new file mode 100644 index 0000000..5acf600 --- /dev/null +++ b/tools/openstudio_ace_runner.py @@ -0,0 +1,585 @@ +#!/usr/bin/env python3 +"""OpenStudio-owned ACE-Step 1.5 graph runner.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import platform +import sys +import wave +from pathlib import Path +from typing import Any + +import numpy as np + + +PHASE_RANGES: dict[str, tuple[float, float]] = { + "loading_text_encoders": (0.08, 0.24), + "encoding_conditioning": (0.24, 0.48), + "loading_diffusion_model": (0.48, 0.56), + "sampling": (0.56, 0.9), + "decoding_audio": (0.9, 0.97), + "writing_output": (0.97, 0.99), + "done": (1.0, 1.0), +} + +CURRENT_PHASE = "loading_text_encoders" +CURRENT_MESSAGE = "" + + +class OpenStudioAceLogFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + message = record.getMessage().lower() + suppressed_messages = ( + "comfy_kitchen", + "comfy kitchen", + "unable to parse pyproject.toml", + "pydantic-settings", + "pydantic_settings", + ) + return not any(item in message for item in suppressed_messages) + + +def install_openstudio_ace_log_filter() -> None: + root_logger = logging.getLogger() + root_logger.setLevel(logging.ERROR) + if not any(isinstance(item, OpenStudioAceLogFilter) for item in root_logger.filters): + root_logger.addFilter(OpenStudioAceLogFilter()) + for handler in root_logger.handlers: + if not any(isinstance(item, OpenStudioAceLogFilter) for item in handler.filters): + handler.addFilter(OpenStudioAceLogFilter()) + + +def emit(payload: dict[str, Any]) -> None: + sys.stdout.write(json.dumps(payload, ensure_ascii=False) + "\n") + sys.stdout.flush() + + +def emit_phase(phase: str, message: str, *, details: dict[str, Any] | None = None) -> None: + global CURRENT_PHASE, CURRENT_MESSAGE + CURRENT_PHASE = phase + CURRENT_MESSAGE = message + start, _end = PHASE_RANGES.get(phase, (0.0, 0.0)) + payload: dict[str, Any] = { + "kind": "phase", + "phase": phase, + "message": message, + "progress": start, + } + if details: + payload["details"] = details + emit(payload) + + +def progress_hook(current: int, total: int, preview: Any = None, **kwargs: Any) -> None: + start, end = PHASE_RANGES.get(CURRENT_PHASE, (0.0, 0.0)) + fraction = 0.0 + if total: + try: + fraction = max(0.0, min(1.0, float(current) / float(total))) + except Exception: + fraction = 0.0 + emit( + { + "kind": "progress", + "phase": CURRENT_PHASE, + "message": CURRENT_MESSAGE, + "current": current, + "total": total, + "fraction": fraction, + "progress": start + ((end - start) * fraction), + "nodeId": kwargs.get("node_id"), + } + ) + + +class OpenStudioAceExecutorServer: + def __init__(self) -> None: + self.client_id = None + self.last_node_id = None + self.messages: list[tuple[str, dict[str, Any]]] = [] + + def send_sync(self, event: str, data: dict[str, Any], client_id: Any | None = None) -> None: + self.messages.append((event, data)) + + +def choose_existing(base_dir: Path, relative_candidates: list[str]) -> str: + for relative in relative_candidates: + candidate = base_dir / relative + if candidate.is_file(): + return Path(relative).name + raise FileNotFoundError( + "OpenStudio ACE is missing required local model files. Run OpenStudio AI setup " + "while connected to the internet, then retry generation offline. Missing one of: " + + ", ".join(relative_candidates) + ) + + +def enable_offline_generation_mode() -> None: + os.environ.setdefault("HF_HUB_OFFLINE", "1") + os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") + os.environ.setdefault("HF_DATASETS_OFFLINE", "1") + + +def normalize_timesignature(value: Any) -> str: + raw = str(value or "").strip() + if not raw: + return "4" + if "/" in raw: + return raw.split("/", 1)[0].strip() + return raw + + +def seed_for_graph(value: Any) -> int: + try: + seed = int(value) + except Exception: + return 0 + return seed if seed >= 0 else 0 + + +def unwrap_executor_output(value: Any) -> Any: + while isinstance(value, (list, tuple)) and len(value) == 1: + value = value[0] + return value + + +def build_openstudio_ace_prompt( + *, + request: dict[str, Any], + unet_name: str, + clip_name1: str, + clip_name2: str, + vae_name: str, +) -> dict[str, dict[str, Any]]: + request_seed = seed_for_graph(request.get("seed", 0)) + return { + "104": { + "class_type": "UNETLoader", + "inputs": { + "unet_name": unet_name, + "weight_dtype": str(request.get("model_mode", "default") or "default"), + }, + }, + "105": { + "class_type": "DualCLIPLoader", + "inputs": { + "clip_name1": clip_name1, + "clip_name2": clip_name2, + "type": "ace", + "device": str(request.get("clip_device", "default") or "default"), + }, + }, + "106": { + "class_type": "VAELoader", + "inputs": { + "vae_name": vae_name, + }, + }, + "78": { + "class_type": "ModelSamplingAuraFlow", + "inputs": { + "model": ["104", 0], + "shift": float(request["shift"]), + }, + }, + "94": { + "class_type": "TextEncodeAceStepAudio1.5", + "inputs": { + "clip": ["105", 0], + "tags": str(request["prompt"]), + "lyrics": str(request["lyrics"]), + "seed": request_seed, + "bpm": int(request["bpm"]), + "duration": float(request["duration"]), + "timesignature": normalize_timesignature(request["timesignature"]), + "language": str(request["language"]), + "keyscale": str(request["keyscale"]), + "generate_audio_codes": bool(request["generate_audio_codes"]), + "cfg_scale": float(request["cfg_scale"]), + "temperature": float(request["temperature"]), + "top_p": float(request["top_p"]), + "top_k": int(request["top_k"]), + "min_p": float(request["min_p"]), + }, + }, + "98": { + "class_type": "EmptyAceStep1.5LatentAudio", + "inputs": { + "seconds": float(request["duration"]), + "batch_size": 1, + }, + }, + "47": { + "class_type": "ConditioningZeroOut", + "inputs": { + "conditioning": ["94", 0], + }, + }, + "3": { + "class_type": "KSampler", + "inputs": { + "model": ["78", 0], + "seed": request_seed, + "steps": int(request["inferenceSteps"]), + "cfg": float(request["guidance_scale"]), + "sampler_name": str(request.get("sampler_name", "euler") or "euler"), + "scheduler": str(request.get("scheduler", "simple") or "simple"), + "positive": ["94", 0], + "negative": ["47", 0], + "latent_image": ["98", 0], + "denoise": float(request.get("denoise", 1.0) or 1.0), + }, + }, + "18": { + "class_type": "VAEDecodeAudio", + "inputs": { + "samples": ["3", 0], + "vae": ["106", 0], + }, + }, + } + + +async def init_openstudio_ace_nodes(nodes_module: Any, backend_root: Path) -> None: + if hasattr(nodes_module, "load_custom_node"): + await nodes_module.load_custom_node( + str(backend_root / "comfy_extras" / "nodes_ace.py"), + module_parent="comfy_extras", + ) + await nodes_module.load_custom_node( + str(backend_root / "comfy_extras" / "nodes_audio.py"), + module_parent="comfy_extras", + ) + elif hasattr(nodes_module, "init_extra_nodes"): + await nodes_module.init_extra_nodes(init_custom_nodes=False, init_api_nodes=False) + + from comfy_extras.nodes_model_advanced import NODE_CLASS_MAPPINGS as advanced_node_mappings + + nodes_module.NODE_CLASS_MAPPINGS.update( + { + key: value + for key, value in advanced_node_mappings.items() + if key == "ModelSamplingAuraFlow" + } + ) + + +def get_backend_root() -> Path: + backend_root = Path(__file__).with_name("openstudio_ace_backend") / "vendor_runtime" + required = ( + backend_root / "nodes.py", + backend_root / "folder_paths.py", + backend_root / "comfy" / "sd.py", + backend_root / "comfy_extras" / "nodes_ace.py", + ) + missing = [str(path) for path in required if not path.exists()] + if missing: + raise FileNotFoundError( + "OpenStudio ACE split backend is missing packaged runtime files: " + + ", ".join(missing) + ) + return backend_root + + +def get_runtime_workspace(output_path: Path) -> Path: + local_app_data = os.environ.get("LOCALAPPDATA", "").strip() + if local_app_data: + workspace = ( + Path(local_app_data).expanduser().resolve() + / "OpenStudio" + / "runtime" + / "ace-split" + ) + else: + workspace = output_path.parent / ".openstudio-ace-split-runtime" + for folder_name in ("output", "temp", "input"): + (workspace / folder_name).mkdir(parents=True, exist_ok=True) + return workspace + + +def configure_vendor_paths(*, checkpoint_root: Path, workspace_root: Path) -> None: + import folder_paths + + folder_paths.add_model_folder_path( + "diffusion_models", str(checkpoint_root / "diffusion_models"), True + ) + folder_paths.add_model_folder_path( + "text_encoders", str(checkpoint_root / "text_encoders"), True + ) + folder_paths.add_model_folder_path("vae", str(checkpoint_root / "vae"), True) + folder_paths.set_output_directory(str(workspace_root / "output")) + folder_paths.set_temp_directory(str(workspace_root / "temp")) + folder_paths.set_input_directory(str(workspace_root / "input")) + folder_paths.filename_list_cache.clear() + + +def save_audio_to_wav(audio: dict[str, Any], output_path: Path) -> None: + waveform = audio["waveform"][0].cpu().numpy() + sample_rate = int(audio["sample_rate"]) + waveform = np.clip(waveform, -1.0, 1.0) + pcm = (waveform.T * 32767.0).astype(np.int16) + output_path.parent.mkdir(parents=True, exist_ok=True) + with wave.open(str(output_path), "wb") as handle: + handle.setnchannels(pcm.shape[1]) + handle.setsampwidth(2) + handle.setframerate(sample_rate) + handle.writeframes(pcm.tobytes()) + + +def build_runtime_fingerprint( + *, + backend_root: Path, + checkpoint_root: Path, + latent: dict[str, Any] | None = None, +) -> dict[str, Any]: + fingerprint: dict[str, Any] = { + "pythonExecutable": sys.executable, + "pythonVersion": sys.version, + "platform": platform.platform(), + "backendRoot": str(backend_root), + "checkpointRoot": str(checkpoint_root), + "decodeMode": "full", + } + try: + import torch + import comfy.model_management + + intermediate_dtype = comfy.model_management.intermediate_dtype() + intermediate_device = comfy.model_management.intermediate_device() + fingerprint.update( + { + "torchVersion": str(getattr(torch, "__version__", "")), + "cudaVersion": str(getattr(torch.version, "cuda", "") or ""), + "cudaAvailable": bool(torch.cuda.is_available()), + "intermediateDtype": str(intermediate_dtype), + "intermediateDevice": str(intermediate_device), + } + ) + if torch.cuda.is_available(): + device_index = torch.cuda.current_device() + fingerprint["cudaDeviceName"] = torch.cuda.get_device_name(device_index) + fingerprint["cudaDeviceIndex"] = int(device_index) + except Exception as exc: + fingerprint["runtimeFingerprintError"] = f"{type(exc).__name__}: {exc}" + + if isinstance(latent, dict): + latent_samples = latent.get("samples") + if hasattr(latent_samples, "dtype"): + fingerprint["latentDtype"] = str(latent_samples.dtype) + if hasattr(latent_samples, "device"): + fingerprint["latentDevice"] = str(latent_samples.device) + if hasattr(latent_samples, "shape"): + fingerprint["latentShape"] = list(latent_samples.shape) + return fingerprint + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run the OpenStudio ACE split graph") + parser.add_argument("--checkpoint-root", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--output", required=True) + return parser.parse_args() + + +def run_ace_split_request( + *, + checkpoint_root: Path, + request: dict[str, Any], + output_path: Path, +) -> int: + install_openstudio_ace_log_filter() + backend_root = get_backend_root() + workspace_root = get_runtime_workspace(output_path) + + sys.argv = [ + sys.argv[0], + "--disable-auto-launch", + "--disable-all-custom-nodes", + "--disable-api-nodes", + "--base-directory", + str(workspace_root), + ] + sys.path.insert(0, str(backend_root)) + + unet_name = choose_existing( + checkpoint_root, + [ + "diffusion_models/acestep_v1.5_xl_turbo_bf16.safetensors", + "diffusion_models/acestep_v1.5_turbo.safetensors", + ], + ) + clip_name1 = choose_existing( + checkpoint_root, + ["text_encoders/qwen_0.6b_ace15.safetensors"], + ) + clip_name2 = choose_existing( + checkpoint_root, + [ + "text_encoders/qwen_4b_ace15.safetensors", + "text_encoders/qwen_1.7b_ace15.safetensors", + ], + ) + vae_name = choose_existing( + checkpoint_root, + ["vae/ace_1.5_vae.safetensors"], + ) + enable_offline_generation_mode() + + configure_vendor_paths(checkpoint_root=checkpoint_root, workspace_root=workspace_root) + + import comfy.utils + import execution + import nodes + + comfy.utils.set_progress_bar_global_hook(progress_hook) + asyncio.run(init_openstudio_ace_nodes(nodes, backend_root)) + + try: + emit_phase( + "loading_diffusion_model", + "Preparing the OpenStudio ACE graph...", + details={ + "unetName": unet_name, + "clipName1": clip_name1, + "clipName2": clip_name2, + "vaeName": vae_name, + }, + ) + + emit_phase( + "encoding_conditioning", + "Encoding conditioning with TextEncodeAceStepAudio1.5...", + details={ + "generateAudioCodes": bool(request["generate_audio_codes"]), + "duration": float(request["duration"]), + "bpm": int(request["bpm"]), + "timesignature": normalize_timesignature(request["timesignature"]), + "language": request["language"], + "keyscale": request["keyscale"], + }, + ) + emit( + { + "kind": "diagnostic", + "phase": "encoding_conditioning", + "label": "runtime_fingerprint", + "fingerprint": build_runtime_fingerprint( + backend_root=backend_root, + checkpoint_root=checkpoint_root, + ), + } + ) + + emit_phase( + "sampling", + "Sampling the OpenStudio ACE latent graph...", + details={ + "steps": int(request["inferenceSteps"]), + "samplerCfg": float(request["guidance_scale"]), + "shift": float(request["shift"]), + "samplerName": str(request.get("sampler_name", "euler") or "euler"), + "scheduler": str(request.get("scheduler", "simple") or "simple"), + "denoise": float(request.get("denoise", 1.0) or 1.0), + }, + ) + + prompt = build_openstudio_ace_prompt( + request=request, + unet_name=unet_name, + clip_name1=clip_name1, + clip_name2=clip_name2, + vae_name=vae_name, + ) + server = OpenStudioAceExecutorServer() + executor = execution.PromptExecutor(server, cache_args={"ram": 0.0}) + executor.execute( + prompt, + str(request.get("requestId", "") or "openstudio-ace-request"), + extra_data={}, + execute_outputs=["18"], + ) + if not executor.success: + raise RuntimeError("OpenStudio ACE graph execution failed.") + + decoded_cache_entry = asyncio.run(executor.caches.outputs.get("18")) + if decoded_cache_entry is None or not getattr(decoded_cache_entry, "outputs", None): + raise RuntimeError("OpenStudio ACE graph did not produce decoded audio.") + audio = unwrap_executor_output(decoded_cache_entry.outputs[0]) + if not isinstance(audio, dict) or "waveform" not in audio or "sample_rate" not in audio: + raise RuntimeError("OpenStudio ACE graph produced an invalid decoded audio object.") + + latent_cache_entry = asyncio.run(executor.caches.outputs.get("3")) + latent_shape = None + if latent_cache_entry is not None and getattr(latent_cache_entry, "outputs", None): + latent_output = unwrap_executor_output(latent_cache_entry.outputs[0]) + if isinstance(latent_output, dict): + latent_samples = latent_output.get("samples") + if hasattr(latent_samples, "shape"): + latent_shape = list(latent_samples.shape) + + emit_phase( + "decoding_audio", + "Decoded audio with the OpenStudio ACE graph.", + details={ + "decodeMode": "executor", + "durationSeconds": float(request["duration"]), + }, + ) + + emit_phase("writing_output", "Writing generated audio to WAV...") + save_audio_to_wav(audio, output_path) + + emit( + { + "kind": "result", + "progress": 1.0, + "phase": "done", + "outputPath": str(output_path), + "assets": { + "unetName": unet_name, + "clipName1": clip_name1, + "clipName2": clip_name2, + "vaeName": vae_name, + }, + "conditioningSummary": { + "positiveEntries": 1, + "negativeEntries": 1, + "latentShape": latent_shape, + }, + } + ) + return 0 + except Exception: + raise + + +def main() -> int: + args = parse_args() + checkpoint_root = Path(args.checkpoint_root).expanduser().resolve() + request = json.loads(args.request_json) + output_path = Path(args.output).expanduser().resolve() + return run_ace_split_request( + checkpoint_root=checkpoint_root, + request=request, + output_path=output_path, + ) + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: + emit( + { + "kind": "error", + "phase": CURRENT_PHASE, + "message": str(exc), + "errorType": type(exc).__name__, + } + ) + raise diff --git a/tools/package-ai-runtime.ps1 b/tools/package-ai-runtime.ps1 index 5a06ecd..4f721aa 100644 --- a/tools/package-ai-runtime.ps1 +++ b/tools/package-ai-runtime.ps1 @@ -1,6 +1,6 @@ param( [Parameter(Mandatory = $true)] - [ValidateSet("windows", "macos")] + [ValidateSet("windows", "macos", "linux")] [string]$Platform, [Parameter(Mandatory = $true)] diff --git a/tools/package-linux-release.sh b/tools/package-linux-release.sh new file mode 100644 index 0000000..4385a4d --- /dev/null +++ b/tools/package-linux-release.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# package-linux-release.sh — Build a portable AppImage for OpenStudio on Linux. +# +# Usage: +# bash tools/package-linux-release.sh [version] [build_dir] +# +# Defaults: +# version = 0.0.0 +# build_dir = build-release-linux +# +# Prerequisites: +# - Release build already compiled (python build.py prod, or cmake manually) +# - wget available (for downloading linuxdeploy on first run) +# +set -euo pipefail + +VERSION="${1:-0.0.0}" +BUILD_DIR="${2:-build-release-linux}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(dirname "$SCRIPT_DIR")" + +BINARY="$ROOT_DIR/$BUILD_DIR/OpenStudio_artefacts/Release/OpenStudio" +APPDIR="$ROOT_DIR/dist/linux/OpenStudio.AppDir" +OUT_DIR="$ROOT_DIR/dist/linux" +TOOLS_DIR="$ROOT_DIR/tools" + +echo "=== OpenStudio Linux AppImage packaging ===" +echo "Version : $VERSION" +echo "Build dir : $BUILD_DIR" +echo "Binary : $BINARY" + +# ── Validate binary ──────────────────────────────────────────────────────────── +if [ ! -f "$BINARY" ]; then + echo "ERROR: Release binary not found at $BINARY" + echo "Run the release build first:" + echo " python build.py prod" + exit 1 +fi + +# ── Create AppDir skeleton ───────────────────────────────────────────────────── +rm -rf "$APPDIR" +mkdir -p "$APPDIR/usr/bin" "$APPDIR/usr/share/applications" "$APPDIR/usr/share/icons/hicolor/256x256/apps" "$OUT_DIR" + +# Copy main binary +cp "$BINARY" "$APPDIR/usr/bin/OpenStudio" +chmod +x "$APPDIR/usr/bin/OpenStudio" + +# Copy all runtime assets that sit beside the binary (models/, scripts/, ffmpeg, webui/, etc.) +RELEASE_ASSET_DIR="$(dirname "$BINARY")" +for item in "$RELEASE_ASSET_DIR"/*/; do + [ -d "$item" ] && cp -r "$item" "$APPDIR/usr/bin/" +done +for item in "$RELEASE_ASSET_DIR"/*; do + [ -f "$item" ] && [ "$(basename "$item")" != "OpenStudio" ] && cp "$item" "$APPDIR/usr/bin/" +done + +# ── Desktop entry + icon ─────────────────────────────────────────────────────── +cp "$TOOLS_DIR/OpenStudio.desktop" "$APPDIR/OpenStudio.desktop" +cp "$TOOLS_DIR/OpenStudio.desktop" "$APPDIR/usr/share/applications/OpenStudio.desktop" + +if [ -f "$ROOT_DIR/assets/icon-256x256.png" ]; then + cp "$ROOT_DIR/assets/icon-256x256.png" "$APPDIR/OpenStudio.png" + cp "$ROOT_DIR/assets/icon-256x256.png" "$APPDIR/usr/share/icons/hicolor/256x256/apps/OpenStudio.png" +fi + +# ── Download linuxdeploy if needed ───────────────────────────────────────────── +LINUXDEPLOY="$TOOLS_DIR/linuxdeploy-x86_64.AppImage" +if [ ! -f "$LINUXDEPLOY" ]; then + echo "Downloading linuxdeploy..." + wget -q --show-progress \ + -O "$LINUXDEPLOY" \ + "https://github.com/linuxdeploy/linuxdeploy/releases/latest/download/linuxdeploy-x86_64.AppImage" + chmod +x "$LINUXDEPLOY" +fi + +# ── Build AppImage ───────────────────────────────────────────────────────────── +export APPIMAGE_EXTRACT_AND_RUN=1 + +cd "$ROOT_DIR" +"$LINUXDEPLOY" \ + --appdir "$APPDIR" \ + --executable "$APPDIR/usr/bin/OpenStudio" \ + --desktop-file "$APPDIR/OpenStudio.desktop" \ + --icon-file "$APPDIR/OpenStudio.png" \ + --output appimage + +# linuxdeploy names the output using the AppDir Name field — move it to dist/ +APPIMAGE_PATTERN="OpenStudio-*.AppImage" +BUILT_APPIMAGE=$(ls $APPIMAGE_PATTERN 2>/dev/null | head -1 || true) +if [ -z "$BUILT_APPIMAGE" ]; then + # Fallback: linuxdeploy may have placed it differently + BUILT_APPIMAGE=$(ls *.AppImage 2>/dev/null | grep -i openstudio | head -1 || true) +fi + +if [ -n "$BUILT_APPIMAGE" ]; then + OUTPUT_NAME="OpenStudio-${VERSION}-linux-x86_64.AppImage" + mv "$BUILT_APPIMAGE" "$OUT_DIR/$OUTPUT_NAME" + echo "" + echo "AppImage created: dist/linux/$OUTPUT_NAME" +else + echo "WARNING: Could not locate built AppImage. Check linuxdeploy output above." + exit 1 +fi diff --git a/tools/package-macos-release.sh b/tools/package-macos-release.sh index a478505..1d1d66f 100644 --- a/tools/package-macos-release.sh +++ b/tools/package-macos-release.sh @@ -23,12 +23,20 @@ trap 'rm -rf "$STAGING_DIR"' EXIT APP_NAME="$(basename "$APP_PATH")" STAGED_APP="$STAGING_DIR/$APP_NAME" DMG_PATH="$OUTPUT_DIR/OpenStudio-macOS.dmg" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(dirname "$SCRIPT_DIR")" +ENTITLEMENTS_PATH="${MACOS_ENTITLEMENTS_PATH:-$ROOT_DIR/packaging/macos/OpenStudio.entitlements}" ditto "$APP_PATH" "$STAGED_APP" ln -s /Applications "$STAGING_DIR/Applications" if [[ -n "${MACOS_CODESIGN_IDENTITY:-}" ]]; then - codesign --force --deep --timestamp --options runtime --sign "$MACOS_CODESIGN_IDENTITY" "$STAGED_APP" + if [[ -f "$ENTITLEMENTS_PATH" ]]; then + codesign --force --deep --timestamp --options runtime --entitlements "$ENTITLEMENTS_PATH" --sign "$MACOS_CODESIGN_IDENTITY" "$STAGED_APP" + else + echo "macOS entitlements file not found at $ENTITLEMENTS_PATH; signing without extra runtime permissions." >&2 + codesign --force --deep --timestamp --options runtime --sign "$MACOS_CODESIGN_IDENTITY" "$STAGED_APP" + fi codesign --verify --deep --strict "$STAGED_APP" spctl --assess --type execute "$STAGED_APP" else diff --git a/tools/prepare-ai-runtime.ps1 b/tools/prepare-ai-runtime.ps1 index 2371357..7bdf709 100644 --- a/tools/prepare-ai-runtime.ps1 +++ b/tools/prepare-ai-runtime.ps1 @@ -1,6 +1,6 @@ param( [Parameter(Mandatory = $true)] - [ValidateSet("windows", "macos")] + [ValidateSet("windows", "macos", "linux")] [string]$Platform, [Parameter(Mandatory = $true)] @@ -17,10 +17,13 @@ param( [string]$RuntimeFamily = "", [Parameter(Mandatory = $false)] - [string]$WindowsTorchIndexUrl = "https://download.pytorch.org/whl/cu121", + [string]$WindowsTorchIndexUrl = "https://download.pytorch.org/whl/cu128", [Parameter(Mandatory = $false)] - [string[]]$WindowsTorchPackages = @("torch", "torchvision", "torchaudio"), + [string[]]$WindowsTorchPackages = @("torch==2.8.0", "torchvision==0.23.0", "torchaudio==2.8.0"), + + [Parameter(Mandatory = $false)] + [string]$WindowsAccelerationManifestPath = "tools/windows-ai-acceleration-manifest.json", [Parameter(Mandatory = $false)] [string]$ExpectedRuntimeVersion = "", @@ -29,7 +32,7 @@ param( [string]$StandaloneReleaseTag = "20260325", [Parameter(Mandatory = $false)] - [string]$StandalonePythonVersion = "3.10.20", + [string]$StandalonePythonVersion = "3.11.9", [Parameter(Mandatory = $false)] [ValidateSet("install_only", "install_only_stripped")] @@ -210,7 +213,7 @@ function Resolve-RuntimePython { function Get-TargetArchitecture { param( [Parameter(Mandatory = $true)] - [ValidateSet("windows", "macos")] + [ValidateSet("windows", "macos", "linux")] [string]$TargetPlatform, [Parameter(Mandatory = $false)] @@ -235,7 +238,7 @@ function Get-TargetArchitecture { function Get-StandaloneTargetTriple { param( [Parameter(Mandatory = $true)] - [ValidateSet("windows", "macos")] + [ValidateSet("windows", "macos", "linux")] [string]$TargetPlatform, [Parameter(Mandatory = $true)] @@ -244,10 +247,12 @@ function Get-StandaloneTargetTriple { ) switch ("$TargetPlatform/$TargetArchitecture") { - "windows/x64" { return "x86_64-pc-windows-msvc" } + "windows/x64" { return "x86_64-pc-windows-msvc" } "windows/arm64" { return "aarch64-pc-windows-msvc" } - "macos/x64" { return "x86_64-apple-darwin" } - "macos/arm64" { return "aarch64-apple-darwin" } + "macos/x64" { return "x86_64-apple-darwin" } + "macos/arm64" { return "aarch64-apple-darwin" } + "linux/x64" { return "x86_64-unknown-linux-gnu" } + "linux/arm64" { return "aarch64-unknown-linux-gnu" } default { throw "Unsupported runtime target combination '$TargetPlatform/$TargetArchitecture'." } } } @@ -343,7 +348,7 @@ function Expand-StandaloneArchive { function Get-PipInstallArguments { param( [Parameter(Mandatory = $true)] - [ValidateSet("windows", "macos")] + [ValidateSet("windows", "macos", "linux")] [string]$TargetPlatform, [Parameter(Mandatory = $true)] @@ -378,7 +383,7 @@ function Get-PipInstallArguments { function Resolve-RequirementsFile { param( [Parameter(Mandatory = $true)] - [ValidateSet("windows", "macos")] + [ValidateSet("windows", "macos", "linux")] [string]$TargetPlatform, [Parameter(Mandatory = $true)] @@ -393,6 +398,7 @@ function Resolve-RequirementsFile { $preferredFile = switch ($TargetPlatform) { "windows" { Resolve-AbsolutePath -PathValue "tools/ai-runtime-requirements-windows.txt" } "macos" { Resolve-AbsolutePath -PathValue "tools/ai-runtime-requirements-macos.txt" } + "linux" { Resolve-AbsolutePath -PathValue "tools/ai-runtime-requirements-linux.txt" } default { $resolved } } @@ -406,7 +412,7 @@ function Resolve-RequirementsFile { function Get-DefaultRuntimeFamily { param( [Parameter(Mandatory = $true)] - [ValidateSet("windows", "macos")] + [ValidateSet("windows", "macos", "linux")] [string]$TargetPlatform, [Parameter(Mandatory = $true)] @@ -428,6 +434,12 @@ function Get-DefaultRuntimeFamily { } "macos/arm64" { return "macos-arm64" } "macos/x64" { return "macos-x64" } + "linux/x64" { + if ($requirementsName -like "*linux-cuda*") { return "linux-cuda-x64" } + if ($requirementsName -like "*linux-rocm*") { return "linux-rocm-x64" } + return "linux-x64" + } + "linux/arm64" { return "linux-arm64" } default { return "$TargetPlatform-$TargetArchitecture" } } } @@ -493,6 +505,15 @@ function Optimize-RuntimePayload { if (-not (Test-Path $sitePackages)) { $sitePackages = Join-Path $RuntimeRootPath "lib/python3.10/site-packages" } + if (-not (Test-Path $sitePackages)) { + $pythonSitePackages = Get-ChildItem -LiteralPath (Join-Path $RuntimeRootPath "lib") -Directory -Filter "python*" -ErrorAction SilentlyContinue | + ForEach-Object { Join-Path $_.FullName "site-packages" } | + Where-Object { Test-Path $_ } | + Select-Object -First 1 + if ($pythonSitePackages) { + $sitePackages = $pythonSitePackages + } + } if (-not (Test-Path $sitePackages)) { return @@ -713,6 +734,14 @@ try { Invoke-LoggedStep -Description "Installing AI runtime requirements into standalone runtime" -Command $pipInstallArguments } + $windowsAccelerationManifest = $null + if ($Platform -eq "windows" -and -not [string]::IsNullOrWhiteSpace($WindowsAccelerationManifestPath)) { + $resolvedWindowsAccelerationManifestPath = Resolve-AbsolutePath -PathValue $WindowsAccelerationManifestPath + if (Test-Path $resolvedWindowsAccelerationManifestPath) { + $windowsAccelerationManifest = ConvertFrom-JsonCompat -Json (Get-Content -Path $resolvedWindowsAccelerationManifestPath -Raw) -AsHashtable + } + } + if ($resolvedRuntimeFamily -eq "windows-cuda-x64") { $windowsTorchInstallArguments = @( $runtimePython, @@ -724,6 +753,74 @@ try { $WindowsTorchIndexUrl ) + $WindowsTorchPackages Invoke-LoggedStep -Description "Installing CUDA-capable PyTorch wheels for Windows AI runtime" -Command $windowsTorchInstallArguments + + if ($null -ne $windowsAccelerationManifest) { + $targetNode = $windowsAccelerationManifest["target"] + $tritonNode = $targetNode["tritonWindows"] + $flashAttnNode = $targetNode["flashAttn"] + $tritonPackage = [string]$tritonNode["package"] + if (-not [string]::IsNullOrWhiteSpace($tritonPackage)) { + Invoke-LoggedStep -Description "Installing Triton for the Windows ACE-Step runtime" -Command @( + $runtimePython, + "-m", + "pip", + "install", + "--upgrade", + $tritonPackage + ) + } + + $flashAttnUrl = [string]$flashAttnNode["url"] + $flashAttnSha256 = ([string]$flashAttnNode["sha256"]).ToLowerInvariant() + $flashAttnFileName = [string]$flashAttnNode["fileName"] + if (-not [string]::IsNullOrWhiteSpace($flashAttnUrl) -and -not [string]::IsNullOrWhiteSpace($flashAttnFileName)) { + $pinnedWheelDir = Join-Path $resolvedRuntimeRoot ".openstudio-pinned-wheels" + New-Item -ItemType Directory -Force -Path $pinnedWheelDir | Out-Null + $flashAttnWheelPath = Join-Path $pinnedWheelDir $flashAttnFileName + Invoke-DownloadFile -Url $flashAttnUrl -DestinationPath $flashAttnWheelPath + $actualFlashAttnHash = (Get-FileHash -Algorithm SHA256 -Path $flashAttnWheelPath).Hash.ToLowerInvariant() + if ($actualFlashAttnHash -ne $flashAttnSha256) { + throw "Flash Attention wheel checksum mismatch. Expected '$flashAttnSha256' but found '$actualFlashAttnHash'." + } + + Invoke-LoggedStep -Description "Installing Flash Attention for the Windows ACE-Step runtime" -Command @( + $runtimePython, + "-m", + "pip", + "install", + "--no-deps", + $flashAttnWheelPath + ) + } + } + } + + if ($Platform -eq "windows" -and $resolvedRuntimeFamily -ne "windows-base-x64") { + $aceStepSourceRoot = Join-Path $resolvedRuntimeRoot "ace-step-v1.5-source" + Invoke-LoggedStep -Description "Preparing the ACE-Step 1.5 runtime source" -Command @( + $runtimePython, + "-c", + @" +from pathlib import Path +from huggingface_hub import snapshot_download +source_root = Path(r'$aceStepSourceRoot') +source_root.mkdir(parents=True, exist_ok=True) +snapshot_download( + repo_id=r'ACE-Step/Ace-Step-v1.5', + repo_type='space', + local_dir=str(source_root), +) +print('ok') +"@ + ) + Invoke-LoggedStep -Description "Installing the ACE-Step 1.5 runtime bridge into the standalone runtime" -Command @( + $runtimePython, + "-m", + "pip", + "install", + "--no-deps", + $aceStepSourceRoot + ) } Optimize-RuntimePayload -RuntimeRootPath $resolvedRuntimeRoot -TargetPlatform $Platform -ResolvedRuntimeFamily $resolvedRuntimeFamily diff --git a/tools/prepare-release-publish-assets.ps1 b/tools/prepare-release-publish-assets.ps1 index efcecc0..7853d0e 100644 --- a/tools/prepare-release-publish-assets.ps1 +++ b/tools/prepare-release-publish-assets.ps1 @@ -66,6 +66,7 @@ $copyPlan = @( @{ Source = "releases/stable/latest.json"; Target = "OpenStudio-release-stable-latest.json"; Required = $true }, @{ Source = "appcast/windows-stable.xml"; Target = "OpenStudio-appcast-windows-stable.xml"; Required = $true }, @{ Source = "appcast/macos-stable.xml"; Target = "OpenStudio-appcast-macos-stable.xml"; Required = $true }, + @{ Source = "appcast/linux-stable.xml"; Target = "OpenStudio-appcast-linux-stable.xml"; Required = $false }, @{ Source = "OpenStudio-checksums.txt"; Target = "OpenStudio-checksums.txt"; Required = $true }, @{ Source = "releases/ai-runtime/latest.json"; Target = "OpenStudio-ai-runtime-latest.json"; Required = $false }, @{ Source = "releases/ai-runtime/stable/latest.json"; Target = "OpenStudio-ai-runtime-stable-latest.json"; Required = $false } diff --git a/tools/probe_ai_worker_runtime.py b/tools/probe_ai_worker_runtime.py new file mode 100644 index 0000000..a90a970 --- /dev/null +++ b/tools/probe_ai_worker_runtime.py @@ -0,0 +1,491 @@ +#!/usr/bin/env python3 +""" +Local probe for the OpenStudio ACE-Step runtime bridge. + +This script can: +- launch the persistent worker and verify the ready handshake +- submit a framed generate request and inspect the ack +- run one-shot generation with optional LM-mismatch / decode-stall debug flags +""" + +from __future__ import annotations + +import argparse +import json +import os +import socket +import struct +import subprocess +import sys +import threading +import time +import uuid +from copy import deepcopy +from pathlib import Path +from queue import Empty, Queue +from typing import Any + +WORKER_PROTOCOL_VERSION = 2 +DEFAULT_TIMEOUT_SEC = 120.0 + + +def resolve_trace_root(explicit_root: str | None = None) -> Path: + override = (explicit_root or os.environ.get("OPENSTUDIO_AI_TRACE_ROOT", "")).strip() + if override: + return Path(override).expanduser().resolve() + + if sys.platform == "win32": + local_app_data = os.environ.get("LOCALAPPDATA", "").strip() + if local_app_data: + return Path(local_app_data).expanduser().resolve() / "OpenStudio" / "logs" / "ai" / "music-generation" + if sys.platform == "darwin": + return Path.home() / "Library" / "Application Support" / "OpenStudio" / "logs" / "ai" / "music-generation" + + xdg_state_home = os.environ.get("XDG_STATE_HOME", "").strip() + if xdg_state_home: + return Path(xdg_state_home).expanduser().resolve() / "OpenStudio" / "logs" / "ai" / "music-generation" + return Path.home() / ".local" / "state" / "OpenStudio" / "logs" / "ai" / "music-generation" + + +def recv_exact(connection: socket.socket, byte_count: int) -> bytes: + chunks: list[bytes] = [] + remaining = byte_count + while remaining > 0: + data = connection.recv(remaining) + if not data: + raise ConnectionError("Socket closed before the framed payload was fully received.") + chunks.append(data) + remaining -= len(data) + return b"".join(chunks) + + +def recv_framed_json(connection: socket.socket) -> dict[str, Any]: + header = recv_exact(connection, 4) + payload_length = struct.unpack(">I", header)[0] + payload = recv_exact(connection, payload_length) + parsed = json.loads(payload.decode("utf-8")) + if not isinstance(parsed, dict): + raise ValueError("Expected a JSON object response from the worker.") + parsed["_framedPayloadLength"] = payload_length + return parsed + + +def send_framed_json(connection: socket.socket, payload: dict[str, Any]) -> int: + encoded = json.dumps(payload, ensure_ascii=False).encode("utf-8") + connection.sendall(struct.pack(">I", len(encoded)) + encoded) + return len(encoded) + + +def stream_lines( + process: subprocess.Popen[str], + ready_queue: Queue[dict[str, Any]], + event_queue: Queue[dict[str, Any]], +) -> None: + assert process.stdout is not None + for raw_line in process.stdout: + line = raw_line.rstrip() + if not line: + continue + print(f"[worker] {line}") + try: + parsed = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict): + event_queue.put(parsed) + if parsed.get("event") == "ready": + ready_queue.put(parsed) + + +def build_params(args: argparse.Namespace) -> dict[str, Any]: + return { + "prompt": args.prompt, + "lyrics": args.lyrics, + "negativePrompt": "", + "seed": args.seed, + "duration": args.duration, + "bpm": args.bpm, + "timesignature": args.time_signature, + "language": args.language, + "keyscale": args.key_scale, + "cfg_scale": args.cfg_scale, + "temperature": args.temperature, + "top_p": args.top_p, + "top_k": args.top_k, + "min_p": args.min_p, + "runtimeProfile": args.runtime_profile, + "lmModel": args.lm_model, + "generationMode": args.generation_mode, + "auto_metas": args.auto_metas, + "guidance_scale": args.guidance_scale, + "inferenceSteps": args.steps, + "inferMethod": args.infer_method, + "debugForceLmShapeMismatch": args.force_lm_shape_mismatch, + "debugDecodeStallSeconds": args.debug_decode_stall_seconds, + } + + +def wait_for_ready(ready_queue: Queue[dict[str, Any]], timeout_sec: float) -> dict[str, Any]: + try: + return ready_queue.get(timeout=timeout_sec) + except Empty as exc: + raise TimeoutError("Timed out waiting for the worker ready handshake.") from exc + + +def wait_for_terminal_event( + event_queue: Queue[dict[str, Any]], + timeout_sec: float, +) -> dict[str, Any]: + deadline = time.monotonic() + timeout_sec + last_event: dict[str, Any] | None = None + while time.monotonic() < deadline: + remaining = max(0.1, deadline - time.monotonic()) + try: + event = event_queue.get(timeout=min(1.0, remaining)) + except Empty: + continue + last_event = event + if event.get("state") in {"done", "error", "cancelled"}: + return event + raise TimeoutError(f"Timed out waiting for a terminal worker event. Last event: {last_event}") + + +def make_output_path(args: argparse.Namespace, suffix: str) -> Path: + output_dir = Path(args.output_dir).expanduser().resolve() + output_dir.mkdir(parents=True, exist_ok=True) + return output_dir / f"probe_{suffix}_{int(time.time())}.wav" + + +def run_persistent_probe(args: argparse.Namespace) -> dict[str, Any]: + python = Path(args.python).expanduser().resolve() + script = Path(args.script).expanduser().resolve() + checkpoint_root = Path(args.checkpoint_root).expanduser().resolve() + request_id = str(uuid.uuid4()) + + command = [ + str(python), + str(script), + "--worker", + "--checkpoint-root", + str(checkpoint_root), + "--music-gen-model", + args.music_gen_model, + ] + print(f"[probe] launching worker: {' '.join(command)}") + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + + ready_queue: Queue[dict[str, Any]] = Queue() + event_queue: Queue[dict[str, Any]] = Queue() + ready: dict[str, Any] = {} + reader = threading.Thread( + target=stream_lines, + args=(process, ready_queue, event_queue), + name="ProbeWorkerReader", + daemon=True, + ) + reader.start() + + try: + ready = wait_for_ready(ready_queue, args.ready_timeout_sec) + print( + "[probe] ready handshake:", + json.dumps( + { + "port": ready.get("port"), + "pid": ready.get("pid"), + "protocolVersion": ready.get("protocolVersion"), + "scriptVersion": ready.get("scriptVersion"), + "scriptPath": ready.get("scriptPath"), + }, + indent=2, + ), + ) + + output_path = make_output_path(args, "persistent") + request_payload = { + "command": "generate", + "workflow": "text-to-music", + "params": json.dumps(build_params(args), ensure_ascii=False), + "output": str(output_path), + "requestId": request_id, + "protocolVersion": WORKER_PROTOCOL_VERSION, + "scriptVersion": str(ready.get("scriptVersion") or ""), + } + + with socket.create_connection(("127.0.0.1", int(ready["port"])), timeout=10.0) as connection: + payload_bytes = send_framed_json(connection, request_payload) + ack = recv_framed_json(connection) + + print( + "[probe] worker ack:", + json.dumps( + { + "requestId": ack.get("requestId"), + "accepted": ack.get("accepted"), + "protocolVersion": ack.get("protocolVersion"), + "scriptVersion": ack.get("scriptVersion"), + "pid": ack.get("pid"), + "framedPayloadLength": ack.get("framedPayloadLength"), + "sentPayloadBytes": payload_bytes, + }, + indent=2, + ), + ) + + terminal = wait_for_terminal_event(event_queue, args.timeout_sec) + print("[probe] terminal event:", json.dumps(terminal, indent=2)) + if output_path.exists(): + print(f"[probe] output file: {output_path}") + return terminal + finally: + try: + with socket.create_connection(("127.0.0.1", int(ready.get("port", 0))), timeout=3.0) as connection: + send_framed_json( + connection, + { + "command": "shutdown", + "requestId": str(uuid.uuid4()), + "protocolVersion": WORKER_PROTOCOL_VERSION, + "scriptVersion": str(ready.get("scriptVersion") or ""), + }, + ) + print("[probe] shutdown ack:", json.dumps(recv_framed_json(connection), indent=2)) + except Exception: + pass + process.terminate() + try: + process.wait(timeout=5.0) + except subprocess.TimeoutExpired: + process.kill() + + +def run_one_shot_probe(args: argparse.Namespace) -> int: + python = Path(args.python).expanduser().resolve() + script = Path(args.script).expanduser().resolve() + checkpoint_root = Path(args.checkpoint_root).expanduser().resolve() + request_id = str(uuid.uuid4()) + output_path = make_output_path(args, "oneshot") + + command = [ + str(python), + str(script), + "--workflow", + "text-to-music", + "--params", + json.dumps(build_params(args), ensure_ascii=False), + "--output", + str(output_path), + "--request-id", + request_id, + "--checkpoint-root", + str(checkpoint_root), + "--music-gen-model", + args.music_gen_model, + "--session-mode", + args.session_mode, + ] + print(f"[probe] launching one-shot: {' '.join(command)}") + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + assert process.stdout is not None + for raw_line in process.stdout: + print(f"[oneshot] {raw_line.rstrip()}") + return_code = process.wait() + print(f"[probe] one-shot exit code: {return_code}") + if output_path.exists(): + print(f"[probe] output file: {output_path}") + return return_code + + +def read_trace_terminal_summary(trace_path: Path) -> dict[str, Any]: + last_payload: dict[str, Any] = {} + if not trace_path.exists(): + return {} + for raw_line in trace_path.read_text(encoding="utf-8").splitlines(): + raw_line = raw_line.strip() + if not raw_line: + continue + try: + parsed = json.loads(raw_line) + except json.JSONDecodeError: + continue + if parsed.get("event") == "progress_payload": + payload = parsed.get("payload") + if isinstance(payload, dict): + last_payload = payload + return { + "requestId": last_payload.get("requestId"), + "state": last_payload.get("state"), + "failureKind": last_payload.get("failureKind"), + "tracePath": str(trace_path), + "summaryPath": str(trace_path.with_suffix(".txt")), + "lmStage": last_payload.get("lmStage"), + "lmBackend": last_payload.get("lmBackend"), + "runtimeProfile": last_payload.get("runtimeProfile"), + "lmModel": last_payload.get("lmModel"), + "error": last_payload.get("error") or last_payload.get("message"), + } + + +def run_trace_summary(args: argparse.Namespace) -> None: + trace_root = resolve_trace_root(args.trace_root) + if not trace_root.exists(): + print(f"[probe] trace root does not exist: {trace_root}") + return + + summaries: list[dict[str, Any]] = [] + for trace_path in sorted(trace_root.glob("*.jsonl"), key=lambda path: path.stat().st_mtime, reverse=True): + summary = read_trace_terminal_summary(trace_path) + if not summary: + continue + if summary.get("state") != "error": + continue + summaries.append(summary) + if len(summaries) >= args.summary_limit: + break + + print(f"[probe] trace root: {trace_root}") + if not summaries: + print("[probe] no failing traces found") + return + + for index, summary in enumerate(summaries, start=1): + print( + json.dumps( + { + "index": index, + "requestId": summary.get("requestId"), + "failureKind": summary.get("failureKind"), + "lmStage": summary.get("lmStage"), + "lmBackend": summary.get("lmBackend"), + "runtimeProfile": summary.get("runtimeProfile"), + "lmModel": summary.get("lmModel"), + "tracePath": summary.get("tracePath"), + "summaryPath": summary.get("summaryPath"), + "error": summary.get("error"), + }, + ensure_ascii=True, + ) + ) + + +def run_lm_diagnostics(args: argparse.Namespace) -> None: + lm_models = [ + item.strip() + for item in str(args.diagnostic_lm_models or "").split(",") + if item.strip() + ] or ["acestep-5Hz-lm-0.6B", "acestep-5Hz-lm-1.7B"] + + for auto_metas in (True, False): + for lm_model in lm_models: + run_args = argparse.Namespace(**deepcopy(vars(args))) + run_args.scenario = "persistent" + run_args.generation_mode = "lm_first" + run_args.auto_metas = auto_metas + run_args.lm_model = lm_model + print( + f"[probe] lm-diagnostics combo auto_metas={auto_metas} lm_model={lm_model}" + ) + try: + terminal = run_persistent_probe(run_args) + print( + "[probe] lm-diagnostics result:", + json.dumps( + { + "requestId": terminal.get("requestId"), + "state": terminal.get("state"), + "failureKind": terminal.get("failureKind"), + "lmStage": terminal.get("lmStage"), + "lmBackend": terminal.get("lmBackend"), + "tracePath": terminal.get("tracePath"), + }, + ensure_ascii=False, + ), + ) + except Exception as exc: + print( + f"[probe] lm-diagnostics combo failed auto_metas={auto_metas} " + f"lm_model={lm_model}: {type(exc).__name__}: {exc}" + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Probe the OpenStudio ACE-Step runtime bridge.") + parser.add_argument("--python", help="Python interpreter to use") + parser.add_argument("--script", default="tools/generate_music.py", help="Path to generate_music.py") + parser.add_argument("--checkpoint-root", help="Pinned ACE-Step checkpoint root") + parser.add_argument("--music-gen-model", default="acestep-v15-xl-turbo") + parser.add_argument("--output-dir", default=str(Path.cwd() / "tmp")) + parser.add_argument( + "--scenario", + choices=("persistent", "oneshot", "all", "lm-diagnostics", "summary"), + default="all", + ) + parser.add_argument("--session-mode", default="oneshot-probe") + parser.add_argument("--ready-timeout-sec", type=float, default=15.0) + parser.add_argument("--timeout-sec", type=float, default=DEFAULT_TIMEOUT_SEC) + parser.add_argument("--trace-root", help="Override the OpenStudio AI trace root") + parser.add_argument("--summary-limit", type=int, default=5) + parser.add_argument( + "--diagnostic-lm-models", + default="acestep-5Hz-lm-0.6B,acestep-5Hz-lm-1.7B", + help="Comma-separated LM checkpoint ids for lm-diagnostics mode", + ) + parser.add_argument("--prompt", default="Warm cinematic synthwave with driving drums and airy pads") + parser.add_argument("--lyrics", default="[Verse]\\nOpenStudio lights the scene\\n") + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--duration", type=float, default=30.0) + parser.add_argument("--bpm", type=int, default=120) + parser.add_argument("--time-signature", default="4/4") + parser.add_argument("--language", default="en") + parser.add_argument("--key-scale", default="C major") + parser.add_argument("--cfg-scale", type=float, default=2.0) + parser.add_argument("--temperature", type=float, default=0.85) + parser.add_argument("--top-p", type=float, default=0.9) + parser.add_argument("--top-k", type=int, default=0) + parser.add_argument("--min-p", type=float, default=0.0) + parser.add_argument("--steps", type=int, default=8) + parser.add_argument("--runtime-profile", default="native-xl-turbo") + parser.add_argument("--lm-model", default="auto") + parser.add_argument( + "--generation-mode", + choices=("lm_first", "dit_manual"), + default="lm_first", + ) + parser.add_argument("--auto-metas", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--guidance-scale", type=float, default=7.0) + parser.add_argument("--infer-method", choices=("ode", "sde"), default="ode") + parser.add_argument("--force-lm-shape-mismatch", action="store_true") + parser.add_argument("--debug-decode-stall-seconds", type=float, default=0.0) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if args.scenario == "summary": + run_trace_summary(args) + return + + if args.scenario in {"persistent", "oneshot", "all", "lm-diagnostics"}: + if not args.python: + raise SystemExit("--python is required for this scenario.") + if not args.checkpoint_root: + raise SystemExit("--checkpoint-root is required for this scenario.") + + if args.scenario == "lm-diagnostics": + run_lm_diagnostics(args) + return + + if args.scenario in {"persistent", "all"}: + run_persistent_probe(args) + if args.scenario in {"oneshot", "all"}: + run_one_shot_probe(args) + + +if __name__ == "__main__": + main() diff --git a/tools/run-audio-regression-suite.ps1 b/tools/run-audio-regression-suite.ps1 deleted file mode 100644 index 9abe37a..0000000 --- a/tools/run-audio-regression-suite.ps1 +++ /dev/null @@ -1,25 +0,0 @@ -param( - [switch]$SkipFrontend -) - -$ErrorActionPreference = "Stop" - -$repoRoot = Split-Path -Parent $PSScriptRoot -$frontendRoot = Join-Path $repoRoot "frontend" - -Write-Host "[regression] Building native target..." -cmake --build (Join-Path $repoRoot "build") --config Debug --target OpenStudio - -if (-not $SkipFrontend) { - Write-Host "[regression] Building frontend..." - Push-Location $frontendRoot - try { - npm.cmd run build - } - finally { - Pop-Location - } -} - -Write-Host "[regression] Build gates complete." -Write-Host "[regression] For the in-app backend suite, call nativeBridge.runAutomatedRegressionSuite() from the native frontend shell." diff --git a/tools/run-pitch-headless-regression.ps1 b/tools/run-pitch-headless-regression.ps1 new file mode 100644 index 0000000..b4a641a --- /dev/null +++ b/tools/run-pitch-headless-regression.ps1 @@ -0,0 +1,193 @@ +param( + [string]$JobPath, + [string]$SourceAudioPath, + [string]$ReferenceAudioPath, + [string]$NotesJsonPath, + [string]$FramesJsonPath, + [string]$TrackId = "pitch-regression-track-1", + [string]$ClipId = "pitch-regression-clip-1", + [ValidateSet("single", "full_clip_hq", "note_hq")] + [string]$RenderMode = "note_hq", + [double]$GlobalFormantSemitones = 0, + [double]$TargetShiftSemitones = [double]::NaN, + [double]$WindowStart = -1, + [double]$WindowEnd = -1, + [string]$Label = "pitch-headless-regression", + [string]$AppPath, + [string]$OutputRoot, + [ValidateSet("default", "pitch_only_vocal_source_filter_hq")] + [string]$RendererBranch = "default", + [int]$TimeoutSeconds = 180, + [switch]$SkipBuild +) + +$ErrorActionPreference = "Stop" + +$repoRoot = Split-Path -Parent $PSScriptRoot + +function Resolve-AppPath { + if ($AppPath -and (Test-Path $AppPath)) { + return (Resolve-Path $AppPath).Path + } + + $candidates = @( + (Join-Path $repoRoot "build\OpenStudio_artefacts\Debug\OpenStudio.exe"), + (Join-Path $repoRoot "build-check\OpenStudio_artefacts\Release\OpenStudio.exe"), + (Join-Path $repoRoot "build\OpenStudio_artefacts\Release\OpenStudio.exe") + ) + + foreach ($candidate in $candidates) { + if (Test-Path $candidate) { + return (Resolve-Path $candidate).Path + } + } + + throw "OpenStudio.exe not found. Build first or pass -AppPath." +} + +function New-RunDirectory { + $root = if ($OutputRoot) { $OutputRoot } else { Join-Path $repoRoot "tmp_pitch_runs" } + $safeLabel = ($Label -replace '[^A-Za-z0-9._-]', '_') + $stamp = Get-Date -Format "yyyyMMdd_HHmmss" + $dir = Join-Path $root "${stamp}_${safeLabel}" + New-Item -ItemType Directory -Force -Path $dir | Out-Null + return $dir +} + +function Convert-ToJsonLiteral { + param($Value) + + if ($null -eq $Value) { + return "null" + } + if ($Value -is [string]) { + $escaped = $Value.Replace('\', '\\').Replace('"', '\"').Replace("`r", '\r').Replace("`n", '\n').Replace("`t", '\t') + return '"' + $escaped + '"' + } + if ($Value -is [bool]) { + return $(if ($Value) { "true" } else { "false" }) + } + if ($Value -is [int] -or $Value -is [long] -or $Value -is [double] -or $Value -is [decimal] -or $Value -is [single]) { + return [string]::Format([System.Globalization.CultureInfo]::InvariantCulture, "{0}", $Value) + } + return ($Value | ConvertTo-Json -Compress -Depth 80) +} + +function New-HeadlessJob { + param([Parameter(Mandatory = $true)][string]$RunDirectory) + + if (-not $SourceAudioPath) { + throw "Provide -SourceAudioPath or -JobPath." + } + if (-not $NotesJsonPath) { + throw "Provide -NotesJsonPath or -JobPath." + } + if (-not (Test-Path $SourceAudioPath)) { + throw "Source audio not found: $SourceAudioPath" + } + if (-not (Test-Path $NotesJsonPath)) { + throw "Notes JSON not found: $NotesJsonPath" + } + if ($FramesJsonPath -and -not (Test-Path $FramesJsonPath)) { + throw "Frames JSON not found: $FramesJsonPath" + } + + $notesJson = (Get-Content -Raw -Path $NotesJsonPath).Trim() + $fields = [ordered]@{ + jobType = "render" + sourceAudioPath = (Resolve-Path $SourceAudioPath).Path + referenceAudioPath = if ($ReferenceAudioPath -and (Test-Path $ReferenceAudioPath)) { (Resolve-Path $ReferenceAudioPath).Path } else { $ReferenceAudioPath } + trackId = $TrackId + clipId = $ClipId + renderMode = $RenderMode + globalFormantSemitones = $GlobalFormantSemitones + resultJsonPath = (Join-Path $RunDirectory "pitch_regression_result.json") + label = $Label + } + + if (-not [double]::IsNaN($TargetShiftSemitones)) { + $fields.targetShiftSemitones = $TargetShiftSemitones + } + if ($WindowStart -ge 0 -and $WindowEnd -gt $WindowStart) { + $fields.windowStartSec = $WindowStart + $fields.windowEndSec = $WindowEnd + } + + $jobPathOut = Join-Path $RunDirectory "pitch_regression_job.json" + $items = @() + foreach ($key in $fields.Keys) { + $items += (' "{0}": {1}' -f $key, (Convert-ToJsonLiteral $fields[$key])) + } + $items += (' "notes": {0}' -f $notesJson) + if ($FramesJsonPath) { + $framesJson = (Get-Content -Raw -Path $FramesJsonPath).Trim() + $items += (' "frames": {0}' -f $framesJson) + } + $jobJson = "{`n" + ($items -join ",`n") + "`n}" + Set-Content -Path $jobPathOut -Value $jobJson -Encoding UTF8 + return $jobPathOut +} + +if (-not $SkipBuild) { + & cmake --build (Join-Path $repoRoot "build") --config Debug + if ($LASTEXITCODE -ne 0) { + throw "Debug build failed." + } +} + +$runDir = New-RunDirectory +if (-not $JobPath) { + $JobPath = New-HeadlessJob -RunDirectory $runDir +} else { + if (-not (Test-Path $JobPath)) { + throw "Job file not found: $JobPath" + } + $JobPath = (Resolve-Path $JobPath).Path +} + +$resolvedAppPath = Resolve-AppPath +$processStartInfo = [System.Diagnostics.ProcessStartInfo]::new() +$processStartInfo.FileName = $resolvedAppPath +$processStartInfo.Arguments = "--pitch-regression-headless `"$JobPath`"" +$processStartInfo.UseShellExecute = $false +$processStartInfo.CreateNoWindow = $true +$processStartInfo.WindowStyle = [System.Diagnostics.ProcessWindowStyle]::Hidden +$processStartInfo.Environment["OPENSTUDIO_PITCH_DEBUG"] = "1" +$processStartInfo.Environment["OPENSTUDIO_PITCH_APP_FINAL_CAPTURE_DISABLE"] = "1" +if ($RendererBranch -and $RendererBranch -ne "default") { + $processStartInfo.Environment["OPENSTUDIO_PITCH_RENDERER_BRANCH"] = $RendererBranch +} + +$process = [System.Diagnostics.Process]::Start($processStartInfo) +if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { + $process.Kill() + throw "Headless pitch regression timed out after $TimeoutSeconds seconds." +} +if ($process.ExitCode -ne 0) { + Write-Warning "Headless pitch regression exited with code $($process.ExitCode). Reading result if available." +} + +$job = Get-Content -Raw -Path $JobPath | ConvertFrom-Json +$resultPath = $job.resultJsonPath +if (-not (Test-Path $resultPath)) { + throw "Headless pitch regression did not write result JSON: $resultPath" +} + +$result = Get-Content -Raw -Path $resultPath | ConvertFrom-Json +Write-Host "Headless pitch regression result: $resultPath" +Write-Host "Objective gate status: $($result.objectiveGateStatus)" +Write-Host "Subjective quality: $($result.subjectiveQuality)" +Write-Host "Completion claim: $($result.completionClaim)" +if ($result.outputFile) { + Write-Host "Output file: $($result.outputFile)" +} +if ($result.checks) { + $result.checks | ForEach-Object { + Write-Host ("[{0}] {1} - {2}" -f $_.status, $_.id, $_.detail) + } +} + +if ($result.objectiveGateStatus -eq "fail") { + exit 2 +} +exit 0 diff --git a/tools/run-windows-rc.ps1 b/tools/run-windows-rc.ps1 index c610c18..d7774d9 100644 --- a/tools/run-windows-rc.ps1 +++ b/tools/run-windows-rc.ps1 @@ -102,8 +102,12 @@ function Invoke-StartupSelfTest { Remove-Item -LiteralPath $ReportPath -Force -ErrorAction SilentlyContinue } - & $ExePath --startup-self-test --report $ReportPath - if ($LASTEXITCODE -ne 0) { + $process = Start-Process ` + -FilePath $ExePath ` + -ArgumentList @("--startup-self-test", "--report", "`"$ReportPath`"") ` + -Wait ` + -PassThru + if ($process.ExitCode -ne 0) { $reportContent = if (Test-Path $ReportPath) { Get-Content -LiteralPath $ReportPath -Raw } else { "No self-test report was written." } throw "Startup self-test failed for '$ExePath'.`n$reportContent" } diff --git a/tools/setup-linux-prereqs.sh b/tools/setup-linux-prereqs.sh new file mode 100644 index 0000000..6b55f37 --- /dev/null +++ b/tools/setup-linux-prereqs.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# setup-linux-prereqs.sh — Install system packages required to build and run OpenStudio on Ubuntu/Debian. +# Run once before your first build: bash tools/setup-linux-prereqs.sh +set -euo pipefail + +echo "=== OpenStudio Linux prerequisites setup ===" + +sudo apt-get update + +sudo apt-get install -y \ + build-essential \ + cmake \ + ninja-build \ + pkg-config \ + git \ + \ + libasound2-dev \ + libjack-jackd2-dev \ + \ + libwebkit2gtk-4.1-dev \ + libgtk-3-dev \ + \ + libgl1-mesa-dev \ + libglu1-mesa-dev \ + libfreetype6-dev \ + libfontconfig1-dev \ + libcurl4-openssl-dev \ + \ + libx11-dev \ + libxext-dev \ + libxrandr-dev \ + libxi-dev \ + libxinerama-dev \ + libxcursor-dev \ + libxcomposite-dev \ + \ + ffmpeg \ + python3 \ + python3-venv + +# webkit2gtk-4.1 may not be available on Ubuntu 22.04 — fall back to 4.0 +if ! dpkg -l libwebkit2gtk-4.1-dev &>/dev/null; then + echo "webkit2gtk-4.1 not found, installing 4.0 fallback..." + sudo apt-get install -y libwebkit2gtk-4.0-dev +fi + +echo "" +echo "All prerequisites installed." +echo "You can now run: python build.py dev --run" diff --git a/tools/spectral_quality_analysis.py b/tools/spectral_quality_analysis.py deleted file mode 100644 index 175afbb..0000000 --- a/tools/spectral_quality_analysis.py +++ /dev/null @@ -1,377 +0,0 @@ -""" -Spectral quality analysis: original vs RePitch vs Studio13 SMS engine -""" - -import numpy as np -import soundfile as sf -from scipy import signal -from scipy.fft import rfft, rfftfreq -import warnings -warnings.filterwarnings('ignore') - -# ─── File loading ──────────────────────────────────────────────────────────── - -FILE_ORIG = r"D:\test projects\untitled3.wav" -FILE_REPITCH = r"D:\test projects\untitled3-001.wav" -FILE_S13 = r"D:\test projects\Untitled Project26.wav" - -def load_mono(path): - data, sr = sf.read(path, always_2d=True) - mono = data.mean(axis=1) - print(f" {path.split(chr(92))[-1]}: {len(mono)/sr:.3f}s sr={sr} channels={data.shape[1]}") - return mono, sr - -print("=== Loading files ===") -orig, sr_o = load_mono(FILE_ORIG) -rep, sr_r = load_mono(FILE_REPITCH) -s13, sr_s = load_mono(FILE_S13) - -# Normalise all to same sample rate (use original as reference) -assert sr_o == sr_r == sr_s, f"Sample rates differ: {sr_o} {sr_r} {sr_s}" -SR = sr_o -print(f" Common SR: {SR} Hz\n") - -# S13 amplitude correction (normalize was ON → 3.48× higher) -S13_AMP_SCALE = 1.0 / 3.48 -s13_corrected = s13 * S13_AMP_SCALE - -# ─── Helpers ───────────────────────────────────────────────────────────────── - -def frame(sig, sr, t_start, t_end): - return sig[int(t_start*sr):int(t_end*sr)] - -def hnr_autocorr(sig, sr, frame_ms=30, hop_ms=10): - """ - HNR via autocorrelation (Praat-style). - Returns median HNR in dB over all voiced frames. - """ - frame_len = int(sr * frame_ms / 1000) - hop_len = int(sr * hop_ms / 1000) - hnrs = [] - for start in range(0, len(sig) - frame_len, hop_len): - frm = sig[start:start+frame_len] - frm = frm * np.hanning(len(frm)) - ac = np.correlate(frm, frm, mode='full') - ac = ac[len(ac)//2:] # keep positive lags - ac /= (ac[0] + 1e-12) # normalise to 1 at lag=0 - - # search for peak in lag range corresponding to 50–600 Hz - lag_min = int(SR / 600) - lag_max = int(SR / 50) - lag_min = max(lag_min, 1) - lag_max = min(lag_max, len(ac)-1) - - peak_val = ac[lag_min:lag_max].max() - peak_val = min(peak_val, 0.9999) # clamp for log - if peak_val > 0.1: # voiced frame only - hnr = 10 * np.log10(peak_val / (1 - peak_val)) - hnrs.append(hnr) - return np.median(hnrs) if hnrs else float('nan'), np.std(hnrs) if hnrs else float('nan') - -def lpc_formants(sig, sr, order=12): - """ - Estimate formants via LPC (autocorrelation method). - Returns sorted list of formant frequencies. - """ - # Pre-emphasis - pre = np.append(sig[0], sig[1:] - 0.97*sig[:-1]) - pre = pre * np.hanning(len(pre)) - - # Autocorrelation LPC - from scipy.signal import lfilter - # Build autocorrelation matrix - r = np.array([np.dot(pre[i:], pre[:len(pre)-i]) for i in range(order+1)]) - # Levinson-Durbin - a = np.zeros(order+1) - e = r[0] - a[0] = 1.0 - for i in range(1, order+1): - lam = -sum(a[j]*r[i-j] for j in range(i)) / (e + 1e-12) - a_new = a.copy() - for j in range(1, i): - a_new[j] = a[j] + lam * a[i-j] - a_new[i] = lam - e *= (1 - lam**2) - a = a_new - - # Roots of LPC polynomial - roots = np.roots(a) - roots = roots[np.imag(roots) > 0] # upper half plane only - angles = np.angle(roots) - freqs = angles * sr / (2 * np.pi) - freqs = np.sort(freqs) - bw = -np.log(np.abs(roots[np.argsort(angles)])) * sr / np.pi - # Keep only formants with bandwidth < 500 Hz and freq 90–5000 Hz - formants = [(f, b) for f, b in zip(freqs, bw) if 90 < f < 5000 and b < 500] - return formants[:4] # F1..F4 - -def estimate_f0_harmonics(sig, sr, expected_f0=None): - """ - Estimate F0 using autocorrelation and extract first 4 harmonic amplitudes - from the magnitude spectrum. - """ - win = sig * np.hanning(len(sig)) - ac = np.correlate(win, win, mode='full') - ac = ac[len(ac)//2:] - ac /= (ac[0] + 1e-12) - - lag_min = int(sr / 600) - lag_max = int(sr / 50) - peak_lag = lag_min + np.argmax(ac[lag_min:lag_max]) - f0 = sr / peak_lag - - # Magnitude spectrum for harmonic amplitudes - N = len(sig) - spec = np.abs(rfft(sig * np.hanning(N))) * 2 / N - freqs = rfftfreq(N, 1/sr) - - def peak_at(target_hz, tol_hz=15): - mask = np.abs(freqs - target_hz) < tol_hz - if mask.any(): - return spec[mask].max() - return 0.0 - - harmonics = [peak_at(f0 * k) for k in range(1, 5)] - return f0, harmonics - -def snr_dB(ref, deg): - """ - SNR between reference and degraded signal (same length trimmed). - SNR = 10*log10( power(ref) / power(ref-deg) ) - """ - n = min(len(ref), len(deg)) - r = ref[:n] - d = deg[:n] - noise = r - d - p_sig = np.mean(r**2) - p_noise = np.mean(noise**2) - if p_noise < 1e-20: - return float('inf') - return 10 * np.log10(p_sig / p_noise) - -def spectral_distortion_dB(ref, deg, sr, n_fft=2048): - """ - Log-spectral distance (LSD) averaged over frames in dB. - """ - n = min(len(ref), len(deg)) - r = ref[:n]; d = deg[:n] - hop = n_fft // 2 - lsds = [] - for start in range(0, n - n_fft, hop): - fr = np.abs(rfft(r[start:start+n_fft] * np.hanning(n_fft))) + 1e-10 - fd = np.abs(rfft(d[start:start+n_fft] * np.hanning(n_fft))) + 1e-10 - lsd = np.sqrt(np.mean((20*np.log10(fr/fd))**2)) - lsds.append(lsd) - return np.mean(lsds), np.std(lsds) - -# ─── 1. HNR at t=2.0–2.8s ──────────────────────────────────────────────────── - -print("=" * 60) -print("1. HNR (Harmonics-to-Noise Ratio) at t=2.0–2.8s") -print("=" * 60) - -T_HNR_S, T_HNR_E = 2.0, 2.8 - -seg_orig_hnr = frame(orig, SR, T_HNR_S, T_HNR_E) -seg_rep_hnr = frame(rep, SR, T_HNR_S, T_HNR_E) -seg_s13_hnr = frame(s13_corrected, SR, T_HNR_S, T_HNR_E) - -hnr_o, hnr_o_std = hnr_autocorr(seg_orig_hnr, SR) -hnr_r, hnr_r_std = hnr_autocorr(seg_rep_hnr, SR) -hnr_s, hnr_s_std = hnr_autocorr(seg_s13_hnr, SR) - -print(f" Original : {hnr_o:+.2f} dB (std={hnr_o_std:.2f})") -print(f" RePitch : {hnr_r:+.2f} dB (std={hnr_r_std:.2f})") -print(f" Studio13 : {hnr_s:+.2f} dB (std={hnr_s_std:.2f})") -delta_s = hnr_s - hnr_o -delta_r = hnr_r - hnr_o -print(f" Δ Studio13 vs original : {delta_s:+.2f} dB") -print(f" Δ RePitch vs original : {delta_r:+.2f} dB") - -# ─── 2. Formant analysis at t=2.3–2.5s ────────────────────────────────────── - -print() -print("=" * 60) -print("2. Formant analysis (LPC) at t=2.3–2.5s (pitch-shifted region)") -print("=" * 60) - -T_FORM_S, T_FORM_E = 2.3, 2.5 - -seg_orig_f = frame(orig, SR, T_FORM_S, T_FORM_E) -seg_rep_f = frame(rep, SR, T_FORM_S, T_FORM_E) -seg_s13_f = frame(s13_corrected, SR, T_FORM_S, T_FORM_E) - -fm_o = lpc_formants(seg_orig_f, SR, order=14) -fm_r = lpc_formants(seg_rep_f, SR, order=14) -fm_s = lpc_formants(seg_s13_f, SR, order=14) - -def fmt_formants(flist): - parts = [] - for i, (f, b) in enumerate(flist[:3]): - parts.append(f"F{i+1}={f:.0f}Hz(BW={b:.0f})") - return " ".join(parts) - -print(f" Original : {fmt_formants(fm_o)}") -print(f" RePitch : {fmt_formants(fm_r)}") -print(f" Studio13 : {fmt_formants(fm_s)}") - -# Compare F1/F2/F3 shifts relative to original -print() -print(" Formant shift vs original (+ = higher):") -for label, fms in [("RePitch", fm_r), ("Studio13", fm_s)]: - shifts = [] - for i in range(min(3, len(fm_o), len(fms))): - diff = fms[i][0] - fm_o[i][0] - shifts.append(f"ΔF{i+1}={diff:+.0f}Hz") - print(f" {label}: {', '.join(shifts)}") - -note = " Note: ideal formant-preserving pitch shift = formants stay near original position" -print(note) - -# ─── 3. Harmonic structure at t=2.3s ───────────────────────────────────────── - -print() -print("=" * 60) -print("3. Harmonic structure at t=2.3–2.5s (F0 and first 4 harmonics)") -print("=" * 60) - -ANALYSIS_DURATION = 0.2 # 200ms window - -seg_orig_h = frame(orig, SR, 2.3, 2.3 + ANALYSIS_DURATION) -seg_rep_h = frame(rep, SR, 2.3, 2.3 + ANALYSIS_DURATION) -seg_s13_h = frame(s13_corrected, SR, 2.3, 2.3 + ANALYSIS_DURATION) - -f0_o, harm_o = estimate_f0_harmonics(seg_orig_h, SR) -f0_r, harm_r = estimate_f0_harmonics(seg_rep_h, SR) -f0_s, harm_s = estimate_f0_harmonics(seg_s13_h, SR) - -print(f" Original : F0={f0_o:.1f}Hz harmonics (rel) = {[f'{h/harm_o[0]*100:.1f}%' for h in harm_o]}") -print(f" RePitch : F0={f0_r:.1f}Hz harmonics (rel) = {[f'{h/harm_r[0]*100:.1f}%' for h in harm_r]}") -print(f" Studio13 : F0={f0_s:.1f}Hz harmonics (rel) = {[f'{h/harm_s[0]*100:.1f}%' for h in harm_s]}") - -# Convert to semitones from D4 (293.66 Hz) -D4 = 293.66 -def to_semitones(f, ref=D4): - return 12 * np.log2(f / ref) if f > 0 else float('nan') - -print(f"\n Pitch relative to D4 (293.66 Hz):") -print(f" Original : {to_semitones(f0_o):+.2f} semitones ({f0_o:.1f} Hz)") -print(f" RePitch : {to_semitones(f0_r):+.2f} semitones ({f0_r:.1f} Hz)") -print(f" Studio13 : {to_semitones(f0_s):+.2f} semitones ({f0_s:.1f} Hz)") - -# Harmonic amplitude drop-off (indicator of tonal clarity — exponential decay = clean) -print(f"\n Harmonic amplitude absolute values (normalized to orig H1=1.0):") -h_ref = harm_o[0] + 1e-12 -for label, harms in [("Original", harm_o), ("RePitch", harm_r), ("Studio13", harm_s)]: - norm = [h/h_ref for h in harms] - print(f" {label:10s}: {['H'+str(i+1)+'='+f'{n:.3f}' for i,n in enumerate(norm)]}") - -# ─── 4. Identity region SNR: t=0–2.0s ──────────────────────────────────────── - -print() -print("=" * 60) -print("4. Identity region SNR (t=0.0–2.0s) — unmodified region") -print("=" * 60) - -T_ID_S, T_ID_E = 0.0, 2.0 - -seg_orig_id = frame(orig, SR, T_ID_S, T_ID_E) -seg_rep_id = frame(rep, SR, T_ID_S, T_ID_E) -seg_s13_id = frame(s13_corrected, SR, T_ID_S, T_ID_E) - -snr_rep = snr_dB(seg_orig_id, seg_rep_id) -snr_s13 = snr_dB(seg_orig_id, seg_s13_id) - -lsd_rep, lsd_rep_std = spectral_distortion_dB(seg_orig_id, seg_rep_id, SR) -lsd_s13, lsd_s13_std = spectral_distortion_dB(seg_orig_id, seg_s13_id, SR) - -print(f" Time-domain SNR vs original:") -print(f" RePitch : {snr_rep:+.1f} dB") -print(f" Studio13 : {snr_s13:+.1f} dB") -print(f" Log-Spectral Distance (LSD, lower = better):") -print(f" RePitch : {lsd_rep:.2f} dB (σ={lsd_rep_std:.2f})") -print(f" Studio13 : {lsd_s13:.2f} dB (σ={lsd_s13_std:.2f})") - -# ─── 5. Bonus: Spectral flatness (noise proxy) in shifted region ────────────── - -print() -print("=" * 60) -print("5. Bonus: Spectral flatness at t=2.0–2.8s (noise proxy)") -print(" Lower = more tonal/harmonic; higher = noisier/more noise-like") -print("=" * 60) - -def spectral_flatness(sig, n_fft=2048): - """Geometric mean / arithmetic mean of power spectrum (averaged over frames).""" - hop = n_fft // 2 - sfs = [] - for start in range(0, len(sig) - n_fft, hop): - chunk = sig[start:start+n_fft] - win = chunk * np.hanning(n_fft) - spec = np.abs(rfft(win))**2 + 1e-10 - geo = np.exp(np.mean(np.log(spec))) - ari = np.mean(spec) - sfs.append(geo / ari) - return float(np.mean(sfs)) if sfs else 0.0 - -sf_o = spectral_flatness(seg_orig_hnr) -sf_r = spectral_flatness(seg_rep_hnr) -sf_s = spectral_flatness(seg_s13_hnr) - -print(f" Original : {sf_o:.6f}") -print(f" RePitch : {sf_r:.6f}") -print(f" Studio13 : {sf_s:.6f}") -print(f" (ratio S13/orig: {sf_s/sf_o:.3f}x, ratio RePitch/orig: {sf_r/sf_o:.3f}x)") - -# ─── 6. Spectral continuity across edit boundary ───────────────────────────── - -print() -print("=" * 60) -print("6. Spectral continuity across edit boundary (t=2.0–2.4s vs t=2.4–2.8s)") -print(" LSD between adjacent halves — high = discontinuity/artifact") -print("=" * 60) - -def spectral_centroid_and_rolloff(sig, sr, n_fft=2048): - chunk = sig[:n_fft] - win = chunk * np.hanning(len(chunk)) - spec = np.abs(rfft(win))**2 - freqs = rfftfreq(n_fft, 1/sr) - centroid = np.sum(freqs * spec) / (np.sum(spec) + 1e-12) - cum = np.cumsum(spec) - rolloff_idx = np.searchsorted(cum, 0.85 * cum[-1]) - rolloff = freqs[min(rolloff_idx, len(freqs)-1)] - return centroid, rolloff - -for label, sig_arr in [("Original", orig), ("RePitch", rep), ("Studio13", s13_corrected)]: - seg_pre = frame(sig_arr, SR, 2.0, 2.4) - seg_post = frame(sig_arr, SR, 2.4, 2.8) - lsd, _ = spectral_distortion_dB(seg_pre, seg_post, SR) - c_pre, r_pre = spectral_centroid_and_rolloff(seg_pre, SR) - c_post, r_post = spectral_centroid_and_rolloff(seg_post, SR) - print(f" {label:10s}: LSD(pre vs post) = {lsd:.2f} dB " - f"centroid: {c_pre:.0f}→{c_post:.0f}Hz rolloff: {r_pre:.0f}→{r_post:.0f}Hz") - -# ─── Summary ───────────────────────────────────────────────────────────────── - -print() -print("=" * 60) -print("SUMMARY") -print("=" * 60) -print(f""" -Quality metric Original RePitch Studio13 -─────────────────────────────────────────────────────────── -HNR (dB, t=2.0-2.8s) {hnr_o:+6.2f} {hnr_r:+6.2f} {hnr_s:+6.2f} -HNR Δ vs original -- {delta_r:+6.2f} {delta_s:+6.2f} -Spectral flatness {sf_o:.6f} {sf_r:.6f} {sf_s:.6f} - (noise proxy) -Identity SNR (0-2s) -- {snr_rep:+6.1f} {snr_s13:+6.1f} dB -Identity LSD (0-2s) -- {lsd_rep:6.2f} {lsd_s13:6.2f} dB - -F0 at t=2.3s {f0_o:.1f}Hz {f0_r:.1f}Hz {f0_s:.1f}Hz - semitones vs D4 {to_semitones(f0_o):+.2f}st {to_semitones(f0_r):+.2f}st {to_semitones(f0_s):+.2f}st - -Formants (F1/F2/F3) at t=2.3–2.5s: - F1 {fm_o[0][0] if fm_o else 0:.0f}Hz {fm_r[0][0] if fm_r else 0:.0f}Hz {fm_s[0][0] if fm_s else 0:.0f}Hz - F2 {fm_o[1][0] if len(fm_o)>1 else 0:.0f}Hz {fm_r[1][0] if len(fm_r)>1 else 0:.0f}Hz {fm_s[1][0] if len(fm_s)>1 else 0:.0f}Hz - F3 {fm_o[2][0] if len(fm_o)>2 else 0:.0f}Hz {fm_r[2][0] if len(fm_r)>2 else 0:.0f}Hz {fm_s[2][0] if len(fm_s)>2 else 0:.0f}Hz -""") -print("Analysis complete.") diff --git a/tools/validate-ai-runtime-package.ps1 b/tools/validate-ai-runtime-package.ps1 index d154d19..58de603 100644 --- a/tools/validate-ai-runtime-package.ps1 +++ b/tools/validate-ai-runtime-package.ps1 @@ -1,6 +1,6 @@ param( [Parameter(Mandatory = $true)] - [ValidateSet("windows", "macos")] + [ValidateSet("windows", "macos", "linux")] [string]$Platform, [Parameter(Mandatory = $true)] @@ -84,6 +84,33 @@ function Assert-True { } } +function Restore-UnixExecuteBits { + param([string]$RuntimeRoot) + + if ($Platform -notin @("macos", "linux")) { + return + } + + $candidateRoots = @( + (Join-Path $RuntimeRoot "bin"), + (Join-Path $RuntimeRoot "lib"), + (Join-Path $RuntimeRoot "python/bin"), + (Join-Path $RuntimeRoot "python/lib") + ) + + foreach ($root in $candidateRoots) { + if (Test-Path $root) { + Get-ChildItem -LiteralPath $root -File -Recurse -ErrorAction SilentlyContinue | + ForEach-Object { & chmod +x $_.FullName } + } + } + + $pythonExe = Resolve-PythonExecutable -Root $RuntimeRoot + if (-not [string]::IsNullOrWhiteSpace($pythonExe)) { + & chmod +x $pythonExe + } +} + $resolvedArchive = if ([System.IO.Path]::IsPathRooted($ArchivePath)) { $ArchivePath } else { @@ -120,6 +147,8 @@ try { throw "AI runtime archive did not contain a Python executable." } + Restore-UnixExecuteBits -RuntimeRoot $runtimeRoot + $venvMarker = Join-Path $runtimeRoot "pyvenv.cfg" Assert-True (-not (Test-Path $venvMarker)) "AI runtime archive contains pyvenv.cfg and is not relocatable." @@ -182,7 +211,7 @@ try { $probe = ($probeJson | Select-Object -Last 1) | ConvertFrom-Json Assert-True ($probe.baseRuntimeReady -eq $true) "AI runtime capability probe did not report a valid base runtime." - Assert-True ($probe.selectedBackend -in @("cuda", "directml", "coreml", "mps", "cpu")) "AI runtime capability probe returned an unexpected selectedBackend '$($probe.selectedBackend)'." + Assert-True ($probe.selectedBackend -in @("cuda", "directml", "coreml", "mps", "rocm", "cpu")) "AI runtime capability probe returned an unexpected selectedBackend '$($probe.selectedBackend)'." Assert-True ($probe.supportedBackends.Count -ge 1) "AI runtime capability probe did not report supportedBackends." if ($runtimeMode -eq "base") { @@ -199,6 +228,10 @@ try { Assert-True (($probe.packagedBackends -contains "coreml") -or ($probe.packagedBackends -contains "mps") -or ($probe.packagedBackends -contains "cpu")) "macOS AI runtime probe did not report packaged backends." } + if ($Platform -eq "linux") { + Assert-True (($probe.packagedBackends -contains "cuda") -or ($probe.packagedBackends -contains "rocm") -or ($probe.packagedBackends -contains "cpu")) "Linux AI runtime probe did not report packaged backends." + } + foreach ($expectedBackend in $ExpectedPackagedBackends) { Assert-True (($probe.packagedBackends -contains $expectedBackend)) "AI runtime probe did not report expected packaged backend '$expectedBackend'." } diff --git a/tools/validate-release-metadata.ps1 b/tools/validate-release-metadata.ps1 index 3693674..9d16b7c 100644 --- a/tools/validate-release-metadata.ps1 +++ b/tools/validate-release-metadata.ps1 @@ -37,7 +37,19 @@ param( [string]$MacArm64AiRuntimeAssetPath = "", [Parameter(Mandatory = $false)] - [string]$MacX64AiRuntimeAssetPath = "" + [string]$MacX64AiRuntimeAssetPath = "", + + [Parameter(Mandatory = $false)] + [string]$LinuxAiRuntimeAssetPath = "", + + [Parameter(Mandatory = $false)] + [string]$LinuxX64AiRuntimeAssetPath = "", + + [Parameter(Mandatory = $false)] + [string]$LinuxArm64AiRuntimeAssetPath = "", + + [Parameter(Mandatory = $false)] + [string]$LinuxAssetPath = "" ) $ErrorActionPreference = "Stop" @@ -262,6 +274,7 @@ $rootAiRuntimeManifestPath = Join-Path $resolvedMetadataDir "releases/ai-runtime $channelAiRuntimeManifestPath = Join-Path $resolvedMetadataDir ("releases/ai-runtime/{0}/latest.json" -f $Channel) $windowsAppcastPath = Join-Path $resolvedMetadataDir ("appcast/windows-{0}.xml" -f $Channel) $macosAppcastPath = Join-Path $resolvedMetadataDir ("appcast/macos-{0}.xml" -f $Channel) +$linuxAppcastPath = Join-Path $resolvedMetadataDir ("appcast/linux-{0}.xml" -f $Channel) $checksums = Parse-Checksums $checksumsPath $rootManifest = Load-Json $rootManifestPath @@ -276,6 +289,7 @@ Assert-True ($rootJson -eq $channelJson) "Root latest.json and channel latest.js $windowsAsset = Get-AssetInfo $WindowsAssetPath $macosAsset = Get-AssetInfo $MacAssetPath +$linuxAsset = Get-AssetInfo $LinuxAssetPath $windowsAiRuntimeAsset = Get-AssetInfo $WindowsAiRuntimeAssetPath $windowsBaseAiRuntimeAsset = Get-AssetInfo $WindowsBaseAiRuntimeAssetPath $windowsDirectmlAiRuntimeAsset = Get-AssetInfo $WindowsDirectmlAiRuntimeAssetPath @@ -283,14 +297,21 @@ $windowsCudaAiRuntimeAsset = Get-AssetInfo $WindowsCudaAiRuntimeAssetPath $macosAiRuntimeAsset = Get-AssetInfo $MacAiRuntimeAssetPath $macosArm64AiRuntimeAsset = Get-AssetInfo $MacArm64AiRuntimeAssetPath $macosX64AiRuntimeAsset = Get-AssetInfo $MacX64AiRuntimeAssetPath +$linuxAiRuntimeAsset = Get-AssetInfo $LinuxAiRuntimeAssetPath +$linuxX64AiRuntimeAsset = Get-AssetInfo $LinuxX64AiRuntimeAssetPath +$linuxArm64AiRuntimeAsset = Get-AssetInfo $LinuxArm64AiRuntimeAssetPath $windowsCudaInstallPlan = Load-OptionalJson $WindowsCudaInstallPlanPath $windowsDirectmlInstallPlan = Load-OptionalJson $WindowsDirectmlInstallPlanPath Validate-PlatformEntry -PlatformName "windows" -PlatformNode $rootManifest.platforms.windows -Checksums $checksums -AssetInfo $windowsAsset Validate-PlatformEntry -PlatformName "macos" -PlatformNode $rootManifest.platforms.macos -Checksums $checksums -AssetInfo $macosAsset +Validate-PlatformEntry -PlatformName "linux" -PlatformNode $rootManifest.platforms.linux -Checksums $checksums -AssetInfo $linuxAsset Validate-Appcast -PlatformName "windows" -AppcastPath $windowsAppcastPath -Manifest $rootManifest -PlatformNode $rootManifest.platforms.windows Validate-Appcast -PlatformName "macos" -AppcastPath $macosAppcastPath -Manifest $rootManifest -PlatformNode $rootManifest.platforms.macos +if ($null -ne $rootManifest.platforms.linux) { + Validate-Appcast -PlatformName "linux" -AppcastPath $linuxAppcastPath -Manifest $rootManifest -PlatformNode $rootManifest.platforms.linux +} if ((Test-Path $rootAiRuntimeManifestPath) -or (Test-Path $channelAiRuntimeManifestPath)) { $rootAiRuntimeManifest = Load-Json $rootAiRuntimeManifestPath @@ -350,6 +371,15 @@ if ((Test-Path $rootAiRuntimeManifestPath) -or (Test-Path $channelAiRuntimeManif else { Validate-PlatformEntry -PlatformName "macos AI runtime" -PlatformNode $macAiNode -Checksums $checksums -AssetInfo $macosAiRuntimeAsset } + + $linuxAiNode = $rootAiRuntimeManifest.platforms.linux + if ($null -ne $linuxAiNode -and (($null -ne $linuxAiNode.x64) -or ($null -ne $linuxAiNode.arm64))) { + Validate-PlatformEntry -PlatformName "linux x64 AI runtime" -PlatformNode $linuxAiNode.x64 -Checksums $checksums -AssetInfo $linuxX64AiRuntimeAsset + Validate-PlatformEntry -PlatformName "linux arm64 AI runtime" -PlatformNode $linuxAiNode.arm64 -Checksums $checksums -AssetInfo $linuxArm64AiRuntimeAsset + } + elseif ($null -ne $linuxAiNode) { + Validate-PlatformEntry -PlatformName "linux AI runtime" -PlatformNode $linuxAiNode -Checksums $checksums -AssetInfo $linuxAiRuntimeAsset + } } Write-Host "Release metadata validation passed for channel '$Channel'." diff --git a/tools/validate-runtime-bundle.ps1 b/tools/validate-runtime-bundle.ps1 index 4c3d4a3..b56e194 100644 --- a/tools/validate-runtime-bundle.ps1 +++ b/tools/validate-runtime-bundle.ps1 @@ -1,6 +1,6 @@ param( [Parameter(Mandatory = $true)] - [ValidateSet("windows", "macos")] + [ValidateSet("windows", "macos", "linux")] [string]$Platform, [Parameter(Mandatory = $true)] @@ -117,6 +117,11 @@ switch ($Platform) { Assert-Exists -Path $runtimeRoot -Description "OpenStudio app resources directory" Assert-Exists -Path $plistPath -Description "OpenStudio app Info.plist" + $microphoneUsageText = Get-PlistStringValue -PlistPath $plistPath -Key "NSMicrophoneUsageDescription" + if ([string]::IsNullOrWhiteSpace($microphoneUsageText)) { + throw "macOS bundle Info.plist is missing NSMicrophoneUsageDescription, so microphone permission prompts will not work." + } + if (-not [string]::IsNullOrWhiteSpace($ExpectedVersion)) { $bundleVersion = Get-PlistStringValue -PlistPath $plistPath -Key "CFBundleShortVersionString" if ([string]::IsNullOrWhiteSpace($bundleVersion)) { @@ -128,6 +133,23 @@ switch ($Platform) { } } } + "linux" { + # Linux: flat output directory (AppImage contents or raw build output) + $binaryPath = Join-Path $resolvedBundlePath "OpenStudio" + Assert-Exists -Path $binaryPath -Description "OpenStudio binary" + + if (-not [string]::IsNullOrWhiteSpace($ExpectedVersion)) { + # Read version from embedded string via --version flag if the binary is executable + try { + $versionOutput = & "$binaryPath" --version 2>&1 | Select-Object -First 1 + if ($versionOutput -notmatch [regex]::Escape($ExpectedVersion)) { + Write-Warning "Linux bundle version check: expected '$ExpectedVersion', binary reported '$versionOutput'. Continuing." + } + } catch { + Write-Warning "Could not query Linux binary version: $_" + } + } + } } $shellCriticalRuntimeEntries = @( @@ -140,7 +162,8 @@ $bundledFeatureEntries = @( @{ Source = "scripts"; Target = "scripts"; Description = "stock scripts bundle" }, @{ Source = "resources/models/basic_pitch_nmp.onnx"; Target = "models/basic_pitch_nmp.onnx"; Description = "polyphonic pitch model" }, @{ Source = "tools/install_ai_tools.py"; Target = "scripts/install_ai_tools.py"; Description = "AI tools installer script" }, - @{ Source = "tools/ai_runtime_probe.py"; Target = "scripts/ai_runtime_probe.py"; Description = "AI runtime capability probe script" } + @{ Source = "tools/ai_runtime_probe.py"; Target = "scripts/ai_runtime_probe.py"; Description = "AI runtime capability probe script" }, + @{ Source = "tools/generate_music.py"; Target = "scripts/generate_music.py"; Description = "music generation helper script" } ) foreach ($entry in $shellCriticalRuntimeEntries) { diff --git a/tools/windows-ai-acceleration-manifest.json b/tools/windows-ai-acceleration-manifest.json new file mode 100644 index 0000000..8f299ec --- /dev/null +++ b/tools/windows-ai-acceleration-manifest.json @@ -0,0 +1,32 @@ +{ + "schemaVersion": 1, + "platform": "windows", + "pythonFamily": "3.11", + "target": { + "cuda": "cu128", + "pytorch": { + "indexUrl": "https://download.pytorch.org/whl/cu128", + "packages": [ + "torch==2.8.0", + "torchvision==0.23.0", + "torchaudio==2.8.0" + ] + }, + "tritonWindows": { + "package": "triton-windows==3.6.0.post26", + "version": "3.6.0.post26", + "url": "https://files.pythonhosted.org/packages/5e/20/acab7b4f50abe68d93f632b2e29cfb9e76284eaf9b4041a6eabbbd4afdc7/triton_windows-3.6.0.post26-cp311-cp311-win_amd64.whl", + "sha256": "369a47ed3ca25f6d405387a058b8fbc62e6ff4592bbf87b0c5f6de372ab68931" + }, + "flashAttn": { + "distribution": "flash-attn", + "version": "2.8.3", + "fileName": "flash_attn-2.8.3+cu128torch2.8.0cxx11abiFALSEfullbackward-cp311-cp311-win_amd64.whl", + "url": "https://github.com/sdbds/flash-attention-for-windows/releases/download/2.8.3/flash_attn-2.8.3%2Bcu128torch2.8.0cxx11abiFALSEfullbackward-cp311-cp311-win_amd64.whl", + "sha256": "9850d5010e62dde2bf1ec415bd267da23c996f05939501e316fd2e0a7c729076", + "pythonTag": "cp311", + "torchVersion": "2.8.0", + "cuda": "cu128" + } + } +}