diff --git a/README.md b/README.md index c479319..524e121 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ writing the processed audio to an output file. | Option | Description | Required | | ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------- | | `--plugin=` | Path to, or identifier of the plugin to use. | Yes | -| `--input=` | Path to an audio input file.
To supply multiple inputs, provide the `--input` argument multiple times. | Yes, unless `--midiInput` is set | -| `--generatorInput=` | Path to a JSON generator config file or a JSON generator config string. See [Generators](#generators) for specification.
To supply multiple inputs, provide the `--input` argument multiple times. | Yes, unless `--midiInput` is set | +| `--input=` | Path to an audio input file.
To supply multiple inputs, provide the `--input` argument multiple times. | Yes, unless `--midiInput` is set | +| `--generatorInput=` | Path to a JSON generator config file or a JSON generator config string. See [Generators](#generators) for specification.
To supply multiple inputs, provide the `--input` argument multiple times. | Yes, unless `--midiInput` is set | | `--midiInput=` | Path to a MIDI input file. | No | | `--output=` | Path to write the processed audio to. | Yes | | `--overwrite` | Overwrite the output file if it exists.
If this option is not set, processing is aborted if the output file exists. | No | @@ -164,21 +164,29 @@ You can pass the path to a JSON file or a JSON string. ## Compare audio files The `audioDiff` command takes two input files, compares the values of each sample and returns the RMS of the difference. It can be used to compare the output of two plugins, or two versions of the same plugin for regression testing. -| Option | Description | Required | -| -------------------- | -------------------------------------------------------------------------------------------- | -------- | -| `--test ` | Path to an audio file to compare - the output being tested. | Yes | -| `--reference ` | Path to an audio file to compare - the reference audio against which the output is compared. | Yes | +| Option | Description | Required | +| ------------------------------- | ------------------------------------------------------------------------------------------------ | -------- | +| `--test ` | Path to an audio file to compare - the output being tested. | Yes | +| `--reference ` | Path to an audio file to compare - the reference audio against which the output is compared. | Yes | +| `--tolerance ` | The volume of the difference at which the two files will be considered different. Default -50dB. | No | +Example usage: +```shell +plugalyzer audioDiff \ + --test=new_audio.wav \ + --reference=audio_from_plugin_v1.wav \ + --tolerance=-30dB +``` ## List plugin parameters The `listParameters` command lists all available plugin parameters and their value range, as well as whether they support parameter values in text form. -| Option | Description | Required | -| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -| `--plugin=` | Path to the plugin to use. | Yes | -| `--output=` | Path to output the automation.
If not supplied, will be output to stdout. | No | -| `--overwrite` | Overwrite the output file if it exists.
If this option is not set and the file exists, a new file with a different name (eg. 'outputfile2.json') will be created. | No | -| `--format=` | The format in which to output the parameter list. Default text. | No | +| Option | Description | Required | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| `--plugin=` | Path to the plugin to use. | Yes | +| `--output=` | Path to output the automation.
If not supplied, will be output to stdout. | No | +| `--overwrite` | Overwrite the output file if it exists.
If this option is not set and the file exists, a new file with a different name (eg. 'outputfile2.json') will be created. | No | +| `--format=` | The format in which to output the parameter list. Default text. | No | Example usage: ```shell @@ -217,12 +225,12 @@ Plugin parameters: The `busLayouts` command outputs all of the plugin's supported bus layouts. This can be useful in development to make sure you won't be provided bus layouts you cannot handle, and are being provided the bus layouts you intend to handle. -| Option | Description | Required | -| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -| `--plugin=` | Path to the plugin to use. | Yes | -| `--output=` | Path to output the bus layouts.
If not supplied, will be output to stdout. | No | -| `--overwrite` | Overwrite the output file if it exists.
If this option is not set and the file exists, a new file with a different name (eg. 'outputfile2.json') will be created. | No | -| `--format=` | The format in which to output the parameter list. Default text. | No | +| Option | Description | Required | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| `--plugin=` | Path to the plugin to use. | Yes | +| `--output=` | Path to output the bus layouts.
If not supplied, will be output to stdout. | No | +| `--overwrite` | Overwrite the output file if it exists.
If this option is not set and the file exists, a new file with a different name (eg. 'outputfile2.json') will be created. | No | +| `--format=` | The format in which to output the parameter list. Default text. | No | Example usage: diff --git a/Source/Validators.cpp b/Source/Validators.cpp index 589d2ed..886e3a5 100644 --- a/Source/Validators.cpp +++ b/Source/Validators.cpp @@ -10,6 +10,7 @@ #include #include + // Validators for CLI11 // Signature: // std::string validate(const std::string& arg) @@ -111,4 +112,21 @@ std::string generator(const std::string& str) { return ""; } +std::string amplitude(const std::string& str) { + try { + auto [value, unit] = parse::numberAndUnits(str); + if (unit != "" && string_utils::lowerCase(unit) != "db") { + return "If units are provided, it must be in dB, e.g. '1.5dB'"; + } + } catch (const std::invalid_argument& e) { + return std::format("Can't get a number from: {}, error: {}", str, e.what()); + } catch (const std::out_of_range& e) { + return std::format("Number too large: {}, error: {}", str, e.what()); + } catch (const std::exception& e) { + return std::format("Couldn't parse: {}, error: {}", str, e.what()); + } + + return ""; +} + } // namespace validate diff --git a/Source/Validators.h b/Source/Validators.h index 4250dfe..2b142ff 100644 --- a/Source/Validators.h +++ b/Source/Validators.h @@ -62,4 +62,13 @@ std::string outputFormat(const std::string& str); */ std::string generator(const std::string& str); +/** + * Validates an amplitude. + * Can be linear or with a dB suffix. + * + * @param str The amplitude argument + * @return Empty string if valid, or an error message + */ +std::string amplitude(const std::string& str); + } // namespace validate \ No newline at end of file diff --git a/Source/commands/AudioDiffCommand.cpp b/Source/commands/AudioDiffCommand.cpp index c3e4f29..2de905f 100644 --- a/Source/commands/AudioDiffCommand.cpp +++ b/Source/commands/AudioDiffCommand.cpp @@ -1,14 +1,26 @@ #include "AudioDiffCommand.h" +#include "Errors.h" +#include "Parsers.h" +#include "Validators.h" + +#include #include + std::shared_ptr AudioDiffCommand::createApp() { - // don't break these lines, please // clang-format off std::shared_ptr app = std::make_shared("Compares two audio files", "audioDiff"); - app->add_option("-t,--test", inputFilePaths.at(AudioFileRole::test), "Audio to test")->required()->check(CLI::ExistingFile); - app->add_option("-r,--reference", inputFilePaths.at(AudioFileRole::reference), "Reference audio")->required()->check(CLI::ExistingFile); + app->add_option("-t,--test", inputFilePaths.at(AudioFileRole::test), "Audio to test") + ->required() + ->check(CLI::ExistingFile); + app->add_option("-r,--reference", inputFilePaths.at(AudioFileRole::reference), "Reference audio") + ->required() + ->check(CLI::ExistingFile); + app->add_option("-d,--tolerance", argThreshold, "How different the audio can be before it's considered a failure, in RMS.") + ->check(validate::amplitude) + ->each([&](std::string arg){ rmsThreshold = parse::amplitude(arg); }); return app; // clang-format on @@ -25,31 +37,33 @@ void AudioDiffCommand::execute() { verifiedInPath = juce::File::getCurrentWorkingDirectory().getChildFile(inPath); } - audioFiles.insert({role, juce::File{verifiedInPath}}); + audioFiles.insert({ role, juce::File{ verifiedInPath } }); } - std::println("Reading audio files"); - auto differ = AudioDiff::create(audioFiles); if (!differ) { for (const auto& [role, errorMessage] : differ.error()) { std::println(std::cerr, "{}: {}", stringFromRole(role), errorMessage.toStdString()); } - throw FileReadError("Couldn't read files. Aborting.", 1); + throw FileLoadError("Couldn't read files. Aborting.", 2); } - const double rmsThresh = 0.005; - const auto actualRMS = differ->getDifferenceRMS(); + const auto actualRMSdB = juce::Decibels::gainToDecibels(actualRMS, -96.f); + const auto actualRMSdBReadout = + juce::Decibels::toString(actualRMSdB, 6, -96.f, true, "-inf").toStdString(); + const auto rmsThresholddB = juce::Decibels::gainToDecibels(rmsThreshold, -96.0); + const auto rmsThresholddBReadout = + juce::Decibels::toString(rmsThresholddB, 6, -96.0, true, "-inf").toStdString(); - std::println("Difference in RMS between test audio and reference: {}", actualRMS); + std::println("{}", actualRMSdBReadout); - if (actualRMS > rmsThresh) { - std::println("Cancellation test failed: detected SNR of {}, which exceeds threshold of {}", - actualRMS, rmsThresh); - throw FailedDiffError("Cancellation test failed", 2); + if (actualRMS > rmsThreshold) { + std::println( + stderr, "Detected SNR of {}, which exceeds threshold of {}", actualRMSdBReadout, + rmsThresholddBReadout + ); + throw FailedDiffError("Cancellation test failed", 1); } - - std::println("Cancellation test successful."); } diff --git a/Source/commands/AudioDiffCommand.h b/Source/commands/AudioDiffCommand.h index 3e11271..f28de85 100644 --- a/Source/commands/AudioDiffCommand.h +++ b/Source/commands/AudioDiffCommand.h @@ -2,13 +2,8 @@ #include "AudioDiff.h" #include "CLICommand.h" -#include -class FileReadError : public CLI::Error { - public: - FileReadError(std::string msg, int exit_code) - : CLI::Error("FileReadError", std::move(msg), exit_code) {} -}; +#include class FailedDiffError : public CLI::Error { public: @@ -23,8 +18,12 @@ class AudioDiffCommand : public CLICommand { void execute() override; private: + // String from CLI to be parsed into a double + std::string argThreshold; + + double rmsThreshold{ juce::Decibels::decibelsToGain(-50.0, -96.0) }; std::map inputFilePaths{ - {AudioFileRole::test, {}}, - {AudioFileRole::reference, {}}, + { AudioFileRole::test, {} }, + { AudioFileRole::reference, {} }, }; }; diff --git a/test/e2e_tests/run_tests.py b/test/e2e_tests/run_tests.py index cb5d89b..030d307 100644 --- a/test/e2e_tests/run_tests.py +++ b/test/e2e_tests/run_tests.py @@ -1,4 +1,3 @@ -import os import subprocess import shutil from pathlib import Path @@ -94,12 +93,12 @@ def main(): for fail in failures: logstring = f"Failed: {fail.description}\n" - if isinstance(fail.correct_output, str): - logstring += f"Expected: {fail.correct_output}\n" - logstring += f"Got: {fail.output}\n" - elif isinstance(fail.correct_output, bytes): + if isinstance(fail.correct_output, bytes): logstring += f"Expected: {fail.correct_output.hex()}\n" logstring += f"Got: {fail.output.hex()}\n" + else: + logstring += f"Expected: {fail.correct_output}\n" + logstring += f"Got: {fail.output}\n" if fail.correct_exit_code != 0: logstring += f"Expected exit code {fail.correct_exit_code}, got {fail.exit_code}\n" diff --git a/test/e2e_tests/test_cases.py b/test/e2e_tests/test_cases.py index 8505682..50800a4 100644 --- a/test/e2e_tests/test_cases.py +++ b/test/e2e_tests/test_cases.py @@ -360,7 +360,7 @@ def __init__(self, failures: FailureLogger, paths: TestPaths) -> None: "-t", f"{prep.prepped_data}", "-r", f"{prep.prepped_data}" ], - "Reading audio files\nDifference in RMS between test audio and reference: 0\nCancellation test successful.\n" + "-inf dB\n" ) self.prep = prep @@ -375,12 +375,28 @@ def __init__(self, failures: FailureLogger, paths: TestPaths) -> None: "-r", f"{prep.prepped_data_r}" ], re.compile( - r"^Reading audio files\sDifference in RMS between test audio and reference.*?\sCancellation test failed.*?\s$", - re.DOTALL | re.MULTILINE + r"^-?\d*\.\d* dB\s$" + ) + ) + self.prep = prep + self.correct_exit_code = 1 + +class AudiodiffSucceedWithTolerance(TestCase): + def __init__(self, failures: FailureLogger, paths: TestPaths) -> None: + prep = generate_test_data.AudioDiffFailPrep(paths) + super().__init__(failures, paths, + "Audiodiff: succeed using custom tolerance", + [ + "audioDiff", + "-t", f"{prep.prepped_data_t}", + "-r", f"{prep.prepped_data_r}", + "-d", "-20dB" + ], + re.compile( + r"^-?\d*\.\d* dB\s$" ) ) self.prep = prep - self.correct_exit_code = 2 class ProcessWithGenerator(TestCase): def __init__(self, failures: FailureLogger, paths: TestPaths) -> None: @@ -642,6 +658,7 @@ def get_all_test_cases(failures: FailureLogger, paths: TestPaths) -> List[TestCa BusLayoutsJsonFile(failures, paths), AudiodiffSucceed(failures, paths), AudiodiffFail(failures, paths), + AudiodiffSucceedWithTolerance(failures, paths), ProcessWithGenerator(failures, paths), ProcessWithGeneratorTextInput(failures, paths), ProcessWithAudioAndGeneratorSidechain(failures, paths),