diff --git a/Directory.Packages.props b/Directory.Packages.props index 0758fef79..77b964981 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -9,6 +9,7 @@ + diff --git a/fallout.slnx b/fallout.slnx index 0c05b8183..178ea5ab1 100644 --- a/fallout.slnx +++ b/fallout.slnx @@ -43,6 +43,7 @@ + diff --git a/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/Fallout.Persistence.Solution.Benchmarks.csproj b/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/Fallout.Persistence.Solution.Benchmarks.csproj new file mode 100644 index 000000000..dd8c70687 --- /dev/null +++ b/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/Fallout.Persistence.Solution.Benchmarks.csproj @@ -0,0 +1,22 @@ + + + + net10.0 + Exe + false + Fallout.Persistence.Solution.Benchmarks + + + + + + + + + + + + + diff --git a/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/Program.cs b/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/Program.cs new file mode 100644 index 000000000..a0105a4fe --- /dev/null +++ b/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/Program.cs @@ -0,0 +1,8 @@ +using BenchmarkDotNet.Running; + +namespace Fallout.Persistence.Solution.Benchmarks; + +public class Program +{ + public static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); +} diff --git a/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/README.md b/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/README.md new file mode 100644 index 000000000..ffdf90557 --- /dev/null +++ b/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/README.md @@ -0,0 +1,50 @@ +# Fallout.Persistence.Solution.Benchmarks + +BenchmarkDotNet measurements for the `.slnx` parser inlined from Microsoft's `vs-solutionpersistence` (see #248). Establishes a **baseline** before any optimisation work — so we can later prove (with numbers, not vibes) whether replacing the inlined parser with a proper `XmlSerializer` implementation actually buys anything. + +## Scenarios + +| Dimension | Values | +|---|---| +| Project count | 1, 10, 100, 1000 | +| Layout | flat (`WithFolders=false`) and grouped into folders of 10 (`WithFolders=true`) | + +Cartesian product = 8 measurements per benchmark method. Fixtures are generated programmatically in `[GlobalSetup]` (deterministic, no committed binary blobs). + +## Running locally + +```sh +dotnet run --project tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/ -c Release --filter '*' +``` + +Release config matters: Debug numbers are meaningless. BenchmarkDotNet enforces this by default and refuses to run a Debug build, but the explicit `-c Release` keeps invocation muscle-memory aligned with what's required. + +Common filters: + +```sh +# Just the small cases: +dotnet run --project tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/ -c Release --filter '*ProjectCount=1*' '*ProjectCount=10*' + +# Just one benchmark: +dotnet run --project tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/ -c Release --filter '*SlnxParseBenchmarks.ParseSlnx*' + +# Help on the full invocation surface: +dotnet run --project tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/ -c Release -- --help +``` + +## What's measured + +- **`SlnxParseBenchmarks.ParseSlnx`** — `SolutionSerializers.GetSerializerByMoniker(...).OpenAsync(...)` end-to-end on a temp `.slnx` file matching the parameter combination. Includes XML reading, model construction, folder hierarchy resolution. Excludes the Fallout-facade `path.ReadSolution()` wrapping (trivial) so the numbers attribute purely to the parser. + +`[MemoryDiagnoser]` is on, so allocations + Gen0/Gen1/Gen2 counts are reported alongside time. Allocation pressure is often the bigger win in parser rewrites than wall-time. + +## CI integration + +**Not run in CI.** Benchmarks in CI are noisy (shared-runner variance), slow (a sweep takes ~3-5 minutes), and the numbers from a GitHub-Actions VM rarely compare meaningfully across runs. The CI check on this project is just `dotnet build` (does the benchmark project compile against the current API surface). + +If/when we want continuous benchmark tracking, a dedicated machine + a results-database pattern is the right shape — out of scope here. + +## Related + +- **#248** — the inlining PR that made us own this code in the first place. +- **#258** — broader BenchmarkDotNet coverage tracking issue (tool wrapper, source generators, globbing, etc.). diff --git a/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/SlnxFixtureBuilder.cs b/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/SlnxFixtureBuilder.cs new file mode 100644 index 000000000..54a498b62 --- /dev/null +++ b/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/SlnxFixtureBuilder.cs @@ -0,0 +1,74 @@ +using System.Globalization; +using System.IO; +using System.Text; + +namespace Fallout.Persistence.Solution.Benchmarks; + +// Generates `.slnx` content of a given shape (project count, with or without +// folders) so benchmarks can isolate scaling behaviour without committing +// binary fixture blobs to the repo. The structure mirrors what Visual Studio +// would emit: +// +// +// +// +// ... +// +// +// +// +// All projects point at the same Project.csproj stub (which doesn't need to +// exist on disk for the parser benchmark — the serializer reads the .slnx +// only, not the referenced csprojs). +internal static class SlnxFixtureBuilder +{ + public static string Build(int projectCount, bool withFolders) + { + var sb = new StringBuilder(capacity: 256 + projectCount * 96); + sb.AppendLine(""); + + if (withFolders) + { + // Group projects into folders of 10 to exercise the folder + // parsing path; for 1-project counts a single folder is created. + const int projectsPerFolder = 10; + var folderCount = (projectCount + projectsPerFolder - 1) / projectsPerFolder; + for (var f = 0; f < folderCount; f++) + { + sb.Append(" "); + var start = f * projectsPerFolder; + var end = (start + projectsPerFolder).LessOrEqual(projectCount); + for (var p = start; p < end; p++) + { + AppendProject(sb, p, indent: " "); + } + sb.AppendLine(" "); + } + } + else + { + for (var p = 0; p < projectCount; p++) + { + AppendProject(sb, p, indent: " "); + } + } + + sb.AppendLine(""); + return sb.ToString(); + } + + private static void AppendProject(StringBuilder sb, int index, string indent) + { + var n = index.ToString(CultureInfo.InvariantCulture); + sb.Append(indent).Append(""); + } + + public static string WriteToTempFile(string content) + { + var path = Path.Combine(Path.GetTempPath(), $"fallout-bench-{Path.GetRandomFileName()}.slnx"); + File.WriteAllText(path, content); + return path; + } + + private static int LessOrEqual(this int value, int max) => value <= max ? value : max; +} diff --git a/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/SlnxParseBenchmarks.cs b/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/SlnxParseBenchmarks.cs new file mode 100644 index 000000000..714c4e293 --- /dev/null +++ b/tests/Benchmarks/Fallout.Persistence.Solution.Benchmarks/SlnxParseBenchmarks.cs @@ -0,0 +1,53 @@ +using System.IO; +using System.Threading; +using BenchmarkDotNet.Attributes; +using Fallout.Persistence.Solution.Serializer; + +namespace Fallout.Persistence.Solution.Benchmarks; + +// Baseline benchmarks for the .slnx parser inlined from Microsoft's +// vs-solutionpersistence (see #248). Sweeps project count × folder layout +// to characterise the scaling shape. Numbers from these inform whether a +// proper XmlSerializer-based rewrite would be worth the effort. +// +// Each iteration measures `serializer.OpenAsync(path, ct)` end-to-end on a +// pre-staged temp .slnx file. Fixture generation is in `[GlobalSetup]` so +// its cost is excluded from measurements. +[MemoryDiagnoser] // alloc + GC pressure are interesting alongside time +[BenchmarkCategory("slnx", "parse")] +public class SlnxParseBenchmarks +{ + [Params(1, 10, 100, 1000)] + public int ProjectCount; + + [Params(false, true)] + public bool WithFolders; + + private string _fixturePath = string.Empty; + + [GlobalSetup] + public void Setup() + { + var content = SlnxFixtureBuilder.Build(ProjectCount, WithFolders); + _fixturePath = SlnxFixtureBuilder.WriteToTempFile(content); + } + + [GlobalCleanup] + public void Cleanup() + { + if (File.Exists(_fixturePath)) + File.Delete(_fixturePath); + } + + [Benchmark] + public object ParseSlnx() + { + // Equivalent to Fallout.Solutions.SolutionModelExtensions.ReadSolution(), + // minus the AbsolutePath + facade-wrap layer — measures the parser path + // in isolation. .GetAwaiter().GetResult() rather than AsyncHelper because + // AsyncHelper is internal to Fallout.Utilities and we don't want to + // [InternalsVisibleTo] a benchmark project just for one call. + var serializer = SolutionSerializers.GetSerializerByMoniker(_fixturePath); + return serializer!.OpenAsync(_fixturePath, CancellationToken.None).GetAwaiter().GetResult(); + } +} diff --git a/tests/Fallout.SourceGenerators.Tests/StronglyTypedSolutionGeneratorTest.Test#Solution.g.verified.cs b/tests/Fallout.SourceGenerators.Tests/StronglyTypedSolutionGeneratorTest.Test#Solution.g.verified.cs index 06eae0c4d..975b7a67e 100644 --- a/tests/Fallout.SourceGenerators.Tests/StronglyTypedSolutionGeneratorTest.Test#Solution.g.verified.cs +++ b/tests/Fallout.SourceGenerators.Tests/StronglyTypedSolutionGeneratorTest.Test#Solution.g.verified.cs @@ -25,6 +25,7 @@ internal class Solution(SolutionModel model, AbsolutePath path) : Fallout.Soluti public Fallout.Solutions.Project Fallout_Migrate_Tests => this.GetProject("Fallout.Migrate.Tests"); public Fallout.Solutions.Project Fallout_MSBuildTasks => this.GetProject("Fallout.MSBuildTasks"); public Fallout.Solutions.Project Fallout_Persistence_Solution => this.GetProject("Fallout.Persistence.Solution"); + public Fallout.Solutions.Project Fallout_Persistence_Solution_Benchmarks => this.GetProject("Fallout.Persistence.Solution.Benchmarks"); public Fallout.Solutions.Project Fallout_ProjectModel => this.GetProject("Fallout.ProjectModel"); public Fallout.Solutions.Project Fallout_ProjectModel_Tests => this.GetProject("Fallout.ProjectModel.Tests"); public Fallout.Solutions.Project Fallout_Solution => this.GetProject("Fallout.Solution");