From ae08c816a60a18dbd949f20da5b6c5b6fe575284 Mon Sep 17 00:00:00 2001 From: ocots <66357348+ocots@users.noreply.github.com> Date: Thu, 15 Jan 2026 00:25:18 +0000 Subject: [PATCH] :robot: Format .jl files --- benchmarks/core-midpoint-trapeze.jl | 23 ++-- benchmarks/core-moonshot-gpu.jl | 8 +- docs/make.jl | 3 +- .../core-midpoint-trapeze.jl | 23 ++-- .../core-moonshot-gpu/core-moonshot-gpu.jl | 8 +- docs/src/docutils/CTBenchmarksDocUtils.jl | 1 - docs/src/docutils/Core/FigureEngine.jl | 4 +- docs/src/docutils/Core/ProfileEngine.jl | 4 +- docs/src/docutils/Core/TemplateEngine.jl | 25 +++- docs/src/docutils/Core/TextEngine.jl | 5 +- docs/src/docutils/Handlers/DefaultProfiles.jl | 21 ++-- src/CTBenchmarks.jl | 3 +- src/performance_profile.jl | 31 ++--- src/plot_solutions.jl | 13 +- test/test_performance_profile.jl | 76 +++++------- test/test_performance_profile_internals.jl | 111 +++++++++++++---- test/test_plot_styling.jl | 114 ++++++++---------- 17 files changed, 258 insertions(+), 215 deletions(-) diff --git a/benchmarks/core-midpoint-trapeze.jl b/benchmarks/core-midpoint-trapeze.jl index dd679e1b5..130b4bdc4 100644 --- a/benchmarks/core-midpoint-trapeze.jl +++ b/benchmarks/core-midpoint-trapeze.jl @@ -1,7 +1,7 @@ # Modeler: :exa function run() - results = CTBenchmarks.benchmark( + results = CTBenchmarks.benchmark(; problems=[ :beam, :chain, @@ -20,19 +20,14 @@ function run() ], # solver × modeler - solver_models = [ - :madnlp => [:exa], - :ipopt => [:exa], - ], - - disc_methods = [:trapeze, :midpoint], - - grid_sizes = [200], - tol = 1e-8, - ipopt_mu_strategy = "adaptive", - print_trace = false, - max_iter = 2000, - max_wall_time = 600.0, + solver_models=[:madnlp => [:exa], :ipopt => [:exa]], + disc_methods=[:trapeze, :midpoint], + grid_sizes=[200], + tol=1e-8, + ipopt_mu_strategy="adaptive", + print_trace=false, + max_iter=2000, + max_wall_time=600.0, ) println("✅ Benchmark midpoint vs trapeze completed successfully!") return results diff --git a/benchmarks/core-moonshot-gpu.jl b/benchmarks/core-moonshot-gpu.jl index a2d7d6f59..6610687c9 100644 --- a/benchmarks/core-moonshot-gpu.jl +++ b/benchmarks/core-moonshot-gpu.jl @@ -10,13 +10,13 @@ function run() # :ducted_fan, :electric_vehicle, :glider, - ## :insurance, # to be re-added (unstable / sincos issue) + ## :insurance, # to be re-added (unstable / sincos issue) :jackson, :robbins, - ## :robot, # to be re-added (unstable / sincos issue) + ## :robot, # to be re-added (unstable / sincos issue) :rocket, - ## :space_shuttle, # to be re-added (unstable / sincos issue) - ## :steering, # to be re-added (unstable / sincos issue) + ## :space_shuttle, # to be re-added (unstable / sincos issue) + ## :steering, # to be re-added (unstable / sincos issue) :vanderpol, ], solver_models=[:madnlp => [:exa, :exa_gpu]], diff --git a/docs/make.jl b/docs/make.jl index 09b27fa80..78461ecf1 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -120,7 +120,8 @@ with_processed_template_problems( "Developers Guidelines" => [ "Add a New Benchmark" => "add_benchmark.md", "Add a Custom Profile" => "add_performance_profile.md", - "DocUtils Template System" => joinpath("developers", "docutils_templates.md"), + "DocUtils Template System" => + joinpath("developers", "docutils_templates.md"), "Documentation Process" => "documentation_process.md", ], ], diff --git a/docs/src/assets/benchmarks/core-midpoint-trapeze/core-midpoint-trapeze.jl b/docs/src/assets/benchmarks/core-midpoint-trapeze/core-midpoint-trapeze.jl index dd679e1b5..130b4bdc4 100644 --- a/docs/src/assets/benchmarks/core-midpoint-trapeze/core-midpoint-trapeze.jl +++ b/docs/src/assets/benchmarks/core-midpoint-trapeze/core-midpoint-trapeze.jl @@ -1,7 +1,7 @@ # Modeler: :exa function run() - results = CTBenchmarks.benchmark( + results = CTBenchmarks.benchmark(; problems=[ :beam, :chain, @@ -20,19 +20,14 @@ function run() ], # solver × modeler - solver_models = [ - :madnlp => [:exa], - :ipopt => [:exa], - ], - - disc_methods = [:trapeze, :midpoint], - - grid_sizes = [200], - tol = 1e-8, - ipopt_mu_strategy = "adaptive", - print_trace = false, - max_iter = 2000, - max_wall_time = 600.0, + solver_models=[:madnlp => [:exa], :ipopt => [:exa]], + disc_methods=[:trapeze, :midpoint], + grid_sizes=[200], + tol=1e-8, + ipopt_mu_strategy="adaptive", + print_trace=false, + max_iter=2000, + max_wall_time=600.0, ) println("✅ Benchmark midpoint vs trapeze completed successfully!") return results diff --git a/docs/src/assets/benchmarks/core-moonshot-gpu/core-moonshot-gpu.jl b/docs/src/assets/benchmarks/core-moonshot-gpu/core-moonshot-gpu.jl index a2d7d6f59..6610687c9 100644 --- a/docs/src/assets/benchmarks/core-moonshot-gpu/core-moonshot-gpu.jl +++ b/docs/src/assets/benchmarks/core-moonshot-gpu/core-moonshot-gpu.jl @@ -10,13 +10,13 @@ function run() # :ducted_fan, :electric_vehicle, :glider, - ## :insurance, # to be re-added (unstable / sincos issue) + ## :insurance, # to be re-added (unstable / sincos issue) :jackson, :robbins, - ## :robot, # to be re-added (unstable / sincos issue) + ## :robot, # to be re-added (unstable / sincos issue) :rocket, - ## :space_shuttle, # to be re-added (unstable / sincos issue) - ## :steering, # to be re-added (unstable / sincos issue) + ## :space_shuttle, # to be re-added (unstable / sincos issue) + ## :steering, # to be re-added (unstable / sincos issue) :vanderpol, ], solver_models=[:madnlp => [:exa, :exa_gpu]], diff --git a/docs/src/docutils/CTBenchmarksDocUtils.jl b/docs/src/docutils/CTBenchmarksDocUtils.jl index d1bcd0ab2..2d8301c34 100644 --- a/docs/src/docutils/CTBenchmarksDocUtils.jl +++ b/docs/src/docutils/CTBenchmarksDocUtils.jl @@ -129,7 +129,6 @@ using .DocumenterReference # Wrapper Functions (no src_dir parameter needed in templates) # ═══════════════════════════════════════════════════════════════════════════════ - # ═══════════════════════════════════════════════════════════════════════════════ # Exports # ═══════════════════════════════════════════════════════════════════════════════ diff --git a/docs/src/docutils/Core/FigureEngine.jl b/docs/src/docutils/Core/FigureEngine.jl index f6a5fd2d9..4d7359297 100644 --- a/docs/src/docutils/Core/FigureEngine.jl +++ b/docs/src/docutils/Core/FigureEngine.jl @@ -50,9 +50,7 @@ plt = call_figure_function("_plot_profile_default_cpu", ["core-ubuntu-latest"]) ``` """ function call_figure_function( - function_name::AbstractString, - args::Vector{<:AbstractString}, - extra_args::Tuple=() + function_name::AbstractString, args::Vector{<:AbstractString}, extra_args::Tuple=() ) if !haskey(FIGURE_FUNCTIONS, function_name) available = join(sort(collect(keys(FIGURE_FUNCTIONS))), ", ") diff --git a/docs/src/docutils/Core/ProfileEngine.jl b/docs/src/docutils/Core/ProfileEngine.jl index 04b06bde5..a6ce0330e 100644 --- a/docs/src/docutils/Core/ProfileEngine.jl +++ b/docs/src/docutils/Core/ProfileEngine.jl @@ -33,7 +33,7 @@ function plot_profile_from_registry( name::String, bench_id::AbstractString, src_dir::AbstractString; - combos::Union{Nothing,Vector{<:Tuple}}=nothing + combos::Union{Nothing,Vector{<:Tuple}}=nothing, ) config = CTBenchmarks.get_config(PROFILE_REGISTRY, name) json_path = joinpath(src_dir, "assets", "benchmarks", bench_id, bench_id * ".json") @@ -67,7 +67,7 @@ function analyze_profile_from_registry( name::String, bench_id::AbstractString, src_dir::AbstractString; - combos::Union{Nothing,Vector{<:Tuple}}=nothing + combos::Union{Nothing,Vector{<:Tuple}}=nothing, ) config = CTBenchmarks.get_config(PROFILE_REGISTRY, name) json_path = joinpath(src_dir, "assets", "benchmarks", bench_id, bench_id * ".json") diff --git a/docs/src/docutils/Core/TemplateEngine.jl b/docs/src/docutils/Core/TemplateEngine.jl index c968a19d2..5dbb845cb 100644 --- a/docs/src/docutils/Core/TemplateEngine.jl +++ b/docs/src/docutils/Core/TemplateEngine.jl @@ -404,8 +404,17 @@ function replace_profile_plot_blocks( plt = plot_profile_from_registry(profile_name, bench_id, SRC_DIR; combos=combos) # Generate unique basename for the figure - args_str = bench_id * (isnothing(combos) ? "" : "_" * join(["$(m)_$(s)" for (m, s) in combos], "_")) - basename = generate_figure_basename(template_filename, "profile_plot_$(profile_name)", args_str) + args_str = + bench_id * ( + if isnothing(combos) + "" + else + "_" * join(["$(m)_$(s)" for (m, s) in combos], "_") + end + ) + basename = generate_figure_basename( + template_filename, "profile_plot_$(profile_name)", args_str + ) # Create output directory if it doesn't exist mkpath(figures_output_dir) @@ -442,7 +451,9 @@ function replace_profile_plot_blocks( catch e if DOC_DEBUG[] - @error " ✗ Failed to generate profile plot" exception = (e, catch_backtrace()) + @error " ✗ Failed to generate profile plot" exception = ( + e, catch_backtrace() + ) else @error " ✗ Failed to generate profile plot: $(e)" end @@ -528,13 +539,17 @@ function replace_profile_analysis_blocks(content::String) end try - text_md = analyze_profile_from_registry(profile_name, bench_id, SRC_DIR; combos=combos) + text_md = analyze_profile_from_registry( + profile_name, bench_id, SRC_DIR; combos=combos + ) DOC_DEBUG[] && @info " ✓ Replaced PROFILE_ANALYSIS block #$block_count: $profile_name for $bench_id" return text_md catch e if DOC_DEBUG[] - @error " ✗ Failed to generate profile analysis" exception = (e, catch_backtrace()) + @error " ✗ Failed to generate profile analysis" exception = ( + e, catch_backtrace() + ) else @error " ✗ Failed to generate profile analysis: $(e)" end diff --git a/docs/src/docutils/Core/TextEngine.jl b/docs/src/docutils/Core/TextEngine.jl index c7f37b81f..971fd868b 100644 --- a/docs/src/docutils/Core/TextEngine.jl +++ b/docs/src/docutils/Core/TextEngine.jl @@ -11,7 +11,6 @@ string that will be inlined directly into the generated documentation. const TEXT_FUNCTIONS = Dict{String,Function}() - """ register_text_handler!(name::String, func::Function) @@ -43,9 +42,7 @@ Arguments are passed as: `func(extra_args..., args...)` so that injected dependencies (like `src_dir`) come first, following the Dependency Inversion Principle. """ function call_text_function( - function_name::AbstractString, - args::Vector{<:AbstractString}, - extra_args::Tuple=() + function_name::AbstractString, args::Vector{<:AbstractString}, extra_args::Tuple=() ) if !haskey(TEXT_FUNCTIONS, function_name) available = join(sort(collect(keys(TEXT_FUNCTIONS))), ", ") diff --git a/docs/src/docutils/Handlers/DefaultProfiles.jl b/docs/src/docutils/Handlers/DefaultProfiles.jl index d90a1d8b5..9f9049983 100644 --- a/docs/src/docutils/Handlers/DefaultProfiles.jl +++ b/docs/src/docutils/Handlers/DefaultProfiles.jl @@ -18,7 +18,8 @@ const IS_SUCCESS_WITH_BENCHMARK = Filter function: accepts rows where success is true and iterations data is available. """ const IS_SUCCESS_WITH_ITERATIONS = - row -> row.success == true && hasproperty(row, :iterations) && !ismissing(row.iterations) + row -> + row.success == true && hasproperty(row, :iterations) && !ismissing(row.iterations) """ Filter function: accepts all rows (no additional filtering). @@ -41,8 +42,7 @@ Extracts the `:time` field from the benchmark object and uses "smaller is better comparison (a <= b). Returns NaN if benchmark data is missing or malformed. """ const CPU_TIME_CRITERION = CTBenchmarks.ProfileCriterion{Float64}( - "CPU time", - row -> begin + "CPU time", row -> begin bench = get(row, :benchmark, nothing) if bench === nothing || ismissing(bench) return NaN @@ -50,8 +50,7 @@ const CPU_TIME_CRITERION = CTBenchmarks.ProfileCriterion{Float64}( time_raw = get(bench, "time", nothing) time_raw === nothing && return NaN return Float64(time_raw) - end, - (a, b) -> a <= b + end, (a, b) -> a <= b ) """ @@ -68,7 +67,7 @@ const ITERATIONS_CRITERION = CTBenchmarks.ProfileCriterion{Float64}( end return Float64(row.iterations) end, - (a, b) -> a <= b + (a, b) -> a <= b, ) # ─────────────────────────────────────────────────────────────────────────────── @@ -95,7 +94,7 @@ function init_default_profiles!() CPU_TIME_CRITERION, IS_SUCCESS_WITH_BENCHMARK, NO_ADDITIONAL_FILTER, - AGGREGATE_MEAN + AGGREGATE_MEAN, ) CTBenchmarks.register!(PROFILE_REGISTRY, "default_cpu", cpu_config) @@ -106,7 +105,7 @@ function init_default_profiles!() ITERATIONS_CRITERION, IS_SUCCESS_WITH_ITERATIONS, NO_ADDITIONAL_FILTER, - AGGREGATE_MEAN + AGGREGATE_MEAN, ) CTBenchmarks.register!(PROFILE_REGISTRY, "default_iter", iter_config) @@ -118,9 +117,11 @@ function init_default_profiles!() CPU_TIME_CRITERION, IS_SUCCESS_WITH_BENCHMARK, NO_ADDITIONAL_FILTER, - AGGREGATE_MEAN + AGGREGATE_MEAN, + ) + CTBenchmarks.register!( + PROFILE_REGISTRY, "midpoint_trapeze_cpu", midpoint_trapeze_config ) - CTBenchmarks.register!(PROFILE_REGISTRY, "midpoint_trapeze_cpu", midpoint_trapeze_config) return nothing end diff --git a/src/CTBenchmarks.jl b/src/CTBenchmarks.jl index 4444291c9..4f6f6bf4c 100644 --- a/src/CTBenchmarks.jl +++ b/src/CTBenchmarks.jl @@ -69,7 +69,8 @@ include(joinpath(@__DIR__, "plot_solutions.jl")) include(joinpath(@__DIR__, "performance_profile.jl")) export run, benchmark, plot_solutions -export analyze_performance_profile, build_profile_from_df, plot_performance_profile, load_benchmark_df +export analyze_performance_profile, + build_profile_from_df, plot_performance_profile, load_benchmark_df export PerformanceProfileRegistry, PerformanceProfileConfig, ProfileCriterion end # module diff --git a/src/performance_profile.jl b/src/performance_profile.jl index a9eb4877a..6bd4f4649 100644 --- a/src/performance_profile.jl +++ b/src/performance_profile.jl @@ -292,7 +292,6 @@ end # Performance Profile Construction # ─────────────────────────────────────────────────────────────────────────────── - """ _filter_benchmark_data(df, cfg, allowed_combos) -> DataFrame @@ -306,8 +305,7 @@ function _filter_benchmark_data(df, cfg, allowed_combos) if !isnothing(allowed_combos) && !isempty(allowed_combos) allowed_set = Set(allowed_combos) df_filtered = filter( - row -> Tuple(row[c] for c in cfg.solver_cols) in allowed_set, - df_filtered, + row -> Tuple(row[c] for c in cfg.solver_cols) in allowed_set, df_filtered ) end @@ -553,7 +551,7 @@ function _marker_indices_for_curve(ratios; M=6) if M <= 1 push!(indices, 1) else - for k in 0:(M-1) + for k in 0:(M - 1) t = k / (M - 1) p = a + t * (b - a) x_target = 2.0^p @@ -632,7 +630,7 @@ function default_plot_config() 1.5, 4, :box, - :bottomright + :bottomright, ) end @@ -689,7 +687,9 @@ end Add a single solver combination series (line + markers) to the plot. """ -function _add_combo_series!(plt, x, y, label, color, marker, cfg::PerformanceProfilePlotConfig) +function _add_combo_series!( + plt, x, y, label, color, marker, cfg::PerformanceProfilePlotConfig +) # Plot the curve plot!(plt, x, y; label="", lw=cfg.linewidth, color=color) @@ -698,7 +698,8 @@ function _add_combo_series!(plt, x, y, label, color, marker, cfg::PerformancePro x_markers = x[marker_indices] y_markers = y[marker_indices] - scatter!(plt, + scatter!( + plt, x_markers, y_markers; color=color, @@ -709,7 +710,8 @@ function _add_combo_series!(plt, x, y, label, color, marker, cfg::PerformancePro ) # Add marker/label entry on the first point of the curve for the legend - plot!(plt, + plot!( + plt, [x[1]], [y[1]]; color=color, @@ -855,15 +857,14 @@ function compute_profile_stats(pp::PerformanceProfile) # Find most efficient combos if !isempty(performances) best_efficient_rate = maximum(p -> p.efficiency, performances) - most_efficient = - [p.combo for p in performances if p.efficiency == best_efficient_rate] + most_efficient = [ + p.combo for p in performances if p.efficiency == best_efficient_rate + ] else most_efficient = String[] end - return ProfileAnalysis( - pp.bench_id, stats, performances, most_robust, most_efficient - ) + return ProfileAnalysis(pp.bench_id, stats, performances, most_robust, most_efficient) end """ @@ -884,7 +885,9 @@ function _format_analysis_markdown(analysis::ProfileAnalysis) # Header print(buf, "!!! info \"Performance Profile Analysis\"\n") print(buf, " **Dataset overview for `$(analysis.bench_id)`:**\n") - print(buf, " - **Problems**: ", stats.n_problems, " unique optimal control problems\n") + print( + buf, " - **Problems**: ", stats.n_problems, " unique optimal control problems\n" + ) print(buf, " - **Instances**: ", stats.n_instances, "\n") print(buf, " - **Solver combos**: ", stats.n_combos, "\n") diff --git a/src/plot_solutions.jl b/src/plot_solutions.jl index 55e4e25d8..88734e55d 100644 --- a/src/plot_solutions.jl +++ b/src/plot_solutions.jl @@ -58,12 +58,12 @@ function get_color(model::T, solver::T, idx::Int) where {T<:Union{String,Symbol} ] fixed = Dict( - ("adnlp", "ipopt") => :steelblue, - ("exa", "ipopt") => :tomato, - ("adnlp", "madnlp") => :seagreen, - ("exa", "madnlp") => :darkorange, - ("jump", "ipopt") => :mediumpurple, - ("jump", "madnlp") => :sienna, + ("adnlp", "ipopt") => :steelblue, + ("exa", "ipopt") => :tomato, + ("adnlp", "madnlp") => :seagreen, + ("exa", "madnlp") => :darkorange, + ("jump", "ipopt") => :mediumpurple, + ("jump", "madnlp") => :sienna, ("exa_gpu", "madnlp") => :mediumturquoise, # Discretization method combinations ("trapeze", "ipopt") => :steelblue, @@ -100,7 +100,6 @@ function get_color(params::Vector, idx::Int) end end - # ----------------------------------- # Helper: left margin for plots # ----------------------------------- diff --git a/test/test_performance_profile.jl b/test/test_performance_profile.jl index 013ab869e..6becb4ac9 100644 --- a/test/test_performance_profile.jl +++ b/test/test_performance_profile.jl @@ -11,9 +11,7 @@ function test_performance_profile() @testset "ProfileCriterion" begin criterion = CTBenchmarks.ProfileCriterion{Float64}( - "Test Criterion", - row -> row.time, - (a, b) -> a <= b + "Test Criterion", row -> row.time, (a, b) -> a <= b ) @test criterion.name == "Test Criterion" @test criterion.value isa Function @@ -24,9 +22,7 @@ function test_performance_profile() @testset "PerformanceProfileConfig" begin criterion = CTBenchmarks.ProfileCriterion{Float64}( - "CPU time", - row -> row.time, - (a, b) -> a <= b + "CPU time", row -> row.time, (a, b) -> a <= b ) config = CTBenchmarks.PerformanceProfileConfig{Float64}( [:problem, :grid_size], @@ -34,7 +30,7 @@ function test_performance_profile() criterion, row -> row.success == true, row -> true, - xs -> mean(xs) + xs -> mean(xs), ) @test config.instance_cols == [:problem, :grid_size] @test config.solver_cols == [:model, :solver] @@ -50,25 +46,13 @@ function test_performance_profile() @test isempty(CTBenchmarks.list_profiles(registry)) criterion = CTBenchmarks.ProfileCriterion{Float64}( - "Test", - row -> row.time, - (a, b) -> a <= b + "Test", row -> row.time, (a, b) -> a <= b ) config1 = CTBenchmarks.PerformanceProfileConfig{Float64}( - [:problem], - [:solver], - criterion, - row -> true, - row -> true, - xs -> mean(xs) + [:problem], [:solver], criterion, row -> true, row -> true, xs -> mean(xs) ) config2 = CTBenchmarks.PerformanceProfileConfig{Float64}( - [:problem], - [:model], - criterion, - row -> true, - row -> true, - xs -> mean(xs) + [:problem], [:model], criterion, row -> true, row -> true, xs -> mean(xs) ) # Basic registration @@ -127,18 +111,20 @@ function test_performance_profile() @testset "build_profile_from_df" begin # Create mock benchmark data df = DataFrame( - problem=["prob1", "prob1", "prob1", "prob1", "prob2", "prob2", "prob2", "prob2"], + problem=[ + "prob1", "prob1", "prob1", "prob1", "prob2", "prob2", "prob2", "prob2" + ], grid_size=[100, 100, 100, 100, 100, 100, 100, 100], model=["exa", "exa", "jump", "jump", "exa", "exa", "jump", "jump"], - solver=["ipopt", "madnlp", "ipopt", "madnlp", "ipopt", "madnlp", "ipopt", "madnlp"], + solver=[ + "ipopt", "madnlp", "ipopt", "madnlp", "ipopt", "madnlp", "ipopt", "madnlp" + ], success=[true, true, true, true, true, true, true, true], - time=[1.0, 1.5, 2.0, 1.8, 1.2, 1.4, 2.2, 2.0] + time=[1.0, 1.5, 2.0, 1.8, 1.2, 1.4, 2.2, 2.0], ) criterion = CTBenchmarks.ProfileCriterion{Float64}( - "CPU time (s)", - row -> row.time, - (a, b) -> a <= b + "CPU time (s)", row -> row.time, (a, b) -> a <= b ) config = CTBenchmarks.PerformanceProfileConfig{Float64}( [:problem, :grid_size], @@ -146,7 +132,7 @@ function test_performance_profile() criterion, row -> row.success == true, row -> true, - xs -> mean(xs) + xs -> mean(xs), ) pp = CTBenchmarks.build_profile_from_df(df, "test_bench", config) @@ -160,7 +146,9 @@ function test_performance_profile() # Test safety contract: Missing columns df_bad = select(df, Not(:solver)) - @test_throws ArgumentError CTBenchmarks.build_profile_from_df(df_bad, "bad_bench", config) + @test_throws ArgumentError CTBenchmarks.build_profile_from_df( + df_bad, "bad_bench", config + ) end @testset "build_profile_from_df with allowed_combos" begin @@ -170,13 +158,11 @@ function test_performance_profile() model=["exa", "exa", "jump"], solver=["ipopt", "madnlp", "ipopt"], success=[true, true, true], - time=[1.0, 1.5, 2.0] + time=[1.0, 1.5, 2.0], ) criterion = CTBenchmarks.ProfileCriterion{Float64}( - "CPU time", - row -> row.time, - (a, b) -> a <= b + "CPU time", row -> row.time, (a, b) -> a <= b ) config = CTBenchmarks.PerformanceProfileConfig{Float64}( [:problem, :grid_size], @@ -184,13 +170,12 @@ function test_performance_profile() criterion, row -> row.success == true, row -> true, - xs -> mean(xs) + xs -> mean(xs), ) # Filter to only exa solvers pp = CTBenchmarks.build_profile_from_df( - df, "test_bench", config; - allowed_combos=[("exa", "ipopt"), ("exa", "madnlp")] + df, "test_bench", config; allowed_combos=[("exa", "ipopt"), ("exa", "madnlp")] ) @test pp !== nothing @@ -208,13 +193,11 @@ function test_performance_profile() model=["exa", "jump"], solver=["ipopt", "ipopt"], success=[true, true], - time=[1.0, 2.0] + time=[1.0, 2.0], ) criterion = CTBenchmarks.ProfileCriterion{Float64}( - "CPU time", - row -> row.time, - (a, b) -> a <= b + "CPU time", row -> row.time, (a, b) -> a <= b ) config = CTBenchmarks.PerformanceProfileConfig{Float64}( [:problem, :grid_size], @@ -222,7 +205,7 @@ function test_performance_profile() criterion, row -> row.success == true, row -> true, - xs -> mean(xs) + xs -> mean(xs), ) pp = CTBenchmarks.build_profile_from_df(df, "test_bench", config) @@ -246,13 +229,11 @@ function test_performance_profile() model=["exa", "jump"], solver=["ipopt", "ipopt"], success=[true, true], - time=[1.0, 2.0] + time=[1.0, 2.0], ) criterion = CTBenchmarks.ProfileCriterion{Float64}( - "CPU time", - row -> row.time, - (a, b) -> a <= b + "CPU time", row -> row.time, (a, b) -> a <= b ) config = CTBenchmarks.PerformanceProfileConfig{Float64}( [:problem, :grid_size], @@ -260,7 +241,7 @@ function test_performance_profile() criterion, row -> row.success == true, row -> true, - xs -> mean(xs) + xs -> mean(xs), ) pp = CTBenchmarks.build_profile_from_df(df, "test_bench", config) @@ -268,5 +249,4 @@ function test_performance_profile() @test plt isa Plots.Plot end - end diff --git a/test/test_performance_profile_internals.jl b/test/test_performance_profile_internals.jl index e453d620e..0a42a38b4 100644 --- a/test/test_performance_profile_internals.jl +++ b/test/test_performance_profile_internals.jl @@ -8,23 +8,70 @@ function test_performance_profile_internals() # Mock data data = Dict( "results" => [ - Dict("problem" => "p1", "grid_size" => 10, "model" => "exa", "solver" => "ipopt", "time" => 1.0, "success" => true), - Dict("problem" => "p1", "grid_size" => 10, "model" => "exa", "solver" => "ab", "time" => 2.0, "success" => true), - Dict("problem" => "p1", "grid_size" => 10, "model" => "exb", "solver" => "ipopt", "time" => 0.5, "success" => true), - Dict("problem" => "p1", "grid_size" => 20, "model" => "exa", "solver" => "ipopt", "time" => 10.0, "success" => true), - Dict("problem" => "p1", "grid_size" => 20, "model" => "exa", "solver" => "ab", "time" => NaN, "success" => false), + Dict( + "problem" => "p1", + "grid_size" => 10, + "model" => "exa", + "solver" => "ipopt", + "time" => 1.0, + "success" => true, + ), + Dict( + "problem" => "p1", + "grid_size" => 10, + "model" => "exa", + "solver" => "ab", + "time" => 2.0, + "success" => true, + ), + Dict( + "problem" => "p1", + "grid_size" => 10, + "model" => "exb", + "solver" => "ipopt", + "time" => 0.5, + "success" => true, + ), + Dict( + "problem" => "p1", + "grid_size" => 20, + "model" => "exa", + "solver" => "ipopt", + "time" => 10.0, + "success" => true, + ), + Dict( + "problem" => "p1", + "grid_size" => 20, + "model" => "exa", + "solver" => "ab", + "time" => NaN, + "success" => false, + ), # Duplicate run for aggregation test - Dict("problem" => "p2", "grid_size" => 10, "model" => "exa", "solver" => "ipopt", "time" => 1.0, "success" => true), - Dict("problem" => "p2", "grid_size" => 10, "model" => "exa", "solver" => "ipopt", "time" => 1.2, "success" => true), - ] + Dict( + "problem" => "p2", + "grid_size" => 10, + "model" => "exa", + "solver" => "ipopt", + "time" => 1.0, + "success" => true, + ), + Dict( + "problem" => "p2", + "grid_size" => 10, + "model" => "exa", + "solver" => "ipopt", + "time" => 1.2, + "success" => true, + ), + ], ) df = DataFrame(data["results"]) # Config cpu_criterion = CTBenchmarks.ProfileCriterion{Float64}( - "CPU time (s)", - row -> get(row, "time", NaN), - (a, b) -> a <= b + "CPU time (s)", row -> get(row, "time", NaN), (a, b) -> a <= b ) config = CTBenchmarks.PerformanceProfileConfig{Float64}( [:problem, :grid_size], @@ -32,7 +79,7 @@ function test_performance_profile_internals() cpu_criterion, row -> row.success == true, row -> true, - xs -> Statistics.mean(skipmissing(xs)) + xs -> Statistics.mean(skipmissing(xs)), ) @testset "_filter_benchmark_data" begin @@ -44,7 +91,9 @@ function test_performance_profile_internals() allowed = [("exa", "ipopt")] filtered_restricted = CTBenchmarks._filter_benchmark_data(df, config, allowed) @test nrow(filtered_restricted) == 4 # Only (exa, ipopt) rows - @test all(r -> (r.model == "exa" && r.solver == "ipopt"), eachrow(filtered_restricted)) + @test all( + r -> (r.model == "exa" && r.solver == "ipopt"), eachrow(filtered_restricted) + ) # Test generic filtering with different solver_cols config_gen = CTBenchmarks.PerformanceProfileConfig{Float64}( @@ -53,7 +102,7 @@ function test_performance_profile_internals() config.criterion, row -> true, row -> true, - xs -> Statistics.mean(xs) + xs -> Statistics.mean(xs), ) allowed_gen = [("exb",)] filtered_gen = CTBenchmarks._filter_benchmark_data(df, config_gen, allowed_gen) @@ -81,7 +130,14 @@ function test_performance_profile_internals() @test p2_row[1, :metric] ≈ 1.1 # Check non-duplicated rows exist - p1_row = filter(r -> r.problem == "p1" && r.grid_size == 10 && r.solver == "ipopt" && r.model == "exa", aggregated) + p1_row = filter( + r -> + r.problem == "p1" && + r.grid_size == 10 && + r.solver == "ipopt" && + r.model == "exa", + aggregated, + ) @test nrow(p1_row) == 1 @test p1_row[1, :metric] == 1.0 end @@ -115,7 +171,7 @@ function test_performance_profile_internals() grid_size=[10, 10], model=["m1", "m2"], solver=["s1", "s2"], - metric=[1.0, 0.0] + metric=[1.0, 0.0], ) ratios0 = CTBenchmarks._compute_dolan_more_ratios(df0, config) @test ratios0[1, :ratio] == Inf @@ -127,7 +183,7 @@ function test_performance_profile_internals() grid_size=[10, 10], model=["m1", "m2"], solver=["s1", "s2"], - metric=[Inf, Inf] + metric=[Inf, Inf], ) ratiosinf = CTBenchmarks._compute_dolan_more_ratios(dfinf, config) @test all(isnan.(ratiosinf.ratio)) @@ -186,14 +242,17 @@ function test_performance_profile_internals() solver=["s1", "s1", "s1"], ratio=[1.0, 1.0, 2.0], best_metric=[1.0, 1.0, 1.0], - combo=["(m1, s1)", "(m2, s1)", "(m3, s1)"] + combo=["(m1, s1)", "(m2, s1)", "(m3, s1)"], ) pp_ties = CTBenchmarks.PerformanceProfile( "ties", DataFrame(problem=["p1"], grid_size=[10]), df_ties, ["(m1, s1)", "(m2, s1)", "(m3, s1)"], - 1, 1.0, 2.0, config + 1, + 1.0, + 2.0, + config, ) stats = CTBenchmarks.compute_profile_stats(pp_ties) @@ -206,15 +265,23 @@ function test_performance_profile_internals() # 2. Total failure: one instance, no success # Note: build_profile_from_df would return nothing, but we test the stats logic df_fail_ratios = DataFrame( - problem=["p1"], grid_size=[10], model=["m1"], solver=["s1"], - ratio=[NaN], best_metric=[Inf], combo=["(m1, s1)"] + problem=["p1"], + grid_size=[10], + model=["m1"], + solver=["s1"], + ratio=[NaN], + best_metric=[Inf], + combo=["(m1, s1)"], ) pp_fail = CTBenchmarks.PerformanceProfile( "fail", DataFrame(problem=["p1"], grid_size=[10]), DataFrame(problem=String[], grid_size=Int[]), # Empty but has columns ["(m1, s1)"], - 1, 1.0, 1.0, config + 1, + 1.0, + 1.0, + config, ) stats_fail = CTBenchmarks.compute_profile_stats(pp_fail) diff --git a/test/test_plot_styling.jl b/test/test_plot_styling.jl index a9ce6ed7a..fa830f524 100644 --- a/test/test_plot_styling.jl +++ b/test/test_plot_styling.jl @@ -11,71 +11,70 @@ different solver column configurations like [:model, :solver] or [:disc_method, :solver]. """ function test_plot_styling() - @testset "get_color variadic versions" begin # Test 2-parameter version (original) @test CTBenchmarks.get_color(:exa, :ipopt, 1) == :tomato @test CTBenchmarks.get_color(:adnlp, :madnlp, 1) == :seagreen @test CTBenchmarks.get_color(:trapeze, :ipopt, 1) == :steelblue @test CTBenchmarks.get_color(:midpoint, :madnlp, 1) == :darkorange - + # Test variadic version with 2 parameters @test CTBenchmarks.get_color([:exa, :ipopt], 1) == :tomato @test CTBenchmarks.get_color([:adnlp, :madnlp], 1) == :seagreen @test CTBenchmarks.get_color([:trapeze, :ipopt], 1) == :steelblue @test CTBenchmarks.get_color([:midpoint, :madnlp], 1) == :darkorange - + # Test variadic version with 3+ parameters (uses first two) @test CTBenchmarks.get_color([:exa, :ipopt, :extra], 1) == :tomato @test CTBenchmarks.get_color([:model, :solver, :disc_method], 1) isa Symbol - + # Test variadic version with 1 parameter (fallback to palette) @test CTBenchmarks.get_color([:single], 1) isa Symbol @test CTBenchmarks.get_color([:single], 2) isa Symbol - + # Test error on empty parameters @test_throws ErrorException CTBenchmarks.get_color(Symbol[], 1) end - + @testset "get_marker_style variadic versions" begin # Test 2-parameter version (original) @test CTBenchmarks.get_marker_style(:exa, :ipopt, 1) == :square @test CTBenchmarks.get_marker_style(:adnlp, :madnlp, 1) == :diamond @test CTBenchmarks.get_marker_style(:trapeze, :ipopt, 1) == :circle @test CTBenchmarks.get_marker_style(:midpoint, :madnlp, 1) == :utriangle - + # Test variadic version with 2 parameters @test CTBenchmarks.get_marker_style([:exa, :ipopt], 1) == :square @test CTBenchmarks.get_marker_style([:adnlp, :madnlp], 1) == :diamond @test CTBenchmarks.get_marker_style([:trapeze, :ipopt], 1) == :circle @test CTBenchmarks.get_marker_style([:midpoint, :madnlp], 1) == :utriangle - + # Test variadic version with 3+ parameters @test CTBenchmarks.get_marker_style([:exa, :ipopt, :extra], 1) == :square @test CTBenchmarks.get_marker_style([:model, :solver, :disc_method], 1) isa Symbol - + # Test variadic version with 1 parameter @test CTBenchmarks.get_marker_style([:single], 1) isa Symbol - + # Test error on empty parameters @test_throws ErrorException CTBenchmarks.get_marker_style(Symbol[], 1) end - + @testset "get_marker_style with grid_size" begin # Test 3-parameter version (original) marker, interval = CTBenchmarks.get_marker_style(:exa, :ipopt, 1, 200) @test marker == :square @test interval == 33 # 200 ÷ 6 - + # Test variadic version with grid_size marker2, interval2 = CTBenchmarks.get_marker_style([:exa, :ipopt], 1, 200) @test marker2 == :square @test interval2 == 33 - + # Test with different grid sizes _, interval_small = CTBenchmarks.get_marker_style([:exa, :ipopt], 1, 10) @test interval_small == 1 # max(1, 10 ÷ 6) - + _, interval_large = CTBenchmarks.get_marker_style([:exa, :ipopt], 1, 600) @test interval_large == 100 # 600 ÷ 6 end @@ -94,113 +93,106 @@ These tests verify that the plotting system works correctly with: - Future-proof: 3+ column configurations """ function test_plot_performance_profile_solver_columns() - @testset "plot_performance_profile with [:model, :solver] columns" begin # Create test data with standard columns df = DataFrame( - problem = [:prob1, :prob1, :prob2, :prob2], - grid_size = [100, 100, 100, 100], - model = [:exa, :adnlp, :exa, :adnlp], - solver = [:ipopt, :ipopt, :ipopt, :ipopt], - success = [true, true, true, true], - benchmark = [(time=0.1,), (time=0.15,), (time=0.2,), (time=0.25,)] + problem=[:prob1, :prob1, :prob2, :prob2], + grid_size=[100, 100, 100, 100], + model=[:exa, :adnlp, :exa, :adnlp], + solver=[:ipopt, :ipopt, :ipopt, :ipopt], + success=[true, true, true, true], + benchmark=[(time=0.1,), (time=0.15,), (time=0.2,), (time=0.25,)], ) - + criterion = CTBenchmarks.ProfileCriterion{Float64}( - "CPU time (s)", - row -> get(row.benchmark, :time, NaN), - (a, b) -> a <= b + "CPU time (s)", row -> get(row.benchmark, :time, NaN), (a, b) -> a <= b ) - + config = CTBenchmarks.PerformanceProfileConfig{Float64}( [:problem, :grid_size], [:model, :solver], # Standard columns criterion, row -> row.success == true && !ismissing(row.benchmark), row -> true, - xs -> Statistics.mean(skipmissing(xs)) + xs -> Statistics.mean(skipmissing(xs)), ) - + pp = CTBenchmarks.build_profile_from_df(df, "test", config) @test pp !== nothing - + # Test that plotting doesn't error plt = CTBenchmarks.plot_performance_profile(pp) @test plt isa Plots.Plot end - + @testset "plot_performance_profile with [:disc_method, :solver] columns" begin # Create test data with discretization method columns df = DataFrame( - problem = [:prob1, :prob1, :prob2, :prob2], - grid_size = [200, 200, 200, 200], - disc_method = [:trapeze, :midpoint, :trapeze, :midpoint], - solver = [:ipopt, :ipopt, :madnlp, :madnlp], - model = [:exa, :exa, :exa, :exa], # Present but not used in solver_cols - success = [true, true, true, true], - benchmark = [(time=0.1,), (time=0.12,), (time=0.15,), (time=0.18,)] + problem=[:prob1, :prob1, :prob2, :prob2], + grid_size=[200, 200, 200, 200], + disc_method=[:trapeze, :midpoint, :trapeze, :midpoint], + solver=[:ipopt, :ipopt, :madnlp, :madnlp], + model=[:exa, :exa, :exa, :exa], # Present but not used in solver_cols + success=[true, true, true, true], + benchmark=[(time=0.1,), (time=0.12,), (time=0.15,), (time=0.18,)], ) - + criterion = CTBenchmarks.ProfileCriterion{Float64}( - "CPU time (s)", - row -> get(row.benchmark, :time, NaN), - (a, b) -> a <= b + "CPU time (s)", row -> get(row.benchmark, :time, NaN), (a, b) -> a <= b ) - + config = CTBenchmarks.PerformanceProfileConfig{Float64}( [:problem, :grid_size], [:disc_method, :solver], # Alternative columns criterion, row -> row.success == true && !ismissing(row.benchmark), row -> true, - xs -> Statistics.mean(skipmissing(xs)) + xs -> Statistics.mean(skipmissing(xs)), ) - + pp = CTBenchmarks.build_profile_from_df(df, "midpoint-trapeze", config) @test pp !== nothing @test length(pp.combos) >= 2 # At least trapeze and midpoint combinations - + # Test that plotting doesn't error (this was the bug!) plt = CTBenchmarks.plot_performance_profile(pp) @test plt isa Plots.Plot end - + @testset "Helper functions: _prepare_combo_data and _get_combo_styling" begin # Create minimal test profile df = DataFrame( - problem = [:p1, :p1], - grid_size = [100, 100], - disc_method = [:trapeze, :midpoint], - solver = [:ipopt, :ipopt], - success = [true, true], - benchmark = [(time=0.1,), (time=0.15,)] + problem=[:p1, :p1], + grid_size=[100, 100], + disc_method=[:trapeze, :midpoint], + solver=[:ipopt, :ipopt], + success=[true, true], + benchmark=[(time=0.1,), (time=0.15,)], ) - + criterion = CTBenchmarks.ProfileCriterion{Float64}( - "CPU time (s)", - row -> get(row.benchmark, :time, NaN), - (a, b) -> a <= b + "CPU time (s)", row -> get(row.benchmark, :time, NaN), (a, b) -> a <= b ) - + config = CTBenchmarks.PerformanceProfileConfig{Float64}( [:problem, :grid_size], [:disc_method, :solver], criterion, row -> row.success == true && !ismissing(row.benchmark), row -> true, - xs -> Statistics.mean(skipmissing(xs)) + xs -> Statistics.mean(skipmissing(xs)), ) - + pp = CTBenchmarks.build_profile_from_df(df, "test", config) @test pp !== nothing - + # Test _prepare_combo_data combo = pp.combos[1] ratios = CTBenchmarks._prepare_combo_data(pp, combo) @test ratios !== nothing @test ratios isa Vector{Float64} @test length(ratios) > 0 - + # Test _get_combo_styling color, marker = CTBenchmarks._get_combo_styling(pp, combo, 1) @test color isa Symbol