diff --git a/.gitignore b/.gitignore index 84b2b8755..9af706d0a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ experiments/ test/test_.*.py +# VS Code environment +.vscode/ + # MacOS .DS_Store diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 438da4d39..7368eb2a0 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,6 +9,5 @@ build: tools: python: miniconda-latest - conda: environment: environment.yml \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 912895bc2..000000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "configurations": [ - { - "name": "Python Debugger: Module", - "type": "debugpy", - "request": "launch", - "module": "simopt", - "justMyCode": true, - } - ] -} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 0b50fcf58..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "python.testing.unittestArgs": [ - "-v", - "-s", - "./test", - "-p", - "test_*.py" - ], - "python.testing.pytestEnabled": false, - "python.testing.unittestEnabled": true, - "python.analysis.typeCheckingMode": "standard", - "editor.codeActionsOnSave": { - "source.unusedImports": "explicit", - "source.organizeImports.ruff": "explicit", - "source.fixAll.ruff": "explicit", - } -} \ No newline at end of file diff --git a/demo/demo_data_farming_model.py b/demo/demo_data_farming_model.py index 6655d4cb4..a693910e4 100644 --- a/demo/demo_data_farming_model.py +++ b/demo/demo_data_farming_model.py @@ -6,7 +6,10 @@ import sys import os.path as o -sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) # type:ignore + +sys.path.append( + o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), "..")) +) # type:ignore from simopt.data_farming_base import DataFarmingExperiment @@ -15,7 +18,12 @@ model_name = "CNTNEWS" # Specify the names of the model factors (in order) that will be varied. -factor_headers = ["purchase_price", "sales_price", "salvage_price", "order_quantity"] +factor_headers = [ + "purchase_price", + "sales_price", + "salvage_price", + "order_quantity", +] # If creating the design, provide the name of a .txt file containing # the following: @@ -52,12 +60,13 @@ # No code beyond this point needs to be edited. # Create DataFarmingExperiment object. -myexperiment = DataFarmingExperiment(model_name=model_name, - factor_settings_filename=factor_settings_filename, - factor_headers=factor_headers, - design_filepath=design_filename, - model_fixed_factors={} - ) +myexperiment = DataFarmingExperiment( + model_name=model_name, + factor_settings_filename=factor_settings_filename, + factor_headers=factor_headers, + design_filepath=design_filename, + model_fixed_factors={}, +) # Run replications and print results to file. myexperiment.run(n_reps=n_reps, crn_across_design_pts=crn_across_design_pts) diff --git a/demo/demo_data_farming_over_solver_and_problem.py b/demo/demo_data_farming_over_solver_and_problem.py index 9aa028d2f..0bf0cbb7f 100644 --- a/demo/demo_data_farming_over_solver_and_problem.py +++ b/demo/demo_data_farming_over_solver_and_problem.py @@ -6,11 +6,15 @@ import sys import os.path as o -sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) # type:ignore + +sys.path.append( + o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), "..")) +) # type:ignore from simopt.experiment_base import create_design, ProblemsSolvers -def main(): + +def main() -> None: # Specify the name of the solver as it appears in directory.py solver_name = "ASTRODF" # Specify the name of the problem as it appears in directory.py @@ -19,13 +23,13 @@ def main(): model_name = "CNTNEWS" # Specify the names of the sovler factors (in order) that will be varied. - solver_factor_headers = ["eta_1", "eta_2", "lambda_min" ] + solver_factor_headers = ["eta_1", "eta_2", "lambda_min"] # Specify the names of the model factors (in order) that will be varied. model_factor_headers = ["purchase_price", "sales_price", "order_quantity"] # OPTIONAL: factors chosen for cross design # factor name followed by list containing factor values to cross design over - solver_cross_design_factors = {'crn_across_solns': [True, False]} + solver_cross_design_factors = {"crn_across_solns": [True, False]} # model_cross_design_factors = {} # OPTIONAL: Provide additional overrides for default factors. @@ -65,38 +69,41 @@ def main(): crn_across_init_opt = True # Default # Create DataFarmingExperiment object for sovler design - solver_design_list = create_design(name=solver_name, - factor_headers=solver_factor_headers, - factor_settings_filename=solver_factor_settings_filename, - n_stacks=solver_n_stacks, - fixed_factors=solver_fixed_factors, #optional - cross_design_factors= solver_cross_design_factors, #optional - ) + solver_design_list = create_design( + name=solver_name, + factor_headers=solver_factor_headers, + factor_settings_filename=solver_factor_settings_filename, + n_stacks=solver_n_stacks, + fixed_factors=solver_fixed_factors, # optional + cross_design_factors=solver_cross_design_factors, # optional + ) # Create DataFarmingExperiment object for model design - model_design_list = create_design(name=model_name, - factor_headers=model_factor_headers, - factor_settings_filename=model_factor_settings_filename, - n_stacks=problem_n_stacks, - fixed_factors=model_fixed_factors, #optional - #cross_design_factors=model_cross_design_factors, #optional - ) + model_design_list = create_design( + name=model_name, + factor_headers=model_factor_headers, + factor_settings_filename=model_factor_settings_filename, + n_stacks=problem_n_stacks, + fixed_factors=model_fixed_factors, # optional + # cross_design_factors=model_cross_design_factors, #optional + ) # create solver name list for ProblemsSolvers (do not edit) solver_names = [] - for i in range(len(solver_design_list)): + for _ in range(len(solver_design_list)): solver_names.append(solver_name) - + # create proble name list for ProblemsSolvers (do not edit) problem_names = [] - for i in range(len(model_design_list)): + for _ in range(len(model_design_list)): problem_names.append(problem_name) # Create ProblemsSovlers experiment with solver and model design - experiment = ProblemsSolvers(solver_factors = solver_design_list, - problem_factors = model_design_list, - solver_names = solver_names, - problem_names = problem_names - ) + experiment = ProblemsSolvers( + solver_factors=solver_design_list, + problem_factors=model_design_list, + solver_names=solver_names, + problem_names=problem_names, + ) # check compatibility of selected solvers and problems experiment.check_compatibility() @@ -105,16 +112,21 @@ def main(): experiment.run(n_macroreps) # Postprocess the experimental results from each design point. - experiment.post_replicate(n_postreps=n_postreps, - crn_across_budget=crn_across_budget, - crn_across_macroreps=crn_across_macroreps) - experiment.post_normalize(n_postreps_init_opt=n_postreps_init_opt, - crn_across_init_opt=crn_across_init_opt) + experiment.post_replicate( + n_postreps=n_postreps, + crn_across_budget=crn_across_budget, + crn_across_macroreps=crn_across_macroreps, + ) + experiment.post_normalize( + n_postreps_init_opt=n_postreps_init_opt, + crn_across_init_opt=crn_across_init_opt, + ) # Record and log results experiment.record_group_experiment_results() experiment.log_group_experiment_results() experiment.report_group_statistics() + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/demo/demo_data_farming_problem.py b/demo/demo_data_farming_problem.py index 084d89f56..f1dfe2a3d 100644 --- a/demo/demo_data_farming_problem.py +++ b/demo/demo_data_farming_problem.py @@ -6,25 +6,28 @@ import sys import os.path as o -sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) -from simopt.experiment_base import create_design, ProblemsSolvers # type:ignore +sys.path.append( + o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), "..")) +) -def main(): +from simopt.experiment_base import create_design, ProblemsSolvers # type:ignore + + +def main() -> None: # Specify the name of the problem as it appears in directory.py problem_name = "CNTNEWS-1" # Specify the name of the model as it appears in directory.py model_name = "CNTNEWS" # Specify the name of the solver as it appears in directory.py - solver_names = ["ASTRODF","RNDSRCH"] + solver_names = ["ASTRODF", "RNDSRCH"] # Specify the names of the model factors (in order) that will be varied. model_factor_headers = ["purchase_price", "sales_price", "order_quantity"] # OPTIONAL: factors chosen for cross design # factor name followed by list containing factor values to cross design over - #model_cross_design_factors = {} - + # model_cross_design_factors = {} # OPTIONAL: Provide additional overrides for model default factors. # If empty, default factor settings are used. @@ -34,10 +37,10 @@ def main(): # If empty, default factor settings are used. # list of dictionaries that provide fixed factors for problems when you don't want to use the default values # if you want to use all default values use empty dictionary, order must match problem names - solver_fixed_factors = [{"eta_1": .5, "eta_2": .4}, {"sample_size": 15}] + solver_fixed_factors = [{"eta_1": 0.5, "eta_2": 0.4}, {"sample_size": 15}] # uncomment this version to run w/ only default solver factors - #sp;ver_fixed_factors = [{},{}] + # sp;ver_fixed_factors = [{},{}] # Provide the name of a file .txt locatated in the datafarming_experiments folder containing # the following: @@ -52,7 +55,6 @@ def main(): # Specify the number stacks to use for ruby design creation problem_n_stacks = 1 - # Specify a common number of macroreplications of each unique solver/problem combination # i.e., the number of runs at each design point. n_macroreps = 3 @@ -70,26 +72,27 @@ def main(): crn_across_init_opt = True # Default # Create DataFarmingExperiment object for model design - model_design_list = create_design(name=model_name, - factor_headers=model_factor_headers, - factor_settings_filename=model_factor_settings_filename, - n_stacks=problem_n_stacks, - fixed_factors=model_fixed_factors, #optional - #cross_design_factors=model_cross_design_factors, #optional - ) - - + model_design_list = create_design( + name=model_name, + factor_headers=model_factor_headers, + factor_settings_filename=model_factor_settings_filename, + n_stacks=problem_n_stacks, + fixed_factors=model_fixed_factors, # optional + # cross_design_factors=model_cross_design_factors, #optional + ) + # create proble name list for ProblemsSolvers (do not edit) problem_names = [] - for i in range(len(model_design_list)): + for _ in range(len(model_design_list)): problem_names.append(problem_name) # Create ProblemsSovlers experiment with solver and model design - experiment = ProblemsSolvers(solver_factors = solver_fixed_factors, - problem_factors = model_design_list, - solver_names = solver_names, - problem_names = problem_names - ) + experiment = ProblemsSolvers( + solver_factors=solver_fixed_factors, + problem_factors=model_design_list, + solver_names=solver_names, + problem_names=problem_names, + ) # check compatibility of selected solvers and problems experiment.check_compatibility() @@ -98,16 +101,21 @@ def main(): experiment.run(n_macroreps) # Postprocess the experimental results from each design point. - experiment.post_replicate(n_postreps=n_postreps, - crn_across_budget=crn_across_budget, - crn_across_macroreps=crn_across_macroreps) - experiment.post_normalize(n_postreps_init_opt=n_postreps_init_opt, - crn_across_init_opt=crn_across_init_opt) + experiment.post_replicate( + n_postreps=n_postreps, + crn_across_budget=crn_across_budget, + crn_across_macroreps=crn_across_macroreps, + ) + experiment.post_normalize( + n_postreps_init_opt=n_postreps_init_opt, + crn_across_init_opt=crn_across_init_opt, + ) # Record and log results experiment.record_group_experiment_results() experiment.log_group_experiment_results() experiment.report_group_statistics() + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/demo/demo_data_farming_solver.py b/demo/demo_data_farming_solver.py index 5a603ba4e..d6af54180 100644 --- a/demo/demo_data_farming_solver.py +++ b/demo/demo_data_farming_solver.py @@ -6,24 +6,27 @@ import sys import os.path as o -sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) # type:ignore + +sys.path.append( + o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), "..")) +) # type:ignore from simopt.experiment_base import create_design, ProblemsSolvers -def main(): + +def main() -> None: # Specify the name of the solver as it appears in directory.py solver_name = "ASTRODF" # list of problem names for solver design to be run on (if more than one version of same problem, repeat name) # Specify the name of the problem as it appears in directory.py - problem_names = ['SSCONT-1', 'SAN-1'] + problem_names = ["SSCONT-1", "SAN-1"] # Specify the names of the sovler factors (in order) that will be varied. - solver_factor_headers = ["eta_1", "eta_2", "lambda_min" ] - + solver_factor_headers = ["eta_1", "eta_2", "lambda_min"] # OPTIONAL: factors chosen for cross design # factor name followed by list containing factor values to cross design over - solver_cross_design_factors = {'crn_across_solns': [True, False]} + solver_cross_design_factors = {"crn_across_solns": [True, False]} # OPTIONAL: Provide additional overrides for solver default factors. # If empty, default factor settings are used. @@ -31,8 +34,11 @@ def main(): # OPTIONAL: Provide additional overrides for problem default factors. # If empty, default factor settings are used. # list of dictionaries that provide fixed factors for problems when you don't want to use the default values - # if you want to use all default values use empty dictionary, order must match problem names - problem_fixed_factors = [{'budget': 2000, 'demand_mean': 90.0, 'fixed_cost':25},{'budget': 500}] + # if you want to use all default values use empty dictionary, order must match problem names + problem_fixed_factors = [ + {"budget": 2000, "demand_mean": 90.0, "fixed_cost": 25}, + {"budget": 500}, + ] # Provide the name of a file .txt locatated in the datafarming_experiments folder containing # the following: @@ -64,26 +70,27 @@ def main(): crn_across_init_opt = True # Default # Create DataFarmingExperiment object for sovler design - solver_design_list = create_design(name=solver_name, - factor_headers=solver_factor_headers, - factor_settings_filename=solver_factor_settings_filename, - n_stacks=solver_n_stacks, - fixed_factors=solver_fixed_factors, #optional - cross_design_factors= solver_cross_design_factors, #optional - ) + solver_design_list = create_design( + name=solver_name, + factor_headers=solver_factor_headers, + factor_settings_filename=solver_factor_settings_filename, + n_stacks=solver_n_stacks, + fixed_factors=solver_fixed_factors, # optional + cross_design_factors=solver_cross_design_factors, # optional + ) # create solver name list for ProblemsSolvers (do not edit) solver_names = [] - for i in range(len(solver_design_list)): + for _ in range(len(solver_design_list)): solver_names.append(solver_name) - # Create ProblemsSovlers experiment with solver and model design - experiment = ProblemsSolvers(solver_factors = solver_design_list, - problem_factors = problem_fixed_factors, - solver_names = solver_names, - problem_names = problem_names - ) + experiment = ProblemsSolvers( + solver_factors=solver_design_list, + problem_factors=problem_fixed_factors, + solver_names=solver_names, + problem_names=problem_names, + ) # check compatibility of selected solvers and problems experiment.check_compatibility() @@ -92,16 +99,21 @@ def main(): experiment.run(n_macroreps) # Postprocess the experimental results from each design point. - experiment.post_replicate(n_postreps=n_postreps, - crn_across_budget=crn_across_budget, - crn_across_macroreps=crn_across_macroreps) - experiment.post_normalize(n_postreps_init_opt=n_postreps_init_opt, - crn_across_init_opt=crn_across_init_opt) + experiment.post_replicate( + n_postreps=n_postreps, + crn_across_budget=crn_across_budget, + crn_across_macroreps=crn_across_macroreps, + ) + experiment.post_normalize( + n_postreps_init_opt=n_postreps_init_opt, + crn_across_init_opt=crn_across_init_opt, + ) # Record and log results experiment.record_group_experiment_results() experiment.log_group_experiment_results() experiment.report_group_statistics() + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/demo/demo_load_solver_design.py b/demo/demo_load_solver_design.py index 4c1d89c44..fe06e3998 100644 --- a/demo/demo_load_solver_design.py +++ b/demo/demo_load_solver_design.py @@ -1,76 +1,88 @@ -# -*- coding: utf-8 -*- - """ This script is intended to help with loading a design of solver factors to run on one or more problems. If more than one problem are used a cross-design of problems -will be run. The design file should be a txt or csv file with headers as factor names and each row -representing an individual design point. Design files generated by the simopt GUI can also be run -using this script. Outputs are printed to the experiments folder in simopt. +will be run. The design file should be a txt or csv file with headers as factor names and each row +representing an individual design point. Design files generated by the simopt GUI can also be run +using this script. Outputs are printed to the experiments folder in simopt. """ - import sys import os.path as o -sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) +sys.path.append( + o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), "..")) +) -from simopt.experiment_base import ProblemsSolvers +from simopt.experiment_base import ProblemsSolvers import pandas as pd -def main(): + +def main() -> None: # run this script in the terminal from the simopt directory - solver_name = 'RNDSRCH' # name of solver that design was created on - problem_names = ['SSCONT-1', 'SAN-1'] # list of problem names for solver design to be run on (if more than one version of same problem, repeat name) + solver_name = "RNDSRCH" # name of solver that design was created on + problem_names = [ + "SSCONT-1", + "SAN-1", + ] # list of problem names for solver design to be run on (if more than one version of same problem, repeat name) # name of file containing design points (csv or excel): column headers must exactly match names of solver factors w/ each row representing a design point (can also use csv's generated by GUI) - design_filename = ".\data_farming_experiments\RNDSRCH_design.csv" + design_filename = ".\data_farming_experiments\RNDSRCH_design.csv" - # list of dictionaries that provide fixed factors for problems when you don't want to use the default values, if you want to use all default values use empty dictionary, order must match problem names - problem_fixed_factors = [{'budget': 2000, 'demand_mean': 90.0, 'fixed_cost':25},{'budget': 500}] + # list of dictionaries that provide fixed factors for problems when you don't want to use the default values, if you want to use all default values use empty dictionary, order must match problem names + problem_fixed_factors = [ + {"budget": 2000, "demand_mean": 90.0, "fixed_cost": 25}, + {"budget": 500}, + ] # uncomment this version to run w/ only default problem factors - #problem_fixed_factors = [{},{}] + # problem_fixed_factors = [{},{}] - solver_fixed_factors = {} # use this dictionary to change any default solver factors that were not included in the design + solver_fixed_factors = {} # use this dictionary to change any default solver factors that were not included in the design - n_macroreps = 2 # number of macroreplication to run and each solver design point - n_postreps = 100 # number of post replications to run on each macro replication - n_postreps_init_opt = 200 # number of normalization postreplications to run at initial solution and optimal solution + n_macroreps = ( + 2 # number of macroreplication to run and each solver design point + ) + n_postreps = ( + 100 # number of post replications to run on each macro replication + ) + n_postreps_init_opt = 200 # number of normalization postreplications to run at initial solution and optimal solution # turn design file into df & retrive dp information design_table = pd.read_csv(design_filename) - design_factor_names = design_table.columns.tolist() + design_factor_names = design_table.columns.tolist() - #remove GUI columns from list if present (ignore if not using design files generated by GUI) - design_factor_names.remove('Design #') - design_factor_names.remove('Solver Name') - design_factor_names.remove('Design Type') - design_factor_names.remove('Number Stacks') + # remove GUI columns from list if present (ignore if not using design files generated by GUI) + design_factor_names.remove("Design #") + design_factor_names.remove("Solver Name") + design_factor_names.remove("Design Type") + design_factor_names.remove("Number Stacks") - dp_list = [] # list of all design points + dp_list = [] # list of all design points - for index, row in design_table.iterrows(): - dp = {} #dictionary of current dp + for _, row in design_table.iterrows(): + dp = {} # dictionary of current dp for factor in design_factor_names: - dp[factor] = row[factor] + dp[factor] = row[factor] dp_list.append(dp) # add fixed solver factors to dps for fixed_factor in solver_fixed_factors: for dp in dp_list: dp[fixed_factor] = solver_fixed_factors[fixed_factor] - + n_dp = len(dp_list) solver_names = [] - for i in range(n_dp): + for _ in range(n_dp): solver_names.append(solver_name) - - experiment = ProblemsSolvers(solver_factors= dp_list, - problem_factors = problem_fixed_factors, - solver_names = solver_names, - problem_names = problem_names) + + experiment = ProblemsSolvers( + solver_factors=dp_list, + problem_factors=problem_fixed_factors, + solver_names=solver_names, + problem_names=problem_names, + ) experiment.run(n_macroreps) experiment.post_replicate(n_postreps) @@ -81,4 +93,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/demo/demo_model.py b/demo/demo_model.py index 277cbca7e..54e792a83 100644 --- a/demo/demo_model.py +++ b/demo/demo_model.py @@ -6,7 +6,10 @@ import sys import os.path as o -sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) # type:ignore + +sys.path.append( + o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), "..")) +) # type:ignore # Import random number generator. from mrg32k3a.mrg32k3a import MRG32k3a @@ -30,6 +33,7 @@ # Working example for MM1 model. # ----------------------------------------------- from simopt.models.mm1queue import MM1Queue + fixed_factors = {"lambda": 3.0, "mu": 8.0} mymodel = MM1Queue(fixed_factors) # ----------------------------------------------- @@ -39,9 +43,13 @@ # Check that all factors describe a simulatable model. # Check fixed factors individually. for key, value in mymodel.factors.items(): - print(f"The factor {key} is set as {value}. Is this simulatable? {bool(mymodel.check_simulatable_factor(key))}.") + print( + f"The factor {key} is set as {value}. Is this simulatable? {bool(mymodel.check_simulatable_factor(key))}." + ) # Check all factors collectively. -print(f"Is the specified model simulatable? {bool(mymodel.check_simulatable_factors())}.") +print( + f"Is the specified model simulatable? {bool(mymodel.check_simulatable_factors())}." +) # Create a list of RNG objects for the simulation model to use when # running replications. diff --git a/demo/demo_plots.py b/demo/demo_plots.py index c611e6c8f..d9e1546fe 100644 --- a/demo/demo_plots.py +++ b/demo/demo_plots.py @@ -6,28 +6,40 @@ import sys import os.path as o -import os -sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) # type:ignore + +sys.path.append( + o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), "..")) +) # type:ignore # Import the ProblemSolver class and other useful functions -from simopt.experiment_base import ProblemSolver, read_experiment_results, post_normalize, plot_terminal_progress, plot_terminal_scatterplots +from simopt.experiment_base import ( + ProblemSolver, + read_experiment_results, + post_normalize, + plot_terminal_progress, +) solver_names = {"RNDSRCH", "ASTRODF", "NELDMD"} -problem_names = {"SAN-1"} # CNTNEWS-1"} #, "SAN-1"} +problem_names = {"SAN-1"} # CNTNEWS-1"} #, "SAN-1"} # solver_name = "RNDSRCH" # Random search solver # problem_name = "CNTNEWS-1" # Continuous newsvendor problem # solver_name = # problem_name = for problem_name in problem_names: - problem_experiments = [] for solver_name in solver_names: print(f"Testing solver {solver_name} on problem {problem_name}.") # Initialize an instance of the experiment class. myexperiment = ProblemSolver(solver_name, problem_name) - file_name_path = "experiments/outputs/" + solver_name + "_on_" + problem_name + ".pickle" + file_name_path = ( + "experiments/outputs/" + + solver_name + + "_on_" + + problem_name + + ".pickle" + ) # Run a fixed number of macroreplications of the solver on the problem. # myexperiment.run(n_macroreps=10) @@ -49,21 +61,38 @@ # Re-compile problem-solver results. myexperiments = [] for solver_name in solver_names: - #solver_experiments = [] + # solver_experiments = [] for problem_name in problem_names: - file_name_path = "experiments/outputs/" + solver_name + "_on_" + problem_name + ".pickle" + file_name_path = ( + "experiments/outputs/" + + solver_name + + "_on_" + + problem_name + + ".pickle" + ) myexperiment = read_experiment_results(file_name_path) myexperiments.append(myexperiment) # solver_experiments.append(myexperiment) - # myexperiments.append(solver_experiments) +# myexperiments.append(solver_experiments) print("Plotting results.") # Produce basic plots. -plot_terminal_progress(experiments=myexperiments, plot_type="box", normalize=False) -plot_terminal_progress(experiments=myexperiments, plot_type="box", normalize=True) -plot_terminal_progress(experiments=myexperiments, plot_type="violin", normalize=False, all_in_one=False) -plot_terminal_progress(experiments=myexperiments, plot_type="violin", normalize=True) -#plot_terminal_scatterplots(experiments = myexperiments, all_in_one=False) +plot_terminal_progress( + experiments=myexperiments, plot_type="box", normalize=False +) +plot_terminal_progress( + experiments=myexperiments, plot_type="box", normalize=True +) +plot_terminal_progress( + experiments=myexperiments, + plot_type="violin", + normalize=False, + all_in_one=False, +) +plot_terminal_progress( + experiments=myexperiments, plot_type="violin", normalize=True +) +# plot_terminal_scatterplots(experiments = myexperiments, all_in_one=False) # Plots will be saved in the folder experiments/plots. print("Finished. Plots can be found in experiments/plots folder.") diff --git a/demo/demo_problem.py b/demo/demo_problem.py index 22c899ff4..a4cd58298 100644 --- a/demo/demo_problem.py +++ b/demo/demo_problem.py @@ -7,7 +7,10 @@ import sys import os.path as o -sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) # type:ignore + +sys.path.append( + o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), "..")) +) # type:ignore # Import random number generator. from mrg32k3a.mrg32k3a import MRG32k3a @@ -43,6 +46,7 @@ # Working example for CntNVMaxProfit problem. # ----------------------------------------------- from simopt.models.cntnv import CntNVMaxProfit + fixed_factors = {"initial_solution": (2,), "budget": 500} myproblem = CntNVMaxProfit(fixed_factors=fixed_factors) x = (3,) @@ -63,7 +67,9 @@ # The rest of this script requires no changes. # Create and attach rngs to solution -rng_list = [MRG32k3a(s_ss_sss_index=[0, ss, 0]) for ss in range(myproblem.model.n_rngs)] +rng_list = [ + MRG32k3a(s_ss_sss_index=[0, ss, 0]) for ss in range(myproblem.model.n_rngs) +] mysolution.attach_rngs(rng_list, copy=False) # Simulate a fixed number of replications (n_reps) at the solution x. @@ -71,23 +77,37 @@ myproblem.simulate(mysolution, num_macroreps=n_reps) # Print results to console. -print(f"Ran {n_reps} replications of the {myproblem.name} problem at solution x = {x}.\n") -print(f"The mean objective estimate was {round(mysolution.objectives_mean[0], 4)} with standard error {round(mysolution.objectives_stderr[0], 4)}.") +print( + f"Ran {n_reps} replications of the {myproblem.name} problem at solution x = {x}.\n" +) +print( + f"The mean objective estimate was {round(mysolution.objectives_mean[0], 4)} with standard error {round(mysolution.objectives_stderr[0], 4)}." +) print("The individual observations of the objective were:") for idx in range(n_reps): print(f"\t {round(mysolution.objectives[idx][0], 4)}") if myproblem.gradient_available: - print("\nThe individual observations of the gradients of the objective were:") + print( + "\nThe individual observations of the gradients of the objective were:" + ) for idx in range(n_reps): - print(f"\t {[round(g, 4) for g in mysolution.objectives_gradients[idx][0]]}") + print( + f"\t {[round(g, 4) for g in mysolution.objectives_gradients[idx][0]]}" + ) else: print("\nThis problem has no known gradients.") if myproblem.n_stochastic_constraints > 0: - print(f"\nThis problem has {myproblem.n_stochastic_constraints} stochastic constraints of the form E[LHS] <= 0.") + print( + f"\nThis problem has {myproblem.n_stochastic_constraints} stochastic constraints of the form E[LHS] <= 0." + ) for stc_idx in range(myproblem.n_stochastic_constraints): - print(f"\tFor stochastic constraint #{stc_idx + 1}, the mean of the LHS was {round(mysolution.stoch_constraints_mean[stc_idx], 4)} with standard error {round(mysolution.stoch_constraints_stderr[stc_idx], 4)}.") + print( + f"\tFor stochastic constraint #{stc_idx + 1}, the mean of the LHS was {round(mysolution.stoch_constraints_mean[stc_idx], 4)} with standard error {round(mysolution.stoch_constraints_stderr[stc_idx], 4)}." + ) print("\tThe observations of the LHSs were:") for idx in range(n_reps): - print(f"\t\t {round(mysolution.stoch_constraints[idx][stc_idx], 4)}") + print( + f"\t\t {round(mysolution.stoch_constraints[idx][stc_idx], 4)}" + ) else: print("\nThis problem has no stochastic constraints.") diff --git a/demo/demo_problem_solver.py b/demo/demo_problem_solver.py index 643c3c37b..70af5fd6e 100644 --- a/demo/demo_problem_solver.py +++ b/demo/demo_problem_solver.py @@ -6,13 +6,21 @@ import sys import os.path as o -import os -sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) # type:ignore + +sys.path.append( + o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), "..")) +) # type:ignore # Import the ProblemSolver class and other useful functions -from simopt.experiment_base import ProblemSolver, read_experiment_results, post_normalize, plot_progress_curves, plot_solvability_cdfs +from simopt.experiment_base import ( + ProblemSolver, + post_normalize, + plot_progress_curves, + plot_solvability_cdfs, +) + -def main(): +def main() -> None: # !! When testing a new solver/problem, first go to directory.py. # There you should add the import statement and an entry in the respective # dictionary (or dictionaries). @@ -32,7 +40,9 @@ def main(): print(f"Testing solver {solver_name} on problem {problem_name}.") # Specify file path name for storing experiment outputs in .pickle file. - file_name_path = "experiments/outputs/" + solver_name + "_on_" + problem_name + ".pickle" + file_name_path = ( + "experiments/outputs/" + solver_name + "_on_" + problem_name + ".pickle" + ) print(f"Results will be stored as {file_name_path}.") # Initialize an instance of the experiment class. @@ -57,13 +67,23 @@ def main(): print("Plotting results.") # Produce basic plots of the solver on the problem. - plot_progress_curves(experiments=[myexperiment], plot_type="all", normalize=False) - plot_progress_curves(experiments=[myexperiment], plot_type="mean", normalize=False) - plot_progress_curves(experiments=[myexperiment], plot_type="quantile", beta=0.90, normalize=False) + plot_progress_curves( + experiments=[myexperiment], plot_type="all", normalize=False + ) + plot_progress_curves( + experiments=[myexperiment], plot_type="mean", normalize=False + ) + plot_progress_curves( + experiments=[myexperiment], + plot_type="quantile", + beta=0.90, + normalize=False, + ) plot_solvability_cdfs(experiments=[myexperiment], solve_tol=0.1) # Plots will be saved in the folder experiments/plots. print("Finished. Plots can be found in experiments/plots folder.") + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/demo/demo_problems_solvers.py b/demo/demo_problems_solvers.py index c6b9a55ce..c4e218b9f 100644 --- a/demo/demo_problems_solvers.py +++ b/demo/demo_problems_solvers.py @@ -6,13 +6,16 @@ import sys import os.path as o -import os -sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) # type:ignore + +sys.path.append( + o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), "..")) +) # type:ignore # Import the ProblemsSolvers class and other useful functions from simopt.experiment_base import ProblemsSolvers, plot_solvability_profiles -def main(): + +def main() -> None: # !! When testing a new solver/problem, first go to directory.py. # There you should add the import statement and an entry in the respective # dictionary (or dictionaries). @@ -24,9 +27,10 @@ def main(): solver_names = ["RNDSRCH", "ASTRODF", "NELDMD"] problem_names = ["CNTNEWS-1", "SAN-1"] - # Initialize an instance of the experiment class. - mymetaexperiment = ProblemsSolvers(solver_names = solver_names, problem_names = problem_names) + mymetaexperiment = ProblemsSolvers( + solver_names=solver_names, problem_names=problem_names + ) # Write to log file. mymetaexperiment.log_group_experiment_results() @@ -42,10 +46,13 @@ def main(): print("Plotting results.") # Produce basic plots of the solvers on the problems. - plot_solvability_profiles(experiments=mymetaexperiment.experiments, plot_type="cdf_solvability") + plot_solvability_profiles( + experiments=mymetaexperiment.experiments, plot_type="cdf_solvability" + ) # Plots will be saved in the folder experiments/plots. print("Finished. Plots can be found in experiments/plots folder.") + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/demo/demo_san-sscont-ironorecont_experiment.py b/demo/demo_san-sscont-ironorecont_experiment.py index f25b869f9..f1ba9a708 100644 --- a/demo/demo_san-sscont-ironorecont_experiment.py +++ b/demo/demo_san-sscont-ironorecont_experiment.py @@ -5,40 +5,55 @@ (s, S) inventory, iron ore, and stochastic activity network). Produces plots appearing in the INFORMS Journal on Computing submission. """ + import sys import os.path as o -import os -sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) # type:ignore -from simopt.experiment_base import ProblemSolver, plot_area_scatterplots, post_normalize, plot_progress_curves, plot_solvability_cdfs, read_experiment_results, plot_solvability_profiles, plot_terminal_scatterplots, plot_terminal_progress +sys.path.append( + o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), "..")) +) # type:ignore + +from simopt.experiment_base import ( + ProblemSolver, + plot_area_scatterplots, + post_normalize, + plot_progress_curves, + plot_solvability_cdfs, + read_experiment_results, + plot_solvability_profiles, + plot_terminal_scatterplots, + plot_terminal_progress, +) + -def main(): +def main() -> None: # Problems factors used in experiments # SAN - all_random_costs = [(1, 2, 2, 7, 17, 7, 2, 13, 1, 9, 18, 16, 7), - (2, 1, 10, 13, 15, 13, 12, 9, 12, 15, 5, 8, 10), - (2, 6, 7, 11, 13, 5, 1, 2, 2, 3, 15, 16, 13), - (3, 4, 18, 8, 10, 17, 14, 19, 15, 15, 7, 10, 6), - (3, 6, 9, 15, 1, 19, 1, 13, 2, 19, 6, 7, 14), - (4, 4, 2, 4, 5, 3, 19, 4, 17, 5, 16, 8, 8), - (5, 14, 14, 7, 10, 14, 16, 16, 8, 7, 14, 11, 17), - (7, 9, 17, 19, 1, 7, 4, 3, 9, 9, 13, 17, 14), - (8, 14, 1, 10, 18, 10, 17, 1, 2, 11, 1, 16, 6), - (8, 17, 5, 17, 4, 14, 2, 5, 5, 5, 8, 8, 16), - (10, 3, 2, 7, 15, 12, 7, 9, 12, 17, 9, 1, 2), - (10, 5, 17, 12, 13, 14, 6, 5, 19, 17, 1, 7, 17), - (10, 16, 10, 13, 9, 1, 1, 16, 5, 7, 7, 12, 15), - (11, 5, 15, 13, 15, 17, 12, 12, 16, 11, 18, 19, 2), - (12, 11, 13, 4, 15, 11, 16, 2, 7, 7, 13, 8, 3), - (13, 3, 14, 2, 15, 18, 17, 13, 5, 17, 17, 5, 18), - (14, 8, 8, 14, 8, 8, 18, 16, 8, 18, 12, 6, 7), - (14, 18, 7, 8, 13, 17, 10, 17, 19, 1, 13, 6, 12), - (15, 1, 2, 6, 14, 18, 11, 19, 15, 18, 15, 1, 4), - (18, 4, 19, 2, 13, 11, 9, 2, 17, 18, 11, 7, 14)] + all_random_costs = [ + (1, 2, 2, 7, 17, 7, 2, 13, 1, 9, 18, 16, 7), + (2, 1, 10, 13, 15, 13, 12, 9, 12, 15, 5, 8, 10), + (2, 6, 7, 11, 13, 5, 1, 2, 2, 3, 15, 16, 13), + (3, 4, 18, 8, 10, 17, 14, 19, 15, 15, 7, 10, 6), + (3, 6, 9, 15, 1, 19, 1, 13, 2, 19, 6, 7, 14), + (4, 4, 2, 4, 5, 3, 19, 4, 17, 5, 16, 8, 8), + (5, 14, 14, 7, 10, 14, 16, 16, 8, 7, 14, 11, 17), + (7, 9, 17, 19, 1, 7, 4, 3, 9, 9, 13, 17, 14), + (8, 14, 1, 10, 18, 10, 17, 1, 2, 11, 1, 16, 6), + (8, 17, 5, 17, 4, 14, 2, 5, 5, 5, 8, 8, 16), + (10, 3, 2, 7, 15, 12, 7, 9, 12, 17, 9, 1, 2), + (10, 5, 17, 12, 13, 14, 6, 5, 19, 17, 1, 7, 17), + (10, 16, 10, 13, 9, 1, 1, 16, 5, 7, 7, 12, 15), + (11, 5, 15, 13, 15, 17, 12, 12, 16, 11, 18, 19, 2), + (12, 11, 13, 4, 15, 11, 16, 2, 7, 7, 13, 8, 3), + (13, 3, 14, 2, 15, 18, 17, 13, 5, 17, 17, 5, 18), + (14, 8, 8, 14, 8, 8, 18, 16, 8, 18, 12, 6, 7), + (14, 18, 7, 8, 13, 17, 10, 17, 19, 1, 13, 6, 12), + (15, 1, 2, 6, 14, 18, 11, 19, 15, 18, 15, 1, 4), + (18, 4, 19, 2, 13, 11, 9, 2, 17, 18, 11, 7, 14), + ] num_problems = len(all_random_costs) - # SSCONT demand_means = [25.0, 50.0, 100.0, 200.0, 400.0] lead_means = [1.0, 3.0, 6.0, 9.0] @@ -49,24 +64,25 @@ def main(): inven_stops = [1000, 10000] # RUNNING AND POST-PROCESSING EXPERIMENTS - M = 10 - N = 100 - L = 200 - + num_macroreps = 10 + num_postreps = 100 + num_postnorms = 200 # Five solvers. solvers = ["RNDSRCH_ss=10", "RNDSRCH_ss=50", "ASTRODF", "NELDMD", "STRONG"] # Two versions of random search with varying sample sizes. - rs_sample_sizes = [10, 50] + # rs_sample_sizes = [10, 50] # ASTRODF factors delta_max = 200.0 - # First problem: SAN # Loop over problem instances. for i in range(num_problems): model_fixed_factors = {} - problem_fixed_factors = {"budget": 10000, "arc_costs": all_random_costs[i]} + problem_fixed_factors = { + "budget": 10000, + "arc_costs": all_random_costs[i], + } problem_rename = f"SAN-1_rc={all_random_costs[i]}" # Temporarily store experiments on the same problem for post-normalization. @@ -85,29 +101,32 @@ def main(): solver_fixed_factors = {"delta_max": delta_max} # Loop over solvers: - new_experiment = ProblemSolver(solver_name=solver_name, - solver_rename=solver, - solver_fixed_factors=solver_fixed_factors, - problem_name="SAN-1", - problem_rename=problem_rename, - problem_fixed_factors=problem_fixed_factors, - model_fixed_factors=model_fixed_factors - ) + new_experiment = ProblemSolver( + solver_name=solver_name, + solver_rename=solver, + solver_fixed_factors=solver_fixed_factors, + problem_name="SAN-1", + problem_rename=problem_rename, + problem_fixed_factors=problem_fixed_factors, + model_fixed_factors=model_fixed_factors, + ) # Run experiment with M. - new_experiment.run(n_macroreps=M) + new_experiment.run(n_macroreps=num_macroreps) # Post replicate experiment with N. - new_experiment.post_replicate(n_postreps=N) + new_experiment.post_replicate(n_postreps=num_postreps) experiments_same_problem.append(new_experiment) # Post-normalize experiments with L. # Provide NO proxies for f(x0), f(x*), or f(x). - post_normalize(experiments=experiments_same_problem, n_postreps_init_opt=L) + post_normalize( + experiments=experiments_same_problem, + n_postreps_init_opt=num_postnorms, + ) # Second problem: SSCONT for dm in demand_means: for lm in lead_means: - model_fixed_factors = {"demand_mean": dm, - "lead_mean": lm} + model_fixed_factors = {"demand_mean": dm, "lead_mean": lm} # Default budget for (s,S) inventory problem = 1000 replications. # RS with sample size of 100 will get through only 10 iterations. problem_fixed_factors = {"budget": 1000} @@ -128,31 +147,37 @@ def main(): solver_fixed_factors = {"delta_max": delta_max} # Loop over solvers: - new_experiment = ProblemSolver(solver_name=solver_name, - solver_rename=solver, - solver_fixed_factors=solver_fixed_factors, - problem_name="SSCONT-1", - problem_rename=problem_rename, - problem_fixed_factors=problem_fixed_factors, - model_fixed_factors=model_fixed_factors - ) + new_experiment = ProblemSolver( + solver_name=solver_name, + solver_rename=solver, + solver_fixed_factors=solver_fixed_factors, + problem_name="SSCONT-1", + problem_rename=problem_rename, + problem_fixed_factors=problem_fixed_factors, + model_fixed_factors=model_fixed_factors, + ) # Run experiment with M. - new_experiment.run(n_macroreps=M) + new_experiment.run(n_macroreps=num_macroreps) # Post replicate experiment with N. - new_experiment.post_replicate(n_postreps=N) + new_experiment.post_replicate(n_postreps=num_postreps) experiments_same_problem.append(new_experiment) # Post-normalize experiments with L. # Provide NO proxies for f(x0), f(x*), or f(x). - post_normalize(experiments=experiments_same_problem, n_postreps_init_opt=L) + post_normalize( + experiments=experiments_same_problem, + n_postreps_init_opt=num_postnorms, + ) # Third problem: IRONORECONT for sd in st_devs: for hc in holding_costs: for inv in inven_stops: - model_fixed_factors = {"st_dev": sd, - "holding_cost": hc, - "inven_stop": inv} + model_fixed_factors = { + "st_dev": sd, + "holding_cost": hc, + "inven_stop": inv, + } problem_fixed_factors = {"budget": 1000} problem_rename = f"IRONORECONT-1_sd={sd}_hc={hc}_inv={inv}" @@ -171,23 +196,27 @@ def main(): solver_fixed_factors = {"delta_max": delta_max} # Loop over solvers: - new_experiment = ProblemSolver(solver_name=solver_name, - solver_rename=solver, - solver_fixed_factors=solver_fixed_factors, - problem_name="IRONORECONT-1", - problem_rename=problem_rename, - problem_fixed_factors=problem_fixed_factors, - model_fixed_factors=model_fixed_factors - ) + new_experiment = ProblemSolver( + solver_name=solver_name, + solver_rename=solver, + solver_fixed_factors=solver_fixed_factors, + problem_name="IRONORECONT-1", + problem_rename=problem_rename, + problem_fixed_factors=problem_fixed_factors, + model_fixed_factors=model_fixed_factors, + ) # Run experiment with M. - new_experiment.run(n_macroreps=M) + new_experiment.run(n_macroreps=num_macroreps) # Post replicate experiment with N. - new_experiment.post_replicate(n_postreps=N) + new_experiment.post_replicate(n_postreps=num_postreps) experiments_same_problem.append(new_experiment) # Post-normalize experiments with L. # Provide NO proxies for f(x0), f(x*), or f(x). - post_normalize(experiments=experiments_same_problem, n_postreps_init_opt=L) + post_normalize( + experiments=experiments_same_problem, + n_postreps_init_opt=num_postnorms, + ) # LOAD DATA FROM .PICKLE FILES TO PREPARE FOR PLOTTING. @@ -196,7 +225,6 @@ def main(): # inner list - index by problem experiments = [] - # Load .pickle files of past results. # Load all experiments for a given solver, for all solvers. # Load experiments belonging to the problems in: @@ -223,9 +251,13 @@ def main(): problem_rename = f"{problem}-1_rc={all_random_costs[i]}" file_name = f"{solver}_on_{problem_rename}" # Load experiment. - new_experiment = read_experiment_results(f"experiments/outputs/{file_name}.pickle") + new_experiment = read_experiment_results( + f"experiments/outputs/{file_name}.pickle" + ) # Rename problem to produce nicer plot labels. - new_experiment.problem.name = f"{problem}-1 with rc={all_random_costs[i]}" + new_experiment.problem.name = ( + f"{problem}-1 with rc={all_random_costs[i]}" + ) new_experiment.solver.name = solver_display experiments_same_solver.append(new_experiment) @@ -236,9 +268,11 @@ def main(): problem_rename = f"{problem}-1_dm={dm}_lm={lm}" file_name = f"{solver}_on_{problem_rename}" # Load experiment. - new_experiment = read_experiment_results(f"experiments/outputs/{file_name}.pickle") + new_experiment = read_experiment_results( + f"experiments/outputs/{file_name}.pickle" + ) # Rename problem to produce nicer plot labels. - new_experiment.problem.name = fr"{problem}-1 with $\mu_D={round(dm)}$ and $\mu_L={round(lm)}$" + new_experiment.problem.name = rf"{problem}-1 with $\mu_D={round(dm)}$ and $\mu_L={round(lm)}$" new_experiment.solver.name = solver_display experiments_same_solver.append(new_experiment) @@ -247,12 +281,16 @@ def main(): for sd in st_devs: for hc in holding_costs: for inv in inven_stops: - problem_rename = f"{problem}-1_sd={sd}_hc={hc}_inv={inv}" + problem_rename = ( + f"{problem}-1_sd={sd}_hc={hc}_inv={inv}" + ) file_name = f"{solver}_on_{problem_rename}" # Load experiment. - new_experiment = read_experiment_results(f"experiments/outputs/{file_name}.pickle") + new_experiment = read_experiment_results( + f"experiments/outputs/{file_name}.pickle" + ) # Rename problem to produce nicer plot labels. - new_experiment.problem.name = fr"{problem}-1 with $\sigma={sd}$ and hc={hc} and inv={inv}" + new_experiment.problem.name = rf"{problem}-1 with $\sigma={sd}$ and hc={hc} and inv={inv}" new_experiment.solver.name = solver_display experiments_same_solver.append(new_experiment) @@ -263,31 +301,109 @@ def main(): n_solvers = len(experiments) n_problems = len(experiments[0]) - CI_param = True + enable_confidence_intervals = True alpha = 0.2 - plot_solvability_profiles(experiments, plot_type="cdf_solvability", solve_tol=alpha, all_in_one=True, plot_conf_ints=CI_param, print_max_hw=CI_param) - plot_solvability_profiles(experiments, plot_type="quantile_solvability", solve_tol=alpha, beta=0.5, all_in_one=True, plot_conf_ints=CI_param, print_max_hw=CI_param) - plot_solvability_profiles(experiments=experiments, plot_type="diff_cdf_solvability", solve_tol=alpha, ref_solver="ASTRO-DF", all_in_one=True, plot_conf_ints=CI_param, print_max_hw=CI_param) - plot_solvability_profiles(experiments=experiments, plot_type="diff_quantile_solvability", solve_tol=alpha, beta=0.5, ref_solver="ASTRO-DF", all_in_one=True, plot_conf_ints=CI_param, print_max_hw=CI_param) - plot_area_scatterplots(experiments, all_in_one=True, plot_conf_ints=CI_param, print_max_hw=CI_param) + plot_solvability_profiles( + experiments, + plot_type="cdf_solvability", + solve_tol=alpha, + all_in_one=True, + plot_conf_ints=enable_confidence_intervals, + print_max_hw=enable_confidence_intervals, + ) + plot_solvability_profiles( + experiments, + plot_type="quantile_solvability", + solve_tol=alpha, + beta=0.5, + all_in_one=True, + plot_conf_ints=enable_confidence_intervals, + print_max_hw=enable_confidence_intervals, + ) + plot_solvability_profiles( + experiments=experiments, + plot_type="diff_cdf_solvability", + solve_tol=alpha, + ref_solver="ASTRO-DF", + all_in_one=True, + plot_conf_ints=enable_confidence_intervals, + print_max_hw=enable_confidence_intervals, + ) + plot_solvability_profiles( + experiments=experiments, + plot_type="diff_quantile_solvability", + solve_tol=alpha, + beta=0.5, + ref_solver="ASTRO-DF", + all_in_one=True, + plot_conf_ints=enable_confidence_intervals, + print_max_hw=enable_confidence_intervals, + ) + plot_area_scatterplots( + experiments, + all_in_one=True, + plot_conf_ints=enable_confidence_intervals, + print_max_hw=enable_confidence_intervals, + ) plot_terminal_scatterplots(experiments, all_in_one=True) for i in range(n_problems): - plot_progress_curves([experiments[solver_idx][i] for solver_idx in range(n_solvers)], plot_type="mean", all_in_one=True, plot_conf_ints=CI_param, print_max_hw=True) - plot_terminal_progress([experiments[solver_idx][i] for solver_idx in range(n_solvers)], plot_type="violin", normalize=True, all_in_one=True) + plot_progress_curves( + [experiments[solver_idx][i] for solver_idx in range(n_solvers)], + plot_type="mean", + all_in_one=True, + plot_conf_ints=enable_confidence_intervals, + print_max_hw=True, + ) + plot_terminal_progress( + [experiments[solver_idx][i] for solver_idx in range(n_solvers)], + plot_type="violin", + normalize=True, + all_in_one=True, + ) # plot_solvability_cdfs([experiments[solver_idx][i] for solver_idx in range(n_solvers)], solve_tol=0.2, all_in_one=True, plot_CIs=True, print_max_hw=True) # Plots for mu_D = 400 and mu_L = 6 (appreared in the paper) - plot_progress_curves([experiments[solver_idx][0] for solver_idx in range(n_solvers)], plot_type="all", all_in_one=True) - - plot_progress_curves([experiments[solver_idx][0] for solver_idx in range(3, 4)], plot_type="all", all_in_one=True, normalize=False) - - plot_progress_curves([experiments[solver_idx][0] for solver_idx in range(n_solvers)], plot_type="mean", all_in_one=True, plot_conf_ints=True, print_max_hw=False, normalize=True) - - plot_solvability_cdfs(experiments=[experiments[solver_idx][0] for solver_idx in range(n_solvers)], solve_tol=0.2, all_in_one=True, plot_conf_ints=True, print_max_hw=False) - - plot_terminal_progress([experiments[solver_idx][0] for solver_idx in range(n_solvers)], plot_type="violin", normalize=False, all_in_one=True) - -if (__name__ == "__main__"): - main() \ No newline at end of file + plot_progress_curves( + [experiments[solver_idx][0] for solver_idx in range(n_solvers)], + plot_type="all", + all_in_one=True, + ) + + plot_progress_curves( + [experiments[solver_idx][0] for solver_idx in range(3, 4)], + plot_type="all", + all_in_one=True, + normalize=False, + ) + + plot_progress_curves( + [experiments[solver_idx][0] for solver_idx in range(n_solvers)], + plot_type="mean", + all_in_one=True, + plot_conf_ints=True, + print_max_hw=False, + normalize=True, + ) + + plot_solvability_cdfs( + experiments=[ + experiments[solver_idx][0] for solver_idx in range(n_solvers) + ], + solve_tol=0.2, + all_in_one=True, + plot_conf_ints=True, + print_max_hw=False, + ) + + plot_terminal_progress( + [experiments[solver_idx][0] for solver_idx in range(n_solvers)], + plot_type="violin", + normalize=False, + all_in_one=True, + ) + + +if __name__ == "__main__": + main() diff --git a/demo/demo_sscont_experiment.py b/demo/demo_sscont_experiment.py index d236e6f99..6413c8a20 100644 --- a/demo/demo_sscont_experiment.py +++ b/demo/demo_sscont_experiment.py @@ -22,7 +22,7 @@ ) -def main(): +def main() -> None: # Default values of the (s, S) model: # "demand_mean": 100.0 # "lead_mean": 6.0 @@ -42,9 +42,9 @@ def main(): rs_sample_sizes = [10, 50] # RUNNING AND POST-PROCESSING EXPERIMENTS - M = 10 - N = 100 - L = 200 + num_macroreps = 10 + num_postreps = 100 + num_postnorms = 200 # Loop over problem instances. for dm in demand_means: for lm in lead_means: @@ -72,9 +72,9 @@ def main(): model_fixed_factors=model_fixed_factors, ) # Run experiment with M. - new_experiment.run(n_macroreps=M) + new_experiment.run(n_macroreps=num_macroreps) # Post replicate experiment with N. - new_experiment.post_replicate(n_postreps=N) + new_experiment.post_replicate(n_postreps=num_postreps) experiments_same_problem.append(new_experiment) # Setup and run ASTRO-DF. @@ -88,9 +88,9 @@ def main(): model_fixed_factors=model_fixed_factors, ) # Run experiment with M. - new_experiment.run(n_macroreps=M) + new_experiment.run(n_macroreps=num_macroreps) # Post replicate experiment with N. - new_experiment.post_replicate(n_postreps=N) + new_experiment.post_replicate(n_postreps=num_postreps) experiments_same_problem.append(new_experiment) # Setup and run Nelder-Mead. @@ -102,9 +102,9 @@ def main(): model_fixed_factors=model_fixed_factors, ) # Run experiment withM. - new_experiment.run(n_macroreps=M) + new_experiment.run(n_macroreps=num_macroreps) # Post replicate experiment with N. - new_experiment.post_replicate(n_postreps=N) + new_experiment.post_replicate(n_postreps=num_postreps) experiments_same_problem.append(new_experiment) # Setup and run STRONG.= @@ -116,15 +116,16 @@ def main(): model_fixed_factors=model_fixed_factors, ) # Run experiment with M. - new_experiment.run(n_macroreps=M) + new_experiment.run(n_macroreps=num_macroreps) # Post replicate experiment with N. - new_experiment.post_replicate(n_postreps=N) + new_experiment.post_replicate(n_postreps=num_postreps) experiments_same_problem.append(new_experiment) # Post-normalize experiments with L. # Provide NO proxies for f(x0), f(x*), or f(x). post_normalize( - experiments=experiments_same_problem, n_postreps_init_opt=L + experiments=experiments_same_problem, + n_postreps_init_opt=num_postnorms, ) # LOAD DATA FROM .PICKLE FILES TO PREPARE FOR PLOTTING. @@ -385,5 +386,6 @@ def main(): # ref_solver="ASTRO-DF" # ) + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/demo/exp_base_testing.py b/demo/exp_base_testing.py index fa1f2f5e7..1f8e4ad89 100644 --- a/demo/exp_base_testing.py +++ b/demo/exp_base_testing.py @@ -6,38 +6,42 @@ import sys import os.path as o -sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) # type:ignore -from simopt.experiment_base import create_design, ProblemsSolvers +sys.path.append( + o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), "..")) +) # type:ignore -def main(): +from simopt.experiment_base import ProblemsSolvers + + +def main() -> None: # Specify the name of the solver as it appears in directory.py - - solver_names = ['RNDSRCH', 'RNDSRCH', 'ASTRODF'] - + + solver_names = ["RNDSRCH", "RNDSRCH", "ASTRODF"] + solver_renames = ["RND_test1", "RND_test2", "AST_test"] - - problem_names = ['EXAMPLE-1', 'CNTNEWS-1'] - - problem_renames = ['EX_test', 'NEWS_test'] - - experiment_name = 'test_exp' - - solver_factors = [{}, {'sample_size':2}, {}] - + + problem_names = ["EXAMPLE-1", "CNTNEWS-1"] + + problem_renames = ["EX_test", "NEWS_test"] + + experiment_name = "test_exp" + + solver_factors = [{}, {"sample_size": 2}, {}] + problem_factors = [{}, {}] - # Create ProblemsSovlers experiment with solver and model design - experiment = ProblemsSolvers(solver_factors = solver_factors, - problem_factors = problem_factors, - solver_names = solver_names, - problem_names = problem_names, - solver_renames = solver_renames, - problem_renames = problem_renames, - experiment_name = experiment_name, - create_pair_pickles=True - ) + experiment = ProblemsSolvers( + solver_factors=solver_factors, + problem_factors=problem_factors, + solver_names=solver_names, + problem_names=problem_names, + solver_renames=solver_renames, + problem_renames=problem_renames, + experiment_name=experiment_name, + create_pair_pickles=True, + ) # check compatibility of selected solvers and problems experiment.check_compatibility() @@ -54,5 +58,6 @@ def main(): experiment.log_group_experiment_results() experiment.report_group_statistics() + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/docs/conf.py b/docs/conf.py index 032938bb9..dc15780c8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,7 +29,7 @@ # -- Project information ----------------------------------------------------- project = "SimOpt" -copyright = "2021, simopt-admin" +copyright = "2021, simopt-admin" # noqa: A001 author = "simopt-admin" # The full version, including alpha/beta/rc tags diff --git a/notebooks/hello_simopt.ipynb b/notebooks/hello_simopt.ipynb index 2ef9ed9db..048ece0f9 100644 --- a/notebooks/hello_simopt.ipynb +++ b/notebooks/hello_simopt.ipynb @@ -6,8 +6,10 @@ "metadata": {}, "outputs": [], "source": [ + "# ruff: noqa: E402\n", "import os\n", - "os.chdir('../') # Move one level up to import simopt\n", + "\n", + "os.chdir(\"../\") # Move one level up to import simopt\n", "\n", "# Import experiment_base module, which contains functions for experimentation.\n", "import simopt.experiment_base" @@ -25,11 +27,15 @@ "\n", "# Post-process the results.\n", "myexperiment.post_replicate(n_postreps=200)\n", - "simopt.experiment_base.post_normalize(experiments=[myexperiment], n_postreps_init_opt=200)\n", + "simopt.experiment_base.post_normalize(\n", + " experiments=[myexperiment], n_postreps_init_opt=200\n", + ")\n", "\n", "# Record the results and plot the mean progress curve.\n", "myexperiment.log_experiment_results()\n", - "simopt.experiment_base.plot_progress_curves(experiments=[myexperiment], plot_type=\"mean\", normalize=False)" + "simopt.experiment_base.plot_progress_curves(\n", + " experiments=[myexperiment], plot_type=\"mean\", normalize=False\n", + ")" ] } ], diff --git a/simopt/gui/data_farming_window.py b/simopt/gui/data_farming_window.py index c52797037..bd328b456 100644 --- a/simopt/gui/data_farming_window.py +++ b/simopt/gui/data_farming_window.py @@ -684,10 +684,10 @@ def show_model_factors(self, *args: tuple) -> None: ].get("description") self.factor_default = self.model_object.specifications[factor].get( "default" - ) - self.factor_isDatafarmable = self.model_object.specifications[factor].get( - "isDatafarmable") + self.factor_isDatafarmable = self.model_object.specifications[ + factor + ].get("isDatafarmable") # Values to help with formatting entry_width = 10 @@ -710,7 +710,10 @@ def show_model_factors(self, *args: tuple) -> None: padx=10, ) - if self.factor_datatype is float and self.factor_isDatafarmable is not False: + if ( + self.factor_datatype is float + and self.factor_isDatafarmable is not False + ): self.factors_frame.grid_rowconfigure( self.factor_que_length, weight=1 ) @@ -845,7 +848,10 @@ def show_model_factors(self, *args: tuple) -> None: self.factor_que_length += 1 - elif self.factor_datatype is int and self.factor_isDatafarmable is not False: + elif ( + self.factor_datatype is int + and self.factor_isDatafarmable is not False + ): self.factors_frame.grid_rowconfigure( self.factor_que_length, weight=1 ) @@ -950,7 +956,10 @@ def show_model_factors(self, *args: tuple) -> None: self.factor_que_length += 1 - elif self.factor_datatype is list or self.factor_isDatafarmable is False: + elif ( + self.factor_datatype is list + or self.factor_isDatafarmable is False + ): self.factors_frame.grid_rowconfigure( self.factor_que_length, weight=1 ) @@ -1219,9 +1228,9 @@ def create_design(self, *args: tuple) -> None: factor_datatype = self.model_object.specifications[factor].get( "datatype" ) - factor_isDatafarmable = self.model_object.specifications[factor].get( - "isDatafarmable" - ) + is_datafarmable_factor = self.model_object.specifications[ + factor + ].get("isDatafarmable") factor_include = check_values[factor_index] # get user inputs for design factors @@ -1229,7 +1238,10 @@ def create_design(self, *args: tuple) -> None: if factor_include: self.factor_names.append(factor) - if factor_datatype in (float, int) and factor_isDatafarmable is not False: + if ( + factor_datatype in (float, int) + and is_datafarmable_factor is not False + ): factor_min = str(min_values[maxmin_index]) factor_max = str(max_values[maxmin_index]) maxmin_index += 1 @@ -1363,12 +1375,15 @@ def include_factor(self, *args: tuple) -> None: self.factor_default = self.model_object.specifications[factor].get( "default" ) - self.factor_isDatafarmable = self.model_object.specifications[factor].get( - "isDatafarmable" - ) + self.factor_isDatafarmable = self.model_object.specifications[ + factor + ].get("isDatafarmable") # Disable / enable experiment option widgets depending on factor type - if self.factor_datatype in (int, float) and self.factor_isDatafarmable is not False: + if ( + self.factor_datatype in (int, float) + and self.factor_isDatafarmable is not False + ): self.current_min_entry = self.min_widgets[factor] self.current_max_entry = self.max_widgets[factor] diff --git a/simopt/gui/experiment_window.py b/simopt/gui/experiment_window.py index 2f103a6eb..5233cb4e0 100644 --- a/simopt/gui/experiment_window.py +++ b/simopt/gui/experiment_window.py @@ -3655,7 +3655,7 @@ class PostProcessingWindow(Toplevel): def __init__( self, root: tk.Tk, - myexperiment, + myexperiment: ProblemSolver, experiment_list: list, main_window: tk.Tk, meta: bool = False, @@ -3866,8 +3866,7 @@ def post_processing_run_function(self) -> list: and self.crn_across_macroreps_var.get() in self.crn_across_macroreps_list and ( - (self.meta - and self.n_norm_postreps_entry.get().isnumeric()) + (self.meta and self.n_norm_postreps_entry.get().isnumeric()) or not self.meta ) ): diff --git a/simopt/gui/main_menu.py b/simopt/gui/main_menu.py index 629f0ae88..538ebc54b 100644 --- a/simopt/gui/main_menu.py +++ b/simopt/gui/main_menu.py @@ -27,7 +27,7 @@ def __init__(self, root: tk.Tk) -> None: ) # Set the size of the window to XX% of the screen size size_percent = 50 - self.center_window(size_percent/100.0) + self.center_window(size_percent / 100.0) self.menu_frame = ttk.Frame(master=self) self.menu_frame.pack(anchor="center", expand=True) diff --git a/simopt/gui/plot_window.py b/simopt/gui/plot_window.py index 85d55c2b1..db501a141 100644 --- a/simopt/gui/plot_window.py +++ b/simopt/gui/plot_window.py @@ -4,7 +4,7 @@ from tkinter import Listbox, Scrollbar, ttk from tkinter.constants import MULTIPLE from tkinter.font import nametofont -from typing import Literal +from typing import Literal, Union from PIL import Image, ImageTk @@ -13,6 +13,7 @@ solver_unabbreviated_directory, ) from simopt.experiment_base import ( + ProblemSolver, plot_area_scatterplots, plot_progress_curves, plot_solvability_cdfs, @@ -42,7 +43,7 @@ def __init__( root: tk.Tk, main_window: tk.Tk, experiment_list: list, - meta_list=None, + meta_list: Union[list[ProblemSolver], None] = None, ) -> None: """Initialize the Plot_Window class. diff --git a/simopt/gui/toplevel_custom.py b/simopt/gui/toplevel_custom.py index cbf5bacf0..765b0a890 100644 --- a/simopt/gui/toplevel_custom.py +++ b/simopt/gui/toplevel_custom.py @@ -48,7 +48,7 @@ def set_style(self) -> None: width = height * (16 / 9) # Otherwise, we're good with just using the width # Target a 1920x1080 screen - scale = width / 1920 + scale = width / 1920 font_medium = int(12 * scale) if sys.platform == "darwin": diff --git a/simopt/models/amusementpark.py b/simopt/models/amusementpark.py index be1ae66d1..a5af04704 100644 --- a/simopt/models/amusementpark.py +++ b/simopt/models/amusementpark.py @@ -79,7 +79,7 @@ def specifications(self) -> dict[str, dict]: "description": "The number of attractions in the park.", "datatype": int, "default": NUM_ATTRACTIONS, - "isDatafarmable": False + "isDatafarmable": False, }, "time_open": { "description": "The number of minutes per day the park is open.", @@ -557,7 +557,7 @@ def specifications(self) -> dict[str, dict]: "description": "Max # of replications for a solver to take.", "datatype": int, "default": 100, - "isDatafarmable": False + "isDatafarmable": False, }, } diff --git a/simopt/models/chessmm.py b/simopt/models/chessmm.py index cd38e929d..cc8a53bcc 100644 --- a/simopt/models/chessmm.py +++ b/simopt/models/chessmm.py @@ -343,7 +343,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 1000, - "isDatafarmable": False + "isDatafarmable": False, }, "upper_time": { "description": "upper bound on wait time", diff --git a/simopt/models/cntnv.py b/simopt/models/cntnv.py index 5a23c26f1..0b285d23e 100644 --- a/simopt/models/cntnv.py +++ b/simopt/models/cntnv.py @@ -348,7 +348,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 1000, - "isDatafarmable": False + "isDatafarmable": False, }, } diff --git a/simopt/models/contam.py b/simopt/models/contam.py index 583a0a9df..4d1ce7980 100644 --- a/simopt/models/contam.py +++ b/simopt/models/contam.py @@ -192,8 +192,8 @@ def replicate(self, rng_list: list[MRG32k3a]) -> tuple[dict, dict]: contam_rng = rng_list[0] restore_rng = rng_list[1] # Generate rates with beta distribution. - X = np.zeros(self.factors["stages"]) - X[0] = restore_rng.betavariate( + levels = np.zeros(self.factors["stages"]) + levels[0] = restore_rng.betavariate( alpha=self.factors["initial_rate_alpha"], beta=self.factors["initial_rate_beta"], ) @@ -207,9 +207,12 @@ def replicate(self, rng_list: list[MRG32k3a]) -> tuple[dict, dict]: alpha=self.factors["restore_rate_alpha"], beta=self.factors["restore_rate_beta"], ) - X[i] = c * (1 - u[i]) * (1 - X[i - 1]) + (1 - r * u[i]) * X[i - 1] + levels[i] = ( + c * (1 - u[i]) * (1 - levels[i - 1]) + + (1 - r * u[i]) * levels[i - 1] + ) # Compose responses and gradients. - responses = {"level": X} + responses = {"level": levels} gradients = { response_key: { factor_key: np.nan for factor_key in self.specifications @@ -737,7 +740,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 10000, - "isDatafarmable": False + "isDatafarmable": False, }, "prev_cost": { "description": "cost of prevention", diff --git a/simopt/models/dualsourcing.py b/simopt/models/dualsourcing.py index fe893e3b4..8c167ca2e 100644 --- a/simopt/models/dualsourcing.py +++ b/simopt/models/dualsourcing.py @@ -92,7 +92,7 @@ def specifications(self) -> dict[str, dict]: "description": "number of days to simulate", "datatype": int, "default": 1000, - "isDatafarmable": False + "isDatafarmable": False, }, "initial_inv": { "description": "initial inventory", @@ -468,7 +468,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 1000, - "isDatafarmable": False + "isDatafarmable": False, }, } diff --git a/simopt/models/dynamnews.py b/simopt/models/dynamnews.py index cda4e4842..9f9c4bb48 100644 --- a/simopt/models/dynamnews.py +++ b/simopt/models/dynamnews.py @@ -374,7 +374,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 1000, - "isDatafarmable": False + "isDatafarmable": False, }, } diff --git a/simopt/models/example.py b/simopt/models/example.py index 06ee9b75c..84292bc47 100644 --- a/simopt/models/example.py +++ b/simopt/models/example.py @@ -242,7 +242,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 1000, - "isDatafarmable": False + "isDatafarmable": False, }, } diff --git a/simopt/models/facilitysizing.py b/simopt/models/facilitysizing.py index 947cef3e9..e0093f0c1 100644 --- a/simopt/models/facilitysizing.py +++ b/simopt/models/facilitysizing.py @@ -87,7 +87,7 @@ def specifications(self) -> dict[str, dict]: "description": "number of facilities", "datatype": int, "default": NUM_FACILITIES, - "isDatafarmable": False + "isDatafarmable": False, }, } @@ -318,7 +318,7 @@ def specifications(self) -> dict[str, dict]: "description": "Max # of replications for a solver to take.", "datatype": int, "default": 10000, - "isDatafarmable": False + "isDatafarmable": False, }, "installation_costs": { "description": "Cost to install a unit of capacity at each facility.", diff --git a/simopt/models/fixedsan.py b/simopt/models/fixedsan.py index 292207264..224587c59 100644 --- a/simopt/models/fixedsan.py +++ b/simopt/models/fixedsan.py @@ -340,7 +340,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 10000, - "isDatafarmable": False + "isDatafarmable": False, }, "arc_costs": { "description": "cost associated to each arc", diff --git a/simopt/models/hotel.py b/simopt/models/hotel.py index 48ebda901..eece5218c 100644 --- a/simopt/models/hotel.py +++ b/simopt/models/hotel.py @@ -898,7 +898,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 100, - "isDatafarmable": False + "isDatafarmable": False, }, } diff --git a/simopt/models/ironore.py b/simopt/models/ironore.py index 7ffe2e19f..9a6deb4d4 100644 --- a/simopt/models/ironore.py +++ b/simopt/models/ironore.py @@ -132,7 +132,7 @@ def specifications(self) -> dict[str, dict]: "description": "number of days to simulate", "datatype": int, "default": 365, - "isDatafarmable": False + "isDatafarmable": False, }, } @@ -450,7 +450,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 1000, - "isDatafarmable": False + "isDatafarmable": False, }, } diff --git a/simopt/models/mm1queue.py b/simopt/models/mm1queue.py index dcb118ca4..eb7465235 100644 --- a/simopt/models/mm1queue.py +++ b/simopt/models/mm1queue.py @@ -366,7 +366,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 1000, - "isDatafarmable": False + "isDatafarmable": False, }, "cost": { "description": "cost for increasing service rate", diff --git a/simopt/models/network.py b/simopt/models/network.py index 2fa45eb02..0a5fd3670 100644 --- a/simopt/models/network.py +++ b/simopt/models/network.py @@ -472,7 +472,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 1000, - "isDatafarmable": False + "isDatafarmable": False, }, } diff --git a/simopt/models/paramesti.py b/simopt/models/paramesti.py index a55fa4ec2..b73f95f3b 100644 --- a/simopt/models/paramesti.py +++ b/simopt/models/paramesti.py @@ -274,7 +274,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 1000, - "isDatafarmable": False + "isDatafarmable": False, }, } diff --git a/simopt/models/rmitd.py b/simopt/models/rmitd.py index 080065938..4f246c139 100644 --- a/simopt/models/rmitd.py +++ b/simopt/models/rmitd.py @@ -399,7 +399,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 10000, - "isDatafarmable": False + "isDatafarmable": False, }, } diff --git a/simopt/models/san.py b/simopt/models/san.py index cffffebf7..6d2045e5c 100644 --- a/simopt/models/san.py +++ b/simopt/models/san.py @@ -68,7 +68,7 @@ def specifications(self) -> dict[str, dict]: "description": "number of nodes", "datatype": int, "default": 9, - "isDatafarmable": False + "isDatafarmable": False, }, "arcs": { "description": "list of arcs", @@ -374,7 +374,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 10000, - "isDatafarmable": False + "isDatafarmable": False, }, "arc_costs": { "description": "Cost associated to each arc.", diff --git a/simopt/models/sscont.py b/simopt/models/sscont.py index c2252ae34..9dbae51f0 100644 --- a/simopt/models/sscont.py +++ b/simopt/models/sscont.py @@ -131,13 +131,13 @@ def specifications(self) -> dict[str, dict]: "description": "number of periods to simulate", "datatype": int, "default": 100, - "isDatafarmable": False + "isDatafarmable": False, }, "warmup": { "description": "number of periods as warmup before collecting statistics", "datatype": int, "default": 20, - "isDatafarmable": False + "isDatafarmable": False, }, } @@ -500,7 +500,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 1000, - "isDatafarmable": False + "isDatafarmable": False, }, } diff --git a/simopt/models/tableallocation.py b/simopt/models/tableallocation.py index a9b5a5628..deb797622 100644 --- a/simopt/models/tableallocation.py +++ b/simopt/models/tableallocation.py @@ -399,7 +399,7 @@ def specifications(self) -> dict[str, dict]: "description": "max # of replications for a solver to take", "datatype": int, "default": 1000, - "isDatafarmable": False + "isDatafarmable": False, }, } diff --git a/simoptlib.egg-info/PKG-INFO b/simoptlib.egg-info/PKG-INFO deleted file mode 100644 index 42443ca97..000000000 --- a/simoptlib.egg-info/PKG-INFO +++ /dev/null @@ -1,292 +0,0 @@ -Metadata-Version: 2.1 -Name: simoptlib -Version: 1.1.1 -Summary: A testbed for simulation-optimization experiments. -Author-email: David Eckman , Shane Henderson , Sara Shashaani , William Grochocinski -License: MIT License - - Copyright (c) 2019 simopt-admin - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. - -Project-URL: Homepage, https://github.com/simopt-admin/simopt -Project-URL: Documentation, https://simopt.readthedocs.io/en/latest/ -Classifier: Programming Language :: Python :: 3 -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: OS Independent -Requires-Python: >=3.8 -Description-Content-Type: text/markdown -License-File: LICENSE -Requires-Dist: numpy>=1.21.6 -Requires-Dist: scipy>=1.7.3 -Requires-Dist: matplotlib>=3.2.2 -Requires-Dist: pandas>=1.3.5 -Requires-Dist: seaborn>=0.11.2 -Requires-Dist: mrg32k3a>=1.0.0 - -![SimOpt Logo](.github/resources/logo_full_magnifying_glass.png) - -## About the Project -SimOpt is a testbed of simulation-optimization problems and solvers. Its purpose is to encourage the development and constructive comparison of simulation-optimization (SO) solvers (algorithms). We are particularly interested in the finite-time performance of solvers, rather than the asymptotic results that one often finds in related literature. - -For the purposes of this project, we define simulation as a very general technique for estimating statistical measures of complex systems. A system is modeled as if the probability distributions of the underlying random variables were known. Realizations of these random variables are then drawn randomly from these distributions. Each replication gives one observation of the system response, i.e., an evaluation of the objective function or stochastic constraints. By simulating a system in this fashion for multiple replications and aggregating the responses, one can compute statistics and use them for evaluation and design. - -Several papers have discussed the development of SimOpt and experiments run on the testbed: -* [Eckman et al. (2024)](https://ieeexplore.ieee.org/document/10408734) studies feasibility metrics for stochastically constrained simulation-optimization problems in preparation for introducing related metrics in SimOpt. -* [Shashaani et al. (2024)](https://dl.acm.org/doi/10.1145/3680282) conducts a large data-farming experiment over solver factors to learn relationships between their settings and a solver's finite-time performance. -* [Eckman et al. (2023)](https://pubsonline.informs.org/doi/10.1287/ijoc.2023.1273) is the most up-to-date publication about SimOpt and describes the code architecture and how users can interact with the library. -* [Eckman et al. (2023)](https://pubsonline.informs.org/doi/10.1287/ijoc.2022.1261) introduces the design of experiments for comparing solvers; this design has been implemented in the latest Python version of SimOpt. For detailed description of the terminology used in the library, e.g., factors, macroreplications, post-processing, solvability plots, etc., see this paper. -* [Eckman et al. (2019)](https://www.informs-sim.org/wsc19papers/374.pdf) describes in detail changes to the architecture of the MATLAB version of SimOpt and the control of random number streams. -* [Dong et al. (2017)](https://www.informs-sim.org/wsc17papers/includes/files/179.pdf) conducts an experimental comparison of several solvers in SimOpt and analyzes their relative performance. -* [Pasupathy and Henderson (2011)](https://www.informs-sim.org/wsc11papers/363.pdf) describes an earlier interface for MATLAB implementations of problems and solvers. -* [Pasupathy and Henderson (2006)](https://www.informs-sim.org/wsc06papers/028.pdf) explains the original motivation for the testbed. - -## Code -### Python -- The [`master branch`](https://github.com/simopt-admin/simopt/tree/master) contains the source code for the latest stable release of the testbed -- The [`development branch`](https://github.com/simopt-admin/simopt/tree/development) contains the latest code for the testbed, but may contain more bugs than the master branch - -### Matlab -> ⚠️ MATLAB versions of this testbed are no longer supported -- The [`matlab branch`](https://github.com/simopt-admin/simopt/tree/matlab) contains a previous stable version of the testbed written in MATLAB - -## Documentation -Full documentation for the source code can be found on our **[readthedocs page](https://simopt.readthedocs.io/en/latest/index.html)**. - -[![Documentation Status](https://readthedocs.org/projects/simopt/badge/?version=latest)](https://simopt.readthedocs.io/en/latest/?badge=latest) - -## Getting Started -### Requirements -- Python >= 3.8 - - To check your Python version, open a terminal and run `python --version`. If you see a message along the lines of `Command not found`, then you likely don't have Python installed. If you know you have it installed but are getting a `Command not found` error, then you may need to [add Python to your PATH](https://realpython.com/add-python-to-path/). - - For new installs, [Miniconda or Anaconda](https://www.anaconda.com/download) is recommended ([read about the differences between Miniconda and Anaconda](https://docs.anaconda.com/distro-or-miniconda/)). If you already have a compatible IDE (such as VS Code), we've found that Miniconda will work fine at 1/10 of the size of Anaconda. It is ***highly recommended*** to check the box during installation to add Python/Miniconda/Anaconda to your system PATH. -- Ruby >= 2.5 (required for datafarming) - - Included on MacOS, but Windows users will need to grab it from [here](https://rubyinstaller.org/). -- `datafarming` gem < 2.0 (required for datafarming) - - This can be installed via `gem install datafarming -v 1.4` once Ruby is installed/configured. - - If experiencing issues, make sure that you are not using version 2.0 or later as those releases does not include the required files. -- Python packages for `numpy`, `scipy`, `matplotlib`, `pandas`, `seaborn`, `sphinx`, `mrg32k3a`, and `jupyter`. - - If using conda/miniconda, run `conda env create -f environment.yml` to create the `simopt` environment. Then run `conda activate simopt` to load the environment. - - Otherwise, run `pip install numpy scipy matplotlib pandas seaborn sphinx mrg32k3a jupyter`. - -### Downloading Source Code -There are two ways to download a copy of the source code onto your machine: -1. Download the code in a zip file by clicking the green `<> Code` button above repo contents and clicking the `Download ZIP` option, then unzip the code to a folder on your computer. This does not require `git` to be installed but makes downloading updates to the repository more challenging. -![image](https://github.com/user-attachments/assets/3c45804c-f8b0-48ed-b32c-a443550c6ef5) - -3. [Clone](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) the branch you'd like to download to a folder on your computer. This requires `git` to be installed but makes downloading updates to the repository much easier. - -If you do not need the source code for SimOpt, you may install the library as a Python package instead. See the [Package](#package) and [Basic Example](#basic-example) sections for more details about this option. - -The `demo` folder contains a handful of useful scripts that can be easily modified, as directed in the comments: - -| File | Description | -| ---------------------------------------- | ----------- | -| `demo_model.py` | Run multiple replications of a simulation model and report its responses | -| `demo_problem.py` | Run multiple replications of a given solution for an SO problem and report its objective function values and left-hand sides of stochastic constraints | -| `demo_problem_solver.py` | Run multiple macroreplications of a solver on a problem, save the outputs to a `.pickle` file in the `experiments/outputs` folder, and save plots of the results to `.png` files in the `experiments/plots` folder | -| `demo_problems_solvers.py` | Run multiple macroreplications of groups of problem-solver pairs and save the outputs and plots | -| `demo_data_farming_model.py` | Create a design over model factors, run multiple replications at each design point, and save the results to a comma separated value (`.csv`) file in the `data_farming_experiments` folder | -| `demo_san-sscont-ironorecont_experiment` | Run multiple solvers on multiple versions of (s, S) inventory, iron ore, and stochastic activiy network problems and produce plots | - -## Graphical User Interface (GUI) - User Guide - -### Main Menu -To open the GUI, run `python -m simopt`. This will launch the main menu. From the menu, you will be given 2 options: -1. Open the `Data Farm Models` window -2. Open the `Simulation Optimization Experiments` window ([read more](#simulation-optimization-experiments)) - -### Simulation Optimization Experiments - -#### Overview -From the GUI, you can create a specified **problem-solver pair** or a **problem-solver group**, run macroreplications, and generate plots. The main page provides ways to create or continue working with experiments: - -1. Create an individual **problem-solver pair** with customized problem and solver factors. -2. Load a .pickle file of a previously created **problem-solver pair**. -3. Create a **problem-solver group**. - -At the bottom of the main page, there is a workspace containing all **problem-solver pair**s and **problem-solver group**s. The first tab lists the **problem-solver pair**s ready to be run or post-replicated, the second tab lists the **problem-solver group**s made from the cross-design or by generating a **problem-solver group** from partial set of **problem-solver pair** in the first tab, and the third tab lists those **problem-solver pair**s that are ready to be post-normalized and prepared for plotting. - -#### 1. Creating a **problem-solver pair** -This is the main way to add **problem-solver pair**s to the queue in the workspace. -1. First, select a solver from the "Solver" dropdown list. Each of the solvers has an abbreviation for the type of problems the solver can handle. Once a solver is selected, the "Problem" list will be sorted and show only the problems that work with the selected solver. -2. Change factors associated with the solver as necessary. The first factor is a customizable name for the solver that use can specify. -3. All solvers with unique combinations of factors must have unique names, i.e., no two solvers can have the same name, but different factors. If you want to use the same solver twice for a problem but with different solver factors, make sure you change the name of the solver accordingly. For example, if you want to create two **problem-solver pair**s with the same problem and solver but with or without CRN for the solver, you can change the name of the solver of choice for each pair to reflect that. This name will appear in the queue within the workspace below. -4. Select a problem from the "Problem" dropdown list. -Each problem has an abbreviation indicating which types of solver is compatible to solve it. The letters in the abbreviation stand for: - -| Objective | Constraint | Variable | Direct Gradient Observations | -| ------------ | ----------------- | -------------- | ---------------------------- | -| Single (S) | Unconstrained (U) | Discrete (D) | Available (G) | -| Multiple (M) | Box (B) | Continuous (C) | Not Available (N) | -| | Deterministic (D) | Mixed (M) | | -| | Stochastic (S) | | | - -6. Change factors associated with the problem and model as necessary. -7. All problems with unique factors must have unique names, i.e., no two problems can have the same name, but different factors. If you want to use the same problem twice for a solver but with different problem or model factors, make sure you change the name of the problem accordingly. This name will appear in the queue within the workspace below. -8. The number of macroreplications can be modified in the top-left corner. The default is 10. -9. Select the "Add **problem-solver pair**" button, which only appears when a solver and problem is selected. The **problem-solver pair** will be added in the "Queue of **problem-solver pair**s." - -#### 2. Loading a **problem-solver pair** from a file -Instead of creating a **problem-solver pair** from scratch, you can load one from a \*.pickle file: -1. In the top left corner, click "Load a **problem-solver pair**". Your file system will pop up, and you can navigate to and select an appropriate \*.pickle file. The GUI will throw an error if the selected file is not a \*.pickle file. -2. Once a **problem-solver pair** object is loaded, it will be added to the "Queue of **problem-solver pair**s". -3. The Run and Post-Process buttons will be updated to accurately reflect whether the **problem-solver pair** has already been run and/or post-processed. - -#### 3. Creating a **problem-solver group** -Currently, **problem-solver group**s can only be created within the GUI or command line; they cannot be loaded from a file. - -You can create a **problem-solver group** and add a new item to the "Queue of **problem-solver group**s" in two ways. The first is a quick grouping of problems and solvers that are compatible with their default factors: -of problems and solvers with their default factors. -1. Click the "Create a **problem-solver group**" button. -2. Check the compatibility of the Problems and Solvers being selected. Note that solvers with deterministic constraint type can not handle problems with stochastic constraints (e.g., ASTRO-DF cannot be run on FACSIZE-2). -3. Specify the number of macroreplications - the default is 10. -4. Click "Confirm Cross-Design **problem-solver group**." -5. The pop-up window will disappear, and the **problem-solver pair**s frame will automatically switch to the "Queue of **problem-solver group**s". -6. To exit out of the **problem-solver group** pop-up without creating a **problem-solver group**, click the red "x" in the top-left corner of the window. - -The second is converting a list of **problem-solver pair**s into a **problem-solver group** by a cross-design: -1. Select the **problem-solver pair**s of interest from the "Queue of **problem-solver pair**s". -2. Clicking the "Convert to a **problem-solver group**" button. This will complete the cross-design for the partial list and create a new row in the "Queue of **problem-solver group**s". - -### Running a **problem-solver pair** or a **problem-solver group** -To run a **problem-solver pair** or a **problem-solver group**, click the "Run" button in the "Queue of **problem-solver pair**s" or "Queue of **problem-solver group**s". Once the **problem-solver pair** or **problem-solver group** has been run, the "Run" button becomes disabled. -**Note:** Running a **problem-solver pair** can take anywhere from a couple of seconds to a couple of minutes depending on the **problem-solver pair** and the number of macroreplications. - -### Post-Processing and Post-Normalization -Post-processing happens before post-normalizing and after the run is complete. You can specify the number of post-replications and the (proxy) optimal solution or function value. After post-normalization is complete, the Plotting window appears. -To exit out of the Post-Process/Normalize pop-up without post-processing or post-normalizing, click the red "x" in the top-left corner of the window. - -#### - **problem-solver pair** -**problem-solver pair**s can be post-processed from the "Queue of **problem-solver pair**s" tab by clicking "Post-Process." Adjust Post-Processing factors as necessary. Only **problem-solver pair**s that have already been run can be post-processed. After post-processing, click the "Post-Normalize by Problem" tab to select which **problem-solver pair**s to post-normalize together. -* Only **problem-solver pair**s with the same problem can be post-normalized together. -* Once all **problem-solver pair**s of interest are selected, click the "Post-Normalize Selected" button at the bottom of the GUI (this button only appears when in the Post-Normalize tab). -* In the new pop-up form, update any values necessary and click "Post-Normalize" when the **problem-solver pair**s are ready to be post-normalized. - -#### - **problem-solver group** -**problem-solver group**s are post-processed and post-normalized at the same time. In the "Queue of **problem-solver group**s" tab, click the "Post-Process" button for the specific **problem-solver group**, then change any values necessary, then click "Post-Process". - -### Plotting -The Plotting page is identical for both **problem-solver pair**s and **problem-solver group**s. Currently, multiple **problem-solver pair**s with the same problem can be plotted together, and any problem-solver pair from a single **problem-solver group** can be plotted together: -1. On the left side, select one or more problems from the problem list. -2. Select solvers from the solver list. -3. On the right side, select a plot type and adjust plot parameters and settings. -There are 5 settings common to most plot types: Confidence Intervals, Number of Bootstrap Samples, Confidence Level, Plot Together, and Print Max HW. -The type of plots that are currently available in the GUI are: Mean Progress Curve, Quantile Progress Curve, Solve Time CDF, Scatter Plot, CDF Solvability, Quantile Solvability, CDF Difference Plot, Quantile Difference Plot, Terminal Box/Violin, and Terminal Scatter. -4. Click "Add." -5. All plots will show in the plotting queue, along with information about their parameters and where the file is saved. -6. To view one plot, click "View Plot." All plots can be viewed together by clicking "See All Plots" at the bottom of the page. -7. To return to the main page, click the red "x" in the top-left corner of the window. - -## Package -The `simoptlib` package is available to download through the Python Packaging Index (PyPI) and can be installed from the terminal with the following command: -``` -python -m pip install simoptlib -``` - -## Basic Example -After installing `simoptlib`, the package's main modules can be imported from the Python console (or in code): -``` -import simopt -from simopt import models, solvers, experiment_base -``` - -The following snippet of code will run 10 macroreplications of the Random Search solver ("RNDSRCH") on the Continuous Newsvendor problem ("CNTNEWS-1"): -``` -myexperiment = simopt.experiment_base.ProblemSolver("RNDSRCH", "CNTNEWS-1") -myexperiment.run(n_macroreps=10) -``` - -The results will be saved to a .pickle file in a folder called `experiments/outputs`. To post-process the results, by taking, for example 200 postreplications at each recommended solution, run the following: -``` -myexperiment.post_replicate(n_postreps=200) -simopt.experiment_base.post_normalize([myexperiment], n_postreps_init_opt=200) -``` - -A .txt file summarizing the progress of the solver on each macroreplication can be produced: -``` -myexperiment.log_experiment_results() -``` - -A .txt file called `RNDSRCH_on_CNTNEWS-1_experiment_results.txt` will be saved in a folder called `experiments/logs`. - -One can then plot the mean progress curve of the solver (with confidence intervals) with the objective function values shown on the y-axis: -``` -simopt.experiment_base.plot_progress_curves(experiments=[myexperiment], plot_type="mean", normalize=False) -``` - -The Python scripts in the `demo` folder provide more guidance on how to run common experiments using the library. - -One can also use the SimOpt graphical user interface by running the following from the terminal: -``` -python -m simopt -``` - -## Contributing -You can contribute problems and solvers to SimOpt (or fix other coding bugs) by [forking](https://docs.github.com/en/get-started/quickstart/fork-a-repo) the repository and initiating [pull requests](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests) in GitHub to request that your changes be integrated. - - -## Authors -The core development team currently consists of -- [**David Eckman**](https://eckman.engr.tamu.edu) (Texas A&M University) -- [**Sara Shashaani**](https://shashaani.wordpress.ncsu.edu) (North Carolina State University) -- [**Shane Henderson**](https://people.orie.cornell.edu/shane/) (Cornell University) -- [**William Grochocinski**](https://github.com/Grochocinski) (North Carolina State University) - -## Citation -To cite this work, please use -``` -@misc{simoptgithub, - author = {D. J. Eckman and S. G. Henderson and S. Shashaani and R. Pasupathy}, - title = {{SimOpt}}, - year = {2024}, - publisher = {GitHub}, - journal = {GitHub repository}, - howpublished = {\url{https://github.com/simopt-admin/simopt}}, - commit = {21802a685ec394fed048820e692209628f40dd4e} -} -``` - -## Acknowledgments -An earlier website for SimOpt ([http://www.simopt.org](http://www.simopt.org)) was developed through work supported by the following grants: -- National Science Foundation - - [DMI-0400287](https://www.nsf.gov/awardsearch/showAward?AWD_ID=0400287) - - [CMMI-0800688](https://www.nsf.gov/awardsearch/showAward?AWD_ID=0800688) - - [CMMI-1200315](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1200315) - -Recent work on the development of SimOpt has been supported by the following grants -- National Science Foundation - - [IIS-1247696](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1247696) - - [CMMI-1254298](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1254298) - - [CMMI-1536895](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1536895) - - [CMMI-1537394](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1537394) - - [DGE-1650441](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1650441) - - [DMS-1839346](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1839346) (TRIPODS+X) - - [CMMI-2206972](https://www.nsf.gov/awardsearch/showAward?AWD_ID=2206972) - - [OAC-2410948](https://www.nsf.gov/awardsearch/showAward?AWD_ID=2410948) - - [OAC-2410949](https://www.nsf.gov/awardsearch/showAward?AWD_ID=2410949) - - [OAC-2410950](https://www.nsf.gov/awardsearch/showAward?AWD_ID=2410950) -- Air Force Office of Scientific Research - - FA9550-12-1-0200 - - FA9550-15-1-0038 - - FA9550-16-1-0046 -- Army Research Office - - W911NF-17-1-0094 - -*Any opinions, findings and conclusions or recommendations expressed in this material are those of the authors and do not necessarily reflect the views of the National Science Foundation (NSF).* diff --git a/simoptlib.egg-info/SOURCES.txt b/simoptlib.egg-info/SOURCES.txt deleted file mode 100644 index 9e934a9ac..000000000 --- a/simoptlib.egg-info/SOURCES.txt +++ /dev/null @@ -1,49 +0,0 @@ -LICENSE -README.md -pyproject.toml -simopt/GUI.py -simopt/__init__.py -simopt/__main__.py -simopt/base.py -simopt/data_farming_base.py -simopt/directory.py -simopt/experiment_base.py -simopt/gui/data_farming_window.py -simopt/gui/df_object.py -simopt/gui/experiment_window.py -simopt/gui/main_menu.py -simopt/gui/new_experiment_window.py -simopt/gui/plot_window.py -simopt/gui/toplevel_custom.py -simopt/models/__init__.py -simopt/models/amusementpark.py -simopt/models/chessmm.py -simopt/models/cntnv.py -simopt/models/contam.py -simopt/models/dualsourcing.py -simopt/models/dynamnews.py -simopt/models/example.py -simopt/models/facilitysizing.py -simopt/models/fixedsan.py -simopt/models/hotel.py -simopt/models/ironore.py -simopt/models/mm1queue.py -simopt/models/network.py -simopt/models/paramesti.py -simopt/models/rmitd.py -simopt/models/san.py -simopt/models/sscont.py -simopt/models/tableallocation.py -simopt/solvers/__init__.py -simopt/solvers/adam.py -simopt/solvers/aloe.py -simopt/solvers/astrodf.py -simopt/solvers/neldmd.py -simopt/solvers/randomsearch.py -simopt/solvers/spsa.py -simopt/solvers/strong.py -simoptlib.egg-info/PKG-INFO -simoptlib.egg-info/SOURCES.txt -simoptlib.egg-info/dependency_links.txt -simoptlib.egg-info/requires.txt -simoptlib.egg-info/top_level.txt \ No newline at end of file diff --git a/simoptlib.egg-info/dependency_links.txt b/simoptlib.egg-info/dependency_links.txt deleted file mode 100644 index 8b1378917..000000000 --- a/simoptlib.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/simoptlib.egg-info/requires.txt b/simoptlib.egg-info/requires.txt deleted file mode 100644 index 6631aa8f1..000000000 --- a/simoptlib.egg-info/requires.txt +++ /dev/null @@ -1,6 +0,0 @@ -numpy>=1.21.6 -scipy>=1.7.3 -matplotlib>=3.2.2 -pandas>=1.3.5 -seaborn>=0.11.2 -mrg32k3a>=1.0.0 diff --git a/simoptlib.egg-info/top_level.txt b/simoptlib.egg-info/top_level.txt deleted file mode 100644 index 409877471..000000000 --- a/simoptlib.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -simopt diff --git a/test/make_tests.py b/test/make_tests.py index 9df10401b..317e8e796 100644 --- a/test/make_tests.py +++ b/test/make_tests.py @@ -94,9 +94,11 @@ def create_test(problem_name: str, solver_name: str) -> None: template = template.replace( b"{{PROGRESS_CURVES}}", str(myexperiment.progress_curves).encode() ) + problem_name_title = problem_name.title() + solver_name_title = solver_name.title() template = template.replace( - b"TEST_PROBLEM_SOLVER", - ("test_" + file_problem_name + "_" + file_solver_name).encode(), + b"TestProblemSolver", + ("Test" + problem_name_title + solver_name_title).encode(), ) # Replace the class name # Write the new test into the new file diff --git a/test/template.py b/test/template.py index 2a037986e..78ddafcf0 100644 --- a/test/template.py +++ b/test/template.py @@ -10,8 +10,8 @@ # with assertEqual as these should not change between runs. -class TEST_PROBLEM_SOLVER(unittest.TestCase): - def setUp(self): +class TestProblemSolver(unittest.TestCase): + def setUp(self) -> None: # Expected values self.expected_problem_name = "{{PROBLEM_NAME}}" self.expected_solver_name = "{{SOLVER_NAME}}" @@ -70,7 +70,7 @@ def setUp(self): + ")", ) - def test_run(self): + def test_run(self) -> None: # Check actual run results against expected self.myexperiment.run(n_macroreps=self.num_macroreps) self.assertEqual( @@ -95,11 +95,15 @@ def test_run(self): + " do not match.", ) # For each list of recommended solutions - for list in range(len(self.myexperiment.all_recommended_xs[mrep])): + for sol_list_idx in range( + len(self.myexperiment.all_recommended_xs[mrep]) + ): # Check to make sure the tuples are the same length self.assertEqual( - len(self.myexperiment.all_recommended_xs[mrep][list]), - len(self.expected_all_recommended_xs[mrep][list]), + len( + self.myexperiment.all_recommended_xs[mrep][sol_list_idx] + ), + len(self.expected_all_recommended_xs[mrep][sol_list_idx]), "Recommended solutions for problem " + self.expected_problem_name + " and solver " @@ -107,16 +111,22 @@ def test_run(self): + " do not match at mrep " + str(mrep) + " and index " - + str(list) + + str(sol_list_idx) + ".", ) # For each tuple of recommended solutions - for tuple in range( - len(self.myexperiment.all_recommended_xs[mrep][list]) + for sol_idx in range( + len( + self.myexperiment.all_recommended_xs[mrep][sol_list_idx] + ) ): self.assertAlmostEqual( - self.myexperiment.all_recommended_xs[mrep][list][tuple], - self.expected_all_recommended_xs[mrep][list][tuple], + self.myexperiment.all_recommended_xs[mrep][ + sol_list_idx + ][sol_idx], + self.expected_all_recommended_xs[mrep][sol_list_idx][ + sol_idx + ], 5, "Recommended solutions for problem " + self.expected_problem_name @@ -125,9 +135,9 @@ def test_run(self): + " do not match at mrep " + str(mrep) + " and index " - + str(list) + + str(sol_list_idx) + " and tuple " - + str(tuple) + + str(sol_idx) + ".", ) # Check to make sure the list lengths are the same @@ -141,13 +151,15 @@ def test_run(self): + " do not match.", ) # For each list of intermediate budgets - for list in range( + for sol_list_idx in range( len(self.myexperiment.all_intermediate_budgets[mrep]) ): # Check the values in the list self.assertAlmostEqual( - self.myexperiment.all_intermediate_budgets[mrep][list], - self.expected_all_intermediate_budgets[mrep][list], + self.myexperiment.all_intermediate_budgets[mrep][ + sol_list_idx + ], + self.expected_all_intermediate_budgets[mrep][sol_list_idx], 5, "Intermediate budgets for problem " + self.expected_problem_name @@ -156,11 +168,11 @@ def test_run(self): + " do not match at mrep " + str(mrep) + " and index " - + str(list) + + str(sol_list_idx) + ".", ) - def test_post_replicate(self): + def test_post_replicate(self) -> None: # Simulate results from the run method self.myexperiment = ProblemSolver( self.expected_solver_name, self.expected_problem_name @@ -195,11 +207,13 @@ def test_post_replicate(self): + " do not match.", ) # For each list in the estimated objectives - for list in range(len(self.myexperiment.all_est_objectives[mrep])): + for objective_idx in range( + len(self.myexperiment.all_est_objectives[mrep]) + ): # Check the values in the list self.assertAlmostEqual( - self.myexperiment.all_est_objectives[mrep][list], - self.expected_all_est_objectives[mrep][list], + self.myexperiment.all_est_objectives[mrep][objective_idx], + self.expected_all_est_objectives[mrep][objective_idx], 5, "Estimated objectives for problem " + self.expected_problem_name @@ -208,11 +222,11 @@ def test_post_replicate(self): + " do not match at mrep " + str(mrep) + " and index " - + str(list) + + str(objective_idx) + ".", ) - def test_post_normalize(self): + def test_post_normalize(self) -> None: # Simulate results from the post_replicate method self.myexperiment = ProblemSolver( self.expected_solver_name, self.expected_problem_name diff --git a/workshop/workshop.ipynb b/workshop/workshop.ipynb index 825f6a53f..9ead4f3b1 100644 --- a/workshop/workshop.ipynb +++ b/workshop/workshop.ipynb @@ -13,12 +13,17 @@ "metadata": {}, "outputs": [], "source": [ + "# Ignore some warnings that pop up since this is running in a Jupyter notebook\n", + "# ruff: noqa: E402, F811\n", + "\n", "# Some setup...\n", "\n", "import os\n", - "os.chdir('../') # Move one level up to import simopt\n", + "\n", + "os.chdir(\"../\") # Move one level up to import simopt\n", "\n", "import sys\n", + "\n", "sys.path.append(\"venv\\\\lib\\\\site-packages\")" ] }, @@ -67,11 +72,17 @@ "# CODE CELL [1]\n", "\n", "# Instantiate the problem and the Random Search solver, with specifications.\n", - "myproblem = ExampleProblem(fixed_factors={\"initial_solution\": (2.0, 2.0), \"budget\": 200})\n", - "myRSsolver = RandomSearch(fixed_factors={\"crn_across_solns\": True, \"sample_size\": 10})\n", + "my_problem = ExampleProblem(\n", + " fixed_factors={\"initial_solution\": (2.0, 2.0), \"budget\": 200}\n", + ")\n", + "my_rand_search_solver = RandomSearch(\n", + " fixed_factors={\"crn_across_solns\": True, \"sample_size\": 10}\n", + ")\n", "\n", "# Pair the problem and solver for experimentation.\n", - "myexperiment = expbase.ProblemSolver(problem=myproblem, solver=myRSsolver)" + "myexperiment = expbase.ProblemSolver(\n", + " problem=my_problem, solver=my_rand_search_solver\n", + ")" ] }, { @@ -104,9 +115,13 @@ "# [Go check out the file called experiments/logs/RNDSRCH_on_EXAMPLE-1_experiment_results.txt.]\n", "\n", "# Plot the (unnormalized) progress curves from the 10 macroreplications.\n", - "expbase.plot_progress_curves(experiments=[myexperiment], plot_type=\"all\", normalize=False)\n", + "expbase.plot_progress_curves(\n", + " experiments=[myexperiment], plot_type=\"all\", normalize=False\n", + ")\n", "# Plot the (unnormalized) mean progress curve with bootstrapped CIs.\n", - "expbase.plot_progress_curves(experiments=[myexperiment], plot_type=\"mean\", normalize=False)\n", + "expbase.plot_progress_curves(\n", + " experiments=[myexperiment], plot_type=\"mean\", normalize=False\n", + ")\n", "# [The plots should be displayed in the output produced below.]" ] }, @@ -162,24 +177,36 @@ "source": [ "# COMBO CODE CELL [0 + 1 + 2]\n", "import os\n", - "os.chdir('../')\n", + "\n", + "os.chdir(\"../\")\n", "import sys\n", + "\n", "sys.path.append(\"venv\\\\lib\\\\site-packages\")\n", "import simopt.experiment_base as expbase\n", "from simopt.models.example import ExampleProblem\n", "from simopt.solvers.randomsearch import RandomSearch\n", "from simopt.solvers.adam import ADAM\n", "\n", - "myproblem = ExampleProblem(fixed_factors={\"initial_solution\": (2.0, 2.0), \"budget\": 200})\n", - "myRSsolver = RandomSearch(fixed_factors={\"crn_across_solns\": True, \"sample_size\": 10})\n", - "myexperiment = expbase.ProblemSolver(problem=myproblem, solver=myRSsolver)\n", + "my_problem = ExampleProblem(\n", + " fixed_factors={\"initial_solution\": (2.0, 2.0), \"budget\": 200}\n", + ")\n", + "my_rand_search_solver = RandomSearch(\n", + " fixed_factors={\"crn_across_solns\": True, \"sample_size\": 10}\n", + ")\n", + "myexperiment = expbase.ProblemSolver(\n", + " problem=my_problem, solver=my_rand_search_solver\n", + ")\n", "\n", "myexperiment.run(n_macroreps=10)\n", "myexperiment.post_replicate(n_postreps=200)\n", "expbase.post_normalize(experiments=[myexperiment], n_postreps_init_opt=200)\n", "myexperiment.log_experiment_results()\n", - "expbase.plot_progress_curves(experiments=[myexperiment], plot_type=\"all\", normalize=False)\n", - "expbase.plot_progress_curves(experiments=[myexperiment], plot_type=\"mean\", normalize=False)" + "expbase.plot_progress_curves(\n", + " experiments=[myexperiment], plot_type=\"all\", normalize=False\n", + ")\n", + "expbase.plot_progress_curves(\n", + " experiments=[myexperiment], plot_type=\"mean\", normalize=False\n", + ")" ] }, { @@ -197,9 +224,11 @@ "source": [ "# CODE CELL [3]\n", "\n", - "myADAMsolver = ADAM(fixed_factors={\"crn_across_solns\": True, \"r\": 10})\n", + "my_adam_solver = ADAM(fixed_factors={\"crn_across_solns\": True, \"r\": 10})\n", "# Create a grouping of Example-RandomSearch and Example-ADAM pairs.\n", - "mygroupexperiment = expbase.ProblemsSolvers(problems=[myproblem], solvers=[myRSsolver, myADAMsolver])\n", + "mygroupexperiment = expbase.ProblemsSolvers(\n", + " problems=[my_problem], solvers=[my_rand_search_solver, my_adam_solver]\n", + ")\n", "\n", "# Run 10 macroreplications of each pair and post-process.\n", "mygroupexperiment.run(n_macroreps=10)\n", @@ -211,7 +240,14 @@ "# [Go check out the file called experiments/logs/group_RNDSRCH_ADAM_on_EXAMPLE-1_group_experiment_results.txt.]\n", "\n", "# Plot the mean progress curve for each solver from the 10 macroreplications.\n", - "expbase.plot_progress_curves(experiments=[mygroupexperiment.experiments[0][0], mygroupexperiment.experiments[1][0]], plot_type=\"mean\", normalize=False)\n", + "expbase.plot_progress_curves(\n", + " experiments=[\n", + " mygroupexperiment.experiments[0][0],\n", + " mygroupexperiment.experiments[1][0],\n", + " ],\n", + " plot_type=\"mean\",\n", + " normalize=False,\n", + ")\n", "# [The plot should be displayed in the output produced below.]" ] }, @@ -263,24 +299,39 @@ "source": [ "# COMBO CODE CELL [0 + 1 + 3]\n", "import os\n", - "os.chdir('../')\n", + "\n", + "os.chdir(\"../\")\n", "import sys\n", + "\n", "sys.path.append(\"venv\\\\lib\\\\site-packages\")\n", "import simopt.experiment_base as expbase\n", "from simopt.models.example import ExampleProblem\n", "from simopt.solvers.randomsearch import RandomSearch\n", "from simopt.solvers.adam import ADAM\n", "\n", - "myproblem = ExampleProblem(fixed_factors={\"initial_solution\": (2.0, 2.0), \"budget\": 200})\n", - "myRSsolver = RandomSearch(fixed_factors={\"crn_across_solns\": True, \"sample_size\": 10})\n", - "myADAMsolver = ADAM(fixed_factors={\"crn_across_solns\": True, \"r\": 10})\n", - "\n", - "mygroupexperiment = expbase.ProblemsSolvers(problems=[myproblem], solvers=[myRSsolver, myADAMsolver])\n", + "my_problem = ExampleProblem(\n", + " fixed_factors={\"initial_solution\": (2.0, 2.0), \"budget\": 200}\n", + ")\n", + "my_rand_search_solver = RandomSearch(\n", + " fixed_factors={\"crn_across_solns\": True, \"sample_size\": 10}\n", + ")\n", + "my_adam_solver = ADAM(fixed_factors={\"crn_across_solns\": True, \"r\": 10})\n", + "\n", + "mygroupexperiment = expbase.ProblemsSolvers(\n", + " problems=[my_problem], solvers=[my_rand_search_solver, my_adam_solver]\n", + ")\n", "mygroupexperiment.run(n_macroreps=10)\n", "mygroupexperiment.post_replicate(n_postreps=200)\n", "mygroupexperiment.post_normalize(n_postreps_init_opt=200)\n", "mygroupexperiment.log_group_experiment_results()\n", - "expbase.plot_progress_curves(experiments=[mygroupexperiment.experiments[0][0], mygroupexperiment.experiments[1][0]], plot_type=\"mean\", normalize=False)" + "expbase.plot_progress_curves(\n", + " experiments=[\n", + " mygroupexperiment.experiments[0][0],\n", + " mygroupexperiment.experiments[1][0],\n", + " ],\n", + " plot_type=\"mean\",\n", + " normalize=False,\n", + ")" ] }, {