From 0537f5f49af18ea14c5ce5530d293199f81f6d1b Mon Sep 17 00:00:00 2001 From: John Siirola Date: Fri, 14 Nov 2025 17:12:29 -0700 Subject: [PATCH 01/41] Add 'SolutionStatus.unknown' --- pyomo/contrib/solver/common/results.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyomo/contrib/solver/common/results.py b/pyomo/contrib/solver/common/results.py index ad5c17c4864..ef81b358aa4 100644 --- a/pyomo/contrib/solver/common/results.py +++ b/pyomo/contrib/solver/common/results.py @@ -112,6 +112,9 @@ class SolutionStatus(enum.Enum): solutions was returned. """ + unknown = 5 + "Solution returned, but feasibility/optimality unknown." + infeasible = 10 "Solution point does not satisfy some domains and/or constraints." From 516685b199f29eb937345974065bc04337715fe5 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Fri, 14 Nov 2025 17:16:49 -0700 Subject: [PATCH 02/41] Rework SOL file parser to be self-contained --- pyomo/contrib/solver/solvers/sol_reader.py | 397 ++++++++++++--------- 1 file changed, 222 insertions(+), 175 deletions(-) diff --git a/pyomo/contrib/solver/solvers/sol_reader.py b/pyomo/contrib/solver/solvers/sol_reader.py index 7d2f613eb6a..f82129e4cad 100644 --- a/pyomo/contrib/solver/solvers/sol_reader.py +++ b/pyomo/contrib/solver/solvers/sol_reader.py @@ -18,9 +18,9 @@ from pyomo.core.expr import value from pyomo.common.collections import ComponentMap from pyomo.core.staleflag import StaleFlagManager -from pyomo.common.errors import DeveloperError, PyomoException from pyomo.repn.plugins.nl_writer import NLWriterInfo from pyomo.core.expr.visitor import replace_expressions +from pyomo.contrib.solver.common.util import SolverError from pyomo.contrib.solver.common.results import ( Results, SolutionStatus, @@ -35,13 +35,18 @@ class SolFileData: """ def __init__(self) -> None: - self.primals: List[float] = [] - self.duals: List[float] = [] - self.var_suffixes: Dict[str, Dict[int, Any]] = {} - self.con_suffixes: Dict[str, Dict[Any]] = {} - self.obj_suffixes: Dict[str, Dict[int, Any]] = {} - self.problem_suffixes: Dict[str, List[Any]] = {} - self.other: List(str) = [] + self.message: str = None + self.objno: int = 0 + self.solve_code: int = None + self.ampl_options: List[int | float] = None + self.primals: List[float] = None + self.duals: List[float] = None + self.var_suffixes: Dict[str, Dict[int, int | float]] = {} + self.con_suffixes: Dict[str, Dict[int, int | float]] = {} + self.obj_suffixes: Dict[str, Dict[int, int | float]] = {} + self.problem_suffixes: Dict[str, int | float] = {} + self.suffix_table: Dict[(int, str), List[int | float, str, ...]] = {} + self.unparsed: str = None class SolSolutionLoader(SolutionLoaderBase): @@ -156,183 +161,225 @@ def get_duals( return res -def parse_sol_file( - sol_file: io.TextIOBase, nl_info: NLWriterInfo, result: Results -) -> Tuple[Results, SolFileData]: +def ampl_solve_code_to_solution_status(sol_data: SolFileData, result: Results) -> None: + # + # This table (the values and the string interpretations) are from + # Chapter 14 in the AMPL Book: + # + code = sol_data.solve_code + status = SolutionStatus.unknown if sol_data.primals else SolutionStatus.noSolution + if code is None: + message = f"AMPL({code}): solver did not generate a SOL file" + term = TerminationCondition.error + elif (code >= 0) and (code <= 99): + # message = f"AMPL({code}:solved): optimal solution found" + message = '' + status = SolutionStatus.optimal + term = TerminationCondition.convergenceCriteriaSatisfied + elif (code >= 100) and (code <= 199): + message = f"AMPL({code}:solved?): optimal solution indicated, but error likely" + status = SolutionStatus.feasible + term = TerminationCondition.error + elif (code >= 200) and (code <= 299): + message = f"AMPL({code}:infeasible): constraints cannot be satisfied" + status = SolutionStatus.infeasible + term = TerminationCondition.locallyInfeasible + elif (code >= 300) and (code <= 399): + message = f"AMPL({code}:unbounded): objective can be improved without limit" + term = TerminationCondition.unbounded + elif (code >= 400) and (code <= 499): + message = f"AMPL({code}:limit): stopped by a limit that you set" + term = TerminationCondition.iterationLimit # this is not always correct + elif (code >= 500) and (code <= 599): + message = f"AMPL({code}:failure): stopped by an error condition in the solver" + term = TerminationCondition.error + else: + message = f"AMPL({code}): unexpected solve code" + term = TerminationCondition.error + + if sol_data.message: + # TBD: [JDS 10/2025]: Why do we convert newlines to semicolons? + result.extra_info.solver_message = sol_data.message.replace('\n', '; ') + if message: + result.extra_info.solver_message += '; ' + message + else: + result.extra_info.solver_message = message + result.solution_status = status + result.termination_condition = term + + +def parse_sol_file(FILE: io.TextIOBase) -> SolFileData: """ Parse a .sol file and populate to Pyomo objects """ sol_data = SolFileData() + # Parse the initial solver message and the AMPL options sections + z = _parse_message_and_options(FILE, sol_data) + + # + # Parse the duals and variable values # + num_duals = z[1] # "m" in writesol.c + assert num_duals == z[0] or not num_duals + sol_data.duals = [float(FILE.readline()) for i in range(num_duals)] + + num_primals = z[3] # "n" in writesol.c + assert num_primals == z[2] or not num_primals + sol_data.primals = [float(FILE.readline()) for i in range(num_primals)] + + # Parse the OBJNO (objective number and solver exit code) + _parse_objno_and_exitcode(FILE, sol_data) + + # Parse the suffix data + _parse_suffixes(FILE, sol_data) + + return sol_data + + +def _parse_message_and_options(FILE: io.TextIOBase, data: SolFileData) -> List[int]: + msg = [] # Some solvers (minto) do not write a message. We will assume - # all non-blank lines up to the 'Options' line is the message. - # For backwards compatibility and general safety, we will parse all - # lines until "Options" appears. Anything before "Options" we will - # consider to be the solver message. - options_found = False - message = [] - model_objects = [] - for line in sol_file: + # all non-blank lines up the 'Options' line is the message. + while True: + line = FILE.readline() if not line: - break + # EOF + raise SolverError("Error reading `sol` file: no 'Options' line found.") line = line.strip() - if "Options" in line: - # Once "Options" appears, we must now read the content under it. - options_found = True - line = sol_file.readline() - number_of_options = int(line) - # We are adding in this DeveloperError to see if the alternative case - # is ever actually hit in the wild. In a previous iteration of the sol - # reader, there was logic to check for the number of options, but it - # was uncovered by tests and unclear if actually necessary. - if number_of_options > 4: - raise DeveloperError( - """ - The sol file reader has hit an unexpected error while parsing. The number of - options recorded is greater than 4. Please report this error to the Pyomo - developers. - """ - ) - for i in range(number_of_options + 4): - line = sol_file.readline() - model_objects.append(int(line)) + if line == 'Options': break - message.append(line) - if not options_found: - raise PyomoException("ERROR READING `sol` FILE. No 'Options' line found.") - message = '\n'.join(message) - # Identify the total number of variables and constraints - number_of_cons = model_objects[number_of_options + 1] - number_of_vars = model_objects[number_of_options + 3] - assert number_of_cons == len(nl_info.constraints) - assert number_of_vars == len(nl_info.variables) - - duals = [float(sol_file.readline()) for i in range(number_of_cons)] - variable_vals = [float(sol_file.readline()) for i in range(number_of_vars)] - - # Parse the exit code line and capture it - exit_code = [0, 0] - line = sol_file.readline() - if line and ('objno' in line): - exit_code_line = line.split() - if len(exit_code_line) != 3: - raise PyomoException( - f"ERROR READING `sol` FILE. Expected two numbers in `objno` line; received {line}." - ) - exit_code = [int(exit_code_line[1]), int(exit_code_line[2])] - else: - raise PyomoException( - f"ERROR READING `sol` FILE. Expected `objno`; received {line}." - ) - result.extra_info.solver_message = message.strip().replace('\n', '; ') - exit_code_message = '' - if (exit_code[1] >= 0) and (exit_code[1] <= 99): - result.solution_status = SolutionStatus.optimal - result.termination_condition = TerminationCondition.convergenceCriteriaSatisfied - elif (exit_code[1] >= 100) and (exit_code[1] <= 199): - exit_code_message = "Optimal solution indicated, but ERROR LIKELY!" - result.solution_status = SolutionStatus.feasible - result.termination_condition = TerminationCondition.error - elif (exit_code[1] >= 200) and (exit_code[1] <= 299): - exit_code_message = "INFEASIBLE SOLUTION: constraints cannot be satisfied!" - result.solution_status = SolutionStatus.infeasible - result.termination_condition = TerminationCondition.locallyInfeasible - elif (exit_code[1] >= 300) and (exit_code[1] <= 399): - exit_code_message = ( - "UNBOUNDED PROBLEM: the objective can be improved without limit!" - ) - result.solution_status = SolutionStatus.noSolution - result.termination_condition = TerminationCondition.unbounded - elif (exit_code[1] >= 400) and (exit_code[1] <= 499): - exit_code_message = ( - "EXCEEDED MAXIMUM NUMBER OF ITERATIONS: the solver " - "was stopped by a limit that you set!" + if line: + msg.append(line) + data.message = "\n".join(msg) + + # WARNING: This appears to be undocumented outside of the ASL + # writesol.c implemention. Before changing this logic, please + # familiarize yourself with that code. + # + # The AMPL options are a sequence of ints, the first of which + # specifies the number of options to expect, followed by the + # options (all ints), followed by the 4 int-elements of "z". + # + n_opts = int(FILE.readline()) + # + # The ASL will occasionally "lie" about the number of options: if + # the second option (not including the number of options) is "3", + # then the ASL will add 2 to the number of options reported, and + # will add *one* option (vbtol, a float) *after* the elements of + # "z". + # + # Because of this, we will read the first two options from the file + # first so we can know how to correctly parse the remaining options. + assert n_opts >= 2 + ampl_options = [int(FILE.readline()) for i in range(2)] + read_vbtol = ampl_options[1] == 3 + if read_vbtol: + n_opts -= 2 + ampl_options.extend(int(FILE.readline()) for i in range(n_opts - 2)) + # Note: "z" comes from the name used for this data structure in + # `writesol.c`. It is unknown to us what motivated that name. + # + # Z: [ #cons; #duals, #vars, #var_vals ] + # #duals will either be #cons or 0 + # #var_vals will either be #vars or 0 + z = [int(FILE.readline()) for i in range(4)] + if read_vbtol: + ampl_options.append(float(FILE.readline())) + + data.ampl_options = ampl_options + return z + + +def _parse_objno_and_exitcode(FILE: io.TextIOBase, data: SolFileData) -> None: + line = FILE.readline().strip() + objno = line.split(maxsplit=2) + if not objno or objno[0] != 'objno': + raise SolverError( + f"Error reading `sol` file: expected 'objno'; received {line!r}." ) - result.solution_status = SolutionStatus.infeasible - result.termination_condition = ( - TerminationCondition.iterationLimit - ) # this is not always correct - elif (exit_code[1] >= 500) and (exit_code[1] <= 599): - exit_code_message = ( - "FAILURE: the solver stopped by an error condition " - "in the solver routines!" + elif len(objno) != 3: + # TBD: [JDS, 10/2025] there are paths where writesol.c will + # generate `objno` lines that contain only the objective number + # and not the solve_code. It is not clear to me that we should + # generate an exception here. + raise SolverError( + "Error reading `sol` file: expected two numbers in 'objno' line; " + f"received {line!r}." ) - result.termination_condition = TerminationCondition.error + data.objno = int(objno[1]) + data.solve_code = int(objno[2]) - if result.extra_info.solver_message: - if exit_code_message: - result.extra_info.solver_message += '; ' + exit_code_message - else: - result.extra_info.solver_message = exit_code_message - - if result.solution_status != SolutionStatus.noSolution: - sol_data.primals = variable_vals - sol_data.duals = duals - ### Read suffixes ### - line = sol_file.readline() - while line: - line = line.strip() - if line == "": - continue - line = line.split() - # Extra solver message processing - if line[0] != 'suffix': - # We assume this is the start of a - # section like kestrel_option, which - # comes after all suffixes. - remaining = '' - line = sol_file.readline() - while line: - remaining += line.strip() + '; ' - line = sol_file.readline() - result.extra_info.solver_message += remaining - break - read_data_type = int(line[1]) - data_type = read_data_type & 3 # 0-var, 1-con, 2-obj, 3-prob - convert_function = int - if (read_data_type & 4) == 4: - convert_function = float - number_of_entries = int(line[2]) - # The third entry is name length, and it is length+1. This is unnecessary - # except for data validation. - # The fourth entry is table "length", e.g., memory size. - number_of_string_lines = int(line[5]) - suffix_name = sol_file.readline().strip() - # Add any arbitrary string lines to the "other" list - for line in range(number_of_string_lines): - sol_data.other.append(sol_file.readline()) - if data_type == 0: # Var - sol_data.var_suffixes[suffix_name] = {} - for cnt in range(number_of_entries): - suf_line = sol_file.readline().split() - var_ndx = int(suf_line[0]) - sol_data.var_suffixes[suffix_name][var_ndx] = convert_function( - suf_line[1] - ) - elif data_type == 1: # Con - sol_data.con_suffixes[suffix_name] = {} - for cnt in range(number_of_entries): - suf_line = sol_file.readline().split() - con_ndx = int(suf_line[0]) - sol_data.con_suffixes[suffix_name][con_ndx] = convert_function( - suf_line[1] - ) - elif data_type == 2: # Obj - sol_data.obj_suffixes[suffix_name] = {} - for cnt in range(number_of_entries): - suf_line = sol_file.readline().split() - obj_ndx = int(suf_line[0]) - sol_data.obj_suffixes[suffix_name][obj_ndx] = convert_function( - suf_line[1] - ) - elif data_type == 3: # Prob - sol_data.problem_suffixes[suffix_name] = [] - for cnt in range(number_of_entries): - suf_line = sol_file.readline().split() - sol_data.problem_suffixes[suffix_name].append( - convert_function(suf_line[1]) - ) - line = sol_file.readline() - - return result, sol_data + +def _parse_suffixes(FILE: io.TextIOBase, data: SolFileData) -> None: + while line := FILE.readline(): + line = line.strip() + if not line: + continue + + line = line.split(maxsplit=6) + if line[0] != 'suffix': + # We assume this is the start of a section (like + # kestrel_option) that comes *after* all suffixes. We + # will capture it (and everything after it) and return + # it as a single "unparsed" text string. + data.unparsed = ' '.join(line) + "\n" + ''.join(FILE) + break + + # Each suffix is introduced by: + # + # 'suffix' + # + # + # Where: + # kind (int): bitmask indicating suffix data type and target + # n (int): number of values returned + # namelen (int): suffix name string length (including NULL termination) + # tablelen (int): length of the "table" string (including NULL) + # tablines (int): number of lines in the table + # sufname (str): suffix name + kind = int(line[1]) + value_converter = float if kind & 4 else int + suffix_target = kind & 3 # 0-var, 1-con, 2-obj, 3-prob + + num_values = int(line[2]) + # Note: we will use namelen to strip off the newline instead of + # strip() in case the suffix name actually ended with whitespace + # (evil, but technically allowed by the NL spec) + suffix_name = FILE.readline()[: int(line[3]) - 1] + + # If the Suffix includes a value <-> string table, parse it. + # The table should be a series of lines of the form: + # + # + # + # The string representation of a suffix value is the row in the + # table whose is the largest value less than or equal to + # the suffix value. The table should be ordered by . + if int(line[4]): + data.suffix_table[suffix_target, suffix_name] = [ + FILE.readline().strip().split(maxsplit=2) for _ in range(int(line[5])) + ] + for entry in data.suffix_table[suffix_target, suffix_name]: + entry[0] = value_converter(entry[0]) + + # Parse the actual suffix values + if suffix_target == 0: # Var + data.var_suffixes[suffix_name] = suffix = {} + elif suffix_target == 1: # Con + data.con_suffixes[suffix_name] = suffix = {} + elif suffix_target == 2: # Obj + data.obj_suffixes[suffix_name] = suffix = {} + elif suffix_target == 3: # Prob + suffix = {} + # else: # Unreachable: kind & 3 can ONLY be 0..3 + + for cnt in range(num_values): + suf_line = FILE.readline().split(maxsplit=1) + suffix[int(suf_line[0])] = value_converter(suf_line[1]) + + if suffix_target == 3 and suffix: + assert len(suffix) == 1 + data.problem_suffixes[suffix_name] = next(iter(suffix.values())) + + return From 4935a63f8f8b909c575e69e649e036aefc39ee9f Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 18 Nov 2025 00:32:31 -0700 Subject: [PATCH 03/41] Rework solution loaders for improved efficiency/consistency --- pyomo/contrib/solver/solvers/ipopt.py | 93 +++++++-------- pyomo/contrib/solver/solvers/sol_reader.py | 129 +++++++++++---------- 2 files changed, 108 insertions(+), 114 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 1bf1fdb7bf9..87ef4435ec1 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -96,65 +96,50 @@ def __init__( ) -class IpoptSolutionLoader(SolSolutionLoader): - def _error_check(self): - if self._nl_info is None: - raise NoSolutionError() - if len(self._nl_info.eliminated_vars) > 0: - raise NotImplementedError( - 'For now, turn presolve off (opt.config.writer_config.linear_presolve=False) ' - 'to get dual variable values.' - ) - if self._sol_data is None: - raise DeveloperError( - "Solution data is empty. This should not " - "have happened. Report this error to the Pyomo Developers." - ) - +class IpoptSolutionLoader(SolFileSolutionLoader): def get_reduced_costs( self, vars_to_load: Optional[Sequence[VarData]] = None ) -> Mapping[VarData, float]: - self._error_check() - # If the NL instance has no objectives, report zeros - if not len(self._nl_info.objectives): - return ComponentMap() - if self._nl_info.scaling is None: - scale_list = [1] * len(self._nl_info.variables) - obj_scale = 1 - else: - scale_list = self._nl_info.scaling.variables - obj_scale = self._nl_info.scaling.objectives[0] - sol_data = self._sol_data - nl_info = self._nl_info - zl_map = sol_data.var_suffixes['ipopt_zL_out'] - zu_map = sol_data.var_suffixes['ipopt_zU_out'] - rc = {} - for ndx, v in enumerate(nl_info.variables): - scale = scale_list[ndx] - v_id = id(v) - rc[v_id] = (v, 0) + if self._nl_info.eliminated_vars: + raise MouseTrap( + 'Complete reduced costs are not available when variables have ' + 'been presolved from the model. Turn presolve off ' + '(solver.config.writer_config.linear_presolve=False) to get ' + 'reduced costs.' + ) + + zl_map = self._sol_data.var_suffixes.get('ipopt_zL_out', {}) + zu_map = self._sol_data.var_suffixes.get('ipopt_zU_out', {}) + # TBD: is it an error if Ipopt fails to return RC info? + # if not (zl_map or zu_map): + # raise? + if self._nl_info.scaling: + # Unscale the zl and zu maps: + inv_obj_scale = 1.0 + if self._nl_info.scaling.objectives: + inv_obj_scale /= self._nl_info.scaling.objectives[self._sol_data.objno] + var_scale = self._nl_info.scaling.variables + zl_map = {k: v * var_scale[k] * inv_obj_scale for k, v in zl_map.items()} + zu_map = {k: v * var_scale[k] * inv_obj_scale for k, v in zu_map.items()} + + rc = ComponentMap() + for ndx, v in enumerate(self._nl_info.variables): + _rc = 0.0 if ndx in zl_map: - zl = zl_map[ndx] * scale / obj_scale - if abs(zl) > abs(rc[v_id][1]): - rc[v_id] = (v, zl) + # Note *any* value in zl has an absolute value at least + # as big as 0. No need to test and just overwrite _rc: + _rc = zl_map[ndx] if ndx in zu_map: - zu = zu_map[ndx] * scale / obj_scale - if abs(zu) > abs(rc[v_id][1]): - rc[v_id] = (v, zu) - - if vars_to_load is None: - res = ComponentMap(rc.values()) - for v, _ in nl_info.eliminated_vars: - res[v] = 0 - else: - res = ComponentMap() - for v in vars_to_load: - if id(v) in rc: - res[v] = rc[id(v)][1] - else: - # eliminated vars - res[v] = 0 - return res + zu = zu_map[ndx] + if abs(zu) > abs(_rc): + _rc = zu + rc[v] = _rc + + if vars_to_load is not None: + # Note vars_to_load could contain variables that were + # eliminated (so use get()): + rc = ComponentMap((v, rc.get(v, 0)) for v in vars_to_load) + return rc ipopt_command_line_options = { diff --git a/pyomo/contrib/solver/solvers/sol_reader.py b/pyomo/contrib/solver/solvers/sol_reader.py index f82129e4cad..f5a149f27ef 100644 --- a/pyomo/contrib/solver/solvers/sol_reader.py +++ b/pyomo/contrib/solver/solvers/sol_reader.py @@ -49,7 +49,7 @@ def __init__(self) -> None: self.unparsed: str = None -class SolSolutionLoader(SolutionLoaderBase): +class SolFileSolutionLoader(SolutionLoaderBase): """ Loader for solvers that create .sol files (e.g., ipopt) """ @@ -59,14 +59,21 @@ def __init__(self, sol_data: SolFileData, nl_info: NLWriterInfo) -> None: self._nl_info = nl_info def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> NoReturn: - if self._nl_info is None: - raise RuntimeError( - 'Solution loader does not currently have a valid solution. Please ' - 'check results.termination_condition and/or results.solution_status.' - ) - if self._sol_data is None: + if vars_to_load is not None: + # If we are given a list of variables to load, it is easiest + # to use the filtering in get_primals and then just set + # those values. + for var, val in self.get_primals(vars_to_load).items(): + var.set_value(val, skip_validation=True) + StaleFlagManager.mark_all_as_stale(delayed=True) + return + + if not self._sol_data.primals: + # SOL file contained no primal values assert len(self._nl_info.variables) == 0 else: + # Load the primals provided by the SOL file (scaling if necessary) + assert len(self._nl_info.variables) == len(self._sol_data.primals) if self._nl_info.scaling: for var, val, scale in zip( self._nl_info.variables, @@ -78,87 +85,89 @@ def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> NoRetur for var, val in zip(self._nl_info.variables, self._sol_data.primals): var.set_value(val, skip_validation=True) + # Compute all variables presolved out of the model for var, v_expr in self._nl_info.eliminated_vars: - var.value = value(v_expr) + var.set_value(value(v_expr), skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) def get_primals( self, vars_to_load: Optional[Sequence[VarData]] = None ) -> Mapping[VarData, float]: - if self._nl_info is None: - raise RuntimeError( - 'Solution loader does not currently have a valid solution. Please ' - 'check results.termination_condition and/or results.solution_status.' - ) - val_map = {} - if self._sol_data is None: + result = ComponentMap() + if not self._sol_data.primals: + # SOL file contained no primal values assert len(self._nl_info.variables) == 0 else: - if self._nl_info.scaling is None: - scale_list = [1] * len(self._nl_info.variables) + # Load the primals provided by the SOL file (scaling if necessary) + assert len(self._nl_info.variables) == len(self._sol_data.primals) + if self._nl_info.scaling: + for var, val, scale in zip( + self._nl_info.variables, + self._sol_data.primals, + self._nl_info.scaling.variables, + ): + result[var] = val / scale else: - scale_list = self._nl_info.scaling.variables - for var, val, scale in zip( - self._nl_info.variables, self._sol_data.primals, scale_list - ): - val_map[id(var)] = val / scale - - for var, v_expr in self._nl_info.eliminated_vars: - val = replace_expressions(v_expr, substitution_map=val_map) - v_id = id(var) - val_map[v_id] = val - - res = ComponentMap() - if vars_to_load is None: - vars_to_load = self._nl_info.variables + [ - var for var, _ in self._nl_info.eliminated_vars - ] - for var in vars_to_load: - res[var] = val_map[id(var)] + for var, val in zip(self._nl_info.variables, self._sol_data.primals): + result[var] = val - return res + # If we have eliminated variables, then we need to compute + # them. Unfortunately, the expressions that we kept are in + # terms of the actual variable values (which we don't want to + # modify). We will make use of an expression replacement + # visitor to perform the substitution and computation. + # + # It would be great if we could do this withough creating the + # entire (unfiltered) result, but we just don't (easily) know + # which variable values we are going to need (either in the + # vars_to_load list, or in any expression that might be needed + # to compute an eliminated variable value. So to keep things + # simple (i.e., fewer bugs), we will go ahead and always compute + # everything. + if self._nl_info.eliminated_vars: + val_map = {id(k): v for k, v in result.items()} + for var, v_expr in self._nl_info.eliminated_vars: + val = value(replace_expressions(v_expr, substitution_map=val_map)) + val_map[id(var)] = val + result[var] = val + + if vars_to_load is not None: + result = ComponentMap((v, result[v]) for v in vars_to_load) + + return result def get_duals( self, cons_to_load: Optional[Sequence[ConstraintData]] = None ) -> Dict[ConstraintData, float]: - if self._nl_info is None: - raise RuntimeError( - 'Solution loader does not currently have a valid solution. Please ' - 'check results.termination_condition and/or results.solution_status.' - ) - # If the NL instance has no objectives, report zeros - if not self._nl_info.objectives: - cons = ( - cons_to_load if cons_to_load is not None else self._nl_info.constraints - ) - return {c: 0.0 for c in cons} if len(self._nl_info.eliminated_vars) > 0: - raise NotImplementedError( - 'For now, turn presolve off (opt.config.writer_config.linear_presolve=False) ' - 'to get dual variable values.' - ) - if self._sol_data is None: - raise DeveloperError( - "Solution data is empty. This should not " - "have happened. Report this error to the Pyomo Developers." + raise MouseTrap( + 'Complete duals are not available when variables have ' + 'been presolved from the model. Turn presolve off ' + '(solver.config.writer_config.linear_presolve=False) to get ' + 'dual variable values.' ) - res = {} + + if not self._sol_data.duals: + return {} + scaling = self._nl_info.scaling if scaling: _iter = zip( self._nl_info.constraints, self._sol_data.duals, scaling.constraints ) - obj_scale = self._nl_info.scaling.objectives[0] + inv_obj_scale = 1.0 + if self._nl_info.scaling.objectives: + inv_obj_scale /= self._nl_info.scaling.objectives[self._sol_data.objno] else: _iter = zip(self._nl_info.constraints, self._sol_data.duals) if cons_to_load is not None: + cons_to_load = set(cons_to_load) _iter = filter(lambda x: x[0] in cons_to_load, _iter) if scaling: - res = {con: val * scale / obj_scale for con, val, scale in _iter} + return {con: val * scale * inv_obj_scale for con, val, scale in _iter} else: - res = {con: val for con, val in _iter} - return res + return {con: val for con, val in _iter} def ampl_solve_code_to_solution_status(sol_data: SolFileData, result: Results) -> None: From ddd525b686890357759983a4d6ced1da2569b7a4 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 18 Nov 2025 00:33:31 -0700 Subject: [PATCH 04/41] Allow constructing "empty" NLWriterInfo objects --- pyomo/repn/plugins/nl_writer.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pyomo/repn/plugins/nl_writer.py b/pyomo/repn/plugins/nl_writer.py index bfe1ca2766b..ff46ae62ec4 100644 --- a/pyomo/repn/plugins/nl_writer.py +++ b/pyomo/repn/plugins/nl_writer.py @@ -138,22 +138,22 @@ class NLWriterInfo: def __init__( self, - var, - con, - obj, - external_libs, - row_labels, - col_labels, - eliminated_vars, - scaling, + var=None, + con=None, + obj=None, + external_libs=None, + row_labels=None, + col_labels=None, + eliminated_vars=None, + scaling=None, ): - self.variables = var - self.constraints = con - self.objectives = obj - self.external_function_libraries = external_libs - self.row_labels = row_labels - self.column_labels = col_labels - self.eliminated_vars = eliminated_vars + self.variables = var or [] + self.constraints = con or [] + self.objectives = obj or [] + self.external_function_libraries = external_libs or [] + self.row_labels = row_labels or [] + self.column_labels = col_labels or [] + self.eliminated_vars = eliminated_vars or [] self.scaling = scaling From 05919bf5f71e8647cdf21aca0099140b52cdd66a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 18 Nov 2025 00:36:18 -0700 Subject: [PATCH 05/41] Refactor Ipopt solve() to simplify logic and break apart monster function --- pyomo/contrib/solver/common/util.py | 5 + pyomo/contrib/solver/solvers/ipopt.py | 252 ++++++++++++-------------- 2 files changed, 125 insertions(+), 132 deletions(-) diff --git a/pyomo/contrib/solver/common/util.py b/pyomo/contrib/solver/common/util.py index 8c62eea3f73..e33f31aefcd 100644 --- a/pyomo/contrib/solver/common/util.py +++ b/pyomo/contrib/solver/common/util.py @@ -15,6 +15,11 @@ from pyomo.core.base.objective import Objective +class SolverError(PyomoException): + """General error raised by Pyomo solver interfaces when processing + Solver results.""" + + class NoFeasibleSolutionError(PyomoException): default_message = ( 'A feasible solution was not found, so no solution can be loaded. ' diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 87ef4435ec1..bc5987f5c94 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -28,12 +28,12 @@ ) from pyomo.common.errors import ( ApplicationError, - DeveloperError, InfeasibleConstraintException, + MouseTrap, ) from pyomo.common.fileutils import to_legal_filename from pyomo.common.tempfiles import TempfileManager -from pyomo.common.timing import HierarchicalTimer +from pyomo.common.timing import HierarchicalTimer, default_timer from pyomo.core.base.var import VarData from pyomo.core.staleflag import StaleFlagManager from pyomo.repn.plugins.nl_writer import NLWriter, NLWriterInfo @@ -45,12 +45,13 @@ TerminationCondition, SolutionStatus, ) -from pyomo.contrib.solver.solvers.sol_reader import parse_sol_file, SolSolutionLoader -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoSolutionError, +from pyomo.contrib.solver.solvers.sol_reader import ( + ampl_solve_code_to_solution_status, + parse_sol_file, + SolFileData, + SolFileSolutionLoader, ) +from pyomo.contrib.solver.common.util import NoOptimalSolutionError, NoSolutionError from pyomo.common.tee import TeeStream from pyomo.core.expr.visitor import replace_expressions from pyomo.core.expr.numvalue import value @@ -328,7 +329,13 @@ def _create_command_line(self, basename: str, config: IpoptConfig) -> List[str]: def solve(self, model, **kwds) -> Results: "Solve a model using Ipopt" # Begin time tracking - start_timestamp = datetime.datetime.now(datetime.timezone.utc) + start_time = default_timer() + # Allocate the results object so we can populate it as we go + results = Results() + results.timing_info.start_timestamp = datetime.datetime.now( + datetime.timezone.utc + ) + # Update configuration options, based on keywords passed to solve config: IpoptConfig = self.config(value=kwds, preserve_implicit=True) # Check if solver is available @@ -344,7 +351,7 @@ def solve(self, model, **kwds) -> Results: f"but this is not used by {self.__class__}.", ) if config.timer is None: - timer = HierarchicalTimer() + timer = config.timer = HierarchicalTimer() else: timer = config.timer StaleFlagManager.mark_all_as_stale() @@ -403,108 +410,29 @@ def solve(self, model, **kwds) -> Results: proven_infeasible = False except InfeasibleConstraintException: proven_infeasible = True + nl_info = NLWriterInfo() timer.stop('write_nl_file') - if not proven_infeasible and len(nl_info.variables) > 0: - # Get a copy of the environment to pass to the subprocess - env = os.environ.copy() - if nl_info.external_function_libraries: - env['AMPLFUNC'] = amplfunc_merge( - env, *nl_info.external_function_libraries - ) - self._verify_ipopt_options(config) - # Write the options file, if there should be one. If - # the file was written, then 'options_file_name' was - # added to config.options (so we can correctly build the - # command line) - self._write_options_file( - filename=basename + '.opt', options=config.solver_options - ) - # Call ipopt - passing the files via the subprocess - cmd = self._create_command_line(basename=basename, config=config) - # this seems silly, but we have to give the subprocess slightly - # longer to finish than ipopt - if config.time_limit is not None: - timeout = config.time_limit + min( - max(1.0, 0.01 * config.time_limit), 100 - ) - else: - timeout = None - - ostreams = [io.StringIO()] + config.tee - timer.start('subprocess') - try: - with TeeStream(*ostreams) as t: - process = subprocess.run( - cmd, - timeout=timeout, - env=env, - universal_newlines=True, - stdout=t.STDOUT, - stderr=t.STDERR, - check=False, - ) - except OSError: - err = sys.exc_info()[1] - msg = 'Could not execute the command: %s\tError message: %s' - raise ApplicationError(msg % (cmd, err)) - finally: - timer.stop('subprocess') - - # This is the data we need to parse to get the iterations - # and time - parsed_output_data = self._parse_ipopt_output(ostreams[0]) if proven_infeasible: - results = Results() results.termination_condition = TerminationCondition.provenInfeasible - results.solution_loader = SolSolutionLoader(None, None) + results.solution_status = SolutionStatus.noSolution results.extra_info.iteration_count = 0 - results.timing_info.total_seconds = 0 - elif len(nl_info.variables) == 0: - if len(nl_info.eliminated_vars) == 0: - results = Results() - results.termination_condition = TerminationCondition.emptyModel - results.solution_loader = SolSolutionLoader(None, None) - else: - results = Results() + elif not nl_info.variables: + if nl_info.eliminated_vars: results.termination_condition = ( TerminationCondition.convergenceCriteriaSatisfied ) results.solution_status = SolutionStatus.optimal - results.solution_loader = SolSolutionLoader(None, nl_info=nl_info) - results.extra_info.iteration_count = 0 - results.timing_info.total_seconds = 0 - else: - if os.path.isfile(basename + '.sol'): - with open(basename + '.sol', 'r', encoding='utf-8') as sol_file: - timer.start('parse_sol') - results = self._parse_solution(sol_file, nl_info) - timer.stop('parse_sol') - else: - results = Results() - if process.returncode != 0: - results.extra_info.return_code = process.returncode - results.termination_condition = TerminationCondition.error - results.solution_loader = SolSolutionLoader(None, None) else: - try: - results.extra_info.iteration_count = parsed_output_data.pop( - 'iters' - ) - cpu_seconds = parsed_output_data.pop('cpu_seconds') - for k, v in cpu_seconds.items(): - results.timing_info[k] = v - results.extra_info = parsed_output_data - iter_log = results.extra_info.get("iteration_log", None) - if iter_log is not None: - iter_log._visibility = ADVANCED_OPTION - except Exception as e: - logger.log( - logging.WARNING, - "The solver output data is empty or incomplete.\n" - f"Full error message: {e}\n" - f"Parsed solver data: {parsed_output_data}\n", - ) + results.termination_condition = TerminationCondition.emptyModel + results.solution_status = SolutionStatus.noSolution + results.extra_info.iteration_count = 0 + results.solution_loader = IpoptSolutionLoader( + sol_data=SolFileData(), nl_info=nl_info + ) + else: + self._run_ipopt(results, config, nl_info, basename, timer) + if ( config.raise_exception_on_nonoptimal_result and results.solution_status != SolutionStatus.optimal @@ -551,24 +479,90 @@ def solve(self, model, **kwds) -> Results: ) results.solver_config = config - if not proven_infeasible and len(nl_info.variables) > 0: - results.solver_log = ostreams[0].getvalue() # Capture/record end-time / wall-time - end_timestamp = datetime.datetime.now(datetime.timezone.utc) - results.timing_info.start_timestamp = start_timestamp - results.timing_info.wall_time = ( - end_timestamp - start_timestamp - ).total_seconds() results.timing_info.timer = timer + results.timing_info.wall_time = default_timer() - start_time return results - def _parse_ipopt_output(self, output: Union[str, io.StringIO]) -> Dict[str, Any]: - parsed_data = {} + def _run_ipopt(self, results, config, nl_info, basename, timer): + # Get a copy of the environment to pass to the subprocess + env = os.environ.copy() + if nl_info.external_function_libraries: + env['AMPLFUNC'] = amplfunc_merge(env, *nl_info.external_function_libraries) + self._verify_ipopt_options(config) + # Write the options file, if there should be one. If + # the file was written, then 'options_file_name' was + # added to config.options (so we can correctly build the + # command line) + self._write_options_file( + filename=basename + '.opt', options=config.solver_options + ) + # Call ipopt - passing the files via the subprocess + cmd = self._create_command_line(basename=basename, config=config) + # this seems silly, but we have to give the subprocess slightly + # longer to finish than ipopt + if config.time_limit is not None: + timeout = config.time_limit + min(max(1.0, 0.01 * config.time_limit), 100) + else: + timeout = None + + ostreams = [io.StringIO()] + config.tee + timer.start('subprocess') + try: + with TeeStream(*ostreams) as t: + process = subprocess.run( + cmd, + timeout=timeout, + env=env, + universal_newlines=True, + stdout=t.STDOUT, + stderr=t.STDERR, + check=False, + ) + except OSError: + err = sys.exc_info()[1] + msg = 'Could not execute the command: %s\tError message: %s' + raise ApplicationError(msg % (cmd, err)) + finally: + timer.stop('subprocess') + + results.solver_log = ostreams[0].getvalue() + results.extra_info.return_code = process.returncode + if process.returncode: + results.termination_condition = TerminationCondition.error + + # This is the data we need to parse to get the iterations + # and time + parsed_output_data = self._parse_ipopt_output(results.solver_log) + results.extra_info.iteration_count = parsed_output_data.pop('iters', None) + _timing = parsed_output_data.pop('cpu_seconds', None) + if _timing: + # results.timing_info.update(_timing) + for k, v in _timing.items(): + results.timing_info[k] = v + results.extra_info = parsed_output_data + iter_log = results.extra_info.get("iteration_log", None) + if iter_log is not None: + iter_log._visibility = ADVANCED_OPTION + + timer.start('parse_sol') + if os.path.isfile(basename + '.sol'): + with open(basename + '.sol', 'r', encoding='utf-8') as sol_file: + sol_data = parse_sol_file(sol_file) + else: + sol_data = SolFileData() + results.solution_loader = IpoptSolutionLoader( + sol_data=sol_data, nl_info=nl_info + ) + timer.stop('parse_sol') - # Convert output to a string so we can parse it - if isinstance(output, io.StringIO): - output = output.getvalue() + # Initialize the solver message, solution loader solution + # status and termination condition: + ampl_solve_code_to_solution_status(sol_data, results) + + def _parse_ipopt_output(self, output: str) -> Dict[str, Any]: + parsed_data = {} # Stop parsing if there is nothing to parse if not output: @@ -659,13 +653,24 @@ def _parse_ipopt_output(self, output: Union[str, io.StringIO]) -> Dict[str, Any] if len(iterations) != iter_num: logger.warning( - f"Total number of iterations parsed {len(iterations)} " - f"does not match the expected iteration number ({iter_num})." + f"Parsed iteration record {len(iterations)} " + f"does not match the expected iteration number ({iter_num})" + f"\n\t{line}" ) + # Things have gotten fouled up. There is no real + # reason to continue to parse the iteration log + break iterations.append(iter_data) parsed_data['iteration_log'] = iterations + if len(iterations) != parsed_data.get('iters', 0): + n_iter = parsed_data.get('iters', 0) + logger.warning( + f"Total number of iteration records parsed {len(iterations)} does " + f"not match the number of iterations ({n_iter})." + ) + # Extract scaled and unscaled table scaled_unscaled_match = re.search( r''' @@ -716,23 +721,6 @@ def _parse_ipopt_output(self, output: Union[str, io.StringIO]) -> Dict[str, Any] return parsed_data - def _parse_solution( - self, instream: io.TextIOBase, nl_info: NLWriterInfo - ) -> Results: - results = Results() - res, sol_data = parse_sol_file( - sol_file=instream, nl_info=nl_info, result=results - ) - - if res.solution_status == SolutionStatus.noSolution: - res.solution_loader = SolSolutionLoader(None, None) - else: - res.solution_loader = IpoptSolutionLoader( - sol_data=sol_data, nl_info=nl_info - ) - - return res - class LegacyIpoptSolver(LegacySolverWrapper, Ipopt): def _verify_ipopt_options(self, config: IpoptConfig) -> None: From f6bdc8248974e177b11fcfbbdbc40bee3c71e729 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 18 Nov 2025 00:38:28 -0700 Subject: [PATCH 06/41] Raise correct exception when Ipopt fails to return a solution --- pyomo/contrib/solver/solvers/ipopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index bc5987f5c94..dec0e4dd853 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -444,7 +444,7 @@ def solve(self, model, **kwds) -> Results: if config.load_solutions: if results.solution_status == SolutionStatus.noSolution: - raise NoFeasibleSolutionError() + raise NoSolutionError() results.solution_loader.load_vars() if ( hasattr(model, 'dual') From 22e6458fbc585c34d63bcfb2d496b851e4d2ab6e Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 18 Nov 2025 00:40:00 -0700 Subject: [PATCH 07/41] NFC: line wrapping --- pyomo/contrib/solver/solvers/ipopt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index dec0e4dd853..0a7d76e9287 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -88,8 +88,8 @@ def __init__( ConfigValue( domain=Executable, default='ipopt', - description="Preferred executable for ipopt. Defaults to searching the " - "``PATH`` for the first available ``ipopt``.", + description="Preferred executable for ipopt. Defaults to searching " + "the ``PATH`` for the first available ``ipopt``.", ), ) self.writer_config: ConfigDict = self.declare( From 732ccc914c840703a3506b0de1ca497475d73122 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 18 Nov 2025 00:41:00 -0700 Subject: [PATCH 08/41] Update / extend Ipopt testing --- .../solver/tests/solvers/test_ipopt.py | 30 +- .../solver/tests/solvers/test_sol_reader.py | 464 ++++++++++++++---- .../contrib/solver/tests/unit/test_results.py | 2 +- .../solver/tests/unit/test_solution.py | 13 - 4 files changed, 371 insertions(+), 138 deletions(-) diff --git a/pyomo/contrib/solver/tests/solvers/test_ipopt.py b/pyomo/contrib/solver/tests/solvers/test_ipopt.py index edd1ce23f36..126c0ac60f4 100644 --- a/pyomo/contrib/solver/tests/solvers/test_ipopt.py +++ b/pyomo/contrib/solver/tests/solvers/test_ipopt.py @@ -17,7 +17,7 @@ from pyomo.common.envvar import is_windows from pyomo.common.fileutils import ExecutableData from pyomo.common.config import ConfigDict, ADVANCED_OPTION -from pyomo.common.errors import DeveloperError +from pyomo.common.errors import MouseTrap from pyomo.common.tee import capture_output import pyomo.contrib.solver.solvers.ipopt as ipopt from pyomo.contrib.solver.common.util import NoSolutionError @@ -25,7 +25,7 @@ from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.common import unittest, Executable from pyomo.common.tempfiles import TempfileManager -from pyomo.repn.plugins.nl_writer import NLWriter +from pyomo.repn.plugins.nl_writer import NLWriter, NLWriterInfo ipopt_available = ipopt.Ipopt().available() @@ -82,24 +82,12 @@ def test_custom_instantiation(self): class TestIpoptSolutionLoader(unittest.TestCase): def test_get_reduced_costs_error(self): - loader = ipopt.IpoptSolutionLoader(None, None) - with self.assertRaises(NoSolutionError): - loader.get_reduced_costs() - - # Set _nl_info to something completely bogus but is not None - class NLInfo: - pass - - loader._nl_info = NLInfo() - loader._nl_info.eliminated_vars = [1, 2, 3] - # This test may need to be altered if we enable returning duals - # when presolve is on - with self.assertRaises(NotImplementedError): - loader.get_reduced_costs() - # Reset _nl_info so we can ensure we get an error - # when _sol_data is None - loader._nl_info.eliminated_vars = [] - with self.assertRaises(DeveloperError): + loader = ipopt.IpoptSolutionLoader( + ipopt.SolFileData(), NLWriterInfo(eliminated_vars=[1]) + ) + with self.assertRaisesRegex( + MouseTrap, "Complete reduced costs are not available" + ): loader.get_reduced_costs() @@ -594,7 +582,7 @@ def test_ipopt_quiet_print_level(self): result = ipopt.Ipopt().solve(model, solver_options={'print_level': 0}) # IPOPT doesn't tell us anything about the iters if the print level # is set to 0 - self.assertFalse(hasattr(result.extra_info, 'iteration_count')) + self.assertEqual(result.extra_info.iteration_count, None) self.assertFalse(hasattr(result.extra_info, 'iteration_log')) model = self.create_model() result = ipopt.Ipopt().solve(model, solver_options={'print_level': 3}) diff --git a/pyomo/contrib/solver/tests/solvers/test_sol_reader.py b/pyomo/contrib/solver/tests/solvers/test_sol_reader.py index 62d77341f65..90220e56ebf 100644 --- a/pyomo/contrib/solver/tests/solvers/test_sol_reader.py +++ b/pyomo/contrib/solver/tests/solvers/test_sol_reader.py @@ -13,144 +13,402 @@ import pyomo.environ as pyo from pyomo.common import unittest -from pyomo.common.errors import PyomoException +from pyomo.common.collections import ComponentMap from pyomo.common.fileutils import this_file_dir -from pyomo.common.tempfiles import TempfileManager from pyomo.contrib.solver.solvers.sol_reader import ( - SolSolutionLoader, + SolFileSolutionLoader, SolFileData, parse_sol_file, ) from pyomo.contrib.solver.common.results import Results - -currdir = this_file_dir() +from pyomo.contrib.solver.common.util import SolverError +from pyomo.repn.plugins.nl_writer import NLWriterInfo, ScalingFactors class TestSolFileData(unittest.TestCase): def test_default_instantiation(self): instance = SolFileData() - self.assertIsInstance(instance.primals, list) - self.assertIsInstance(instance.duals, list) - self.assertIsInstance(instance.var_suffixes, dict) - self.assertIsInstance(instance.con_suffixes, dict) - self.assertIsInstance(instance.obj_suffixes, dict) - self.assertIsInstance(instance.problem_suffixes, dict) - self.assertIsInstance(instance.other, list) + self.assertEqual(instance.message, None) + self.assertEqual(instance.objno, 0) + self.assertEqual(instance.solve_code, None) + self.assertEqual(instance.ampl_options, None) + self.assertEqual(instance.primals, None) + self.assertEqual(instance.duals, None) + self.assertEqual(instance.var_suffixes, {}) + self.assertEqual(instance.con_suffixes, {}) + self.assertEqual(instance.obj_suffixes, {}) + self.assertEqual(instance.problem_suffixes, {}) + self.assertEqual(instance.unparsed, None) class TestSolParser(unittest.TestCase): - def setUp(self): - TempfileManager.push() - - def tearDown(self): - TempfileManager.pop(remove=True) - - class _FakeNLInfo: - def __init__( - self, - variables, - constraints, - objectives=None, - scaling=None, - eliminated_vars=None, - ): - self.variables = variables - self.constraints = constraints - self.objectives = objectives or [] - self.scaling = scaling - self.eliminated_vars = eliminated_vars or [] - - class _FakeSolData: - def __init__(self, primals=None, duals=None): - self.primals = primals or [] - self.duals = duals or [] - self.var_suffixes = {} - self.con_suffixes = {} - self.obj_suffixes = {} - self.problem_suffixes = {} - self.other = [] - - def test_get_duals_no_objective_returns_zeros(self): - # model with 2 cons, no objective - m = pyo.ConcreteModel() - m.x = pyo.Var(initialize=1.0) - m.y = pyo.Var(initialize=2.0) - m.c1 = pyo.Constraint(expr=m.x + m.y >= 0) - m.c2 = pyo.Constraint(expr=m.x - m.y <= 3) - - nl_info = self._FakeNLInfo( - variables=[m.x, m.y], constraints=[m.c1, m.c2], objectives=[], scaling=None - ) - # solver returned some (non-zero) duals, but we should zero them out - sol_data = self._FakeSolData(duals=[123.0, -7.5]) - - loader = SolSolutionLoader(sol_data, nl_info) - duals = loader.get_duals() - self.assertEqual(duals[m.c1], 0.0) - self.assertEqual(duals[m.c2], 0.0) - - def test_parse_sol_file(self): + def test_parse_minimal_sol_file(self): # Build a tiny .sol text stream: - # - "Options" block with number_of_options = 0, then 4 model_object ints + # - "Options" block with number_of_options = 2, then 4 model_object ints # model_objects[1] = #cons, model_objects[3] = #vars # - #cons duals lines # - #vars primals lines # - "objno " n_cons = 2 n_vars = 3 - sol_content = ( - "Solver message preamble\n" - "Options\n" - "0\n" - f"0\n{n_cons}\n0\n{n_vars}\n" # model_objects (4 ints) - "1.5\n-2.25\n" # duals (2 lines) - "10.0\n20.0\n30.0\n" # primals (3 lines) - "objno 0 0\n" # exit code line - ) - stream = io.StringIO(sol_content) - - # Minimal NL info matching sizes - m = pyo.ConcreteModel() - m.v = pyo.Var(range(n_vars)) - m.c = pyo.Constraint(range(n_cons), rule=lambda m, i: m.v[0] >= -100) - nl_info = self._FakeNLInfo( - variables=[m.v[i] for i in range(n_vars)], - constraints=[m.c[i] for i in range(n_cons)], + stream = io.StringIO( + f"""Solver message preamble +Options +2 +1 +2 +{n_cons} +{n_cons} +{n_vars} +{n_vars} +1.5 +-2.25 +10.0 +20.0 +30.0 +objno 0 100""" ) + sol_data = parse_sol_file(stream) - res = Results() - res_out, sol_data = parse_sol_file(stream, nl_info, res) + self.assertEqual("Solver message preamble", sol_data.message) + self.assertEqual(0, sol_data.objno) + self.assertEqual(100, sol_data.solve_code) + self.assertEqual([1, 2], sol_data.ampl_options) + self.assertEqual([10.0, 20.0, 30.0], sol_data.primals) + self.assertEqual([1.5, -2.25], sol_data.duals) + self.assertEqual({}, sol_data.var_suffixes) + self.assertEqual({}, sol_data.con_suffixes) + self.assertEqual({}, sol_data.obj_suffixes) + self.assertEqual({}, sol_data.problem_suffixes) + self.assertEqual(None, sol_data.unparsed) + + def test_parse_vbtol(self): + stream = io.StringIO( + f"""Solver message preamble +Options +2 +1 +3 +2 +0 +3 +0 +1.5 +objno 0 100""" + ) + sol_data = parse_sol_file(stream) + + self.assertEqual("Solver message preamble", sol_data.message) + self.assertEqual(0, sol_data.objno) + self.assertEqual(100, sol_data.solve_code) + self.assertEqual([1, 3, 1.5], sol_data.ampl_options) + self.assertEqual([], sol_data.primals) + self.assertEqual([], sol_data.duals) + self.assertEqual({}, sol_data.var_suffixes) + self.assertEqual({}, sol_data.con_suffixes) + self.assertEqual({}, sol_data.obj_suffixes) + self.assertEqual({}, sol_data.problem_suffixes) + self.assertEqual(None, sol_data.unparsed) + + def test_multiline_message_and_unparsed(self): + stream = io.StringIO( + """CONOPT 3.17A: Optimal; objective 1 +4 iterations; evals: nf = 2, ng = 0, nc = 2, nJ = 0, nH = 0, nHv = 0 + +Options +3 +1 +1 +0 +1 +1 +1 +1 +1 +1 +objno 0 0 +suffix 0 1 8 0 0 +sstatus +0 1 +suffix 1 1 8 0 0 +sstatus +0 3 +extra data here +and here +""" + ) + sol_data = parse_sol_file(stream) - # Check counts populated - self.assertEqual(len(sol_data.duals), n_cons) - self.assertEqual(len(sol_data.primals), n_vars) - # Exit code 0..99 -> optimal + convergenceCriteriaSatisfied - self.assertEqual(res_out.solution_status.name, "optimal") self.assertEqual( - res_out.termination_condition.name, "convergenceCriteriaSatisfied" + "CONOPT 3.17A: Optimal; objective 1\n" + "4 iterations; evals: nf = 2, ng = 0, nc = 2, nJ = 0, nH = 0, nHv = 0", + sol_data.message, + ) + self.assertEqual(0, sol_data.objno) + self.assertEqual(0, sol_data.solve_code) + self.assertEqual([1, 1, 0], sol_data.ampl_options) + self.assertEqual([1.0], sol_data.primals) + self.assertEqual([1.0], sol_data.duals) + self.assertEqual({'sstatus': {0: 1}}, sol_data.var_suffixes) + self.assertEqual({'sstatus': {0: 3}}, sol_data.con_suffixes) + self.assertEqual({}, sol_data.obj_suffixes) + self.assertEqual({}, sol_data.problem_suffixes) + self.assertEqual("extra data here\nand here\n", sol_data.unparsed) + + def test_suffix_table(self): + stream = io.StringIO( + """CONOPT 3.17A: Optimal; objective 1 +4 iterations; evals: nf = 2, ng = 0, nc = 2, nJ = 0, nH = 0, nHv = 0 + +Options +3 +1 +1 +0 +1 +1 +1 +1 +1 +1 +objno 0 0 +suffix 0 1 7 36 3 +custom +1 INT An int field +2 DBL double +3 STR +0 1 +suffix 1 1 8 0 0 +sstatus +0 3 +suffix 2 1 8 0 0 +sstatus +0 2 +suffix 3 1 8 0 0 +sstatus +0 4 + +""" ) + sol_data = parse_sol_file(stream) - # Values preserved - self.assertAlmostEqual(sol_data.duals[0], 1.5) - self.assertAlmostEqual(sol_data.duals[1], -2.25) - self.assertEqual(sol_data.primals, [10.0, 20.0, 30.0]) + self.assertEqual( + "CONOPT 3.17A: Optimal; objective 1\n" + "4 iterations; evals: nf = 2, ng = 0, nc = 2, nJ = 0, nH = 0, nHv = 0", + sol_data.message, + ) + self.assertEqual(0, sol_data.objno) + self.assertEqual(0, sol_data.solve_code) + self.assertEqual([1, 1, 0], sol_data.ampl_options) + self.assertEqual([1.0], sol_data.primals) + self.assertEqual([1.0], sol_data.duals) + self.assertEqual({'custom': {0: 1}}, sol_data.var_suffixes) + self.assertEqual({'sstatus': {0: 3}}, sol_data.con_suffixes) + self.assertEqual({'sstatus': {0: 2}}, sol_data.obj_suffixes) + self.assertEqual({'sstatus': 4}, sol_data.problem_suffixes) + self.assertEqual( + { + (0, 'custom'): [ + [1, 'INT', 'An int field'], + [2, 'DBL', 'double'], + [3, 'STR'], + ] + }, + sol_data.suffix_table, + ) + self.assertEqual(None, sol_data.unparsed) - def test_parse_sol_file_missing_options_raises(self): + def test_error_missing_options(self): # No line contains the substring "Options" bad_text = "Solver message preamble\nNo header here\n" stream = io.StringIO(bad_text) - nl_info = self._FakeNLInfo(variables=[], constraints=[]) - - with self.assertRaises(PyomoException): - parse_sol_file(stream, nl_info, Results()) + with self.assertRaisesRegex( + SolverError, "Error reading `sol` file: no 'Options' line found." + ): + parse_sol_file(stream) - def test_parse_sol_file_malformed_options_raises(self): + def test_error_malformed_options(self): # Contains "Options" but the required integer line is missing/blank bad_text = "Preamble\nOptions\n\n" stream = io.StringIO(bad_text) - nl_info = self._FakeNLInfo(variables=[], constraints=[]) + with self.assertRaisesRegex(ValueError, "invalid literal"): + parse_sol_file(stream) + + def test_error_objno_not_found(self): + stream = io.StringIO( + f"""Solver message preamble +Options +2 +1 +2 +2 +0 +3 +0 +1.5 +objno 0""" + ) + + with self.assertRaisesRegex( + SolverError, + "Error reading `sol` file: expected 'objno'; " "received '1.5'.", + ): + sol_data = parse_sol_file(stream) + + def test_error_objno_bad_format(self): + stream = io.StringIO( + f"""Solver message preamble +Options +2 +1 +2 +2 +0 +3 +0 +objno 0""" + ) + + with self.assertRaisesRegex( + SolverError, + "Error reading `sol` file: expected two numbers in 'objno' line; " + "received 'objno 0'.", + ): + sol_data = parse_sol_file(stream) + + +class TestSolFileSolutionLoader(unittest.TestCase): - with self.assertRaises(ValueError): - parse_sol_file(stream, nl_info, Results()) + def test_member_list(self): + expected_list = ['load_vars', 'get_primals', 'get_duals', 'get_reduced_costs'] + method_list = [ + method + for method in dir(SolFileSolutionLoader) + if not method.startswith('_') + ] + self.assertEqual(sorted(expected_list), sorted(method_list)) + + def test_load_vars(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var([1, 2, 3]) + + nl_info = NLWriterInfo(var=[m.x, m.y[1], m.y[3]]) + sol_data = SolFileData() + sol_data.primals = [3, 7, 5] + loader = SolFileSolutionLoader(sol_data, nl_info) + + loader.load_vars() + self.assertEqual(m.x.value, 3) + self.assertEqual(m.y[1].value, 7) + self.assertEqual(m.y[2].value, None) + self.assertEqual(m.y[3].value, 5) + + sol_data.primals = [13, 17, 15] + loader.load_vars(vars_to_load=[m.y[3], m.x]) + self.assertEqual(m.x.value, 13) + self.assertEqual(m.y[1].value, 7) + self.assertEqual(m.y[2].value, None) + self.assertEqual(m.y[3].value, 15) + + nl_info.scaling = ScalingFactors([1, 5, 10], [], []) + loader.load_vars() + self.assertEqual(m.x.value, 13) + self.assertEqual(m.y[1].value, 3.4) + self.assertEqual(m.y[2].value, None) + self.assertEqual(m.y[3].value, 1.5) + + nl_info.eliminated_vars = [(m.y[2], 2 * m.y[3] + 1)] + loader.load_vars() + self.assertEqual(m.x.value, 13) + self.assertEqual(m.y[1].value, 3.4) + self.assertEqual(m.y[2].value, 4) + self.assertEqual(m.y[3].value, 1.5) + + def test_load_vars_empty_model(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var([1, 2, 3]) + + nl_info = NLWriterInfo( + var=[], eliminated_vars=[(m.y[3], 1.5), (m.y[2], 2 * m.y[3] + 1)] + ) + sol_data = SolFileData() + sol_data.primals = [] + loader = SolFileSolutionLoader(sol_data, nl_info) + + loader.load_vars() + self.assertEqual(m.x.value, None) + self.assertEqual(m.y[1].value, None) + self.assertEqual(m.y[2].value, 4) + self.assertEqual(m.y[3].value, 1.5) + + def test_get_primals(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var([1, 2, 3]) + + nl_info = NLWriterInfo(var=[m.x, m.y[1], m.y[3]]) + sol_data = SolFileData() + sol_data.primals = [3, 7, 5] + loader = SolFileSolutionLoader(sol_data, nl_info) + + self.assertEqual( + loader.get_primals(), ComponentMap([(m.x, 3), (m.y[1], 7), (m.y[3], 5)]) + ) + self.assertEqual(m.x.value, None) + self.assertEqual(m.y[1].value, None) + self.assertEqual(m.y[2].value, None) + self.assertEqual(m.y[3].value, None) + + sol_data.primals = [13, 17, 15] + self.assertEqual( + loader.get_primals(vars_to_load=[m.y[3], m.x]), + ComponentMap([(m.x, 13), (m.y[3], 15)]), + ) + self.assertEqual(m.x.value, None) + self.assertEqual(m.y[1].value, None) + self.assertEqual(m.y[2].value, None) + self.assertEqual(m.y[3].value, None) + + nl_info.scaling = ScalingFactors([1, 5, 10], [], []) + self.assertEqual( + loader.get_primals(), + ComponentMap([(m.x, 13), (m.y[1], 3.4), (m.y[3], 1.5)]), + ) + self.assertEqual(m.x.value, None) + self.assertEqual(m.y[1].value, None) + self.assertEqual(m.y[2].value, None) + self.assertEqual(m.y[3].value, None) + + nl_info.eliminated_vars = [(m.y[2], 2 * m.y[3] + 1)] + self.assertEqual( + loader.get_primals(), + ComponentMap([(m.x, 13), (m.y[1], 3.4), (m.y[2], 4), (m.y[3], 1.5)]), + ) + self.assertEqual(m.x.value, None) + self.assertEqual(m.y[1].value, None) + self.assertEqual(m.y[2].value, None) + self.assertEqual(m.y[3].value, None) + + def test_get_primals_empty_model(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var([1, 2, 3]) + + nl_info = NLWriterInfo( + var=[], eliminated_vars=[(m.y[3], 1.5), (m.y[2], 2 * m.y[3] + 1)] + ) + sol_data = SolFileData() + sol_data.primals = [] + loader = SolFileSolutionLoader(sol_data, nl_info) + + self.assertEqual( + loader.get_primals(), ComponentMap([(m.y[2], 4), (m.y[3], 1.5)]) + ) + self.assertEqual(m.x.value, None) + self.assertEqual(m.y[1].value, None) + self.assertEqual(m.y[2].value, None) + self.assertEqual(m.y[3].value, None) diff --git a/pyomo/contrib/solver/tests/unit/test_results.py b/pyomo/contrib/solver/tests/unit/test_results.py index df7f12c974f..fef4d019489 100644 --- a/pyomo/contrib/solver/tests/unit/test_results.py +++ b/pyomo/contrib/solver/tests/unit/test_results.py @@ -141,7 +141,7 @@ def test_codes(self): class TestSolutionStatus(unittest.TestCase): def test_member_list(self): member_list = results.SolutionStatus._member_names_ - expected_list = ['noSolution', 'infeasible', 'feasible', 'optimal'] + expected_list = ['noSolution', 'unknown', 'infeasible', 'feasible', 'optimal'] self.assertEqual(member_list, expected_list) def test_codes(self): diff --git a/pyomo/contrib/solver/tests/unit/test_solution.py b/pyomo/contrib/solver/tests/unit/test_solution.py index 0453f0e0cb2..7046055093a 100644 --- a/pyomo/contrib/solver/tests/unit/test_solution.py +++ b/pyomo/contrib/solver/tests/unit/test_solution.py @@ -36,19 +36,6 @@ def test_solution_loader_base(self): self.instance.get_reduced_costs() -class TestSolSolutionLoader(unittest.TestCase): - # I am currently unsure how to test this further because it relies heavily on - # SolFileData and NLWriterInfo - def test_member_list(self): - expected_list = ['load_vars', 'get_primals', 'get_duals', 'get_reduced_costs'] - method_list = [ - method - for method in dir(SolutionLoaderBase) - if method.startswith('_') is False - ] - self.assertEqual(sorted(expected_list), sorted(method_list)) - - class TestPersistentSolutionLoader(unittest.TestCase): def test_member_list(self): expected_list = [ From 28328696cfa235915d473f4809a0c4e9ca6b4f94 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 18 Nov 2025 08:04:47 -0700 Subject: [PATCH 09/41] NFC: typos --- pyomo/contrib/solver/solvers/sol_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/solvers/sol_reader.py b/pyomo/contrib/solver/solvers/sol_reader.py index f5a149f27ef..8b6de5e4f2a 100644 --- a/pyomo/contrib/solver/solvers/sol_reader.py +++ b/pyomo/contrib/solver/solvers/sol_reader.py @@ -118,7 +118,7 @@ def get_primals( # modify). We will make use of an expression replacement # visitor to perform the substitution and computation. # - # It would be great if we could do this withough creating the + # It would be great if we could do this without creating the # entire (unfiltered) result, but we just don't (easily) know # which variable values we are going to need (either in the # vars_to_load list, or in any expression that might be needed @@ -263,7 +263,7 @@ def _parse_message_and_options(FILE: io.TextIOBase, data: SolFileData) -> List[i data.message = "\n".join(msg) # WARNING: This appears to be undocumented outside of the ASL - # writesol.c implemention. Before changing this logic, please + # writesol.c implementation. Before changing this logic, please # familiarize yourself with that code. # # The AMPL options are a sequence of ints, the first of which From 6562296dec0881e279b017339e20ee10bebc0b63 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 18 Nov 2025 08:25:21 -0700 Subject: [PATCH 10/41] Fix logic for verifying the length of the iteration log --- pyomo/contrib/solver/solvers/ipopt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 0a7d76e9287..fb17d4d18e1 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -664,11 +664,11 @@ def _parse_ipopt_output(self, output: str) -> Dict[str, Any]: parsed_data['iteration_log'] = iterations - if len(iterations) != parsed_data.get('iters', 0): + if len(iterations) != parsed_data.get('iters', 0) + 1: n_iter = parsed_data.get('iters', 0) logger.warning( f"Total number of iteration records parsed {len(iterations)} does " - f"not match the number of iterations ({n_iter})." + f"not match the number of iterations ({n_iter}) plus one." ) # Extract scaled and unscaled table From 82bd2d5bec5e94c9f98518e1d4ab988f2e663e09 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Fri, 28 Nov 2025 11:08:23 -0700 Subject: [PATCH 11/41] Shorten the temporary file names --- pyomo/contrib/solver/solvers/ipopt.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index fb17d4d18e1..0fb9450a101 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -16,6 +16,7 @@ import io import re import sys +import time import threading from typing import Optional, Tuple, Union, Mapping, List, Dict, Any, Sequence @@ -65,6 +66,18 @@ # in ipopt's output, per https://coin-or.github.io/Ipopt/OUTPUT.html _ALPHA_PR_CHARS = set("fFhHkKnNRwSstTr") +_charlist = '0123456789abcdefghijklmnopqrstuvwxyz' +assert len(_charlist) >= 32 + + +def _encode_int(i: int) -> str: + ans = [] + i = int(i) + while i: + ans.append(_charlist[i & 31]) + i >>= 5 + return ''.join(reversed(ans)) + class IpoptConfig(SolverConfig): def __init__( @@ -380,8 +393,11 @@ def solve(self, model, **kwds) -> Results: # solver interfaces are not formally thread-safe (yet), so # this is a bit of future-proofing. basename = os.path.join( - dname, f"{basename}.{os.getpid()}.{threading.get_ident()}" + dname, + f"{basename}-{_encode_int(time.time()*1e6)}-" + f"{_encode_int(threading.get_native_id())}", ) + results.extra_info.base_file_name = basename for ext in ('.nl', '.row', '.col', '.sol', '.opt'): if os.path.exists(basename + ext): raise RuntimeError( From 0ff622f6020331b137f6e4080fd8706e8546cb53 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Fri, 28 Nov 2025 11:09:16 -0700 Subject: [PATCH 12/41] More updates / simplifications of ippot log parser --- pyomo/contrib/solver/solvers/ipopt.py | 131 +++++++++++++------------- 1 file changed, 65 insertions(+), 66 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 0fb9450a101..0f92c8bf285 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -22,9 +22,10 @@ from pyomo.common import Executable from pyomo.common.config import ( + ConfigDict, + ConfigList, ConfigValue, document_class_CONFIG, - ConfigDict, ADVANCED_OPTION, ) from pyomo.common.errors import ( @@ -557,10 +558,14 @@ def _run_ipopt(self, results, config, nl_info, basename, timer): # results.timing_info.update(_timing) for k, v in _timing.items(): results.timing_info[k] = v - results.extra_info = parsed_output_data - iter_log = results.extra_info.get("iteration_log", None) + iter_log = parsed_output_data.pop('iteration_log', None) if iter_log is not None: - iter_log._visibility = ADVANCED_OPTION + results.extra_info.add( + 'iteration_log', ConfigList(iter_log, visibility=ADVANCED_OPTION) + ) + # results.extra_info.update(parsed_output_data) + for k, v in parsed_output_data.items(): + results.extra_info[k] = v timer.start('parse_sol') if os.path.isfile(basename + '.sol'): @@ -596,86 +601,80 @@ def _parse_ipopt_output(self, output: str) -> Dict[str, Any]: iter_table = re.findall(r'^(?:\s*\d+.*?)$', output, re.MULTILINE) if iter_table: columns = [ - "iter", - "objective", - "inf_pr", - "inf_du", - "lg_mu", - "d_norm", - "lg_rg", - "alpha_du", - "alpha_pr", - "ls", + ("iter", int), + ("objective", float), + ("inf_pr", float), + ("inf_du", float), + ("lg_mu", float), + ("d_norm", float), + ("lg_rg", float), + ("alpha_du", float), + ("alpha_pr", float), + ("ls", int), ] iterations = [] n_expected_columns = len(columns) + iter_idx = columns.index(('iter', int)) + alpha_pr_idx = columns.index(('alpha_pr', float)) for line in iter_table: tokens = line.strip().split() # IPOPT sometimes mashes the first two column values together # (e.g., "2r-4.93e-03"). We need to split them. - try: - idx = tokens[0].index('-') - head = tokens[0][:idx] - if head and head.rstrip('r').isdigit(): - tokens[:1] = (head, tokens[0][idx:]) - except ValueError: - pass - - iter_data = dict(zip(columns, tokens)) - extra_tokens = tokens[n_expected_columns:] + if '-' in tokens[iter_idx]: + # This happens rarely, so we are OK with this + # portion of the parser being a little less + # efficient (e.g., reallocating the tokens list, and + # performing index math) + tkn = tokens[iter_idx] + idx = tkn.index('-') + tokens[iter_idx : iter_idx + 1] = tkn[:idx], tkn[idx:] # Extract restoration flag from 'iter' - iter_num = iter_data.pop("iter") - restoration = iter_num.endswith("r") + restoration = tokens[iter_idx].endswith("r") if restoration: - iter_num = iter_num[:-1] + tokens[iter_idx] = tokens[iter_idx][:-1] + + # Separate alpha_pr into numeric part and optional tag (f, D, R, etc.) + step_acceptance = tokens[alpha_pr_idx][-1] + if step_acceptance in _ALPHA_PR_CHARS: + tokens[alpha_pr_idx] = tokens[alpha_pr_idx][:-1] + else: + step_acceptance = None try: - iter_num = int(iter_num) - except ValueError: - logger.warning( - f"Could not parse Ipopt iteration number: {iter_num}" + iter_data = { + key: None if t == '-' else cast(t) + for (key, cast), t in zip(columns, tokens) + } + except (ValueError, TypeError): + logger.error( + "Error parsing Ipopt log entry:\n" + f"\t{sys.exc_info()[1]}\n\t{line}" ) + # Fall-back on a simpler (but slower) parse: extract + # the fields, and cast to float what we can. The + # point here is the parser should never fail with an + # exception (even if it fails to parse some of the + # log) + iter_data = {} + for (key, cast), t in zip(columns, tokens): + if t == '-': + t = None + else: + try: + t = cast(t) + except: + pass + iter_data[key] = t iter_data["restoration"] = restoration - iter_data["iter"] = iter_num - - # Separate alpha_pr into numeric part and optional tag (f, D, R, etc.) - step_acceptance_tag = iter_data['alpha_pr'][-1] - if step_acceptance_tag in _ALPHA_PR_CHARS: - iter_data['step_acceptance'] = step_acceptance_tag - iter_data['alpha_pr'] = iter_data['alpha_pr'][:-1] - else: - iter_data['step_acceptance'] = None + iter_data["step_acceptance"] = step_acceptance # Capture optional IPOPT diagnostic tags if present - if extra_tokens: - iter_data['diagnostic_tags'] = " ".join(extra_tokens) - - # Attempt to cast all values to float where possible - for key in columns[1:]: - val = iter_data[key] - if val == '-': - iter_data[key] = None - else: - try: - iter_data[key] = float(val) - except (ValueError, TypeError): - logger.warning( - "Error converting Ipopt log entry to " - f"float:\n\t{sys.exc_info()[1]}\n\t{line}" - ) - - if len(iterations) != iter_num: - logger.warning( - f"Parsed iteration record {len(iterations)} " - f"does not match the expected iteration number ({iter_num})" - f"\n\t{line}" - ) - # Things have gotten fouled up. There is no real - # reason to continue to parse the iteration log - break + if len(tokens) > n_expected_columns: + iter_data['diagnostic_tags'] = " ".join(tokens[n_expected_columns:]) + iterations.append(iter_data) parsed_data['iteration_log'] = iterations From a991e9badae9b66214ef9887ccc9403b16561654 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Fri, 28 Nov 2025 11:09:37 -0700 Subject: [PATCH 13/41] Save the Ipopt command line in the results object --- pyomo/contrib/solver/solvers/ipopt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 0f92c8bf285..eb705f025f7 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -517,6 +517,9 @@ def _run_ipopt(self, results, config, nl_info, basename, timer): ) # Call ipopt - passing the files via the subprocess cmd = self._create_command_line(basename=basename, config=config) + results.extra_info.add( + 'command_line', ConfigValue(cmd, visibility=ADVANCED_OPTION) + ) # this seems silly, but we have to give the subprocess slightly # longer to finish than ipopt if config.time_limit is not None: From 8c107d939cbbca5c85702a2991aad82274041846 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Fri, 28 Nov 2025 11:10:35 -0700 Subject: [PATCH 14/41] bugfix: imports (plus sort them) --- pyomo/contrib/solver/solvers/sol_reader.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/solvers/sol_reader.py b/pyomo/contrib/solver/solvers/sol_reader.py index 8b6de5e4f2a..a03edeee5f2 100644 --- a/pyomo/contrib/solver/solvers/sol_reader.py +++ b/pyomo/contrib/solver/solvers/sol_reader.py @@ -13,13 +13,15 @@ from typing import Tuple, Dict, Any, List, Sequence, Optional, Mapping, NoReturn import io +from pyomo.common.collections import ComponentMap +from pyomo.common.errors import MouseTrap from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.var import VarData from pyomo.core.expr import value -from pyomo.common.collections import ComponentMap from pyomo.core.staleflag import StaleFlagManager -from pyomo.repn.plugins.nl_writer import NLWriterInfo from pyomo.core.expr.visitor import replace_expressions +from pyomo.repn.plugins.nl_writer import NLWriterInfo + from pyomo.contrib.solver.common.util import SolverError from pyomo.contrib.solver.common.results import ( Results, From 165794800d80c904a0d568c4a521df231ed60651 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Fri, 28 Nov 2025 11:11:25 -0700 Subject: [PATCH 15/41] NFC: update comment --- pyomo/repn/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/repn/util.py b/pyomo/repn/util.py index eefda267c60..faff49234f3 100644 --- a/pyomo/repn/util.py +++ b/pyomo/repn/util.py @@ -97,7 +97,7 @@ class FileDeterminism(enums.IntEnum): SORT_SYMBOLS = 30 # We will define __str__ and __format__ so that behavior in python - # 3.11 is consistent with 3.7 - 3.10. + # 3.11+ is consistent with 3.7 - 3.10. def __str__(self): return enums.Enum.__str__(self) From 1f176d5c96d601411670460d4db9b041d9dad602 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sat, 29 Nov 2025 10:45:06 -0700 Subject: [PATCH 16/41] Move available/version cache to class method --- pyomo/contrib/solver/solvers/ipopt.py | 84 +++++++++++++-------------- 1 file changed, 39 insertions(+), 45 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index eb705f025f7..8c756cf7587 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -233,51 +233,53 @@ class Ipopt(SolverBase): #: see :ref:`pyomo.contrib.solver.solvers.ipopt.Ipopt::CONFIG`. CONFIG = IpoptConfig() + #: cache of availability / version information + _exec_cache: dict[str : tuple[int] | None] = {} + + #: default timeout to use when attempting to get the ipopt version number + _version_timeout = 2 + def __init__(self, **kwds: Any) -> None: super().__init__(**kwds) self._writer = NLWriter() - self._available_cache = None - self._version_cache = None - self._version_timeout = 2 #: Instance configuration; #: see :ref:`pyomo.contrib.solver.solvers.ipopt.Ipopt::CONFIG`. self.config = self.config - def available(self, config: Optional[IpoptConfig] = None) -> Availability: - if config is None: - config = self.config - pth = config.executable.path() - if self._available_cache is None or self._available_cache[0] != pth: - if pth is None: - self._available_cache = (None, Availability.NotFound) - else: - self._available_cache = (pth, Availability.FullLicense) - return self._available_cache[1] - - def version( - self, config: Optional[IpoptConfig] = None - ) -> Optional[Tuple[int, int, int]]: - if config is None: - config = self.config - pth = config.executable.path() - if self._version_cache is None or self._version_cache[0] != pth: - if pth is None: - self._version_cache = (None, None) - else: - results = subprocess.run( - [str(pth), '--version'], - timeout=self._version_timeout, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - check=False, - ) - version = results.stdout.splitlines()[0] - version = version.split(' ')[1].strip() - version = tuple(int(i) for i in version.split('.')) - self._version_cache = (pth, version) - return self._version_cache[1] + def available(self) -> Availability: + return ( + Availability.NotFound + if self.version() is None + else Availability.FullLicense + ) + + def version(self) -> Optional[tuple[int, int, int]]: + return self._get_version(self.config.executable.path()) + + def _get_version(self, pth): + try: + return self._exec_cache[pth] + except KeyError: + pass + if pth is None: + self._exec_cache[None] = None + return None + results = subprocess.run( + [str(pth), '--version'], + timeout=self._version_timeout, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + check=False, + ) + ipopt, ver, _ = results.stdout.split(maxsplit=2) + if ipopt.lower() != 'ipopt': + ver = None + else: + ver = tuple(int(i) for i in ver.split('.')) + self._exec_cache[pth] = ver + return ver def has_linear_solver(self, linear_solver: str) -> bool: import pyomo.core as AML @@ -353,11 +355,6 @@ def solve(self, model, **kwds) -> Results: # Update configuration options, based on keywords passed to solve config: IpoptConfig = self.config(value=kwds, preserve_implicit=True) # Check if solver is available - avail = self.available(config) - if not avail: - raise ApplicationError( - f'Solver {self.__class__} is not available ({avail}).' - ) if config.threads: logger.log( logging.WARNING, @@ -456,9 +453,6 @@ def solve(self, model, **kwds) -> Results: ): raise NoOptimalSolutionError() - results.solver_name = self.name - results.solver_version = self.version(config) - if config.load_solutions: if results.solution_status == SolutionStatus.noSolution: raise NoSolutionError() From 4c772f4138884733b6a13cddd9525628b8ef20fa Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sat, 29 Nov 2025 11:55:22 -0700 Subject: [PATCH 17/41] Merge _create_command_line, _verify_options, and _write_options_file --- pyomo/contrib/solver/solvers/ipopt.py | 125 +++++++++++++------------- 1 file changed, 64 insertions(+), 61 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 8c756cf7587..40b6ca66d20 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -80,6 +80,22 @@ def _encode_int(i: int) -> str: return ''.join(reversed(ans)) +def _option_to_str(opt, val): + if isinstance(val, str): + if '"' not in val: + return f'{opt}="{val}"' + elif "'" not in val: + return f"{opt}='{val}'" + else: + raise ValueError( + f"solver_option '{opt}' contained value {val!r} with " + "both single and double quotes. Ipopt cannot parse " + "command line options with escaped quote characters." + ) + else: + return f'{opt}={val}' + + class IpoptConfig(SolverConfig): def __init__( self, @@ -296,52 +312,6 @@ def has_linear_solver(self, linear_solver: str) -> bool: ) return 'running with linear solver' in results.solver_log - def _verify_ipopt_options(self, config: IpoptConfig) -> None: - for key, msg in unallowed_ipopt_options.items(): - if key in config.solver_options: - raise ValueError(f"unallowed Ipopt option '{key}': {msg}") - # Map standard Pyomo solver options to Ipopt options: standard - # options override ipopt-specific options. - if config.time_limit is not None: - config.solver_options['max_cpu_time'] = config.time_limit - - def _write_options_file( - self, filename: str, options: Mapping[str, Union[str, int, float]] - ) -> None: - # Look through the solver options and write them to a file. - # If they are command line options, ignore them; they will be - # added to the command line. - options_file_options = [ - opt for opt in options if opt not in ipopt_command_line_options - ] - if not options_file_options: - return - with open(filename, 'w', encoding='utf-8') as OPT_FILE: - OPT_FILE.writelines( - f"{opt} {options[opt]}\n" for opt in options_file_options - ) - options['option_file_name'] = filename - - def _create_command_line(self, basename: str, config: IpoptConfig) -> List[str]: - cmd = [str(config.executable), basename + '.nl', '-AMPL'] - for opt, val in config.solver_options.items(): - if opt not in ipopt_command_line_options: - continue - if isinstance(val, str): - if '"' not in val: - cmd.append(f'{opt}="{val}"') - elif "'" not in val: - cmd.append(f"{opt}='{val}'") - else: - raise ValueError( - f"solver_option '{opt}' contained value {val!r} with " - "both single and double quotes. Ipopt cannot parse " - "command line options with escaped quote characters." - ) - else: - cmd.append(f'{opt}={val}') - return cmd - def solve(self, model, **kwds) -> Results: "Solve a model using Ipopt" # Begin time tracking @@ -496,21 +466,53 @@ def solve(self, model, **kwds) -> Results: results.timing_info.wall_time = default_timer() - start_time return results + def _process_options( + self, option_fname: str, options: dict[str, str | int | float] + ) -> list[str]: + # Look through the solver options and write them to a file. + # If they are command line options, ignore them; they will be + # added to the command line. + options_file_options = [] + cmd_line_options = [] + for key, val in options.items(): + if key in ipopt_command_line_options: + cmd_line_options.append(_option_to_str(key, val)) + elif key in unallowed_ipopt_options: + msg = unallowed_ipopt_options[key] + raise ValueError(f"unallowed Ipopt option '{key}': {msg}") + else: + options_file_options.append((key, val)) + + if options_file_options: + with open(option_fname, 'w', encoding='utf-8') as OPT_FILE: + OPT_FILE.writelines( + f"{opt} {val}\n" for opt, val in options_file_options + ) + cmd_line_options.append(_option_to_str('option_file_name', option_fname)) + return cmd_line_options + def _run_ipopt(self, results, config, nl_info, basename, timer): # Get a copy of the environment to pass to the subprocess env = os.environ.copy() if nl_info.external_function_libraries: env['AMPLFUNC'] = amplfunc_merge(env, *nl_info.external_function_libraries) - self._verify_ipopt_options(config) - # Write the options file, if there should be one. If - # the file was written, then 'options_file_name' was - # added to config.options (so we can correctly build the - # command line) - self._write_options_file( - filename=basename + '.opt', options=config.solver_options - ) - # Call ipopt - passing the files via the subprocess - cmd = self._create_command_line(basename=basename, config=config) + + # Get the Ipopt executable and start building the command line + exe = config.executable.path() + if not exe: + raise ApplicationError('ipopt executable not found') + cmd = [exe, basename + '.nl', '-AMPL'] + + # Process ipopt options (splitting them between command line + # options and those that must be passed through the opt file) + options = config.solver_options.value() + # Map standard Pyomo solver options to Ipopt options: standard + # options override ipopt-specific options. + if config.time_limit is not None: + options['max_cpu_time'] = config.time_limit + cmd.extend(self._process_options(basename + '.opt', options)) + + results.solver_version = self._get_version(exe) results.extra_info.add( 'command_line', ConfigValue(cmd, visibility=ADVANCED_OPTION) ) @@ -735,12 +737,13 @@ def _parse_ipopt_output(self, output: str) -> Dict[str, Any]: class LegacyIpoptSolver(LegacySolverWrapper, Ipopt): - def _verify_ipopt_options(self, config: IpoptConfig) -> None: + def _process_options( + self, option_fname: str, options: dict[str, str | int | float] + ) -> list[str]: # The old Ipopt solver would map solver_options starting with # "OF_" to the options file. That is no longer needed, so we # will strip off any "OF_" that we find - for opt, val in list(config.solver_options.items()): + for opt in list(options): if opt.startswith('OF_'): - config.solver_options[opt[3:]] = val - del config.solver_options[opt] - return super()._verify_ipopt_options(config) + options[opt[3:]] = options.pop(opt) + return super()._process_options(option_fname, options) From 21dca81afce301687c37de91debc10d44507f7b3 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 11:26:09 -0700 Subject: [PATCH 18/41] Add 'delete=' option to context mkstemp/mkdtemp --- pyomo/common/tempfiles.py | 10 ++++++---- pyomo/common/tests/test_tempfile.py | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/pyomo/common/tempfiles.py b/pyomo/common/tempfiles.py index e61802065b7..496a4da2fc6 100644 --- a/pyomo/common/tempfiles.py +++ b/pyomo/common/tempfiles.py @@ -270,7 +270,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.release() - def mkstemp(self, suffix=None, prefix=None, dir=None, text=False): + def mkstemp(self, suffix=None, prefix=None, dir=None, text=False, delete=True): """Create a unique temporary file using :func:`tempfile.mkstemp` Parameters are handled as in :func:`tempfile.mkstemp`, with @@ -289,10 +289,11 @@ def mkstemp(self, suffix=None, prefix=None, dir=None, text=False): dir = self._resolve_tempdir(dir) # Note: ans == (fd, fname) ans = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir, text=text) - self.tempfiles.append(ans) + if delete: + self.tempfiles.append(ans) return ans - def mkdtemp(self, suffix=None, prefix=None, dir=None): + def mkdtemp(self, suffix=None, prefix=None, dir=None, delete=True): """Create a unique temporary directory using :func:`tempfile.mkdtemp` Parameters are handled as in :func:`tempfile.mkdtemp`, with @@ -307,7 +308,8 @@ def mkdtemp(self, suffix=None, prefix=None, dir=None): """ dir = self._resolve_tempdir(dir) dname = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir) - self.tempfiles.append((None, dname)) + if delete: + self.tempfiles.append((None, dname)) return dname def gettempdir(self): diff --git a/pyomo/common/tests/test_tempfile.py b/pyomo/common/tests/test_tempfile.py index 85aa2fca3c4..8ae88d9e967 100644 --- a/pyomo/common/tests/test_tempfile.py +++ b/pyomo/common/tests/test_tempfile.py @@ -284,6 +284,28 @@ def test_mkstemp(self): os.close(fd) context.release() + def test_mktemp_delete(self): + with self.TM.new_context() as context: + dname = context.mkdtemp() + with self.TM.new_context() as subcontext: + _, fname1 = subcontext.mkstemp(dir=dname) + _, fname2 = subcontext.mkstemp(dir=dname, delete=False) + dname1 = subcontext.mkdtemp(dir=dname) + dname2 = subcontext.mkdtemp(dir=dname, delete=False) + + self.assertTrue(os.path.exists(fname1)) + self.assertTrue(os.path.exists(fname2)) + self.assertTrue(os.path.exists(dname1)) + self.assertTrue(os.path.exists(dname2)) + self.assertFalse(os.path.exists(fname1)) + self.assertTrue(os.path.exists(fname2)) + self.assertFalse(os.path.exists(dname1)) + self.assertTrue(os.path.exists(dname2)) + self.assertFalse(os.path.exists(fname1)) + self.assertFalse(os.path.exists(fname2)) + self.assertFalse(os.path.exists(dname1)) + self.assertFalse(os.path.exists(dname2)) + def test_create_tempdir(self): context = self.TM.push() fname = self.TM.create_tempdir("suffix", "prefix") From 1bee4dc4194db5f035a2a7cab4f9ff97b148b9b8 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 11:41:57 -0700 Subject: [PATCH 19/41] Switch to using tempfile to generate solver inputs --- pyomo/contrib/solver/solvers/ipopt.py | 33 ++++----------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 40b6ca66d20..188ef6ffe75 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -67,18 +67,6 @@ # in ipopt's output, per https://coin-or.github.io/Ipopt/OUTPUT.html _ALPHA_PR_CHARS = set("fFhHkKnNRwSstTr") -_charlist = '0123456789abcdefghijklmnopqrstuvwxyz' -assert len(_charlist) >= 32 - - -def _encode_int(i: int) -> str: - ans = [] - i = int(i) - while i: - ans.append(_charlist[i & 31]) - i >>= 5 - return ''.join(reversed(ans)) - def _option_to_str(opt, val): if isinstance(val, str): @@ -351,22 +339,11 @@ def solve(self, model, **kwds) -> Results: # generate a legal base name (unless, of course, the user # put double quotes somewhere else in the path) basename = to_legal_filename(model.name, universal=True) - # Strip off quotes - the command line parser will re-add them - if basename[0] in "'\"" and basename[0] == basename[-1]: - basename = basename[1:-1] - # The base file name for this interface is "model_name + PID - # + thread id", so that this is reasonably unique in both - # parallel and threaded environments (even when working_dir - # is set to a persistent directory). Note that the Pyomo - # solver interfaces are not formally thread-safe (yet), so - # this is a bit of future-proofing. - basename = os.path.join( - dname, - f"{basename}-{_encode_int(time.time()*1e6)}-" - f"{_encode_int(threading.get_native_id())}", + nlfd, nl_fname = tempfile.mkstemp( + suffix='.nl', prefix=basename, dir=dname, text=True, delete=False ) - results.extra_info.base_file_name = basename - for ext in ('.nl', '.row', '.col', '.sol', '.opt'): + results.extra_info.base_file_name = basename = nl_fname[:-3] + for ext in ('.row', '.col', '.sol', '.opt'): if os.path.exists(basename + ext): raise RuntimeError( f"Solver interface file {basename + ext} already exists!" @@ -377,7 +354,7 @@ def solve(self, model, **kwds) -> Results: # disable universal newlines in the NL file to prevent # Python from mapping those '\n' to '\r\n' on Windows. with ( - open(basename + '.nl', 'w', newline='\n', encoding='utf-8') as nl_file, + os.fdopen(nlfd, 'w', newline='\n', encoding='utf-8') as nl_file, open(basename + '.row', 'w', encoding='utf-8') as row_file, open(basename + '.col', 'w', encoding='utf-8') as col_file, ): From c1cb70405529553fd0b9c35fc4868e638275adc7 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 11:43:39 -0700 Subject: [PATCH 20/41] Simplify how we process options (single pass, direct to string) --- pyomo/contrib/solver/solvers/ipopt.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 188ef6ffe75..6f7c44d54c3 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -68,7 +68,8 @@ _ALPHA_PR_CHARS = set("fFhHkKnNRwSstTr") -def _option_to_str(opt, val): +def _option_to_cmd(opt: str, val: str | int | float): + """Convert a opyion / value pair into a valid command line argument.""" if isinstance(val, str): if '"' not in val: return f'{opt}="{val}"' @@ -446,26 +447,25 @@ def solve(self, model, **kwds) -> Results: def _process_options( self, option_fname: str, options: dict[str, str | int | float] ) -> list[str]: - # Look through the solver options and write them to a file. - # If they are command line options, ignore them; they will be - # added to the command line. + # Look through the solver options and separate the command line + # options from the options that must be sent via an options + # file. Raise an exception for any unallowable options. options_file_options = [] cmd_line_options = [] for key, val in options.items(): - if key in ipopt_command_line_options: - cmd_line_options.append(_option_to_str(key, val)) - elif key in unallowed_ipopt_options: + if key in unallowed_ipopt_options: msg = unallowed_ipopt_options[key] raise ValueError(f"unallowed Ipopt option '{key}': {msg}") + elif key in ipopt_command_line_options: + cmd_line_options.append(_option_to_cmd(key, val)) else: - options_file_options.append((key, val)) - + options_file_options.append(f"{key} {val}\n") + # create the options file (if we need it) if options_file_options: with open(option_fname, 'w', encoding='utf-8') as OPT_FILE: - OPT_FILE.writelines( - f"{opt} {val}\n" for opt, val in options_file_options - ) - cmd_line_options.append(_option_to_str('option_file_name', option_fname)) + OPT_FILE.writelines(options_file_options) + cmd_line_options.append(_option_to_cmd('option_file_name', option_fname)) + # Return the (formatted) command ine options return cmd_line_options def _run_ipopt(self, results, config, nl_info, basename, timer): From 9268b3fe76d3375b54cba5d9643c50bbee0bf7ef Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 11:46:09 -0700 Subject: [PATCH 21/41] Make version identification more robust --- pyomo/contrib/solver/solvers/ipopt.py | 32 ++++++++++++++++++++------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 6f7c44d54c3..90fe29ca87e 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -239,7 +239,7 @@ class Ipopt(SolverBase): CONFIG = IpoptConfig() #: cache of availability / version information - _exec_cache: dict[str : tuple[int] | None] = {} + _exe_cache: dict[str : tuple[int] | None] = {} #: default timeout to use when attempting to get the ipopt version number _version_timeout = 2 @@ -259,17 +259,20 @@ def available(self) -> Availability: else Availability.FullLicense ) - def version(self) -> Optional[tuple[int, int, int]]: + def version(self) -> tuple[int, int, int] | None: return self._get_version(self.config.executable.path()) def _get_version(self, pth): try: - return self._exec_cache[pth] + return self._exe_cache[pth] except KeyError: pass if pth is None: - self._exec_cache[None] = None + # No executable (either we couldn't find a matching file, or + # the file is not executable) + self._exe_cache[None] = None return None + # Run the executable and look for the version results = subprocess.run( [str(pth), '--version'], timeout=self._version_timeout, @@ -278,12 +281,25 @@ def _get_version(self, pth): universal_newlines=True, check=False, ) - ipopt, ver, _ = results.stdout.split(maxsplit=2) - if ipopt.lower() != 'ipopt': + # Note that we expect the command to run without error, AND that + # it returns a string starting "ipopt ". That prevents + # us from tryying to use other (even ASL) executables as if they + # were ipopt + fields = results.stdout.split(maxsplit=2) + if results.returncode: + ver = None + elif len(fields) != 3 or fields[0].lower() != 'ipopt': ver = None else: - ver = tuple(int(i) for i in ver.split('.')) - self._exec_cache[pth] = ver + try: + ver = tuple(int(i) for i in fields[1].split('.')) + except (ValueError, TypeError): + ver = None + if ver is None: + logger.warning( + f"Failed parsing Ipopt version: '{exe} --version':\n\n{results.stdout}" + ) + self._exe_cache[pth] = ver return ver def has_linear_solver(self, linear_solver: str) -> bool: From 503c21985eeafa3a7349e5e0d98bc3ecc05dd696 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 11:48:57 -0700 Subject: [PATCH 22/41] NFC: update comments, report log parse time --- pyomo/contrib/solver/solvers/ipopt.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 90fe29ca87e..38452290a49 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -225,7 +225,7 @@ def get_reduced_costs( 'wantsol': 'The solver interface requires the sol file to be created', 'option_file_name': ( 'Pyomo generates the ipopt options file as part of the `solve` ' - 'method. Add all options to ipopt.config.solver_options instead.' + 'method. Add all options to config.solver_options instead.' ), } @@ -340,7 +340,10 @@ def solve(self, model, **kwds) -> Results: timer = config.timer = HierarchicalTimer() else: timer = config.timer + + # As we are about to run a solver, update the stale flag StaleFlagManager.mark_all_as_stale() + with TempfileManager.new_context() as tempfile: if config.working_dir is None: dname = tempfile.mkdtemp() @@ -516,6 +519,8 @@ def _run_ipopt(self, results, config, nl_info, basename, timer): else: timeout = None + + # Call ipopt - passing the files via the subprocess ostreams = [io.StringIO()] + config.tee timer.start('subprocess') try: @@ -543,21 +548,26 @@ def _run_ipopt(self, results, config, nl_info, basename, timer): # This is the data we need to parse to get the iterations # and time + timer.start('parse_log') parsed_output_data = self._parse_ipopt_output(results.solver_log) results.extra_info.iteration_count = parsed_output_data.pop('iters', None) _timing = parsed_output_data.pop('cpu_seconds', None) if _timing: - # results.timing_info.update(_timing) + # TODO: once #3790 is merged, this is just: + # results.timing_info.update(_timing) for k, v in _timing.items(): results.timing_info[k] = v + # Save the iteration log, but mark it as an "advanced" result iter_log = parsed_output_data.pop('iteration_log', None) if iter_log is not None: results.extra_info.add( 'iteration_log', ConfigList(iter_log, visibility=ADVANCED_OPTION) ) - # results.extra_info.update(parsed_output_data) + # TODO: once #3790 is merged, this is just: + # results.extra_info.update(parsed_output_data) for k, v in parsed_output_data.items(): results.extra_info[k] = v + timer.stop('parse_log') timer.start('parse_sol') if os.path.isfile(basename + '.sol'): From 6cd177b46eccb8f617c87113d3ac3c3b588177fd Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 11:49:28 -0700 Subject: [PATCH 23/41] Record solver name earlier --- pyomo/contrib/solver/solvers/ipopt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 38452290a49..3a88c3704ac 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -326,6 +326,7 @@ def solve(self, model, **kwds) -> Results: results.timing_info.start_timestamp = datetime.datetime.now( datetime.timezone.utc ) + results.solver_name = self.name # Update configuration options, based on keywords passed to solve config: IpoptConfig = self.config(value=kwds, preserve_implicit=True) From 6ab1cd5138aa830fc3331983962398b4a5559443 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 11:50:02 -0700 Subject: [PATCH 24/41] Simplify processing timer/timeout options --- pyomo/contrib/solver/solvers/ipopt.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 3a88c3704ac..f9509091a33 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -337,10 +337,9 @@ def solve(self, model, **kwds) -> Results: msg="The `threads` option was specified, " f"but this is not used by {self.__class__}.", ) - if config.timer is None: + timer = config.timer + if timer is None: timer = config.timer = HierarchicalTimer() - else: - timer = config.timer # As we are about to run a solver, update the stale flag StaleFlagManager.mark_all_as_stale() @@ -513,13 +512,14 @@ def _run_ipopt(self, results, config, nl_info, basename, timer): results.extra_info.add( 'command_line', ConfigValue(cmd, visibility=ADVANCED_OPTION) ) - # this seems silly, but we have to give the subprocess slightly - # longer to finish than ipopt - if config.time_limit is not None: - timeout = config.time_limit + min(max(1.0, 0.01 * config.time_limit), 100) - else: - timeout = None + # This seems silly, but we have to give the subprocess slightly + # longer to finish than ipopt, otherwise we may kill the + # subprocess before ipopt has a chance to write the SOL file. + # We will add 1% (with a min of 1 second and max of 100 seconds). + timeout = config.time_limit + if timeout is not None: + timeout = timeout + min(max(1.0, 0.01 * timeout), 100.0) # Call ipopt - passing the files via the subprocess ostreams = [io.StringIO()] + config.tee From 8b6ce80d1bf87172a57f001c40d8e7dff84351fa Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 11:51:59 -0700 Subject: [PATCH 25/41] Make nlwriter instance ephemeral; reduce writer config copies --- pyomo/contrib/solver/solvers/ipopt.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index f9509091a33..af6c2c10039 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -246,7 +246,6 @@ class Ipopt(SolverBase): def __init__(self, **kwds: Any) -> None: super().__init__(**kwds) - self._writer = NLWriter() #: Instance configuration; #: see :ref:`pyomo.contrib.solver.solvers.ipopt.Ipopt::CONFIG`. @@ -379,13 +378,17 @@ def solve(self, model, **kwds) -> Results: open(basename + '.col', 'w', encoding='utf-8') as col_file, ): timer.start('write_nl_file') - self._writer.config.set_value(config.writer_config) try: - nl_info = self._writer.write( + # Note: this is mapping the top-level + # symbolic_solver_labels onto the solver's writer + # config, and then that config is being used (in + # it's entirety) to set the NLWriter's CONFIG. + nl_info = NLWriter().write( model, nl_file, row_file, col_file, + config=config.writer_config, symbolic_solver_labels=config.symbolic_solver_labels, ) proven_infeasible = False From 225e7726cf1d46567097c232548b7e97dc6c4fd2 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 21:03:00 -0700 Subject: [PATCH 26/41] bugfix: local var name --- pyomo/contrib/solver/solvers/ipopt.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index af6c2c10039..4e6f9be936a 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -261,19 +261,19 @@ def available(self) -> Availability: def version(self) -> tuple[int, int, int] | None: return self._get_version(self.config.executable.path()) - def _get_version(self, pth): + def _get_version(self, exe): try: - return self._exe_cache[pth] + return self._exe_cache[exe] except KeyError: pass - if pth is None: + if exe is None: # No executable (either we couldn't find a matching file, or # the file is not executable) self._exe_cache[None] = None return None # Run the executable and look for the version results = subprocess.run( - [str(pth), '--version'], + [str(exe), '--version'], timeout=self._version_timeout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -298,7 +298,7 @@ def _get_version(self, pth): logger.warning( f"Failed parsing Ipopt version: '{exe} --version':\n\n{results.stdout}" ) - self._exe_cache[pth] = ver + self._exe_cache[exe] = ver return ver def has_linear_solver(self, linear_solver: str) -> bool: From 19fe4a937212e9bfbda8fa451527010e7c7272e0 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 21:03:25 -0700 Subject: [PATCH 27/41] Move options processing to one location --- pyomo/contrib/solver/solvers/ipopt.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 4e6f9be936a..6727a2eb1d2 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -329,13 +329,7 @@ def solve(self, model, **kwds) -> Results: # Update configuration options, based on keywords passed to solve config: IpoptConfig = self.config(value=kwds, preserve_implicit=True) - # Check if solver is available - if config.threads: - logger.log( - logging.WARNING, - msg="The `threads` option was specified, " - f"but this is not used by {self.__class__}.", - ) + timer = config.timer if timer is None: timer = config.timer = HierarchicalTimer() @@ -507,6 +501,12 @@ def _run_ipopt(self, results, config, nl_info, basename, timer): options = config.solver_options.value() # Map standard Pyomo solver options to Ipopt options: standard # options override ipopt-specific options. + if config.threads and config.threads != 1: + logger.log( + logging.WARNING, + msg=f"The `threads={config.threads}` option was specified, " + f"but this is not used by {self.__class__.__name__}.", + ) if config.time_limit is not None: options['max_cpu_time'] = config.time_limit cmd.extend(self._process_options(basename + '.opt', options)) From bad7a1abf8b94f97d008e39edfe57347ac672ba4 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 21:03:51 -0700 Subject: [PATCH 28/41] There shouldn't be a solution loader if there is no solution --- pyomo/contrib/solver/solvers/ipopt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 6727a2eb1d2..4e28b28e82c 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -401,13 +401,13 @@ def solve(self, model, **kwds) -> Results: TerminationCondition.convergenceCriteriaSatisfied ) results.solution_status = SolutionStatus.optimal + results.solution_loader = IpoptSolutionLoader( + sol_data=SolFileData(), nl_info=nl_info + ) else: results.termination_condition = TerminationCondition.emptyModel results.solution_status = SolutionStatus.noSolution results.extra_info.iteration_count = 0 - results.solution_loader = IpoptSolutionLoader( - sol_data=SolFileData(), nl_info=nl_info - ) else: self._run_ipopt(results, config, nl_info, basename, timer) From 3e182b8e633a64f4533ec31a61784060ab4c0213 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 22:26:52 -0700 Subject: [PATCH 29/41] Expand ipopt interface testing --- .../solver/tests/solvers/test_ipopt.py | 1730 +++++++++++++++-- 1 file changed, 1537 insertions(+), 193 deletions(-) diff --git a/pyomo/contrib/solver/tests/solvers/test_ipopt.py b/pyomo/contrib/solver/tests/solvers/test_ipopt.py index 126c0ac60f4..e8531773dd6 100644 --- a/pyomo/contrib/solver/tests/solvers/test_ipopt.py +++ b/pyomo/contrib/solver/tests/solvers/test_ipopt.py @@ -9,23 +9,31 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -import os, sys +import datetime +import os +import stat import subprocess +import sys +import time +import threading from contextlib import contextmanager import pyomo.environ as pyo from pyomo.common.envvar import is_windows from pyomo.common.fileutils import ExecutableData from pyomo.common.config import ConfigDict, ADVANCED_OPTION -from pyomo.common.errors import MouseTrap +from pyomo.common.errors import ApplicationError, MouseTrap +from pyomo.common.log import LoggingIntercept from pyomo.common.tee import capture_output +from pyomo.common.timing import HierarchicalTimer import pyomo.contrib.solver.solvers.ipopt as ipopt -from pyomo.contrib.solver.common.util import NoSolutionError +from pyomo.contrib.solver.common.util import NoSolutionError, NoOptimalSolutionError from pyomo.contrib.solver.common.results import TerminationCondition, SolutionStatus from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.common import unittest, Executable from pyomo.common.tempfiles import TempfileManager from pyomo.repn.plugins.nl_writer import NLWriter, NLWriterInfo +from pyomo.repn.util import FileDeterminism ipopt_available = ipopt.Ipopt().available() @@ -47,7 +55,6 @@ def windows_tee_buffer(size=1 << 20): tee._pipe_buffersize = old -@unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") class TestIpoptSolverConfig(unittest.TestCase): def test_default_instantiation(self): config = ipopt.IpoptConfig() @@ -72,8 +79,10 @@ def test_custom_instantiation(self): self.assertEqual(config._description, "A description") self.assertIsNone(config.time_limit) # Default should be `ipopt` - self.assertIsNotNone(str(config.executable)) - self.assertIn('ipopt', str(config.executable)) + self.assertEqual('ipopt', config.executable._registered_name) + if ipopt_available: + self.assertIsNotNone(config.executable.path()) + self.assertIn('ipopt', str(config.executable)) # Set to a totally bogus path config.executable = Executable('/bogus/path') self.assertIsNone(config.executable.executable) @@ -90,12 +99,22 @@ def test_get_reduced_costs_error(self): ): loader.get_reduced_costs() + def test_get_duals_error(self): + loader = ipopt.IpoptSolutionLoader( + ipopt.SolFileData(), NLWriterInfo(eliminated_vars=[1]) + ) + with self.assertRaisesRegex(MouseTrap, "Complete duals are not available"): + loader.get_duals() + -@unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") class TestIpoptInterface(unittest.TestCase): + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") def test_command_line_options(self): result = subprocess.run( - ['ipopt', '-='], capture_output=True, text=True, check=True + [str(ipopt.Ipopt.CONFIG.executable), '-='], + capture_output=True, + text=True, + check=True, ) output = result.stdout options = [] @@ -118,48 +137,138 @@ def test_class_member_list(self): 'version', 'name', ] - method_list = [method for method in dir(opt) if method.startswith('_') is False] + method_list = [method for method in dir(opt) if not method.startswith('_')] self.assertEqual(sorted(expected_list), sorted(method_list)) def test_default_instantiation(self): opt = ipopt.Ipopt() self.assertFalse(opt.is_persistent()) - self.assertIsNotNone(opt.version()) self.assertEqual(opt.name, 'ipopt') self.assertEqual(opt.CONFIG, opt.config) - self.assertTrue(opt.available()) + if ipopt_available: + self.assertIsNotNone(opt.version()) + self.assertTrue(opt.available()) + else: + self.assertIsNone(opt.version()) + self.assertFalse(opt.available()) def test_context_manager(self): with ipopt.Ipopt() as opt: self.assertFalse(opt.is_persistent()) - self.assertIsNotNone(opt.version()) self.assertEqual(opt.name, 'ipopt') self.assertEqual(opt.CONFIG, opt.config) - self.assertTrue(opt.available()) - - def test_available_cache(self): - opt = ipopt.Ipopt() - opt.available() - self.assertTrue(opt._available_cache[1]) - self.assertIsNotNone(opt._available_cache[0]) - # Now we will try with a custom config that has a fake path - config = ipopt.IpoptConfig() - config.executable = Executable('/a/bogus/path') - opt.available(config=config) - self.assertFalse(opt._available_cache[1]) - self.assertIsNone(opt._available_cache[0]) + if ipopt_available: + self.assertIsNotNone(opt.version()) + self.assertTrue(opt.available()) + else: + self.assertIsNone(opt.version()) + self.assertFalse(opt.available()) + + def test_get_version(self): + if ipopt_available: + ver = ipopt.Ipopt().version() + self.assertIsInstance(ver, tuple) + self.assertEqual(len(ver), 3) + self.assertTrue(all(isinstance(_, int) for _ in ver)) + + _cache = ipopt.Ipopt._exe_cache + try: + with TempfileManager.new_context() as TMP: + dname = TMP.mkdtemp() + + ipopt.Ipopt._exe_cache = {} + fname = os.path.join(dname, 'test1') + solver = ipopt.Ipopt(executable=fname) + self.assertEqual({}, ipopt.Ipopt._exe_cache) + self.assertEqual(ipopt.Availability.NotFound, solver.available()) + self.assertIsNone(solver.version()) + self.assertEqual({None: None}, ipopt.Ipopt._exe_cache) + + ipopt.Ipopt._exe_cache = {} + fname = os.path.join(dname, 'test2') + with open(fname, 'w') as F: + F.write(f"#!{sys.executable}\nimport sys\nsys.exit(0)\n") + solver = ipopt.Ipopt(executable=fname) + self.assertEqual({}, ipopt.Ipopt._exe_cache) + self.assertEqual(ipopt.Availability.NotFound, solver.available()) + self.assertIsNone(solver.version()) + self.assertEqual({None: None}, ipopt.Ipopt._exe_cache) + + # the rest of this test is designed to work on *nix: + if sys.platform.startswith("win"): + return + + # Found an executable, but --version errors + ipopt.Ipopt._exe_cache = {} + fname = os.path.join(dname, 'test3') + with open(fname, 'w') as F: + F.write(f"#!{sys.executable}\nimport sys\nsys.exit(1)\n") + os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + solver = ipopt.Ipopt(executable=fname) + self.assertEqual({}, ipopt.Ipopt._exe_cache) + self.assertEqual(ipopt.Availability.NotFound, solver.available()) + self.assertIsNone(solver.version()) + self.assertEqual({fname: None}, ipopt.Ipopt._exe_cache) + + # Found an executable, but --version doesn't return anything + ipopt.Ipopt._exe_cache = {} + fname = os.path.join(dname, 'test4') + with open(fname, 'w') as F: + F.write(f"#!{sys.executable}\nimport sys\nsys.exit(0)\n") + os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + solver = ipopt.Ipopt(executable=fname) + self.assertEqual({}, ipopt.Ipopt._exe_cache) + self.assertEqual(ipopt.Availability.NotFound, solver.available()) + self.assertIsNone(solver.version()) + self.assertEqual({fname: None}, ipopt.Ipopt._exe_cache) + + # Missing "ipopt" + ipopt.Ipopt._exe_cache = {} + fname = os.path.join(dname, 'test5') + with open(fname, 'w') as F: + F.write( + f"#!{sys.executable}\nprint('cbc 1.2.3 ASL')\n" + "import sys\nsys.exit(0)\n" + ) + os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + solver = ipopt.Ipopt(executable=fname) + self.assertEqual({}, ipopt.Ipopt._exe_cache) + self.assertEqual(ipopt.Availability.NotFound, solver.available()) + self.assertIsNone(solver.version()) + self.assertEqual({fname: None}, ipopt.Ipopt._exe_cache) + + # The version doesn't parse correctly + ipopt.Ipopt._exe_cache = {} + fname = os.path.join(dname, 'test6') + with open(fname, 'w') as F: + F.write( + f"#!{sys.executable}\nprint('Ipopt 1.2.3a ASL')\n" + "import sys\nsys.exit(0)\n" + ) + os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + solver = ipopt.Ipopt(executable=fname) + self.assertEqual({}, ipopt.Ipopt._exe_cache) + self.assertEqual(ipopt.Availability.NotFound, solver.available()) + self.assertIsNone(solver.version()) + self.assertEqual({fname: None}, ipopt.Ipopt._exe_cache) + + # This looks like an Ipopt solver... + ipopt.Ipopt._exe_cache = {} + fname = os.path.join(dname, 'test7') + with open(fname, 'w') as F: + F.write( + f"#!{sys.executable}\nprint('Ipopt 1.2.3 ASL')\n" + "import sys\nsys.exit(0)\n" + ) + os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + solver = ipopt.Ipopt(executable=fname) + self.assertEqual({}, ipopt.Ipopt._exe_cache) + self.assertEqual(ipopt.Availability.FullLicense, solver.available()) + self.assertEqual(solver.version(), (1, 2, 3)) + self.assertEqual({fname: (1, 2, 3)}, ipopt.Ipopt._exe_cache) - def test_version_cache(self): - opt = ipopt.Ipopt() - opt.version() - self.assertIsNotNone(opt._version_cache[0]) - self.assertIsNotNone(opt._version_cache[1]) - # Now we will try with a custom config that has a fake path - config = ipopt.IpoptConfig() - config.executable = Executable('/a/bogus/path') - opt.version(config=config) - self.assertIsNone(opt._version_cache[0]) - self.assertIsNone(opt._version_cache[1]) + finally: + ipopt.Ipopt._exe_cache = _cache def test_parse_output(self): # Old ipopt style (<=3.13) @@ -240,12 +349,197 @@ def test_parse_output(self): """ parsed_output = ipopt.Ipopt()._parse_ipopt_output(output) - self.assertEqual(parsed_output["iters"], 11) - self.assertEqual(len(parsed_output["iteration_log"]), 12) - self.assertEqual(parsed_output["incumbent_objective"], 7.0136459513364959e-25) - self.assertIn("final_scaled_results", parsed_output.keys()) - self.assertIn( - 'IPOPT (w/o function evaluations)', parsed_output['cpu_seconds'].keys() + self.assertEqual( + { + 'iters': 11, + 'iteration_log': [ + { + 'iter': 0, + 'objective': 56.5, + 'inf_pr': 0.0, + 'inf_du': 100.0, + 'lg_mu': -1.0, + 'd_norm': 0.0, + 'lg_rg': None, + 'alpha_du': 0.0, + 'alpha_pr': 0.0, + 'ls': 0, + 'restoration': False, + 'step_acceptance': None, + }, + { + 'iter': 1, + 'objective': 0.24669972, + 'inf_pr': 0.0, + 'inf_du': 0.222, + 'lg_mu': -1.0, + 'd_norm': 0.74, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 2, + 'objective': 0.16256267, + 'inf_pr': 0.0, + 'inf_du': 2.04, + 'lg_mu': -1.7, + 'd_norm': 1.48, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 0.25, + 'ls': 3, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 3, + 'objective': 0.086119444, + 'inf_pr': 0.0, + 'inf_du': 1.08, + 'lg_mu': -1.7, + 'd_norm': 0.236, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 4, + 'objective': 0.043223836, + 'inf_pr': 0.0, + 'inf_du': 1.23, + 'lg_mu': -1.7, + 'd_norm': 0.261, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 5, + 'objective': 0.015610508, + 'inf_pr': 0.0, + 'inf_du': 0.354, + 'lg_mu': -1.7, + 'd_norm': 0.118, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 6, + 'objective': 0.0053544798, + 'inf_pr': 0.0, + 'inf_du': 0.551, + 'lg_mu': -1.7, + 'd_norm': 0.167, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 7, + 'objective': 0.00061281576, + 'inf_pr': 0.0, + 'inf_du': 0.0519, + 'lg_mu': -1.7, + 'd_norm': 0.0387, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 8, + 'objective': 2.8893076e-05, + 'inf_pr': 0.0, + 'inf_du': 0.0452, + 'lg_mu': -2.5, + 'd_norm': 0.0453, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 9, + 'objective': 3.4591761e-08, + 'inf_pr': 0.0, + 'inf_du': 0.00038, + 'lg_mu': -2.5, + 'd_norm': 0.00318, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 10, + 'objective': 1.2680803e-13, + 'inf_pr': 0.0, + 'inf_du': 3.02e-06, + 'lg_mu': -5.7, + 'd_norm': 0.000362, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 11, + 'objective': 7.013646e-25, + 'inf_pr': 0.0, + 'inf_du': 1.72e-12, + 'lg_mu': -8.6, + 'd_norm': 2.13e-07, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + ], + 'incumbent_objective': 7.013645951336496e-25, + 'dual_infeasibility': 7.775113886059942e-12, + 'constraint_violation': 0.0, + 'complementarity_error': 0.0, + 'overall_nlp_error': 7.775113886059942e-12, + 'final_scaled_results': { + 'incumbent_objective': 1.5551321399859192e-25, + 'dual_infeasibility': 1.7239720368203862e-12, + 'constraint_violation': 0.0, + 'complementarity_error': 0.0, + 'overall_nlp_error': 1.7239720368203862e-12, + }, + 'cpu_seconds': { + 'IPOPT (w/o function evaluations)': 0.0, + 'NLP function evaluations': 0.0, + }, + }, + parsed_output, ) # New ipopt style (3.14+) @@ -311,11 +605,197 @@ def test_parse_output(self): Ipopt 3.14.17: Optimal Solution Found """ parsed_output = ipopt.Ipopt()._parse_ipopt_output(output) - self.assertEqual(parsed_output["iters"], 11) - self.assertEqual(len(parsed_output["iteration_log"]), 12) - self.assertEqual(parsed_output["incumbent_objective"], 7.0136459513364959e-25) - self.assertIn("final_scaled_results", parsed_output.keys()) - self.assertIn('IPOPT', parsed_output['cpu_seconds'].keys()) + self.assertEqual( + { + 'iters': 11, + 'iteration_log': [ + { + 'iter': 0, + 'objective': 56.5, + 'inf_pr': 0.0, + 'inf_du': 100.0, + 'lg_mu': -1.0, + 'd_norm': 0.0, + 'lg_rg': None, + 'alpha_du': 0.0, + 'alpha_pr': 0.0, + 'ls': 0, + 'restoration': False, + 'step_acceptance': None, + }, + { + 'iter': 1, + 'objective': 0.24669972, + 'inf_pr': 0.0, + 'inf_du': 0.222, + 'lg_mu': -1.0, + 'd_norm': 0.74, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 2, + 'objective': 0.16256267, + 'inf_pr': 0.0, + 'inf_du': 2.04, + 'lg_mu': -1.7, + 'd_norm': 1.48, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 0.25, + 'ls': 3, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 3, + 'objective': 0.086119444, + 'inf_pr': 0.0, + 'inf_du': 1.08, + 'lg_mu': -1.7, + 'd_norm': 0.236, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 4, + 'objective': 0.043223836, + 'inf_pr': 0.0, + 'inf_du': 1.23, + 'lg_mu': -1.7, + 'd_norm': 0.261, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 5, + 'objective': 0.015610508, + 'inf_pr': 0.0, + 'inf_du': 0.354, + 'lg_mu': -1.7, + 'd_norm': 0.118, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 6, + 'objective': 0.0053544798, + 'inf_pr': 0.0, + 'inf_du': 0.551, + 'lg_mu': -1.7, + 'd_norm': 0.167, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 7, + 'objective': 0.00061281576, + 'inf_pr': 0.0, + 'inf_du': 0.0519, + 'lg_mu': -1.7, + 'd_norm': 0.0387, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 8, + 'objective': 2.8893076e-05, + 'inf_pr': 0.0, + 'inf_du': 0.0452, + 'lg_mu': -2.5, + 'd_norm': 0.0453, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 9, + 'objective': 3.4591761e-08, + 'inf_pr': 0.0, + 'inf_du': 0.00038, + 'lg_mu': -2.5, + 'd_norm': 0.00318, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 10, + 'objective': 1.2680803e-13, + 'inf_pr': 0.0, + 'inf_du': 3.02e-06, + 'lg_mu': -5.7, + 'd_norm': 0.000362, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 11, + 'objective': 7.013646e-25, + 'inf_pr': 0.0, + 'inf_du': 1.72e-12, + 'lg_mu': -8.6, + 'd_norm': 2.13e-07, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + ], + 'incumbent_objective': 7.013645951336496e-25, + 'dual_infeasibility': 7.775113886059942e-12, + 'constraint_violation': 0.0, + 'variable_bound_violation': 0.0, + 'complementarity_error': 0.0, + 'overall_nlp_error': 7.775113886059942e-12, + 'final_scaled_results': { + 'incumbent_objective': 1.5551321399859192e-25, + 'dual_infeasibility': 1.7239720368203862e-12, + 'constraint_violation': 0.0, + 'variable_bound_violation': 0.0, + 'complementarity_error': 0.0, + 'overall_nlp_error': 1.7239720368203862e-12, + }, + 'cpu_seconds': {'IPOPT': 0.002}, + }, + parsed_output, + ) def test_empty_output_parsing(self): with self.assertLogs( @@ -411,75 +891,472 @@ def test_parse_output_diagnostic_tags(self): EXIT: Optimal Solution Found. """ parsed_output = ipopt.Ipopt()._parse_ipopt_output(output) - self.assertEqual(parsed_output["iters"], 20) - self.assertEqual(len(parsed_output["iteration_log"]), 21) - self.assertEqual(parsed_output["incumbent_objective"], 3.2274635418964841e01) - self.assertEqual(parsed_output["iteration_log"][3]["diagnostic_tags"], 'Nhj') - self.assertIn("final_scaled_results", parsed_output.keys()) - self.assertIn( - 'IPOPT (w/o function evaluations)', parsed_output['cpu_seconds'].keys() + self.assertEqual( + { + 'iters': 20, + 'iteration_log': [ + { + 'iter': 0, + 'objective': 4.3126674, + 'inf_pr': 1.34, + 'inf_du': 1.0, + 'lg_mu': -5.0, + 'd_norm': 0.0, + 'lg_rg': None, + 'alpha_du': 0.0, + 'alpha_pr': 0.0, + 'ls': 0, + 'restoration': False, + 'step_acceptance': None, + }, + { + 'iter': 1, + 'objective': 4.3126674, + 'inf_pr': 1.34, + 'inf_du': 999.0, + 'lg_mu': 0.1, + 'd_norm': 0.0, + 'lg_rg': -4.0, + 'alpha_du': 0.0, + 'alpha_pr': 3.29e-10, + 'ls': 2, + 'restoration': True, + 'step_acceptance': 'R', + }, + { + 'iter': 2, + 'objective': 305192460.0, + 'inf_pr': 1.13, + 'inf_du': 990.0, + 'lg_mu': 0.1, + 'd_norm': 230.0, + 'lg_rg': None, + 'alpha_du': 0.026, + 'alpha_pr': 0.00932, + 'ls': 1, + 'restoration': True, + 'step_acceptance': 'f', + }, + { + 'iter': 3, + 'objective': 2271259500.0, + 'inf_pr': 1.69, + 'inf_du': 973.0, + 'lg_mu': 0.1, + 'd_norm': 223.0, + 'lg_rg': None, + 'alpha_du': 0.0254, + 'alpha_pr': 0.0171, + 'ls': 1, + 'restoration': True, + 'step_acceptance': 'f', + 'diagnostic_tags': 'Nhj', + }, + { + 'iter': 4, + 'objective': 2271206500.0, + 'inf_pr': 1.69, + 'inf_du': 1370000000.0, + 'lg_mu': -5.0, + 'd_norm': 3080.0, + 'lg_rg': None, + 'alpha_du': 1.32e-05, + 'alpha_pr': 1.17e-05, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + 'diagnostic_tags': 'q', + }, + { + 'iter': 5, + 'objective': 1906298600.0, + 'inf_pr': 1.55, + 'inf_du': 1250000000.0, + 'lg_mu': -5.0, + 'd_norm': 5130.0, + 'lg_rg': None, + 'alpha_du': 0.119, + 'alpha_pr': 0.0838, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 6, + 'objective': 1704159400.0, + 'inf_pr': 1.46, + 'inf_du': 1180000000.0, + 'lg_mu': -5.0, + 'd_norm': 5660.0, + 'lg_rg': None, + 'alpha_du': 0.0706, + 'alpha_pr': 0.0545, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 7, + 'objective': 1476315800.0, + 'inf_pr': 1.36, + 'inf_du': 1100000000.0, + 'lg_mu': -5.0, + 'd_norm': 3940.0, + 'lg_rg': None, + 'alpha_du': 0.23, + 'alpha_pr': 0.0692, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 8, + 'objective': 858731080.0, + 'inf_pr': 1.04, + 'inf_du': 841000000.0, + 'lg_mu': -5.0, + 'd_norm': 238000.0, + 'lg_rg': None, + 'alpha_du': 3.49e-06, + 'alpha_pr': 0.237, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 9, + 'objective': 442155720.0, + 'inf_pr': 0.745, + 'inf_du': 603000000.0, + 'lg_mu': -5.0, + 'd_norm': 1630000.0, + 'lg_rg': None, + 'alpha_du': 0.0797, + 'alpha_pr': 0.282, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 10, + 'objective': 50.251884, + 'inf_pr': 0.165, + 'inf_du': 15700.0, + 'lg_mu': -5.0, + 'd_norm': 1240000.0, + 'lg_rg': None, + 'alpha_du': 3.92e-05, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 11, + 'objective': 49.121733, + 'inf_pr': 0.0497, + 'inf_du': 4680.0, + 'lg_mu': -5.0, + 'd_norm': 81100.0, + 'lg_rg': None, + 'alpha_du': 0.0431, + 'alpha_pr': 0.701, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'h', + }, + { + 'iter': 12, + 'objective': 41.483985, + 'inf_pr': 0.0224, + 'inf_du': 5970.0, + 'lg_mu': -5.0, + 'd_norm': 1150000.0, + 'lg_rg': None, + 'alpha_du': 0.0593, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 13, + 'objective': 35.762585, + 'inf_pr': 0.0175, + 'inf_du': 5000.0, + 'lg_mu': -5.0, + 'd_norm': 1030000.0, + 'lg_rg': None, + 'alpha_du': 0.125, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 14, + 'objective': 32.291014, + 'inf_pr': 0.0108, + 'inf_du': 3510.0, + 'lg_mu': -5.0, + 'd_norm': 825000.0, + 'lg_rg': None, + 'alpha_du': 0.668, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 15, + 'objective': 32.27463, + 'inf_pr': 3.31e-05, + 'inf_du': 1.17, + 'lg_mu': -5.0, + 'd_norm': 42600.0, + 'lg_rg': None, + 'alpha_du': 0.992, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'h', + }, + { + 'iter': 16, + 'objective': 32.274631, + 'inf_pr': 7.45e-09, + 'inf_du': 0.00271, + 'lg_mu': -5.0, + 'd_norm': 611.0, + 'lg_rg': None, + 'alpha_du': 0.897, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'h', + }, + { + 'iter': 17, + 'objective': 32.274635, + 'inf_pr': 7.45e-09, + 'inf_du': 0.00235, + 'lg_mu': -5.0, + 'd_norm': 27100.0, + 'lg_rg': None, + 'alpha_du': 0.132, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 18, + 'objective': 32.274635, + 'inf_pr': 7.45e-09, + 'inf_du': 0.000115, + 'lg_mu': -5.0, + 'd_norm': 5530.0, + 'lg_rg': None, + 'alpha_du': 0.951, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'h', + }, + { + 'iter': 19, + 'objective': 32.274635, + 'inf_pr': 7.45e-09, + 'inf_du': 2.84e-05, + 'lg_mu': -5.0, + 'd_norm': 44100.0, + 'lg_rg': None, + 'alpha_du': 0.754, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 20, + 'objective': 32.274635, + 'inf_pr': 7.45e-09, + 'inf_du': 8.54e-07, + 'lg_mu': -5.0, + 'd_norm': 18300.0, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'h', + }, + ], + 'incumbent_objective': 32.27463541896484, + 'dual_infeasibility': 8.536507867832867e-07, + 'constraint_violation': 7.450580596923828e-09, + 'complementarity_error': 1.227590456641416e-05, + 'overall_nlp_error': 1.227590456641416e-05, + 'final_scaled_results': { + 'incumbent_objective': 32.27463541896484, + 'dual_infeasibility': 8.536507867832867e-07, + 'constraint_violation': 8.078062506860793e-13, + 'complementarity_error': 1.227590456641416e-05, + 'overall_nlp_error': 1.227590456641416e-05, + }, + 'cpu_seconds': { + 'IPOPT (w/o function evaluations)': 10.45, + 'NLP function evaluations': 1.651, + }, + }, + parsed_output, ) - def test_verify_ipopt_options(self): - opt = ipopt.Ipopt(solver_options={'max_iter': 4}) - opt._verify_ipopt_options(opt.config) - self.assertEqual(opt.config.solver_options.value(), {'max_iter': 4}) + def test_parse_output_errors(self): + output = """****************************************************************************** +****************************************************************************** + +This is Ipopt version 3.13.2, running with linear solver ma57. + +Number of nonzeros in equality constraint Jacobian...: 77541 +Number of nonzeros in inequality constraint Jacobian.: 0 +Number of nonzeros in Lagrangian Hessian.............: 51855 - opt = ipopt.Ipopt(solver_options={'max_iter': 4}, time_limit=10) - opt._verify_ipopt_options(opt.config) +Total number of variables............................: 15468 + variables with only lower bounds: 3491 + variables with lower and upper bounds: 5026 + variables with only upper bounds: 186 +Total number of equality constraints.................: 15417 +Total number of inequality constraints...............: 0 + inequality constraints with only lower bounds: 0 + inequality constraints with lower and upper bounds: 0 + inequality constraints with only upper bounds: 0 + +iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls + 0 4.3126674e+00 1.34e+00 1.00e+00 -5.0 0.00e+00 - 0.00e+00 0.00e+00 0 +Reallocating memory for MA57: lfact (2247250) + 1r-4.3126674e+00 1.34e+00 9.99e+02 0.1 0.00e+00 -4.0 0.00e+00 3.29e-10R 2 + 19t 3.2274635e+01 7.45e-09 2.84e-05 -5.0 4.41e+04 - 7.54e-01 1.00e+00f 1 +iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls + 20 3.2274635e+01f 7.45e-09 8.54e-07 -5.0 1.83e+04 - 1.00e+00 1.00e+00h 1 + +Number of Iterations....: 20 + + (scaled) (unscaled) +Objective...............: 3.2274635418964841e+01 3.2274635418964841e+01 +Dual infeasibility......: 8.5365078678328669e-07 8.5365078678328669e-07 +Constraint violation....: 8.0780625068607930e-13 7.4505805969238281e-09 +Complementarity.........: 1.2275904566414160e-05 1.2275904566414160e-05 +Overall NLP error.......: 1.2275904566414160e-05 1.2275904566414160e-05 + + +Number of objective function evaluations = 23 +Number of objective gradient evaluations = 20 +Number of equality constraint evaluations = 23 +Number of inequality constraint evaluations = 0 +Number of equality constraint Jacobian evaluations = 22 +Number of inequality constraint Jacobian evaluations = 0 +Number of Lagrangian Hessian evaluations = 20 +Total CPU secs in IPOPT (w/o function evaluations) = 10.450 +Total CPU secs in NLP function evaluations = 1.651 + +EXIT: Optimal Solution Found. + """ + with LoggingIntercept() as LOG: + parsed_output = ipopt.Ipopt()._parse_ipopt_output(output) self.assertEqual( - opt.config.solver_options.value(), {'max_iter': 4, 'max_cpu_time': 10} + """Error parsing Ipopt log entry: +\tinvalid literal for int() with base 10: '19t' +\t 19t 3.2274635e+01 7.45e-09 2.84e-05 -5.0 4.41e+04 - 7.54e-01 1.00e+00f 1 +Error parsing Ipopt log entry: +\tcould not convert string to float: '3.2274635e+01f' +\t 20 3.2274635e+01f 7.45e-09 8.54e-07 -5.0 1.83e+04 - 1.00e+00 1.00e+00h 1 +Total number of iteration records parsed 4 does not match the number of iterations (20) plus one. +""", + LOG.getvalue(), ) - - # Finally, let's make sure it errors if someone tries to pass option_file_name - opt = ipopt.Ipopt( - solver_options={'max_iter': 4, 'option_file_name': 'myfile.opt'} + self.assertEqual( + { + 'iters': 20, + 'iteration_log': [ + { + 'iter': 0, + 'objective': 4.3126674, + 'inf_pr': 1.34, + 'inf_du': 1.0, + 'lg_mu': -5.0, + 'd_norm': 0.0, + 'lg_rg': None, + 'alpha_du': 0.0, + 'alpha_pr': 0.0, + 'ls': 0, + 'restoration': False, + 'step_acceptance': None, + }, + { + 'iter': 1, + 'objective': -4.3126674, + 'inf_pr': 1.34, + 'inf_du': 999.0, + 'lg_mu': 0.1, + 'd_norm': 0.0, + 'lg_rg': -4.0, + 'alpha_du': 0.0, + 'alpha_pr': 3.29e-10, + 'ls': 2, + 'restoration': True, + 'step_acceptance': 'R', + }, + { + 'iter': '19t', + 'objective': 32.274635, + 'inf_pr': 7.45e-09, + 'inf_du': 2.84e-05, + 'lg_mu': -5.0, + 'd_norm': 44100.0, + 'lg_rg': None, + 'alpha_du': 0.754, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'f', + }, + { + 'iter': 20, + 'objective': '3.2274635e+01f', + 'inf_pr': 7.45e-09, + 'inf_du': 8.54e-07, + 'lg_mu': -5.0, + 'd_norm': 18300.0, + 'lg_rg': None, + 'alpha_du': 1.0, + 'alpha_pr': 1.0, + 'ls': 1, + 'restoration': False, + 'step_acceptance': 'h', + }, + ], + 'incumbent_objective': 32.27463541896484, + 'dual_infeasibility': 8.536507867832867e-07, + 'constraint_violation': 7.450580596923828e-09, + 'complementarity_error': 1.227590456641416e-05, + 'overall_nlp_error': 1.227590456641416e-05, + 'final_scaled_results': { + 'incumbent_objective': 32.27463541896484, + 'dual_infeasibility': 8.536507867832867e-07, + 'constraint_violation': 8.078062506860793e-13, + 'complementarity_error': 1.227590456641416e-05, + 'overall_nlp_error': 1.227590456641416e-05, + }, + 'cpu_seconds': { + 'IPOPT (w/o function evaluations)': 10.45, + 'NLP function evaluations': 1.651, + }, + }, + parsed_output, ) - with self.assertRaisesRegex( - ValueError, - r'Pyomo generates the ipopt options file as part of the `solve` ' - r'method. Add all options to ipopt.config.solver_options instead', - ): - opt._verify_ipopt_options(opt.config) - - def test_write_options_file(self): - # If we have no options, nothing should happen (and no options - # file should be added tot he set of options) - opt = ipopt.Ipopt() - opt._write_options_file('fakename', opt.config.solver_options) - self.assertEqual(opt.config.solver_options.value(), {}) - # Pass it some options that ARE on the command line - opt = ipopt.Ipopt(solver_options={'max_iter': 4}) - opt._write_options_file('myfile', opt.config.solver_options) - self.assertNotIn('option_file_name', opt.config.solver_options) - self.assertFalse(os.path.isfile('myfile.opt')) - # Now we are going to actually pass it some options that are NOT on - # the command line - opt = ipopt.Ipopt(solver_options={'custom_option': 4}) - with TempfileManager.new_context() as temp: - dname = temp.mkdtemp() - if not os.path.exists(dname): - os.mkdir(dname) - filename = os.path.join(dname, 'myfile.opt') - opt._write_options_file(filename, opt.config.solver_options) - self.assertIn('option_file_name', opt.config.solver_options) - self.assertTrue(os.path.isfile(filename)) - # Make sure all options are writing to the file - opt = ipopt.Ipopt(solver_options={'custom_option_1': 4, 'custom_option_2': 3}) - with TempfileManager.new_context() as temp: - dname = temp.mkdtemp() - if not os.path.exists(dname): - os.mkdir(dname) - filename = os.path.join(dname, 'myfile.opt') - opt._write_options_file(filename, opt.config.solver_options) - self.assertIn('option_file_name', opt.config.solver_options) - self.assertTrue(os.path.isfile(filename)) - with open(filename, 'r') as f: - data = f.readlines() - self.assertEqual( - len(data) + 1, len(list(opt.config.solver_options.keys())) - ) + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") def test_has_linear_solver(self): opt = ipopt.Ipopt() self.assertTrue( @@ -503,46 +1380,491 @@ def test_has_linear_solver(self): ) self.assertFalse(opt.has_linear_solver('bogus_linear_solver')) - def test_create_command_line(self): - opt = ipopt.Ipopt() - # No custom options, no file created. Plain and simple. - result = opt._create_command_line('myfile', opt.config) - self.assertEqual(result, [str(opt.config.executable), 'myfile.nl', '-AMPL']) - # Custom command line options - opt = ipopt.Ipopt(solver_options={'max_iter': 4}) - result = opt._create_command_line('myfile', opt.config) + @unittest.skipIf(sys.platform.startswith("win"), "Test requires *nix") + def test_command_line(self): + with TempfileManager.new_context() as tempfile: + dname = tempfile.mkdtemp() + exe = os.path.join(dname, 'mock') + with open(exe, 'w') as F: + F.write( + f"""#!{sys.executable} +import sys +if sys.argv[1] == '--version': + print('ipopt 1.2.3 ASL') +else: + print('\\n'.join(sys.argv[1:])) + sys.exit(1) +""" + ) + os.chmod(exe, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + opt = ipopt.Ipopt(executable=exe) + + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.o = pyo.Objective(expr=m.x**2) + + opts = dict( + raise_exception_on_nonoptimal_result=False, load_solutions=False + ) + # No custom options, no file created. Plain and simple. + with LoggingIntercept() as LOG: + result = opt.solve(m, **opts) + self.assertEqual("", LOG.getvalue()) + cmd = result.solver_log.splitlines() + self.assertTrue(cmd[0].endswith('nl')) + self.assertEqual(cmd[1:], ['-AMPL']) + + # Custom command line options + with LoggingIntercept() as LOG: + result = opt.solve(m, **opts, solver_options={'max_iter': 4}) + self.assertEqual("", LOG.getvalue()) + cmd = result.solver_log.splitlines() + self.assertTrue(cmd[0].endswith('nl')) + self.assertEqual(cmd[1:], ['-AMPL', 'max_iter=4']) + + # Custom command line options; threads generates a warning + with LoggingIntercept() as LOG: + result = opt.solve( + m, **opts, threads=10, solver_options={'max_iter': 4} + ) + self.assertEqual( + "The `threads=10` option was specified, " + "but this is not used by Ipopt.\n", + LOG.getvalue(), + ) + cmd = result.solver_log.splitlines() + self.assertTrue(cmd[0].endswith('nl')) + self.assertEqual(cmd[1:], ['-AMPL', 'max_iter=4']) + + # Custom command line options; threads generates a warning... unless it's 1 + with LoggingIntercept() as LOG: + result = opt.solve(m, **opts, threads=1, solver_options={'max_iter': 4}) + self.assertEqual("", LOG.getvalue()) + cmd = result.solver_log.splitlines() + self.assertTrue(cmd[0].endswith('nl')) + self.assertEqual(cmd[1:], ['-AMPL', 'max_iter=4']) + + # Let's see if we correctly parse config.time_limit + with LoggingIntercept() as LOG: + result = opt.solve( + m, **opts, time_limit=10, solver_options={'max_iter': 4} + ) + self.assertEqual("", LOG.getvalue()) + cmd = result.solver_log.splitlines() + self.assertTrue(cmd[0].endswith('nl')) + self.assertEqual(cmd[1:], ['-AMPL', 'max_iter=4', 'max_cpu_time=10.0']) + + # Now let's do multiple command line options + with LoggingIntercept() as LOG: + result = opt.solve( + m, **opts, solver_options={'max_iter': 4, 'max_cpu_time': 20} + ) + self.assertEqual("", LOG.getvalue()) + cmd = result.solver_log.splitlines() + self.assertTrue(cmd[0].endswith('nl')) + self.assertEqual(cmd[1:], ['-AMPL', 'max_cpu_time=20', 'max_iter=4']) + + # but top-level options override solver_options + with LoggingIntercept() as LOG: + result = opt.solve( + m, + **opts, + time_limit=10, + solver_options={'max_iter': 4, 'max_cpu_time': 20}, + ) + self.assertEqual("", LOG.getvalue()) + cmd = result.solver_log.splitlines() + self.assertTrue(cmd[0].endswith('nl')) + self.assertEqual(cmd[1:], ['-AMPL', 'max_cpu_time=10.0', 'max_iter=4']) + + def test_option_to_str(self): + # int / float / str + self.assertEqual('opt=5', ipopt._option_to_cmd('opt', 5)) + self.assertEqual('opt=5.0', ipopt._option_to_cmd('opt', 5.0)) + self.assertEqual('opt="5"', ipopt._option_to_cmd('opt', '5')) + + # If the string contains a quote, then the name needs to be + # quoted + self.assertEqual("opt=\"'model'\"", ipopt._option_to_cmd('opt', "'model'")) + self.assertEqual("opt='\"model\"'", ipopt._option_to_cmd('opt', '"model"')) + # but if it has both, we will error + with self.assertRaisesRegex(ValueError, 'single and double'): + ipopt._option_to_cmd('opt', '"\'model"') + + def test_process_options(self): + solver = ipopt.Ipopt() + with TempfileManager.new_context() as TMP: + dname = TMP.mkdtemp() + + # test no options + fname = os.path.join(dname, 'test1.txt') + cmd = solver._process_options(fname, {}) + self.assertFalse(os.path.exists(fname)) + self.assertEqual([], cmd) + + # command-line only options + fname = os.path.join(dname, 'test2.txt') + cmd = solver._process_options(fname, {'bound_push': 'no', 'max_iter': 5}) + self.assertFalse(os.path.exists(fname)) + self.assertEqual(['bound_push="no"', 'max_iter=5'], cmd) + + # both command line and options file + fname = os.path.join(dname, 'test3.txt') + cmd = solver._process_options( + fname, {'custom_option_2': 5, 'bound_push': 'no', 'custom_option_1': 3} + ) + self.assertTrue(os.path.exists(fname)) + with open(fname, 'r') as F: + self.assertEqual('custom_option_2 5\ncustom_option_1 3\n', F.read()) + if '"' in fname: + fname = "'" + fname + "'" + else: + fname = '"' + fname + '"' + self.assertEqual(['bound_push="no"', f'option_file_name={fname}'], cmd) + + # only options file + fname = os.path.join(dname, 'test4.txt') + cmd = solver._process_options(fname, {'custom_option_3': 3}) + self.assertTrue(os.path.exists(fname)) + with open(fname, 'r') as F: + self.assertEqual('custom_option_3 3\n', F.read()) + if '"' in fname: + fname = "'" + fname + "'" + else: + fname = '"' + fname + '"' + self.assertEqual([f'option_file_name={fname}'], cmd) + + # illegal options + fname = os.path.join(dname, 'test5.txt') + with self.assertRaisesRegex( + ValueError, + "unallowed Ipopt option 'wantsol': " + "The solver interface requires the sol file to be created", + ): + solver._process_options(fname, {'bogus': 3, 'wantsol': False}) + self.assertFalse(os.path.exists(fname)) + + fname = os.path.join(dname, 'test5.txt') + with self.assertRaisesRegex( + ValueError, + "unallowed Ipopt option 'option_file_name': " + 'Pyomo generates the ipopt options file as part of the `solve` ' + 'method. Add all options to config.solver_options instead', + ): + solver._process_options( + fname, {'bogus': 3, 'option_file_name': 'myfile.opt'} + ) + self.assertFalse(os.path.exists(fname)) + + def test_presolve_prove_infeasible(self): + m = pyo.ConcreteModel() + m.x = pyo.Var(bounds=(0, 5)) + m.c = pyo.Constraint(expr=m.x == 10) + m.obj = pyo.Objective(expr=m.x) + + timer = HierarchicalTimer() + solver = ipopt.Ipopt() + results = solver.solve( + m, + timer=timer, + load_solutions=False, + raise_exception_on_nonoptimal_result=False, + ) + self.assertEqual(results.solution_status, SolutionStatus.noSolution) self.assertEqual( - result, [str(opt.config.executable), 'myfile.nl', '-AMPL', 'max_iter=4'] + results.termination_condition, TerminationCondition.provenInfeasible ) - # Let's see if we correctly parse config.time_limit - opt = ipopt.Ipopt(solver_options={'max_iter': 4}, time_limit=10) - opt._verify_ipopt_options(opt.config) - result = opt._create_command_line('myfile', opt.config) + cfg = results.solver_config + del cfg.executable self.assertEqual( - result, - [ - str(opt.config.executable), - 'myfile.nl', - '-AMPL', - 'max_iter=4', - 'max_cpu_time=10.0', - ], + { + 'load_solutions': False, + 'raise_exception_on_nonoptimal_result': False, + 'solver_options': {}, + 'symbolic_solver_labels': False, + 'tee': [], + 'threads': None, + 'time_limit': None, + 'timer': timer, + 'working_dir': None, + 'writer_config': { + 'column_order': None, + 'export_defined_variables': True, + 'export_nonlinear_variables': None, + 'file_determinism': FileDeterminism.ORDERED, + 'linear_presolve': True, + 'row_order': None, + 'scale_model': True, + 'show_section_timing': False, + 'skip_trivial_constraints': True, + 'symbolic_solver_labels': False, + }, + }, + cfg.value(), + ) + self.assertLess(results.timing_info.wall_time, 0.1) + self.assertEqual(results.timing_info.start_timestamp.tzinfo, datetime.UTC) + self.assertLess( + ( + datetime.datetime.now(datetime.UTC) + - results.timing_info.start_timestamp + ).seconds, + 1, ) - # Now let's do multiple command line options - opt = ipopt.Ipopt(solver_options={'max_iter': 4, 'max_cpu_time': 10}) - opt._verify_ipopt_options(opt.config) - result = opt._create_command_line('myfile', opt.config) + del results.extra_info.base_file_name + del results.solver_config + del results.timing_info.wall_time + del results.timing_info.start_timestamp self.assertEqual( - result, - [ - str(opt.config.executable), - 'myfile.nl', - '-AMPL', - 'max_cpu_time=10', - 'max_iter=4', - ], + { + 'extra_info': {'iteration_count': 0}, + 'incumbent_objective': None, + 'objective_bound': None, + 'solution_loader': None, + 'solution_status': SolutionStatus.noSolution, + 'solver_log': None, + 'solver_name': 'ipopt', + 'solver_version': None, + 'termination_condition': TerminationCondition.provenInfeasible, + 'timing_info': {'timer': timer}, + }, + results.value(), ) + with self.assertRaisesRegex( + NoSolutionError, "Solution loader does not currently have a valid solution." + ): + results = solver.solve( + m, timer=timer, raise_exception_on_nonoptimal_result=False + ) + with self.assertRaisesRegex( + NoOptimalSolutionError, "Solver did not find the optimal solution." + ): + results = solver.solve(m, timer=timer) + + def test_presolve_solveModel(self): + m = pyo.ConcreteModel() + m.x = pyo.Var(bounds=(0, 50)) + m.c = pyo.Constraint(expr=m.x == 10) + m.obj = pyo.Objective(expr=m.x) + + timer = HierarchicalTimer() + solver = ipopt.Ipopt() + results = solver.solve(m, timer=timer) + self.assertEqual(results.solution_status, SolutionStatus.optimal) + self.assertEqual( + results.termination_condition, + TerminationCondition.convergenceCriteriaSatisfied, + ) + cfg = results.solver_config + del results.solver_config + del cfg.executable + self.assertEqual( + { + 'load_solutions': True, + 'raise_exception_on_nonoptimal_result': True, + 'solver_options': {}, + 'symbolic_solver_labels': False, + 'tee': [], + 'threads': None, + 'time_limit': None, + 'timer': timer, + 'working_dir': None, + 'writer_config': { + 'column_order': None, + 'export_defined_variables': True, + 'export_nonlinear_variables': None, + 'file_determinism': FileDeterminism.ORDERED, + 'linear_presolve': True, + 'row_order': None, + 'scale_model': True, + 'show_section_timing': False, + 'skip_trivial_constraints': True, + 'symbolic_solver_labels': False, + }, + }, + cfg.value(), + ) + self.assertLess(results.timing_info.wall_time, 0.1) + del results.timing_info.wall_time + self.assertEqual(results.timing_info.start_timestamp.tzinfo, datetime.UTC) + self.assertLess( + ( + datetime.datetime.now(datetime.UTC) + - results.timing_info.start_timestamp + ).seconds, + 1, + ) + del results.timing_info.start_timestamp + del results.extra_info.base_file_name + self.assertIsNotNone(results.solution_loader) + del results.solution_loader + self.assertEqual( + { + 'extra_info': {'iteration_count': 0}, + 'incumbent_objective': 10.0, + 'objective_bound': None, + 'solution_status': SolutionStatus.optimal, + 'solver_log': None, + 'solver_name': 'ipopt', + 'solver_version': None, + 'termination_condition': TerminationCondition.convergenceCriteriaSatisfied, + 'timing_info': {'timer': timer}, + }, + results.value(), + ) + self.assertEqual(m.x.value, 10) + + def test_presolve_empty(self): + m = pyo.ConcreteModel() + m.x = pyo.Var(bounds=(0, 5)) + m.obj = pyo.Objective(expr=1) + + timer = HierarchicalTimer() + solver = ipopt.Ipopt() + results = solver.solve( + m, + timer=timer, + load_solutions=False, + raise_exception_on_nonoptimal_result=False, + ) + self.assertEqual(results.solution_status, SolutionStatus.noSolution) + self.assertEqual(results.termination_condition, TerminationCondition.emptyModel) + cfg = results.solver_config + del cfg.executable + self.assertEqual( + { + 'load_solutions': False, + 'raise_exception_on_nonoptimal_result': False, + 'solver_options': {}, + 'symbolic_solver_labels': False, + 'tee': [], + 'threads': None, + 'time_limit': None, + 'timer': timer, + 'working_dir': None, + 'writer_config': { + 'column_order': None, + 'export_defined_variables': True, + 'export_nonlinear_variables': None, + 'file_determinism': FileDeterminism.ORDERED, + 'linear_presolve': True, + 'row_order': None, + 'scale_model': True, + 'show_section_timing': False, + 'skip_trivial_constraints': True, + 'symbolic_solver_labels': False, + }, + }, + cfg.value(), + ) + self.assertLess(results.timing_info.wall_time, 0.1) + self.assertEqual(results.timing_info.start_timestamp.tzinfo, datetime.UTC) + self.assertLess( + ( + datetime.datetime.now(datetime.UTC) + - results.timing_info.start_timestamp + ).seconds, + 1, + ) + del results.extra_info.base_file_name + del results.solver_config + del results.timing_info.wall_time + del results.timing_info.start_timestamp + self.assertEqual( + { + 'extra_info': {'iteration_count': 0}, + 'incumbent_objective': None, + 'objective_bound': None, + 'solution_loader': None, + 'solution_status': SolutionStatus.noSolution, + 'solver_log': None, + 'solver_name': 'ipopt', + 'solver_version': None, + 'termination_condition': TerminationCondition.emptyModel, + 'timing_info': {'timer': timer}, + }, + results.value(), + ) + + with self.assertRaisesRegex( + NoSolutionError, "Solution loader does not currently have a valid solution." + ): + results = solver.solve( + m, timer=timer, raise_exception_on_nonoptimal_result=False + ) + with self.assertRaisesRegex( + NoOptimalSolutionError, "Solver did not find the optimal solution." + ): + results = solver.solve(m, timer=timer) + + def test_file_collision(self): + class mock_tempfile: + def new_context(self): + return self + + def __enter__(self): + return self + + def __exit__(self, et, ev, tb): + pass + + def mkstemp(self, suffix, prefix, dir, text, delete): + fname = os.path.join(dir, "testfile" + suffix) + return os.open(fname, os.O_CREAT | os.O_RDWR), fname + + m = pyo.ConcreteModel() + orig_TempfileManager = ipopt.TempfileManager + try: + ipopt.TempfileManager = mock_tempfile() + with TempfileManager.new_context() as tempfile: + solver = ipopt.Ipopt() + dname = tempfile.mkdtemp() + for ext in ('.row', '.col', '.sol', '.opt'): + fname = os.path.join(dname, 'testfile' + ext) + open(fname, 'w').close() + with self.assertRaisesRegex( + RuntimeError, f"Solver interface file {fname} already exists" + ): + solver.solve(m, working_dir=dname) + os.unlink(fname) + finally: + ipopt.TempfileManager = orig_TempfileManager + + def test_bad_executable(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.o = pyo.Objective(expr=m.x**2) + + solver = ipopt.Ipopt() + + with TempfileManager.new_context() as tempfile: + dname = tempfile.mkdtemp() + exe = os.path.join(dname, 'ipopt') + solver.config.executable = exe + with self.assertRaisesRegex(ApplicationError, 'ipopt executable not found'): + solver.solve(m) + + # The following is designed to run on *NIX + if sys.platform.startswith("win"): + return + + _cache = ipopt.Ipopt._exe_cache + ipopt.Ipopt._exe_cache = {exe: (1, 2, 3)} + try: + with open(exe, 'w') as F: + F.write(f"#!{dname}/bad_interpreter\nsys.exit(1)\n") + os.chmod(exe, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + solver.config.executable.rehash() + with self.assertRaisesRegex( + ApplicationError, + f"Could not execute the command: \['{exe}'.*" + f"Error message: .*No such file or directory: '{exe}'", + ): + solver.solve(m) + finally: + ipopt.Ipopt._exe_cache = _cache + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") class TestIpopt(unittest.TestCase): @@ -649,7 +1971,7 @@ def test_ipopt_timer_object(self): # Newer version of IPOPT self.assertIn('IPOPT', timing_info.keys()) - def test_ipopt_options_file(self): + def test_run_ipopt_options_file(self): # Check that the options file is getting to Ipopt: if we give it # an invalid option in the options file, ipopt will fail. This # is important, as ipopt will NOT fail if you pass if an @@ -665,58 +1987,80 @@ def test_ipopt_options_file(self): self.assertEqual(results.solution_status, SolutionStatus.noSolution) self.assertIn('OPTION_INVALID', results.solver_log) - # If the model name contains a quote, then the name needs - # to be quoted - model.name = "test'model'" - results = ipopt.Ipopt().solve( - model, - solver_options={'bogus_option': 5}, - raise_exception_on_nonoptimal_result=False, - load_solutions=False, - ) - self.assertEqual(results.termination_condition, TerminationCondition.error) - self.assertEqual(results.solution_status, SolutionStatus.noSolution) - self.assertIn('OPTION_INVALID', results.solver_log) - - model.name = 'test"model' - results = ipopt.Ipopt().solve( - model, - solver_options={'bogus_option': 5}, - raise_exception_on_nonoptimal_result=False, - load_solutions=False, - ) - self.assertEqual(results.termination_condition, TerminationCondition.error) - self.assertEqual(results.solution_status, SolutionStatus.noSolution) - self.assertIn('OPTION_INVALID', results.solver_log) - - # Because we are using universal=True for to_legal_filename, - # using both single and double quotes will be OK - model.name = 'test"\'model' - results = ipopt.Ipopt().solve( - model, - solver_options={'bogus_option': 5}, - raise_exception_on_nonoptimal_result=False, - load_solutions=False, + # Verify the full command line once + cmd = results.extra_info.command_line + fname = results.extra_info.base_file_name + self.assertEqual( + [ + str(ipopt.Ipopt().config.executable), + f'{fname}.nl', + "-AMPL", + f'option_file_name="{fname}.opt"', + ], + cmd, ) - self.assertEqual(results.termination_condition, TerminationCondition.error) - self.assertEqual(results.solution_status, SolutionStatus.noSolution) - self.assertIn('OPTION_INVALID', results.solver_log) - if not is_windows: - # This test is not valid on Windows, as {"} is not a valid - # character in a directory name. - with TempfileManager.new_context() as temp: - dname = temp.mkdtemp() - working_dir = os.path.join(dname, '"foo"') - os.mkdir(working_dir) - with self.assertRaisesRegex(ValueError, 'single and double'): - results = ipopt.Ipopt().solve( - model, - working_dir=working_dir, - solver_options={'bogus_option': 5}, - raise_exception_on_nonoptimal_result=False, - load_solutions=False, - ) + def test_ipopt_working_dir(self): + m = self.create_model() + with TempfileManager.new_context() as tempfile: + dname = tempfile.mkdtemp() + working_dir = os.path.join(dname, 'testing') + self.assertFalse(os.path.exists(working_dir)) + + results = ipopt.Ipopt().solve(m, working_dir=working_dir) + + self.assertTrue(os.path.exists(working_dir)) + self.assertTrue(results.extra_info.base_file_name.startswith(working_dir)) + self.assertTrue(os.path.exists(results.extra_info.base_file_name + '.nl')) + + def test_load_solution_suffixes(self): + m = self.create_model() + m.x.lb = 0.6 + m.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT) + m.rc = pyo.Suffix(direction=pyo.Suffix.IMPORT) + m.c = pyo.Constraint(expr=m.x == 2 * m.y) + + solver = ipopt.Ipopt() + results = solver.solve(m, writer_config={'linear_presolve': False}) + o1 = results.extra_info.incumbent_objective + + self.assertAlmostEqual(m.x.value, 0.6, delta=1e-5) + self.assertAlmostEqual(m.y.value, 0.3, delta=1e-5) + self.assertEqual(len(m.dual), 1) + self.assertAlmostEqual(m.dual[m.c], 6, delta=1e-5) + self.assertEqual(len(m.rc), 2) + self.assertAlmostEqual(m.rc[m.x], 7.6, delta=1e-5) + self.assertEqual(m.rc[m.y], 0) + + m.scaling_factor = pyo.Suffix(direction=pyo.Suffix.EXPORT) + m.scaling_factor[m.obj] = 10 + m.scaling_factor[m.c] = 5 + m.scaling_factor[m.x] = 7 + m.scaling_factor[m.y] = 3 + results = solver.solve(m, writer_config={'linear_presolve': False}) + + o2 = results.extra_info.incumbent_objective + self.assertAlmostEqual(o1, o2 / 10, delta=1e-5) + + self.assertAlmostEqual(m.x.value, 0.6, delta=1e-5) + self.assertAlmostEqual(m.y.value, 0.3, delta=1e-5) + self.assertEqual(len(m.dual), 1) + self.assertAlmostEqual(m.dual[m.c], 6, delta=1e-5) + self.assertEqual(len(m.rc), 2) + self.assertAlmostEqual(m.rc[m.x], 7.6, delta=1e-5) + self.assertEqual(m.rc[m.y], 0) + + m.x.lb = None + m.y.ub = 0.25 + results = solver.solve(m, writer_config={'linear_presolve': False}) + + self.assertAlmostEqual(m.x.value, 0.5, delta=1e-5) + self.assertAlmostEqual(m.y.value, 0.25, delta=1e-5) + self.assertEqual(len(m.dual), 1) + self.assertAlmostEqual(m.dual[m.c], -1, delta=1e-5) + self.assertEqual(len(m.rc), 2) + self.assertEqual(m.rc[m.x], 0) + self.assertAlmostEqual(m.rc[m.y], -2, delta=1e-5) @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") From a32a07b9f59abc2f9d5cf6abecd09582e4bf1c82 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 22:27:36 -0700 Subject: [PATCH 30/41] Add a standard test for external functions --- .../solver/tests/solvers/test_solvers.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 8aa452444ec..e8c19cbe95c 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -18,6 +18,7 @@ import pyomo.environ as pyo from pyomo import gdp from pyomo.common.dependencies import attempt_import +from pyomo.common.gsl import find_GSL from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.config import SolverConfig from pyomo.contrib.solver.common.factory import SolverFactory @@ -2289,6 +2290,23 @@ def test_node_limit( ) assert res.termination_condition == TerminationCondition.iterationLimit + @parameterized.expand(input=_load_tests(nl_solvers)) + def test_external_function( + self, name: str, opt_class: Type[SolverBase], use_presolve: bool + ): + DLL = find_GSL() + if not DLL: + self.skipTest("Could not find the amplgsl.dll library") + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + model = pyo.ConcreteModel() + model.z_func = pyo.ExternalFunction(library=DLL, function="gsl_sf_gamma") + model.x = pyo.Var(initialize=3, bounds=(1e-5, None)) + model.o = pyo.Objective(expr=model.z_func(model.x)) + res = opt.solve(model) + self.assertAlmostEqual(pyo.value(model.o), 0.885603194411, 7) + class TestLegacySolverInterface(unittest.TestCase): @parameterized.expand(input=all_solvers) From 849e649b0c726b4cfe7f53ea2e1fe1467d4312be Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 22:47:45 -0700 Subject: [PATCH 31/41] Rename sol reader to emphasize it reads ASL sol files --- pyomo/contrib/solver/solvers/ipopt.py | 18 ++++---- pyomo/contrib/solver/solvers/sol_reader.py | 26 ++++++------ .../solver/tests/solvers/test_ipopt.py | 4 +- .../solver/tests/solvers/test_sol_reader.py | 42 +++++++++---------- 4 files changed, 46 insertions(+), 44 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 4e28b28e82c..3fdea4e8131 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -48,10 +48,10 @@ SolutionStatus, ) from pyomo.contrib.solver.solvers.sol_reader import ( - ampl_solve_code_to_solution_status, - parse_sol_file, - SolFileData, - SolFileSolutionLoader, + asl_solve_code_to_solution_status, + parse_asl_sol_file, + ASLSolFileData, + ASLSolFileSolutionLoader, ) from pyomo.contrib.solver.common.util import NoOptimalSolutionError, NoSolutionError from pyomo.common.tee import TeeStream @@ -116,7 +116,7 @@ def __init__( ) -class IpoptSolutionLoader(SolFileSolutionLoader): +class IpoptSolutionLoader(ASLSolFileSolutionLoader): def get_reduced_costs( self, vars_to_load: Optional[Sequence[VarData]] = None ) -> Mapping[VarData, float]: @@ -402,7 +402,7 @@ def solve(self, model, **kwds) -> Results: ) results.solution_status = SolutionStatus.optimal results.solution_loader = IpoptSolutionLoader( - sol_data=SolFileData(), nl_info=nl_info + sol_data=ASLSolFileData(), nl_info=nl_info ) else: results.termination_condition = TerminationCondition.emptyModel @@ -576,9 +576,9 @@ def _run_ipopt(self, results, config, nl_info, basename, timer): timer.start('parse_sol') if os.path.isfile(basename + '.sol'): with open(basename + '.sol', 'r', encoding='utf-8') as sol_file: - sol_data = parse_sol_file(sol_file) + sol_data = parse_asl_sol_file(sol_file) else: - sol_data = SolFileData() + sol_data = ASLSolFileData() results.solution_loader = IpoptSolutionLoader( sol_data=sol_data, nl_info=nl_info ) @@ -586,7 +586,7 @@ def _run_ipopt(self, results, config, nl_info, basename, timer): # Initialize the solver message, solution loader solution # status and termination condition: - ampl_solve_code_to_solution_status(sol_data, results) + asl_solve_code_to_solution_status(sol_data, results) def _parse_ipopt_output(self, output: str) -> Dict[str, Any]: parsed_data = {} diff --git a/pyomo/contrib/solver/solvers/sol_reader.py b/pyomo/contrib/solver/solvers/sol_reader.py index a03edeee5f2..a9e4221e411 100644 --- a/pyomo/contrib/solver/solvers/sol_reader.py +++ b/pyomo/contrib/solver/solvers/sol_reader.py @@ -31,9 +31,9 @@ from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase -class SolFileData: +class ASLSolFileData: """ - Defines the data types found within a .sol file + Defines the data types found within an ASL .sol file """ def __init__(self) -> None: @@ -51,12 +51,12 @@ def __init__(self) -> None: self.unparsed: str = None -class SolFileSolutionLoader(SolutionLoaderBase): +class ASLSolFileSolutionLoader(SolutionLoaderBase): """ - Loader for solvers that create .sol files (e.g., ipopt) + Loader for solvers that create ASL .sol files (e.g., ipopt) """ - def __init__(self, sol_data: SolFileData, nl_info: NLWriterInfo) -> None: + def __init__(self, sol_data: ASLSolFileData, nl_info: NLWriterInfo) -> None: self._sol_data = sol_data self._nl_info = nl_info @@ -172,7 +172,9 @@ def get_duals( return {con: val for con, val in _iter} -def ampl_solve_code_to_solution_status(sol_data: SolFileData, result: Results) -> None: +def asl_solve_code_to_solution_status( + sol_data: ASLSolFileData, result: Results +) -> None: # # This table (the values and the string interpretations) are from # Chapter 14 in the AMPL Book: @@ -219,11 +221,11 @@ def ampl_solve_code_to_solution_status(sol_data: SolFileData, result: Results) - result.termination_condition = term -def parse_sol_file(FILE: io.TextIOBase) -> SolFileData: +def parse_asl_sol_file(FILE: io.TextIOBase) -> ASLSolFileData: """ - Parse a .sol file and populate to Pyomo objects + Parse an ASL .sol file and populate to Pyomo objects """ - sol_data = SolFileData() + sol_data = ASLSolFileData() # Parse the initial solver message and the AMPL options sections z = _parse_message_and_options(FILE, sol_data) @@ -248,7 +250,7 @@ def parse_sol_file(FILE: io.TextIOBase) -> SolFileData: return sol_data -def _parse_message_and_options(FILE: io.TextIOBase, data: SolFileData) -> List[int]: +def _parse_message_and_options(FILE: io.TextIOBase, data: ASLSolFileData) -> List[int]: msg = [] # Some solvers (minto) do not write a message. We will assume # all non-blank lines up the 'Options' line is the message. @@ -302,7 +304,7 @@ def _parse_message_and_options(FILE: io.TextIOBase, data: SolFileData) -> List[i return z -def _parse_objno_and_exitcode(FILE: io.TextIOBase, data: SolFileData) -> None: +def _parse_objno_and_exitcode(FILE: io.TextIOBase, data: ASLSolFileData) -> None: line = FILE.readline().strip() objno = line.split(maxsplit=2) if not objno or objno[0] != 'objno': @@ -322,7 +324,7 @@ def _parse_objno_and_exitcode(FILE: io.TextIOBase, data: SolFileData) -> None: data.solve_code = int(objno[2]) -def _parse_suffixes(FILE: io.TextIOBase, data: SolFileData) -> None: +def _parse_suffixes(FILE: io.TextIOBase, data: ASLSolFileData) -> None: while line := FILE.readline(): line = line.strip() if not line: diff --git a/pyomo/contrib/solver/tests/solvers/test_ipopt.py b/pyomo/contrib/solver/tests/solvers/test_ipopt.py index e8531773dd6..2bf855e9327 100644 --- a/pyomo/contrib/solver/tests/solvers/test_ipopt.py +++ b/pyomo/contrib/solver/tests/solvers/test_ipopt.py @@ -92,7 +92,7 @@ def test_custom_instantiation(self): class TestIpoptSolutionLoader(unittest.TestCase): def test_get_reduced_costs_error(self): loader = ipopt.IpoptSolutionLoader( - ipopt.SolFileData(), NLWriterInfo(eliminated_vars=[1]) + ipopt.ASLSolFileData(), NLWriterInfo(eliminated_vars=[1]) ) with self.assertRaisesRegex( MouseTrap, "Complete reduced costs are not available" @@ -101,7 +101,7 @@ def test_get_reduced_costs_error(self): def test_get_duals_error(self): loader = ipopt.IpoptSolutionLoader( - ipopt.SolFileData(), NLWriterInfo(eliminated_vars=[1]) + ipopt.ASLSolFileData(), NLWriterInfo(eliminated_vars=[1]) ) with self.assertRaisesRegex(MouseTrap, "Complete duals are not available"): loader.get_duals() diff --git a/pyomo/contrib/solver/tests/solvers/test_sol_reader.py b/pyomo/contrib/solver/tests/solvers/test_sol_reader.py index 90220e56ebf..e59526a55db 100644 --- a/pyomo/contrib/solver/tests/solvers/test_sol_reader.py +++ b/pyomo/contrib/solver/tests/solvers/test_sol_reader.py @@ -16,9 +16,9 @@ from pyomo.common.collections import ComponentMap from pyomo.common.fileutils import this_file_dir from pyomo.contrib.solver.solvers.sol_reader import ( - SolFileSolutionLoader, - SolFileData, - parse_sol_file, + ASLSolFileSolutionLoader, + ASLSolFileData, + parse_asl_sol_file, ) from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import SolverError @@ -27,7 +27,7 @@ class TestSolFileData(unittest.TestCase): def test_default_instantiation(self): - instance = SolFileData() + instance = ASLSolFileData() self.assertEqual(instance.message, None) self.assertEqual(instance.objno, 0) self.assertEqual(instance.solve_code, None) @@ -68,7 +68,7 @@ def test_parse_minimal_sol_file(self): 30.0 objno 0 100""" ) - sol_data = parse_sol_file(stream) + sol_data = parse_asl_sol_file(stream) self.assertEqual("Solver message preamble", sol_data.message) self.assertEqual(0, sol_data.objno) @@ -96,7 +96,7 @@ def test_parse_vbtol(self): 1.5 objno 0 100""" ) - sol_data = parse_sol_file(stream) + sol_data = parse_asl_sol_file(stream) self.assertEqual("Solver message preamble", sol_data.message) self.assertEqual(0, sol_data.objno) @@ -137,7 +137,7 @@ def test_multiline_message_and_unparsed(self): and here """ ) - sol_data = parse_sol_file(stream) + sol_data = parse_asl_sol_file(stream) self.assertEqual( "CONOPT 3.17A: Optimal; objective 1\n" @@ -190,7 +190,7 @@ def test_suffix_table(self): """ ) - sol_data = parse_sol_file(stream) + sol_data = parse_asl_sol_file(stream) self.assertEqual( "CONOPT 3.17A: Optimal; objective 1\n" @@ -226,7 +226,7 @@ def test_error_missing_options(self): with self.assertRaisesRegex( SolverError, "Error reading `sol` file: no 'Options' line found." ): - parse_sol_file(stream) + parse_asl_sol_file(stream) def test_error_malformed_options(self): # Contains "Options" but the required integer line is missing/blank @@ -234,7 +234,7 @@ def test_error_malformed_options(self): stream = io.StringIO(bad_text) with self.assertRaisesRegex(ValueError, "invalid literal"): - parse_sol_file(stream) + parse_asl_sol_file(stream) def test_error_objno_not_found(self): stream = io.StringIO( @@ -255,7 +255,7 @@ def test_error_objno_not_found(self): SolverError, "Error reading `sol` file: expected 'objno'; " "received '1.5'.", ): - sol_data = parse_sol_file(stream) + sol_data = parse_asl_sol_file(stream) def test_error_objno_bad_format(self): stream = io.StringIO( @@ -276,7 +276,7 @@ def test_error_objno_bad_format(self): "Error reading `sol` file: expected two numbers in 'objno' line; " "received 'objno 0'.", ): - sol_data = parse_sol_file(stream) + sol_data = parse_asl_sol_file(stream) class TestSolFileSolutionLoader(unittest.TestCase): @@ -285,7 +285,7 @@ def test_member_list(self): expected_list = ['load_vars', 'get_primals', 'get_duals', 'get_reduced_costs'] method_list = [ method - for method in dir(SolFileSolutionLoader) + for method in dir(ASLSolFileSolutionLoader) if not method.startswith('_') ] self.assertEqual(sorted(expected_list), sorted(method_list)) @@ -296,9 +296,9 @@ def test_load_vars(self): m.y = pyo.Var([1, 2, 3]) nl_info = NLWriterInfo(var=[m.x, m.y[1], m.y[3]]) - sol_data = SolFileData() + sol_data = ASLSolFileData() sol_data.primals = [3, 7, 5] - loader = SolFileSolutionLoader(sol_data, nl_info) + loader = ASLSolFileSolutionLoader(sol_data, nl_info) loader.load_vars() self.assertEqual(m.x.value, 3) @@ -335,9 +335,9 @@ def test_load_vars_empty_model(self): nl_info = NLWriterInfo( var=[], eliminated_vars=[(m.y[3], 1.5), (m.y[2], 2 * m.y[3] + 1)] ) - sol_data = SolFileData() + sol_data = ASLSolFileData() sol_data.primals = [] - loader = SolFileSolutionLoader(sol_data, nl_info) + loader = ASLSolFileSolutionLoader(sol_data, nl_info) loader.load_vars() self.assertEqual(m.x.value, None) @@ -351,9 +351,9 @@ def test_get_primals(self): m.y = pyo.Var([1, 2, 3]) nl_info = NLWriterInfo(var=[m.x, m.y[1], m.y[3]]) - sol_data = SolFileData() + sol_data = ASLSolFileData() sol_data.primals = [3, 7, 5] - loader = SolFileSolutionLoader(sol_data, nl_info) + loader = ASLSolFileSolutionLoader(sol_data, nl_info) self.assertEqual( loader.get_primals(), ComponentMap([(m.x, 3), (m.y[1], 7), (m.y[3], 5)]) @@ -401,9 +401,9 @@ def test_get_primals_empty_model(self): nl_info = NLWriterInfo( var=[], eliminated_vars=[(m.y[3], 1.5), (m.y[2], 2 * m.y[3] + 1)] ) - sol_data = SolFileData() + sol_data = ASLSolFileData() sol_data.primals = [] - loader = SolFileSolutionLoader(sol_data, nl_info) + loader = ASLSolFileSolutionLoader(sol_data, nl_info) self.assertEqual( loader.get_primals(), ComponentMap([(m.y[2], 4), (m.y[3], 1.5)]) From 6088d310115128c99b7507ea0edd76897b321f7b Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 22:53:03 -0700 Subject: [PATCH 32/41] Rename sol_reader -> asl_sol_reader --- pyomo/contrib/solver/__init__.py | 6 ++++++ .../solver/solvers/{sol_reader.py => asl_sol_reader.py} | 0 pyomo/contrib/solver/solvers/ipopt.py | 2 +- .../solvers/{test_sol_reader.py => test_asl_sol_reader.py} | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) rename pyomo/contrib/solver/solvers/{sol_reader.py => asl_sol_reader.py} (100%) rename pyomo/contrib/solver/tests/solvers/{test_sol_reader.py => test_asl_sol_reader.py} (99%) diff --git a/pyomo/contrib/solver/__init__.py b/pyomo/contrib/solver/__init__.py index 9dc5c4b7b03..82e7524d6df 100644 --- a/pyomo/contrib/solver/__init__.py +++ b/pyomo/contrib/solver/__init__.py @@ -37,4 +37,10 @@ version='6.9.2', ) +moved_module( + 'pyomo.contrib.solver.solvers.sol_reader', + 'pyomo.contrib.solver.solvers.asl_sol_reader', + version='6.10.0.dev0', +) + del _module, moved_module diff --git a/pyomo/contrib/solver/solvers/sol_reader.py b/pyomo/contrib/solver/solvers/asl_sol_reader.py similarity index 100% rename from pyomo/contrib/solver/solvers/sol_reader.py rename to pyomo/contrib/solver/solvers/asl_sol_reader.py diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 3fdea4e8131..f9c875eb4c8 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -47,7 +47,7 @@ TerminationCondition, SolutionStatus, ) -from pyomo.contrib.solver.solvers.sol_reader import ( +from pyomo.contrib.solver.solvers.asl_sol_reader import ( asl_solve_code_to_solution_status, parse_asl_sol_file, ASLSolFileData, diff --git a/pyomo/contrib/solver/tests/solvers/test_sol_reader.py b/pyomo/contrib/solver/tests/solvers/test_asl_sol_reader.py similarity index 99% rename from pyomo/contrib/solver/tests/solvers/test_sol_reader.py rename to pyomo/contrib/solver/tests/solvers/test_asl_sol_reader.py index e59526a55db..38db7d3fafd 100644 --- a/pyomo/contrib/solver/tests/solvers/test_sol_reader.py +++ b/pyomo/contrib/solver/tests/solvers/test_asl_sol_reader.py @@ -15,7 +15,7 @@ from pyomo.common import unittest from pyomo.common.collections import ComponentMap from pyomo.common.fileutils import this_file_dir -from pyomo.contrib.solver.solvers.sol_reader import ( +from pyomo.contrib.solver.solvers.asl_sol_reader import ( ASLSolFileSolutionLoader, ASLSolFileData, parse_asl_sol_file, From 85ff4e1078838ac56c46f87a5416a84855528dfb Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 23:39:44 -0700 Subject: [PATCH 33/41] Fix test failures on Windows --- pyomo/common/tests/test_tempfile.py | 8 ++++++-- pyomo/contrib/solver/tests/solvers/test_ipopt.py | 13 ++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/pyomo/common/tests/test_tempfile.py b/pyomo/common/tests/test_tempfile.py index 8ae88d9e967..bab84fabc56 100644 --- a/pyomo/common/tests/test_tempfile.py +++ b/pyomo/common/tests/test_tempfile.py @@ -288,8 +288,12 @@ def test_mktemp_delete(self): with self.TM.new_context() as context: dname = context.mkdtemp() with self.TM.new_context() as subcontext: - _, fname1 = subcontext.mkstemp(dir=dname) - _, fname2 = subcontext.mkstemp(dir=dname, delete=False) + fd, fname1 = subcontext.mkstemp(dir=dname) + fd, fname2 = subcontext.mkstemp(dir=dname, delete=False) + # Note: because the context manager isn't going to + # delete fname2, we need to explicitly close the file + # here... + os.close(fd) dname1 = subcontext.mkdtemp(dir=dname) dname2 = subcontext.mkdtemp(dir=dname, delete=False) diff --git a/pyomo/contrib/solver/tests/solvers/test_ipopt.py b/pyomo/contrib/solver/tests/solvers/test_ipopt.py index 2bf855e9327..011ffbe78d8 100644 --- a/pyomo/contrib/solver/tests/solvers/test_ipopt.py +++ b/pyomo/contrib/solver/tests/solvers/test_ipopt.py @@ -184,6 +184,10 @@ def test_get_version(self): self.assertIsNone(solver.version()) self.assertEqual({None: None}, ipopt.Ipopt._exe_cache) + # the rest of this test is designed to work on *nix: + if sys.platform.startswith("win"): + return + ipopt.Ipopt._exe_cache = {} fname = os.path.join(dname, 'test2') with open(fname, 'w') as F: @@ -194,10 +198,6 @@ def test_get_version(self): self.assertIsNone(solver.version()) self.assertEqual({None: None}, ipopt.Ipopt._exe_cache) - # the rest of this test is designed to work on *nix: - if sys.platform.startswith("win"): - return - # Found an executable, but --version errors ipopt.Ipopt._exe_cache = {} fname = os.path.join(dname, 'test3') @@ -1824,7 +1824,10 @@ def mkstemp(self, suffix, prefix, dir, text, delete): fname = os.path.join(dname, 'testfile' + ext) open(fname, 'w').close() with self.assertRaisesRegex( - RuntimeError, f"Solver interface file {fname} already exists" + RuntimeError, + f"Solver interface file " + + fname.replace('\\', '\\\\') + + " already exists", ): solver.solve(m, working_dir=dname) os.unlink(fname) From 9937f42771b0043e3e77ba416eb18ba88aa449f3 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 23:41:03 -0700 Subject: [PATCH 34/41] Remove legacy typing classes --- .../contrib/solver/solvers/asl_sol_reader.py | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/pyomo/contrib/solver/solvers/asl_sol_reader.py b/pyomo/contrib/solver/solvers/asl_sol_reader.py index a9e4221e411..a9b7ce06516 100644 --- a/pyomo/contrib/solver/solvers/asl_sol_reader.py +++ b/pyomo/contrib/solver/solvers/asl_sol_reader.py @@ -9,9 +9,8 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ - -from typing import Tuple, Dict, Any, List, Sequence, Optional, Mapping, NoReturn import io +from typing import Sequence, Optional, Mapping from pyomo.common.collections import ComponentMap from pyomo.common.errors import MouseTrap @@ -40,14 +39,14 @@ def __init__(self) -> None: self.message: str = None self.objno: int = 0 self.solve_code: int = None - self.ampl_options: List[int | float] = None - self.primals: List[float] = None - self.duals: List[float] = None - self.var_suffixes: Dict[str, Dict[int, int | float]] = {} - self.con_suffixes: Dict[str, Dict[int, int | float]] = {} - self.obj_suffixes: Dict[str, Dict[int, int | float]] = {} - self.problem_suffixes: Dict[str, int | float] = {} - self.suffix_table: Dict[(int, str), List[int | float, str, ...]] = {} + self.ampl_options: list[int | float] = None + self.primals: list[float] = None + self.duals: list[float] = None + self.var_suffixes: dict[str, dict[int, int | float]] = {} + self.con_suffixes: dict[str, dict[int, int | float]] = {} + self.obj_suffixes: dict[str, dict[int, int | float]] = {} + self.problem_suffixes: dict[str, int | float] = {} + self.suffix_table: dict[(int, str), list[int | float, str, ...]] = {} self.unparsed: str = None @@ -60,7 +59,7 @@ def __init__(self, sol_data: ASLSolFileData, nl_info: NLWriterInfo) -> None: self._sol_data = sol_data self._nl_info = nl_info - def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> NoReturn: + def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> None: if vars_to_load is not None: # If we are given a list of variables to load, it is easiest # to use the filtering in get_primals and then just set @@ -141,7 +140,7 @@ def get_primals( def get_duals( self, cons_to_load: Optional[Sequence[ConstraintData]] = None - ) -> Dict[ConstraintData, float]: + ) -> dict[ConstraintData, float]: if len(self._nl_info.eliminated_vars) > 0: raise MouseTrap( 'Complete duals are not available when variables have ' @@ -250,7 +249,7 @@ def parse_asl_sol_file(FILE: io.TextIOBase) -> ASLSolFileData: return sol_data -def _parse_message_and_options(FILE: io.TextIOBase, data: ASLSolFileData) -> List[int]: +def _parse_message_and_options(FILE: io.TextIOBase, data: ASLSolFileData) -> list[int]: msg = [] # Some solvers (minto) do not write a message. We will assume # all non-blank lines up the 'Options' line is the message. From 80745933b29087fcb090063b462d5745a2008d76 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 23:41:45 -0700 Subject: [PATCH 35/41] Minor cleanup to make logic more clear --- pyomo/contrib/solver/solvers/asl_sol_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/asl_sol_reader.py b/pyomo/contrib/solver/solvers/asl_sol_reader.py index a9b7ce06516..90a47d1d5ab 100644 --- a/pyomo/contrib/solver/solvers/asl_sol_reader.py +++ b/pyomo/contrib/solver/solvers/asl_sol_reader.py @@ -284,7 +284,7 @@ def _parse_message_and_options(FILE: io.TextIOBase, data: ASLSolFileData) -> lis # Because of this, we will read the first two options from the file # first so we can know how to correctly parse the remaining options. assert n_opts >= 2 - ampl_options = [int(FILE.readline()) for i in range(2)] + ampl_options = [int(FILE.readline()), int(FILE.readline())] read_vbtol = ampl_options[1] == 3 if read_vbtol: n_opts -= 2 From e2ac142d318cc2608baa22cb0e4273a04a450b8d Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 23:42:11 -0700 Subject: [PATCH 36/41] NFC: update comments --- .../contrib/solver/solvers/asl_sol_reader.py | 23 +++++++++++++++++-- pyomo/contrib/solver/solvers/ipopt.py | 23 ++++++++++++++++--- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/pyomo/contrib/solver/solvers/asl_sol_reader.py b/pyomo/contrib/solver/solvers/asl_sol_reader.py index 90a47d1d5ab..d3b0e9d2d9a 100644 --- a/pyomo/contrib/solver/solvers/asl_sol_reader.py +++ b/pyomo/contrib/solver/solvers/asl_sol_reader.py @@ -174,6 +174,16 @@ def get_duals( def asl_solve_code_to_solution_status( sol_data: ASLSolFileData, result: Results ) -> None: + """Convert the ASL "solve code" integer into a Pyomo status + + The ASL returns an indication of the solution status and termination + condition using a single "solve code" integer. This function + implements the translation of the numeric value into the Pyomo + equivalents (:class:`TerminationCondition` and + :class:`SolutionStatus`), as well as a general string description, + using the table from Section 14.2 in the AMPL Book [FGK02]_. + + """ # # This table (the values and the string interpretations) are from # Chapter 14 in the AMPL Book: @@ -221,8 +231,17 @@ def asl_solve_code_to_solution_status( def parse_asl_sol_file(FILE: io.TextIOBase) -> ASLSolFileData: - """ - Parse an ASL .sol file and populate to Pyomo objects + """Parse an ASL .sol file. + + This is a standalone routine to parse the AMPL Solver Library (ASL) + "``.sol``" file format. The resulting :class:`ASLSolFileData` + object is a faithful representation of the data from the file. + Translating the parsed data back into the context of a Pyomo model + requires additional information (at the very least, the Pyomo model + and the :class:`NLWriterInfo` data structure generated by the writer + that originally created the ``.nl`` file that was sent ot the + solver. + """ sol_data = ASLSolFileData() diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index f9c875eb4c8..de46b3ab836 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -25,6 +25,7 @@ ConfigDict, ConfigList, ConfigValue, + document_configdict, document_class_CONFIG, ADVANCED_OPTION, ) @@ -69,7 +70,7 @@ def _option_to_cmd(opt: str, val: str | int | float): - """Convert a opyion / value pair into a valid command line argument.""" + """Convert a option / value pair into a valid command line argument.""" if isinstance(val, str): if '"' not in val: return f'{opt}="{val}"' @@ -85,6 +86,7 @@ def _option_to_cmd(opt: str, val: str | int | float): return f'{opt}={val}' +@document_configdict() class IpoptConfig(SolverConfig): def __init__( self, @@ -162,6 +164,7 @@ def get_reduced_costs( return rc +#: The set of all ipopt options that can be passed to Ipopt on the command line ipopt_command_line_options = { 'acceptable_compl_inf_tol', 'acceptable_constr_viol_tol', @@ -221,6 +224,7 @@ def get_reduced_costs( 'watchdog_shortened_iter_trigger', } +#: The set of options we forbid the user from setting (with reasons) unallowed_ipopt_options = { 'wantsol': 'The solver interface requires the sol file to be created', 'option_file_name': ( @@ -282,7 +286,7 @@ def _get_version(self, exe): ) # Note that we expect the command to run without error, AND that # it returns a string starting "ipopt ". That prevents - # us from tryying to use other (even ASL) executables as if they + # us from trying to use other (even ASL) executables as if they # were ipopt fields = results.stdout.split(maxsplit=2) if results.returncode: @@ -302,6 +306,19 @@ def _get_version(self, exe): return ver def has_linear_solver(self, linear_solver: str) -> bool: + """Determine if Ipopt has access to the specified linear solver + + This solves a small problem to detect if the Ipopt executable + has access to the specified linear solver. + + Parameters + ---------- + linear_solver : str + + The linear solver to test. Accepts any string that is valid + for the ``linear_solver`` Ipopt option. + + """ import pyomo.core as AML m = AML.ConcreteModel() @@ -481,7 +498,7 @@ def _process_options( with open(option_fname, 'w', encoding='utf-8') as OPT_FILE: OPT_FILE.writelines(options_file_options) cmd_line_options.append(_option_to_cmd('option_file_name', option_fname)) - # Return the (formatted) command ine options + # Return the (formatted) command line options return cmd_line_options def _run_ipopt(self, results, config, nl_info, basename, timer): From c465356de36e7986c425f64ed1612e0068ca8a75 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 1 Dec 2025 23:55:45 -0700 Subject: [PATCH 37/41] NFC: typo --- pyomo/contrib/solver/solvers/asl_sol_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/asl_sol_reader.py b/pyomo/contrib/solver/solvers/asl_sol_reader.py index d3b0e9d2d9a..60e8ed1189d 100644 --- a/pyomo/contrib/solver/solvers/asl_sol_reader.py +++ b/pyomo/contrib/solver/solvers/asl_sol_reader.py @@ -239,7 +239,7 @@ def parse_asl_sol_file(FILE: io.TextIOBase) -> ASLSolFileData: Translating the parsed data back into the context of a Pyomo model requires additional information (at the very least, the Pyomo model and the :class:`NLWriterInfo` data structure generated by the writer - that originally created the ``.nl`` file that was sent ot the + that originally created the ``.nl`` file that was sent to the solver. """ From ad255482ed4d2bcc917e01a96c775da5d7ea4282 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 2 Dec 2025 07:14:25 -0700 Subject: [PATCH 38/41] Update timezone references from UTC to timezone.utc --- pyomo/contrib/solver/tests/solvers/test_ipopt.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyomo/contrib/solver/tests/solvers/test_ipopt.py b/pyomo/contrib/solver/tests/solvers/test_ipopt.py index 011ffbe78d8..87c760f9b3d 100644 --- a/pyomo/contrib/solver/tests/solvers/test_ipopt.py +++ b/pyomo/contrib/solver/tests/solvers/test_ipopt.py @@ -1603,10 +1603,10 @@ def test_presolve_prove_infeasible(self): cfg.value(), ) self.assertLess(results.timing_info.wall_time, 0.1) - self.assertEqual(results.timing_info.start_timestamp.tzinfo, datetime.UTC) + self.assertEqual(results.timing_info.start_timestamp.tzinfo, datetime.timezone.utc) self.assertLess( ( - datetime.datetime.now(datetime.UTC) + datetime.datetime.now(datetime.timezone.utc) - results.timing_info.start_timestamp ).seconds, 1, @@ -1687,10 +1687,10 @@ def test_presolve_solveModel(self): ) self.assertLess(results.timing_info.wall_time, 0.1) del results.timing_info.wall_time - self.assertEqual(results.timing_info.start_timestamp.tzinfo, datetime.UTC) + self.assertEqual(results.timing_info.start_timestamp.tzinfo, datetime.timezone.utc) self.assertLess( ( - datetime.datetime.now(datetime.UTC) + datetime.datetime.now(datetime.timezone.utc) - results.timing_info.start_timestamp ).seconds, 1, @@ -1759,10 +1759,10 @@ def test_presolve_empty(self): cfg.value(), ) self.assertLess(results.timing_info.wall_time, 0.1) - self.assertEqual(results.timing_info.start_timestamp.tzinfo, datetime.UTC) + self.assertEqual(results.timing_info.start_timestamp.tzinfo, datetime.timezone.utc) self.assertLess( ( - datetime.datetime.now(datetime.UTC) + datetime.datetime.now(datetime.timezone.utc) - results.timing_info.start_timestamp ).seconds, 1, From ac35d764dc701fea54fec9039fc2cc19ccadb8db Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 2 Dec 2025 08:34:31 -0700 Subject: [PATCH 39/41] NFC: apply black --- pyomo/contrib/solver/tests/solvers/test_ipopt.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/tests/solvers/test_ipopt.py b/pyomo/contrib/solver/tests/solvers/test_ipopt.py index 87c760f9b3d..71d7a9be534 100644 --- a/pyomo/contrib/solver/tests/solvers/test_ipopt.py +++ b/pyomo/contrib/solver/tests/solvers/test_ipopt.py @@ -1603,7 +1603,9 @@ def test_presolve_prove_infeasible(self): cfg.value(), ) self.assertLess(results.timing_info.wall_time, 0.1) - self.assertEqual(results.timing_info.start_timestamp.tzinfo, datetime.timezone.utc) + self.assertEqual( + results.timing_info.start_timestamp.tzinfo, datetime.timezone.utc + ) self.assertLess( ( datetime.datetime.now(datetime.timezone.utc) @@ -1687,7 +1689,9 @@ def test_presolve_solveModel(self): ) self.assertLess(results.timing_info.wall_time, 0.1) del results.timing_info.wall_time - self.assertEqual(results.timing_info.start_timestamp.tzinfo, datetime.timezone.utc) + self.assertEqual( + results.timing_info.start_timestamp.tzinfo, datetime.timezone.utc + ) self.assertLess( ( datetime.datetime.now(datetime.timezone.utc) @@ -1759,7 +1763,9 @@ def test_presolve_empty(self): cfg.value(), ) self.assertLess(results.timing_info.wall_time, 0.1) - self.assertEqual(results.timing_info.start_timestamp.tzinfo, datetime.timezone.utc) + self.assertEqual( + results.timing_info.start_timestamp.tzinfo, datetime.timezone.utc + ) self.assertLess( ( datetime.datetime.now(datetime.timezone.utc) From 4c9e4d136ee32aa502d9dd28f8289ac770a2ef27 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 2 Dec 2025 09:53:56 -0700 Subject: [PATCH 40/41] Resolve Windows test failure (open file handle) --- pyomo/contrib/solver/tests/solvers/test_ipopt.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/tests/solvers/test_ipopt.py b/pyomo/contrib/solver/tests/solvers/test_ipopt.py index 71d7a9be534..684cfc14568 100644 --- a/pyomo/contrib/solver/tests/solvers/test_ipopt.py +++ b/pyomo/contrib/solver/tests/solvers/test_ipopt.py @@ -1806,6 +1806,9 @@ def test_presolve_empty(self): def test_file_collision(self): class mock_tempfile: + def __init__(self): + self.fd = None + def new_context(self): return self @@ -1813,11 +1816,13 @@ def __enter__(self): return self def __exit__(self, et, ev, tb): - pass + if self.fd is not None: + os.close(self.fd) def mkstemp(self, suffix, prefix, dir, text, delete): fname = os.path.join(dir, "testfile" + suffix) - return os.open(fname, os.O_CREAT | os.O_RDWR), fname + self.fd = os.open(fname, os.O_CREAT | os.O_RDWR) + return self.fd, fname m = pyo.ConcreteModel() orig_TempfileManager = ipopt.TempfileManager From 88816e07698461920a610b0d745fd1764642b219 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 2 Dec 2025 09:54:07 -0700 Subject: [PATCH 41/41] Resolve bad regex --- pyomo/contrib/solver/tests/solvers/test_ipopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/tests/solvers/test_ipopt.py b/pyomo/contrib/solver/tests/solvers/test_ipopt.py index 684cfc14568..0a5936f19b2 100644 --- a/pyomo/contrib/solver/tests/solvers/test_ipopt.py +++ b/pyomo/contrib/solver/tests/solvers/test_ipopt.py @@ -1872,7 +1872,7 @@ def test_bad_executable(self): solver.config.executable.rehash() with self.assertRaisesRegex( ApplicationError, - f"Could not execute the command: \['{exe}'.*" + f"Could not execute the command: \\['{exe}'.*" f"Error message: .*No such file or directory: '{exe}'", ): solver.solve(m)