Merge pull request #4 from fmfn/master

update from main

Author: bwheelz36

.github/workflows/run_tests.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
 
     steps:
     - uses: actions/checkout@v3
@@ -32,9 +32,11 @@ jobs:
         python -m pip install --upgrade pip
         pip install pytest
         pip install pytest-cov
-        pip install codecov
+        pip install coverage
         pip install -e .
+
     - name: Test with pytest
       run: |
-        pytest --cov-config .coveragerc --cov-report html --cov=bayes_opt
-        codecov 
+        pytest --cov-report xml --cov=bayes_opt/
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v3

.gitignore

@@ -27,4 +27,5 @@ env/
 venv/
 ENV/
 env.bak/
-venv.bak/
+venv.bak/
+*temp*

README.md

@@ -323,7 +323,7 @@ If you used this package in your research and is interested in citing it here's
 * Numpy
 * Scipy
 * Scikit-learn
-
+ 
 # References:
 * http://papers.nips.cc/paper/4522-practical-bayesian-optimization-of-machine-learning-algorithms.pdf
 * http://arxiv.org/pdf/1012.2599v1.pdf

bayes_opt/bayesian_optimization.py

@@ -91,6 +91,12 @@ class BayesianOptimization(Observable):
     bounds_transformer: DomainTransformer, optional(default=None)
         If provided, the transformation is applied to the bounds.
 
+    allow_duplicate_points: bool, optional (default=False)
+        If True, the optimizer will allow duplicate points to be registered.
+        This behavior may be desired in high noise situations where repeatedly probing
+        the same point will give different answers. In other situations, the acquisition
+        may occasionaly generate a duplicate point.
+
     Methods
     -------
     probe()
@@ -111,9 +117,10 @@ def __init__(self,
                  constraint=None,
                  random_state=None,
                  verbose=2,
-                 bounds_transformer=None):
+                 bounds_transformer=None,
+                 allow_duplicate_points=False):
         self._random_state = ensure_rng(random_state)
-
+        self._allow_duplicate_points = allow_duplicate_points
         self._queue = Queue()
 
         # Internal GP regressor
@@ -129,7 +136,8 @@ def __init__(self,
             # Data structure containing the function to be optimized, the
             # bounds of its domain, and a record of the evaluations we have
             # done so far
-            self._space = TargetSpace(f, pbounds, random_state=random_state)
+            self._space = TargetSpace(f, pbounds, random_state=random_state,
+                                      allow_duplicate_points=self._allow_duplicate_points)
             self.is_constrained = False
         else:
             constraint_ = ConstraintModel(
@@ -242,12 +250,14 @@ def _prime_subscriptions(self):
     def maximize(self,
                  init_points=5,
                  n_iter=25,
-                 acq='ucb',
-                 kappa=2.576,
-                 kappa_decay=1,
-                 kappa_decay_delay=0,
-                 xi=0.0,
+                 acquisition_function=None,
+                 acq=None,
+                 kappa=None,
+                 kappa_decay=None,
+                 kappa_decay_delay=None,
+                 xi=None,
                  **gp_params):
+
         """
         Probes the target space to find the parameters that yield the maximum
         value for the given function.
@@ -262,38 +272,33 @@ def maximize(self,
             Number of iterations where the method attempts to find the maximum
             value.
 
-        acq: {'ucb', 'ei', 'poi'}
-            The acquisition method used.
-                * 'ucb' stands for the Upper Confidence Bounds method
-                * 'ei' is the Expected Improvement method
-                * 'poi' is the Probability Of Improvement criterion.
-
-        kappa: float, optional(default=2.576)
-            Parameter to indicate how closed are the next parameters sampled.
-                Higher value = favors spaces that are least explored.
-                Lower value = favors spaces where the regression function is
-                the highest.
+        acquisition_function: object, optional
+            An instance of bayes_opt.util.UtilityFunction.
+            If nothing is passed, a default using ucb is used
 
-        kappa_decay: float, optional(default=1)
-            `kappa` is multiplied by this factor every iteration.
-
-        kappa_decay_delay: int, optional(default=0)
-            Number of iterations that must have passed before applying the
-            decay to `kappa`.
-
-        xi: float, optional(default=0.0)
-            [unused]
+        All other parameters are unused, and are only available to ensure backwards compatability - these
+        will be removed in a future release
         """
         self._prime_subscriptions()
         self.dispatch(Events.OPTIMIZATION_START)
         self._prime_queue(init_points)
-        self.set_gp_params(**gp_params)
 
-        util = UtilityFunction(kind=acq,
-                               kappa=kappa,
-                               xi=xi,
-                               kappa_decay=kappa_decay,
-                               kappa_decay_delay=kappa_decay_delay)
+        old_params_used = any([param is not None for param in [acq, kappa, kappa_decay, kappa_decay_delay, xi]])
+        if old_params_used or gp_params:
+            raise Exception('\nPassing acquisition function parameters or gaussian process parameters to maximize'
+                                     '\nis no longer supported. Instead,please use the "set_gp_params" method to set'
+                                     '\n the gp params, and pass an instance of bayes_opt.util.UtilityFunction'
+                                     '\n using the acquisition_function argument\n')
+
+        if acquisition_function is None:
+            util = UtilityFunction(kind='ucb',
+                                   kappa=2.576,
+                                   xi=0.0,
+                                   kappa_decay=1,
+                                   kappa_decay_delay=0)
+        else:
+            util = acquisition_function
+
         iteration = 0
         while not self._queue.empty or iteration < n_iter:
             try:

bayes_opt/domain_reduction.py

@@ -68,6 +68,9 @@ def initialize(self, target_space: TargetSpace) -> None:
 
         self.r = self.contraction_rate * self.r
 
+        # check if the minimum window fits in the orignal bounds
+        self._window_bounds_compatiblity(self.original_bounds)
+
     def _update(self, target_space: TargetSpace) -> None:
 
         # setting the previous
@@ -105,11 +108,27 @@ def _trim(self, new_bounds: np.array, global_bounds: np.array) -> np.array:
                 new_bounds[i, 1] = entry[0]
             window_width = abs(entry[0] - entry[1])
             if window_width < self.minimum_window[i]:
-                new_bounds[i, 0] -= (self.minimum_window[i] - window_width) / 2.0
-                new_bounds[i, 1] += (self.minimum_window[i] - window_width) / 2.0
-
+                dw = (self.minimum_window[i] - window_width) / 2.0
+                left_expansion_space = abs(global_bounds[i, 0] - entry[0]) # should be non-positive
+                right_expansion_space = abs(global_bounds[i, 1] - entry[1]) # should be non-negative
+                # conservative
+                dw_l = min(dw, left_expansion_space)
+                dw_r = min(dw, right_expansion_space)
+                # this crawls towards the edge
+                ddw_r = dw_r + max(dw - dw_l, 0)
+                ddw_l = dw_l + max(dw - dw_r, 0)
+                new_bounds[i, 0] -= ddw_l
+                new_bounds[i, 1] += ddw_r
         return new_bounds
 
+    def _window_bounds_compatiblity(self, global_bounds: np.array) -> bool:
+        """Checks if global bounds are compatible with the minimum window sizes."""
+        for i, entry in enumerate(global_bounds):
+            global_window_width = abs(entry[1] - entry[0])
+            if global_window_width < self.minimum_window[i]:
+                raise ValueError(
+                    "Global bounds are not compatible with the minimum window size.")
+
     def _create_bounds(self, parameters: dict, bounds: np.array) -> dict:
         return {param: bounds[i, :] for i, param in enumerate(parameters)}
 

bayes_opt/target_space.py

@@ -1,5 +1,8 @@
+import warnings
+
 import numpy as np
 from .util import ensure_rng, NotUniqueError
+from .util import Colours
 
 
 def _hashable(x):
@@ -22,7 +25,9 @@ class TargetSpace(object):
     >>> y = space.register_point(x)
     >>> assert self.max_point()['max_val'] == y
     """
-    def __init__(self, target_func, pbounds, constraint=None, random_state=None):
+
+    def __init__(self, target_func, pbounds, constraint=None, random_state=None,
+                 allow_duplicate_points=False):
         """
         Parameters
         ----------
@@ -35,8 +40,16 @@ def __init__(self, target_func, pbounds, constraint=None, random_state=None):
 
         random_state : int, RandomState, or None
             optionally specify a seed for a random number generator
+
+        allow_duplicate_points: bool, optional (default=False)
+            If True, the optimizer will allow duplicate points to be registered.
+            This behavior may be desired in high noise situations where repeatedly probing
+            the same point will give different answers. In other situations, the acquisition
+            may occasionaly generate a duplicate point.
         """
         self.random_state = ensure_rng(random_state)
+        self._allow_duplicate_points = allow_duplicate_points
+        self.n_duplicate_points = 0
 
         # The function to be optimized
         self.target_func = target_func
@@ -56,7 +69,6 @@ def __init__(self, target_func, pbounds, constraint=None, random_state=None):
         # keep track of unique points we have seen so far
         self._cache = {}
 
-
         self._constraint = constraint
 
         if constraint is not None:
@@ -96,7 +108,7 @@ def keys(self):
     @property
     def bounds(self):
         return self._bounds
-    
+
     @property
     def constraint(self):
         return self._constraint
@@ -176,8 +188,13 @@ def register(self, params, target, constraint_value=None):
         """
         x = self._as_array(params)
         if x in self:
-            raise NotUniqueError('Data point {} is not unique'.format(x))
-
+            if self._allow_duplicate_points:
+                self.n_duplicate_points = self.n_duplicate_points + 1
+                print(f'{Colours.RED}Data point {x} is not unique. {self.n_duplicate_points} duplicates registered.'
+                              f' Continuing ...{Colours.END}')
+            else:
+                raise NotUniqueError(f'Data point {x} is not unique. You can set "allow_duplicate_points=True" to '
+                                     f'avoid this error')
 
         self._params = np.concatenate([self._params, x.reshape(1, -1)])
         self._target = np.concatenate([self._target, [target]])
@@ -188,12 +205,12 @@ def register(self, params, target, constraint_value=None):
         else:
             if constraint_value is None:
                 msg = ("When registering a point to a constrained TargetSpace" +
-                    " a constraint value needs to be present.")
+                       " a constraint value needs to be present.")
                 raise ValueError(msg)
             # Insert data into unique dictionary
             self._cache[_hashable(x.ravel())] = (target, constraint_value)
             self._constraint_values = np.concatenate([self._constraint_values,
-                                                 [constraint_value]])
+                                                      [constraint_value]])
 
     def probe(self, params):
         """
@@ -215,21 +232,16 @@ def probe(self, params):
             target function value.
         """
         x = self._as_array(params)
+        params = dict(zip(self._keys, x))
+        target = self.target_func(**params)
 
-        try:
-            return self._cache[_hashable(x)]
-        except KeyError:
-            params = dict(zip(self._keys, x))
-            target = self.target_func(**params)
-
-            if self._constraint is None:
-                self.register(x, target)
-                return target
-            else:
-                constraint_value = self._constraint.eval(**params)
-                self.register(x, target, constraint_value)
-                return target, constraint_value
-
+        if self._constraint is None:
+            self.register(x, target)
+            return target
+        else:
+            constraint_value = self._constraint.eval(**params)
+            self.register(x, target, constraint_value)
+            return target, constraint_value
 
     def random_sample(self):
         """
@@ -317,7 +329,7 @@ def res(self):
                     self._constraint_values,
                     params,
                     self._constraint.allowed(self._constraint_values)
-                    )
+                )
             ]
 
     def set_bounds(self, new_bounds):

bayes_opt/util.py

@@ -2,6 +2,7 @@
 import numpy as np
 from scipy.stats import norm
 from scipy.optimize import minimize
+from colorama import just_fix_windows_console
 
 
 def acq_max(ac, gp, y_max, bounds, random_state, constraint=None, n_warmup=10000, n_iter=10):
@@ -96,9 +97,29 @@ def to_minimize(x):
 class UtilityFunction(object):
     """
     An object to compute the acquisition functions.
+
+    kind: {'ucb', 'ei', 'poi'}
+        * 'ucb' stands for the Upper Confidence Bounds method
+        * 'ei' is the Expected Improvement method
+        * 'poi' is the Probability Of Improvement criterion.
+
+    kappa: float, optional(default=2.576)
+            Parameter to indicate how closed are the next parameters sampled.
+            Higher value = favors spaces that are least explored.
+            Lower value = favors spaces where the regression function is
+            the highest.
+
+    kappa_decay: float, optional(default=1)
+        `kappa` is multiplied by this factor every iteration.
+
+    kappa_decay_delay: int, optional(default=0)
+        Number of iterations that must have passed before applying the
+        decay to `kappa`.
+
+    xi: float, optional(default=0.0)
     """
 
-    def __init__(self, kind, kappa, xi, kappa_decay=1, kappa_decay_delay=0):
+    def __init__(self, kind='ucb', kappa=2.576, xi=0, kappa_decay=1, kappa_decay_delay=0):
 
         self.kappa = kappa
         self._kappa_decay = kappa_decay
@@ -275,3 +296,6 @@ def underline(cls, s):
     def yellow(cls, s):
         """Wrap text in yellow."""
         return cls._wrap_colour(s, cls.YELLOW)
+
+
+just_fix_windows_console()