From 4543f997edeffd86e4b77604b7201bdf7a2d069d Mon Sep 17 00:00:00 2001 From: tommoral Date: Wed, 14 Jan 2026 18:02:16 +0100 Subject: [PATCH 1/6] MNT update CI and clean unused files --- .github/workflows/clean_template.yml | 52 +++++++++++++++ .github/workflows/lint_benchmarks.yml | 21 ------ .github/workflows/main.yml | 4 +- .github/workflows/test_benchmarks.yml | 95 --------------------------- 4 files changed, 54 insertions(+), 118 deletions(-) create mode 100644 .github/workflows/clean_template.yml delete mode 100644 .github/workflows/lint_benchmarks.yml delete mode 100644 .github/workflows/test_benchmarks.yml diff --git a/.github/workflows/clean_template.yml b/.github/workflows/clean_template.yml new file mode 100644 index 0000000..4a3c214 --- /dev/null +++ b/.github/workflows/clean_template.yml @@ -0,0 +1,52 @@ +name: Benchmark creation + +# This step triggers after a user creates a new repository from the template +# It replaces the placeholder with the correct names and +# cleans up template scripts + +# This will run every time we create push a commit to `main` +# Reference https://docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows +on: + workflow_dispatch: + push: + branches: + - main + +permissions: + # Need `contents: read` to checkout the repository + # Need `contents: write` to update the step metadata + contents: write + +jobs: + clean_template: + name: Clean up template + runs-on: ubuntu-latest + # Only run this action when this repository isn't the template repo + if: ${{ !github.event.repository.is_template}} + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + - name: Clean template scripts + run: | + python clean_template.py + + # Make a branch, file, commit, and pull request for the learner + - name: Commit clean up on the repository + run: | + echo "Configure git" + git config user.name github-actions[bot] + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + echo "Commit all changes" + git commit -am "CLN remove template scripts" + + echo "Push to main" + git push + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/lint_benchmarks.yml b/.github/workflows/lint_benchmarks.yml deleted file mode 100644 index 3fe367c..0000000 --- a/.github/workflows/lint_benchmarks.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Lint - -on: - workflow_call: - - -jobs: - linter-flake8: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Lint with flake8 - run: | - pip install flake8 - flake8 . diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f2a61a1..c2fe03a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,8 +11,8 @@ on: branches: - main schedule: - # Run every day at 7:42am UTC. - - cron: '42 7 * * *' + # Run every 1st of the month at 7:42am UTC. + - cron: '42 7 1 * *' jobs: benchopt_dev: diff --git a/.github/workflows/test_benchmarks.yml b/.github/workflows/test_benchmarks.yml deleted file mode 100644 index 0c54c70..0000000 --- a/.github/workflows/test_benchmarks.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Test Benchmark - -on: - workflow_call: - inputs: - benchopt_branch: - description: Branch of benchopt to install to test the benchmark. - default: benchopt@main - required: false - type: string - benchopt_version: - description: | - If set, use a specific version of benchopt for the tests, - thus ignoring the benchopt_branch input. - default: git - required: false - type: string - -jobs: - test-benchmark: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ['ubuntu-latest', 'macos-latest'] - exclude: - # Only run OSX test on version==git, not on the release ones. - - os: ${{ inputs.benchopt_version == 'git' || 'macos-latest' }} - env: - CONDA_ENV: 'test_env' - BENCHOPT_BRANCH: ${{ inputs.benchopt_branch }} - BENCHOPT_VERSION: ${{ inputs.benchopt_version }} - BENCHOPT_DEBUG: 1 - BENCHOPT_CONDA_CMD: mamba - - defaults: - run: - # Need to use this shell to get conda working properly. - # See https://github.com/marketplace/actions/setup-miniconda#important - shell: bash -l {0} - - steps: - - uses: actions/checkout@v2 - - name: Setup Conda - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniforge-variant: Mambaforge - use-mamba: true - channels: conda-forge - python-version: 3.9 - activate-environment: ${{ env.CONDA_ENV }} - - - run: conda info - - - name: Install benchopt and its dependencies - run: | - conda info - mamba install -yq pip - - # Get the correct branch of benchopt - if [[ "$BENCHOPT_VERSION" == "git" ]] - then - user=${BENCHOPT_BRANCH%@*} - branch=${BENCHOPT_BRANCH##*@} - pip install -U git+https://github.com/$user/benchopt@$branch - elif [[ "$BENCHOPT_VERSION" == "latest" ]] - then - pip install -U benchopt - else - pip install -U benchopt==$BENCHOPT_VERSION - fi - - name: Check if benchopt is compatible with this benchmark - id: check_min_version - run: | - min_version=$(grep -Po 'min_benchopt_version = \K.*' objective.py || echo "0.0") - if [[ "$BENCHOPT_VERSION" == "git" ]] - then - # Always test dev version - benchopt_version="99.0" - else - benchopt_version=$(benchopt --version) - fi - echo "$benchopt_version and $min_version" - if [[ "$benchopt_version" < "$min_version" ]] - then - echo "compatible=false" >> $GITHUB_OUTPUT - else - echo "compatible=true" >> $GITHUB_OUTPUT - fi - - - name: Test - if: ${{ steps.check_min_version.outputs.compatible == 'true' }} - run: | - benchopt test . --env-name bench_test_env -vl - benchopt test . --env-name bench_test_env -vl --skip-install From aeeaec93ef36d60852864393c857f7a03a597a92 Mon Sep 17 00:00:00 2001 From: tommoral Date: Wed, 14 Jan 2026 18:07:12 +0100 Subject: [PATCH 2/6] ENH remove safe_import_context --- datasets/simulated.py | 9 ++------- objective.py | 14 +++++--------- solvers/svm.py | 10 ++-------- 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/datasets/simulated.py b/datasets/simulated.py index 3873f91..b5226e3 100644 --- a/datasets/simulated.py +++ b/datasets/simulated.py @@ -1,11 +1,6 @@ -from benchopt import BaseDataset, safe_import_context +from benchopt import BaseDataset - -# Protect the import with `safe_import_context()`. This allows: -# - skipping import to speed up autocompletion in CLI. -# - getting requirements info when all dependencies are not installed. -with safe_import_context() as import_ctx: - from sklearn.datasets import make_classification +from sklearn.datasets import make_classification # All datasets must be named `Dataset` and inherit from `BaseDataset` diff --git a/objective.py b/objective.py index f7986b8..3d85663 100644 --- a/objective.py +++ b/objective.py @@ -1,13 +1,9 @@ from benchopt import BaseObjective, safe_import_context -# Protect the import with `safe_import_context()`. This allows: -# - skipping import to speed up autocompletion in CLI. -# - getting requirements info when all dependencies are not installed. -with safe_import_context() as import_ctx: - import numpy as np - from sklearn.model_selection import KFold - from sklearn.dummy import DummyClassifier - from sklearn.metrics import accuracy_score +import numpy as np +from sklearn.model_selection import KFold +from sklearn.dummy import DummyClassifier +from sklearn.metrics import accuracy_score # The benchmark objective must be named `Objective` and # inherit from `BaseObjective` for `benchopt` to work properly. @@ -38,7 +34,7 @@ class Objective(BaseObjective): # Minimal version of benchopt required to run this benchmark. # Bump it up if the benchmark depends on a new feature of benchopt. - min_benchopt_version = "1.6" + min_benchopt_version = "1.8" def set_data(self, X, y): # The keyword arguments of this function are the keys of the dictionary diff --git a/solvers/svm.py b/solvers/svm.py index 301f045..5e8a45b 100644 --- a/solvers/svm.py +++ b/solvers/svm.py @@ -1,12 +1,6 @@ -from benchopt import BaseSolver, safe_import_context +from benchopt import BaseSolver -# Protect the import with `safe_import_context()`. This allows: -# - skipping import to speed up autocompletion in CLI. -# - getting requirements info when all dependencies are not installed. -with safe_import_context() as import_ctx: - - # import your reusable functions here - from sklearn.svm import SVC +from sklearn.svm import SVC # The benchmark solvers must be named `Solver` and From 9135574f89194238cd6eaf6f55684abe2f7791a4 Mon Sep 17 00:00:00 2001 From: tommoral Date: Wed, 14 Jan 2026 18:36:24 +0100 Subject: [PATCH 3/6] CLN improve instruction for ML benchmark --- datasets/simulated.py | 10 +++++++--- objective.py | 31 +++++++++++++++++++++---------- solvers/svm.py | 13 +++++++------ 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/datasets/simulated.py b/datasets/simulated.py index b5226e3..92179c4 100644 --- a/datasets/simulated.py +++ b/datasets/simulated.py @@ -29,12 +29,16 @@ def get_data(self): # to `Objective.set_data`. This defines the benchmark's # API to pass data. It is customizable for each benchmark. # - # Data splitting is handled by the `Objective.get_objective` method and `Objective.cv` property + # Data splitting is handled by the `Objective.get_objective` method + # and `Objective.cv` property # Generate pseudorandom data using `sklearn` for classification. # Generating synthetic dataset - X, y = make_classification(n_samples=self.n_samples, n_features=self.n_features, n_informative=1, - n_redundant=0, n_clusters_per_class=1, random_state=self.random_state) + X, y = make_classification( + n_samples=self.n_samples, n_features=self.n_features, + n_informative=1, n_redundant=0, n_clusters_per_class=1, + random_state=self.random_state + ) # The dictionary defines the keyword arguments for `Objective.set_data` return dict(X=X, y=y) diff --git a/objective.py b/objective.py index 3d85663..eec1d23 100644 --- a/objective.py +++ b/objective.py @@ -1,9 +1,8 @@ -from benchopt import BaseObjective, safe_import_context +from benchopt import BaseObjective -import numpy as np from sklearn.model_selection import KFold from sklearn.dummy import DummyClassifier -from sklearn.metrics import accuracy_score + # The benchmark objective must be named `Objective` and # inherit from `BaseObjective` for `benchopt` to work properly. @@ -36,6 +35,11 @@ class Objective(BaseObjective): # Bump it up if the benchmark depends on a new feature of benchopt. min_benchopt_version = "1.8" + # Disable performance curves - each solver runs once to completion + # See https://benchopt.github.io/stable/user_guide/performance_curves.html + # for more details. + sampling_strategy = "run_once" + def set_data(self, X, y): # The keyword arguments of this function are the keys of the dictionary # returned by `Dataset.get_data`. This defines the benchmark's @@ -45,7 +49,9 @@ def set_data(self, X, y): # Specify a cross-validation splitter as the `cv` attribute. # This will be automatically used in `self.get_split` to split # the arrays provided. - self.cv = KFold(n_splits=5, shuffle=True, random_state=self.random_state) + self.cv = KFold( + n_splits=5, shuffle=True, random_state=self.random_state + ) # If the cross-validation requires some metadata, it can be # provided in the `cv_metadata` attribute. This will be passed @@ -57,13 +63,16 @@ def evaluate_result(self, model): # dictionary returned by `Solver.get_result`. This defines the # benchmark's API to pass the solvers' result. This can be # customized for each benchmark. - y_pred = model.predict(self.X_test) - accuracy = accuracy_score(self.y_test, y_pred) + # + # Here, the solver returns a trained model, + # with which we can call ``score`` to get the accurcay. + accuracy_train = model.score(self.X_train, self.y_train) + accuracy_test = model.score(self.X_test, self.y_test) - # This method can return many metrics in a dictionary. One of these - # metrics needs to be `value` for convergence detection purposes. + # This method can return many metrics in a dictionary. return dict( - value=accuracy, + accuracy_test=accuracy_test, + accuracy_train=accuracy_train, ) def get_one_result(self): @@ -80,7 +89,9 @@ def get_objective(self): # benchmark's API for passing the objective to the solver. # This can be customized in each benchmark. - self.X_train, self.X_test, self.y_train, self.y_test = self.get_split(self.X, self.y) + self.X_train, self.X_test, self.y_train, self.y_test = self.get_split( + self.X, self.y + ) return dict( X_train=self.X_train, diff --git a/solvers/svm.py b/solvers/svm.py index 5e8a45b..06df367 100644 --- a/solvers/svm.py +++ b/solvers/svm.py @@ -23,9 +23,6 @@ class Solver(BaseSolver): # so no need to add it again. requirements = [] - # Force the solver to run only once if you don't want to record training steps - sampling_strategy = "run_once" - def set_objective(self, X_train, y_train): # Define the information received by each solver from the objective. # The arguments of this function are the results of the @@ -36,10 +33,14 @@ def set_objective(self, X_train, y_train): self.clf = SVC(kernel=self.kernel) def run(self, _): + """Run the solver. + + Parameters + ---------- + _ : ignored + With sampling_strategy="run_once", this parameter is unused. + """ # This is the method that is called to fit the model. - # The input param is only defined if you change the sampling strategy - # to value different than "run_once". - # See https://benchopt.github.io/performance_curves.html self.clf.fit(self.X_train, self.y_train) def get_result(self): From f193edf652e646f3a3217a06e2a78be4c9c9d63e Mon Sep 17 00:00:00 2001 From: tommoral Date: Wed, 14 Jan 2026 19:11:00 +0100 Subject: [PATCH 4/6] CI trigger From 2c9515dadde9d4953f0715f15a28088b1db32c04 Mon Sep 17 00:00:00 2001 From: tommoral Date: Thu, 15 Jan 2026 01:41:09 +0100 Subject: [PATCH 5/6] CI trigger From 0b2cbe6d002b2433a5de46c5842f308c26096e62 Mon Sep 17 00:00:00 2001 From: tommoral Date: Thu, 15 Jan 2026 09:06:17 +0100 Subject: [PATCH 6/6] CI trigger