Skip to content

Commit 167672f

Browse files
NicorettiNicoretti
authored
Add support for dynamic build matrix generation (#313)
1 parent 1a77d9f commit 167672f

File tree

17 files changed

+337
-37
lines changed

17 files changed

+337
-37
lines changed

‎.github/workflows/checks.yml‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,17 @@ jobs:
4040
run: |
4141
poetry run python -m nox -s docs:build
4242
43+
build-matrix:
44+
name: Generate Build Matrix
45+
uses: ./.github/workflows/matrix-python.yml
46+
4347
Lint:
4448
name: Linting (Python-${{ matrix.python-version }})
45-
needs: [ Version-Check ]
49+
needs: [ Version-Check, build-matrix ]
4650
runs-on: ubuntu-latest
4751
strategy:
4852
fail-fast: false
49-
matrix:
50-
python-version: [ "3.9", "3.10", "3.11", "3.12" ]
53+
matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }}
5154

5255
steps:
5356
- name: SCM Checkout
@@ -70,12 +73,11 @@ jobs:
7073

7174
Type-Check:
7275
name: Type Checking (Python-${{ matrix.python-version }})
73-
needs: [ Version-Check ]
76+
needs: [ Version-Check, build-matrix ]
7477
runs-on: ubuntu-latest
7578
strategy:
7679
fail-fast: false
77-
matrix:
78-
python-version: [ "3.9", "3.10", "3.11", "3.12" ]
80+
matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }}
7981

8082
steps:
8183
- name: SCM Checkout
@@ -91,12 +93,11 @@ jobs:
9193

9294
Security:
9395
name: Security Checks (Python-${{ matrix.python-version }})
94-
needs: [ Version-Check ]
96+
needs: [ Version-Check, build-matrix ]
9597
runs-on: ubuntu-latest
9698
strategy:
9799
fail-fast: false
98-
matrix:
99-
python-version: [ "3.9", "3.10", "3.11", "3.12" ]
100+
matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }}
100101

101102
steps:
102103
- name: SCM Checkout
@@ -134,16 +135,14 @@ jobs:
134135
run: poetry run nox -s project:format
135136

136137
Tests:
137-
name: Unit-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}})
138-
needs: [ Documentation, Lint, Type-Check, Security, Format]
138+
name: Unit-Tests (Python-${{ matrix.python-version }})
139+
needs: [ Documentation, Lint, Type-Check, Security, Format, build-matrix ]
139140
runs-on: ubuntu-latest
140141
env:
141142
GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
142143
strategy:
143144
fail-fast: false
144-
matrix:
145-
python-version: [ "3.9", "3.10", "3.11", "3.12" ]
146-
exasol-version: [ "7.1.9" ]
145+
matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }}
147146

148147
steps:
149148
- name: SCM Checkout
@@ -155,7 +154,7 @@ jobs:
155154
python-version: ${{ matrix.python-version }}
156155

157156
- name: Run Tests and Collect Coverage
158-
run: poetry run nox -s test:unit -- -- --coverage --db-version ${{ matrix.exasol-version }}
157+
run: poetry run nox -s test:unit -- -- --coverage
159158

160159
- name: Upload Artifacts
161160
uses: actions/upload-artifact@v4.4.0

‎.github/workflows/matrix-all.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Build Matrix (All Versions)
2+
3+
on:
4+
workflow_call:
5+
outputs:
6+
matrix:
7+
description: "Generates the all versions build matrix"
8+
value: ${{ jobs.all_versions.outputs.matrix }}
9+
10+
jobs:
11+
all_versions:
12+
13+
runs-on: ubuntu-latest
14+
15+
steps:
16+
- name: SCM Checkout
17+
uses: actions/checkout@v4
18+
19+
- name: Setup Python & Poetry Environment
20+
uses: ./.github/actions/python-environment
21+
22+
- name: Generate matrix
23+
run: poetry run nox -s matrix:all
24+
25+
- id: set-matrix
26+
run: |
27+
echo "matrix=$(poetry run nox -s matrix:all)" >> $GITHUB_OUTPUT
28+
29+
outputs:
30+
matrix: ${{ steps.set-matrix.outputs.matrix }}

‎.github/workflows/matrix-exasol.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Build Matrix (Exasol)
2+
3+
on:
4+
workflow_call:
5+
outputs:
6+
matrix:
7+
description: "Generates the exasol version build matrix"
8+
value: ${{ jobs.exasol_versions.outputs.matrix }}
9+
10+
jobs:
11+
exasol_versions:
12+
13+
runs-on: ubuntu-latest
14+
15+
steps:
16+
- name: SCM Checkout
17+
uses: actions/checkout@v4
18+
19+
- name: Setup Python & Poetry Environment
20+
uses: ./.github/actions/python-environment
21+
22+
- name: Generate matrix
23+
run: poetry run nox -s matrix:exasol
24+
25+
- id: set-matrix
26+
run: |
27+
echo "matrix=$(poetry run nox -s matrix:exasol)" >> $GITHUB_OUTPUT
28+
29+
outputs:
30+
matrix: ${{ steps.set-matrix.outputs.matrix }}

‎.github/workflows/matrix-python.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Build Matrix (Python)
2+
3+
on:
4+
workflow_call:
5+
outputs:
6+
matrix:
7+
description: "Generates the python version build matrix"
8+
value: ${{ jobs.python_versions.outputs.matrix }}
9+
10+
jobs:
11+
python_versions:
12+
13+
runs-on: ubuntu-latest
14+
15+
steps:
16+
- name: SCM Checkout
17+
uses: actions/checkout@v4
18+
19+
- name: Setup Python & Poetry Environment
20+
uses: ./.github/actions/python-environment
21+
22+
- name: Generate matrix
23+
run: poetry run nox -s matrix:python
24+
25+
- id: set-matrix
26+
run: |
27+
echo "matrix=$(poetry run nox -s matrix:python)" >> $GITHUB_OUTPUT
28+
29+
outputs:
30+
matrix: ${{ steps.set-matrix.outputs.matrix }}

‎.github/workflows/slow-checks.yml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,13 @@ on:
88

99
jobs:
1010

11+
build-matrix:
12+
name: Generate Build Matrix
13+
uses: ./.github/workflows/matrix-all.yml
14+
1115
Tests:
1216
name: Integration-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}})
17+
needs: [ build-matrix ]
1318
runs-on: ubuntu-latest
1419
# Even though the environment "manual-approval" will be created automatically,
1520
# it still needs to be configured to require interactive review.
@@ -19,9 +24,7 @@ jobs:
1924
GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
2025
strategy:
2126
fail-fast: false
22-
matrix:
23-
python-version: [ "3.9", "3.10", "3.11", "3.12" ]
24-
exasol-version: [ "7.1.9" ]
27+
matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }}
2528

2629
steps:
2730
- name: SCM Checkout

‎doc/changes/unreleased.md‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,42 @@
11
# Unreleased
22

3+
## 🚨 Breaking Changes
4+
* **Matrices in CI/CD workflows will be generated automatically now**
5+
6+
Make sure you have installed all the latest workflow files, especially the newly added ones:
7+
8+
- `matrix-all.yml`
9+
- `matrix-python.yml`
10+
- `matrix-exasol.yml`
11+
12+
13+
## ✨ Added
14+
* Added support for dynamically generated workflow matrices.
15+
16+
This feature allows you to easily change the test matrices in one place: `noxconfig.py`.
17+
18+
Note: As usual, there are different ways a user can adjust or change the behavior. In the case of the build matrices, there are three obvious ways:
19+
20+
- Set the appropriate fields in the `noxconfig.py` project configuration (`PROJECT_CONFIG`):
21+
* `python_versions = [ ... ]`
22+
* `exasol_versions = [ ... ]`
23+
- Overwrite the nox tasks:
24+
* `matrix:all`
25+
* `matrix:python`
26+
* `matrix:exasol`
27+
- Overwrite/replace the matrix generation workflows:
28+
* `matrix-all.yml`
29+
* `matrix-python.yml`
30+
* `matrix-exasol.yml`
31+
32+
Among all of the above, the safest way is to set the matrix-related fields in your project config object in `noxconfig.py`.
33+
34+
335
## 📚 Documentation
436

537
* Added new entries to the frequently asked questions regarding `multiversion documentation`
638

39+
740
## 🐞 Fixed
841

942
* Added multi-version extension to Sphinx configuration of the project template

‎exasol/toolbox/nox/_ci.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import json
2+
import logging
3+
4+
import nox
5+
from nox import Session
6+
7+
from noxconfig import (
8+
PROJECT_CONFIG,
9+
Config,
10+
)
11+
12+
_log = logging.getLogger(__name__)
13+
14+
_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12"]
15+
_EXASOL_VERSIONS = ["7.1.9"]
16+
17+
18+
def _python_matrix(config: Config):
19+
attr = "python_versions"
20+
python_versions = getattr(config, attr, _PYTHON_VERSIONS)
21+
if not hasattr(config, attr):
22+
_log.warning(
23+
"Config does not contain '%s' setting. Using default: %s",
24+
attr,
25+
_PYTHON_VERSIONS,
26+
)
27+
return {"python-version": python_versions}
28+
29+
30+
def _exasol_matrix(config: Config):
31+
attr = "exasol_versions"
32+
exasol_versions = getattr(config, attr, _EXASOL_VERSIONS)
33+
if not hasattr(config, attr):
34+
_log.warning(
35+
"Config does not contain '%s' setting. Using default: %s",
36+
attr,
37+
_EXASOL_VERSIONS,
38+
)
39+
return {"exasol-version": exasol_versions}
40+
41+
42+
@nox.session(name="matrix:python", python=False)
43+
def python_matrix(session: Session) -> None:
44+
"""Output the build matrix for Python versions as JSON."""
45+
print(json.dumps(_python_matrix(PROJECT_CONFIG)))
46+
47+
48+
@nox.session(name="matrix:exasol", python=False)
49+
def exasol_matrix(session: Session) -> None:
50+
"""Output the build matrix for Exasol versions as JSON."""
51+
print(json.dumps(_exasol_matrix(PROJECT_CONFIG)))
52+
53+
54+
@nox.session(name="matrix:all", python=False)
55+
def full_matrix(session: Session) -> None:
56+
"""Output the full build matrix for Python & Exasol versions as JSON."""
57+
matrix = _python_matrix(PROJECT_CONFIG)
58+
matrix.update(_exasol_matrix(PROJECT_CONFIG))
59+
print(json.dumps(matrix))

‎exasol/toolbox/nox/tasks.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,14 +59,20 @@ def check(session: Session) -> None:
5959
clean_docs,
6060
open_docs,
6161
)
62-
from exasol.toolbox.nox._release import prepare_release
6362
from exasol.toolbox.nox._shared import (
6463
Mode,
6564
_context,
6665
_version,
6766
python_files,
6867
)
6968

69+
from exasol.toolbox.nox._ci import (
70+
python_matrix,
71+
exasol_matrix,
72+
full_matrix,
73+
)
74+
75+
from exasol.toolbox.nox._release import prepare_release
7076

7177
# isort: on
7278
# fmt: on

0 commit comments

Comments
 (0)