Feature/#426 (#457)

* #426: Allowed configuring the python version used for coverage
* Updated workflows and templates
* Added user guide
* Moved MINIMUM_PYTHON_VERSION to _shared.py

Co-authored-by: Ariel Schulz <43442541+ArBridgeman@users.noreply.github.com>

Author: ckunki

.github/workflows/report.yml

@@ -27,14 +27,7 @@ jobs:
           path: ./artifacts
 
       - name: Copy Artifacts into Root Folder
-        working-directory: ./artifacts
-        run: |
-          poetry run -- coverage combine --keep coverage-python3.9*/.coverage
-          # Errors during copying are ignored because they are checked in the next step
-          cp .coverage ../ || true
-          cp lint-python3.9/.lint.txt ../ || true
-          cp lint-python3.9/.lint.json ../ || true
-          cp security-python3.9/.security.json ../ || true
+        run: poetry run -- nox -s artifacts:copy -- artifacts
 
       - name: Validate Artifacts
         run: poetry run -- nox -s artifacts:validate

doc/changes/unreleased.md

@@ -1 +1,7 @@
 # Unreleased
+
+## Summary
+
+## ✨ Features
+
+* #426: Allowed configuring the python version used for coverage

doc/user_guide/collecting_metrics.rst

@@ -0,0 +1,35 @@
+Collecting Metrics
+==================
+
+PTB allows you to collect various metrics on the quality of your project
+regarding Coverage, Security, and Static Code Analysis.
+
+For each metric, there is a dedicated nox task, generating one or multiple
+files and based on a selected external Python tool.
+
++-----------------------------+-----------------------------+--------------+
+| Nox Task                    | Generated Files             | Based on     |
++=============================+=============================+==============+
+| ``lint:code``               | ``lint.txt``, ``lint.json`` | ``pylint``   |
++-----------------------------+-----------------------------+--------------+
+| ``lint:security``           | ``.security.json``          | ``bandit``   |
++-----------------------------+-----------------------------+--------------+
+| ``test:unit -- --coverage`` | ``.coverage``               | ``coverage`` |
++-----------------------------+-----------------------------+--------------+
+
+The metrics are computed for each point in your build matrix, e.g. for each
+Python version defined in file ``noxconfig.py``:
+
+.. code-block:: python
+
+    @dataclass(frozen=True)
+    class Config:
+        python_versions = ["3.9", "3.10", "3.11", "3.12", "3.13"]
+
+The GitHub workflows of your project can:
+
+* Use a build matrix, e.g. using different Python versions as shown above
+* Define multiple test sessions, e.g. for distinguishing fast vs. slow or expensive tests.
+
+PTB combines the coverage data of all test sessions but using only the Python
+version named first in attribute ``python_versions`` of class ``Config``.

doc/user_guide/user_guide.rst

@@ -12,3 +12,4 @@
     customization
     migrating
     how_to_release
+    collecting_metrics

exasol/toolbox/nox/_artifacts.py

@@ -1,13 +1,16 @@
 import json
 import pathlib
 import re
+import shutil
 import sqlite3
 import sys
+from collections.abc import Iterable
 from pathlib import Path
 
 import nox
 from nox import Session
 
+from exasol.toolbox.nox._shared import MINIMUM_PYTHON_VERSION
 from noxconfig import PROJECT_CONFIG
 
 
@@ -118,3 +121,49 @@ def _validate_coverage(path: Path) -> str:
             f"Invalid database, the database is missing the following tables {missing}"
         )
     return ""
+
+
+@nox.session(name="artifacts:copy", python=False)
+def copy_artifacts(session: Session) -> None:
+    """
+    Copy artifacts to the current directory
+    """
+
+    dir = Path(session.posargs[0])
+    suffix = _python_version_suffix()
+    _combine_coverage(session, dir, f"coverage{suffix}*/.coverage")
+    _copy_artifacts(
+        dir,
+        dir.parent,
+        [
+            f"lint{suffix}/.lint.txt",
+            f"lint{suffix}/.lint.json",
+            f"security{suffix}/.security.json",
+        ],
+    )
+
+
+def _python_version_suffix() -> str:
+    versions = getattr(PROJECT_CONFIG, "python_versions", None)
+    pivot = versions[0] if versions else MINIMUM_PYTHON_VERSION
+    return f"-python{pivot}"
+
+
+def _combine_coverage(session: Session, dir: Path, pattern: str):
+    """
+    pattern: glob pattern, e.g. "*.coverage"
+    """
+    if args := [f for f in dir.glob(pattern) if f.exists()]:
+        session.run("coverage", "combine", "--keep", *sorted(args))
+    else:
+        print(f"Could not find any file {dir}/{pattern}", file=sys.stderr)
+
+
+def _copy_artifacts(source: Path, dest: Path, files: Iterable[str]):
+    for file in files:
+        path = source / file
+        if path.exists():
+            print(f"Copying file {path}", file=sys.stderr)
+            shutil.copy(path, dest)
+        else:
+            print(f"File not found {path}", file=sys.stderr)

exasol/toolbox/nox/_shared.py

@@ -20,6 +20,8 @@
 DEFAULT_PATH_FILTERS = {"dist", ".eggs", "venv", ".poetry"}
 DOCS_OUTPUT_DIR = ".html-documentation"
 
+MINIMUM_PYTHON_VERSION = "3.9"
+
 
 class Mode(Enum):
     Fix = auto()

exasol/toolbox/templates/github/workflows/report.yml

@@ -27,14 +27,7 @@ jobs:
           path: ./artifacts
 
       - name: Copy Artifacts into Root Folder
-        working-directory: ./artifacts
-        run: |
-          poetry run -- coverage combine --keep coverage-python3.9*/.coverage
-          # Errors during copying are ignored because they are checked in the next step
-          cp .coverage ../ || true
-          cp lint-python3.9/.lint.txt ../ || true
-          cp lint-python3.9/.lint.json ../ || true
-          cp security-python3.9/.security.json ../ || true
+        run: poetry run -- nox -s artifacts:copy -- artifacts
 
       - name: Validate Artifacts
         run: poetry run -- nox -s artifacts:validate

test/unit/artifacts_test.py

@@ -0,0 +1,89 @@
+import contextlib
+import re
+from dataclasses import dataclass
+from inspect import cleandoc
+from pathlib import Path
+from unittest.mock import (
+    Mock,
+    call,
+    patch,
+)
+
+import pytest
+
+from exasol.toolbox.nox._artifacts import copy_artifacts
+
+
+@contextlib.contextmanager
+def mock_session(path: Path, python_version: str, *files: str):
+    with patch("exasol.toolbox.nox._artifacts.PROJECT_CONFIG") as config:
+        config.python_versions = [python_version]
+        for rel in files:
+            file = path / rel
+            file.parent.mkdir(parents=True, exist_ok=True)
+            file.write_text(rel)
+        yield Mock(posargs=[str(path)])
+
+
+def test_missing_files(tmp_path, capsys):
+    with mock_session(tmp_path, "9.9") as session:
+        copy_artifacts(session)
+    captured = capsys.readouterr()
+    assert re.match(
+        cleandoc(
+            f"""
+            Could not find any file .*/coverage-python9.9\\*/.coverage
+            File not found .*/lint-python9.9/.lint.txt
+            File not found .*/lint-python9.9/.lint.json
+            File not found .*/security-python9.9/.security.json
+            """
+        ),
+        captured.err,
+    )
+
+
+@dataclass
+class endswith:
+    """
+    Assert that the str representation of the argument ends with the
+    specified suffix.
+    """
+
+    suffix: str
+
+    def __eq__(self, actual):
+        return str(actual).endswith(self.suffix)
+
+
+def test_all_files(tmp_path, capsys):
+    with mock_session(
+        tmp_path / "artifacts",
+        "9.9",
+        "coverage-python9.9-fast/.coverage",
+        "coverage-python9.9-slow/.coverage",
+        "lint-python9.9/.lint.txt",
+        "lint-python9.9/.lint.json",
+        "security-python9.9/.security.json",
+    ) as session:
+        copy_artifacts(session)
+
+    captured = capsys.readouterr()
+    assert session.run.call_args == call(
+        "coverage",
+        "combine",
+        "--keep",
+        endswith("coverage-python9.9-fast/.coverage"),
+        endswith("coverage-python9.9-slow/.coverage"),
+    )
+    assert re.match(
+        cleandoc(
+            f"""
+            Copying file .*/lint-python9.9/.lint.txt
+            Copying file .*/lint-python9.9/.lint.json
+            Copying file .*/security-python9.9/.security.json
+            """
+        ),
+        captured.err,
+    )
+    for f in [".lint.txt", ".lint.json", ".security.json"]:
+        assert (tmp_path / f).exists()

test/unit/nox/_package_version_test.py

@@ -8,9 +8,7 @@
     write_version_module,
 )
 from exasol.toolbox.util.version import Version
-from noxconfig import (
-    Config,
-)
+from noxconfig import Config
 
 DEFAULT_VERSION = Version(major=0, minor=1, patch=0)
 ALTERNATE_VERSION = Version(major=0, minor=2, patch=0)

test/unit/util/dependencies/licenses_test.py

@@ -7,9 +7,7 @@
     _packages_from_json,
     packages_to_markdown,
 )
-from exasol.toolbox.util.dependencies.poetry_dependencies import (
-    PoetryGroup,
-)
+from exasol.toolbox.util.dependencies.poetry_dependencies import PoetryGroup
 from exasol.toolbox.util.dependencies.shared_models import Package
 
 MAIN_GROUP = PoetryGroup(name="main", toml_section="project.dependencies")