Feature/382 clean up for nox task dependency changes (#446)

* Remove `tool.poetry.dev.dependencies` from nox task `dependency:licenses` as not poetry 2.x+ compatible

* Get groups for direct dependencies from pyproject.toml

* Extract all dependencies from poetry show to divide into direct & transitive

* Use updated pydantic models in license code

* Split up licenses and shared model to appropriate test and files

* Move some of the mappings to be used in the pydantic model

* Add pydantic to main dependencies

* Update to pyupgrades to 3.9 but skip typing changes as incorrectly putting Optional[str], etc. to str | None for python3.9 and fix linting issues

Author: ArBridgeman

doc/changes/unreleased.md

@@ -5,9 +5,21 @@
 With #441, please ensure that the location of the `version.py` is given for `Config.version_file`,
 which is specified in the `noxconfig.py`
 
+With #449, it's possible to customize what arguments are being using with `pyupgrade`
+via the `noxconfig.Config`:
+```python
+pyupgrade_args = ("--py310-plus",)
+```
+
 ## 📚 Documentation
 * Updated getting_started.rst for allowing tag-based releases
 
 ## ✨ Features
 
-* [#441](https://github.com/exasol/python-toolbox/issues/441): Switched nox task for `version:check` to use the config value of `version_file` to specify the location of the `version.py`
+* [#441](https://github.com/exasol/python-toolbox/issues/441): Switched nox task for `version:check` to use the config value of `version_file` to specify the location of the `version.py`
+
+## ⚒️ Refactorings
+
+* [#449](https://github.com/exasol/python-toolbox/issues/449): Refactored `dependency:licenses`:
+    * to use pydantic models & `poetry show` for dependencies
+    * to updated reading the `pyproject.toml` to be compatible with poetry 2.x+

exasol/toolbox/nox/_dependencies.py

@@ -3,214 +3,19 @@
 import argparse
 import json
 import subprocess
-import tempfile
-from dataclasses import dataclass
-from inspect import cleandoc
-from json import loads
 from pathlib import Path
 
 import nox
-import tomlkit
 from nox import Session
 
-
-@dataclass(frozen=True)
-class Package:
-    name: str
-    package_link: str
-    version: str
-    license: str
-    license_link: str
-
-
-def _dependencies(toml_str: str) -> dict[str, list]:
-    toml = tomlkit.loads(toml_str)
-    poetry = toml.get("tool", {}).get("poetry", {})
-    dependencies: dict[str, list] = {}
-
-    packages = poetry.get("dependencies", {})
-    if packages:
-        dependencies["project"] = []
-    for package in packages:
-        dependencies["project"].append(package)
-
-    packages = poetry.get("dev", {}).get("dependencies", {})
-    if packages:
-        dependencies["dev"] = []
-    for package in packages:
-        dependencies["dev"].append(package)
-
-    groups = poetry.get("group", {})
-    for group in groups:
-        packages = groups.get(group, {}).get("dependencies")
-        if packages and not dependencies.get(group, {}):
-            dependencies[group] = []
-        for package in packages:
-            dependencies[group].append(package)
-    return dependencies
-
-
-def _normalize(_license: str) -> str:
-    def is_multi_license(l):
-        return ";" in l
-
-    def select_most_restrictive(licenses: list) -> str:
-        _max = 0
-        lic = "Unknown"
-        _mapping = {
-            "Unknown": -1,
-            "Unlicensed": 0,
-            "BSD": 1,
-            "MIT": 2,
-            "MPLv2": 3,
-            "LGPLv2": 4,
-            "GPLv2": 5,
-            "GPLv3": 6,
-        }
-        for l in licenses:
-            if l in _mapping:
-                if _mapping[l] > _mapping[lic]:
-                    lic = l
-            else:
-                return "<br>".join(licenses)
-        return lic
-
-    mapping = {
-        "BSD License": "BSD",
-        "MIT License": "MIT",
-        "The Unlicensed (Unlicensed)": "Unlicensed",
-        "Mozilla Public License 2.0 (MPL 2.0)": "MPLv2",
-        "GNU General Public License (GPL)": "GPL",
-        "GNU Lesser General Public License v2 (LGPLv2)": "LGPLv2",
-        "GNU General Public License v2 (GPLv2)": "GPLv2",
-        "GNU General Public License v2 or later (GPLv2+)": "GPLv2+",
-        "GNU General Public License v3 (GPLv3)": "GPLv3",
-        "Apache Software License": "Apache",
-    }
-
-    if is_multi_license(_license):
-        items = []
-        for item in _license.split(";"):
-            item = str(item).strip()
-            if item in mapping:
-                items.append(mapping[item])
-            else:
-                items.append(item)
-        return select_most_restrictive(items)
-
-    if _license not in mapping:
-        return _license
-
-    return mapping[_license]
-
-
-def _packages_from_json(json: str) -> list[Package]:
-    packages = loads(json)
-    packages_list = []
-    mapping = {
-        "GPLv1": "https://www.gnu.org/licenses/old-licenses/gpl-1.0.html",
-        "GPLv2": "https://www.gnu.org/licenses/old-licenses/gpl-2.0.html",
-        "LGPLv2": "https://www.gnu.org/licenses/old-licenses/lgpl-2.0.html",
-        "GPLv3": "https://www.gnu.org/licenses/gpl-3.0.html",
-        "LGPLv3": "https://www.gnu.org/licenses/lgpl-3.0.html",
-        "Apache": "https://www.apache.org/licenses/LICENSE-2.0",
-        "MIT": "https://mit-license.org/",
-        "BSD": "https://opensource.org/license/bsd-3-clause",
-    }
-    for package in packages:
-        package_license = _normalize(package["License"])
-        packages_list.append(
-            Package(
-                name=package["Name"],
-                package_link="" if package["URL"] == "UNKNOWN" else package["URL"],
-                version=package["Version"],
-                license=package_license,
-                license_link=(
-                    "" if package_license not in mapping else mapping[package_license]
-                ),
-            )
-        )
-    return packages_list
-
-
-def _licenses() -> list[Package]:
-    with tempfile.NamedTemporaryFile() as file:
-        subprocess.run(
-            [
-                "poetry",
-                "run",
-                "pip-licenses",
-                "--format=json",
-                "--output-file=" + file.name,
-                "--with-system",
-                "--with-urls",
-            ],
-            capture_output=True,
-        )
-        result = _packages_from_json(file.read().decode())
-    return result
-
-
-def _packages_to_markdown(
-    dependencies: dict[str, list], packages: list[Package]
-) -> str:
-    def heading():
-        text = "# Dependencies\n"
-        return text
-
-    def dependency(group: str, group_packages: list, packages: list[Package]) -> str:
-        def _header(_group: str):
-            _group = "".join([word.capitalize() for word in _group.strip().split()])
-            text = f"## {_group} Dependencies\n"
-            text += "|Package|version|Licence|\n"
-            text += "|---|---|---|\n"
-            return text
-
-        def _rows(_group_packages: list, _packages: list[Package]) -> str:
-            def _normalize_package_name(name: str) -> str:
-                _name = name.lower()
-                while "_" in _name:
-                    _name = _name.replace("_", "-")
-                return _name
-
-            text = ""
-            for package in _group_packages:
-                consistent = filter(
-                    lambda elem: (_normalize_package_name(elem.name) == package),
-                    _packages,
-                )
-                for content in consistent:
-                    if content.package_link:
-                        text += f"|[{content.name}]({content.package_link})"
-                    else:
-                        text += f"|{content.name}"
-                    text += f"|{content.version}"
-                    if content.license_link:
-                        text += f"|[{content.license}]({content.license_link})|\n"
-                    else:
-                        text += f"|{content.license}|\n"
-            text += "\n"
-            return text
-
-        _template = cleandoc(
-            """
-            {header}{rows}
-        """
-        )
-        return _template.format(
-            header=_header(group), rows=_rows(group_packages, packages)
-        )
-
-    template = cleandoc(
-        """
-        {heading}{rows}
-    """
-    )
-
-    rows = ""
-    for group in dependencies:
-        rows += dependency(group, dependencies[group], packages)
-    return template.format(heading=heading(), rows=rows)
+from exasol.toolbox.util.dependencies.licenses import (
+    licenses,
+    packages_to_markdown,
+)
+from exasol.toolbox.util.dependencies.poetry_dependencies import (
+    PoetryDependencies,
+    PoetryToml,
+)
 
 
 class Audit:
@@ -282,10 +87,13 @@ def run(self, session: Session) -> None:
 @nox.session(name="dependency:licenses", python=False)
 def dependency_licenses(session: Session) -> None:
     """returns the packages and their licenses"""
-    toml = Path("pyproject.toml")
-    dependencies = _dependencies(toml.read_text())
-    package_infos = _licenses()
-    print(_packages_to_markdown(dependencies=dependencies, packages=package_infos))
+    working_directory = Path()
+    poetry_dep = PoetryToml.load_from_toml(working_directory=working_directory)
+    dependencies = PoetryDependencies(
+        groups=poetry_dep.groups, working_directory=working_directory
+    ).direct_dependencies
+    package_infos = licenses()
+    print(packages_to_markdown(dependencies=dependencies, packages=package_infos))
 
 
 @nox.session(name="dependency:audit", python=False)

exasol/toolbox/nox/_format.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Iterable
+from collections.abc import Iterable
 
 import nox
 from nox import Session
@@ -10,7 +10,12 @@
     _version,
     python_files,
 )
-from noxconfig import PROJECT_CONFIG
+from noxconfig import (
+    PROJECT_CONFIG,
+    Config,
+)
+
+_PYUPGRADE_ARGS = ("--py39-plus",)
 
 
 def _code_format(session: Session, mode: Mode, files: Iterable[str]) -> None:
@@ -21,12 +26,11 @@ def command(*args: str) -> Iterable[str]:
     session.run(*command("black"), *files)
 
 
-def _pyupgrade(session: Session, files: Iterable[str]) -> None:
+def _pyupgrade(session: Session, config: Config, files: Iterable[str]) -> None:
+    pyupgrade_args = getattr(config, "pyupgrade_args", _PYUPGRADE_ARGS)
     session.run(
-        "poetry",
-        "run",
         "pyupgrade",
-        "--py38-plus",
+        *pyupgrade_args,
         "--exit-zero-even-if-changed",
         *files,
     )
@@ -37,7 +41,7 @@ def fix(session: Session) -> None:
     """Runs all automated fixes on the code base"""
     py_files = [f"{file}" for file in python_files(PROJECT_CONFIG.root)]
     _version(session, Mode.Fix)
-    _pyupgrade(session, py_files)
+    _pyupgrade(session, config=PROJECT_CONFIG, files=py_files)
     _code_format(session, Mode.Fix, py_files)
 
 

exasol/toolbox/nox/_lint.py

@@ -2,12 +2,8 @@
 
 import argparse
 import sys
+from collections.abc import Iterable
 from pathlib import Path
-from typing import (
-    Dict,
-    Iterable,
-    List,
-)
 
 import nox
 import rich.console
@@ -20,10 +16,6 @@
 
 def _pylint(session: Session, files: Iterable[str]) -> None:
     session.run(
-        "poetry",
-        "run",
-        "python",
-        "-m",
         "pylint",
         "--output-format",
         "colorized,json:.lint.json,text:.lint.txt",
@@ -33,8 +25,6 @@ def _pylint(session: Session, files: Iterable[str]) -> None:
 
 def _type_check(session: Session, files: Iterable[str]) -> None:
     session.run(
-        "poetry",
-        "run",
         "mypy",
         "--explicit-package-bases",
         "--namespace-packages",
@@ -49,8 +39,6 @@ def _type_check(session: Session, files: Iterable[str]) -> None:
 
 def _security_lint(session: Session, files: Iterable[str]) -> None:
     session.run(
-        "poetry",
-        "run",
         "bandit",
         "--severity-level",
         "low",
@@ -63,8 +51,6 @@ def _security_lint(session: Session, files: Iterable[str]) -> None:
         *files,
     )
     session.run(
-        "poetry",
-        "run",
         "bandit",
         "--severity-level",
         "low",

exasol/toolbox/nox/tasks.py

@@ -21,7 +21,6 @@
 
 from exasol.toolbox.nox._format import (
     _code_format,
-    _pyupgrade,
     fix,
 )
 

exasol/toolbox/tools/security.py

@@ -18,6 +18,7 @@
 from functools import partial
 from inspect import cleandoc
 from pathlib import Path
+from typing import Optional
 
 import typer
 
@@ -269,7 +270,9 @@ def as_markdown_listing(elements: Iterable[str]):
     )
 
 
-def create_security_issue(issue: Issue, project: str | None = None) -> tuple[str, str]:
+def create_security_issue(
+    issue: Issue, project: Optional[str] = None
+) -> tuple[str, str]:
     # fmt: off
     command = [
         "gh", "issue", "create",