Feature/382 update release prepare to add on dependency changes (#505)

* Remove pylint addition which is included already in lint:code

* Remove unused secrets.GITHUB_TOKEN from checks.yml

* Add tracking types for DependencyChange with tests

Co-authored-by: Christoph Kuhnke <christoph.kuhnke@exasol.com>

* Switch dependencies from list to dict of packages for O(1) retrieval

* Update licenses to work with dictionary and have shared special string type for better readability

* Switch to class for cleaner look than function in function

* Add dependency updates to changelog

* Ignore typing as if-elif safeguard None from getting to those lines

* Add changelog entry

* Pass on OrderedDict type

* Pass on OrderedDict type

* Restore order to licenses as dict keys to set loses order

* Fix indent and make date variable; could alternately add freeze gun or mock datetime

* Add types for track_changes.py

* Rename old to previous for dependencies

* Use group to be consistent with what we use in dependency changes and pyproject.toml

* Remove comment as not needed or useful

* Switch source_path to be correctly named root_path

* Add output types to methods

* Simplify output for _format_table_row

* Simplify output for _format_group_table_header

* Use normalized name from object instead of import

* Switch from usage of string with += to using append in a list and joining

* Switch from old_version to previous_version

* Switch from old_package to previous_package

* Switch from str to NormalizedPackageStr as that's what the dict keys are

* Switch Union[*, None] with Optional[*]

* Add comment to make clearer what the operation does

* Add on description of Changelogs arguments

* Improve terminology & adapt _describe_dependency_changes to be simpler

* Add deterministic sorting to versioned changes file

* Simplify _create_versioned_changelog and fix moved type

* Remove unneeded, second cleancode() usage

* Switch text from changes to contents

---------

Co-authored-by: Christoph Kuhnke <christoph.kuhnke@exasol.com>

Author: ArBridgeman

.github/workflows/checks.yml

@@ -168,8 +168,6 @@ jobs:
     runs-on: ubuntu-24.04
     permissions:
       contents: read
-    env:
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }}

doc/changes/unreleased.md

@@ -1,5 +1,9 @@
 # Unreleased
 
+## Feature
+
+* #382: Added onto nox session `release:prepare` to append dependency changes between current and latest tag
+
 ## Refactoring
 
 * #498: Centralized changelog code relevant for `release:trigger` & robustly tested

exasol/toolbox/nox/_artifacts.py

@@ -7,10 +7,7 @@
 import sys
 from collections.abc import Iterable
 from pathlib import Path
-from typing import (
-    Optional,
-    Union,
-)
+from typing import Optional
 
 import nox
 from nox import Session
@@ -188,7 +185,7 @@ def _copy_artifacts(source: Path, dest: Path, files: Iterable[str]):
 
 
 def _prepare_coverage_xml(
-    session: Session, source: Path, cwd: Union[Path, None] = None
+    session: Session, source: Path, cwd: Optional[Path] = None
 ) -> None:
     """
     Prepare the coverage XML for input into Sonar

exasol/toolbox/nox/_dependencies.py

@@ -9,8 +9,8 @@
 from nox import Session
 
 from exasol.toolbox.util.dependencies.licenses import (
-    licenses,
-    packages_to_markdown,
+    PackageLicenseReport,
+    get_licenses,
 )
 from exasol.toolbox.util.dependencies.poetry_dependencies import get_dependencies
 
@@ -85,8 +85,11 @@ def run(self, session: Session) -> None:
 def dependency_licenses(session: Session) -> None:
     """Return the packages with their licenses"""
     dependencies = get_dependencies(working_directory=Path())
-    package_infos = licenses()
-    print(packages_to_markdown(dependencies=dependencies, packages=package_infos))
+    licenses = get_licenses()
+    license_markdown = PackageLicenseReport(
+        dependencies=dependencies, licenses=licenses
+    )
+    print(license_markdown.to_markdown())
 
 
 @nox.session(name="dependency:audit", python=False)

exasol/toolbox/nox/_release.py

@@ -119,7 +119,9 @@ def prepare_release(session: Session) -> None:
     _ = _update_project_version(session, new_version)
 
     changelogs = Changelogs(
-        changes_path=PROJECT_CONFIG.doc / "changes", version=new_version
+        changes_path=PROJECT_CONFIG.doc / "changes",
+        root_path=PROJECT_CONFIG.root,
+        version=new_version,
     )
     changelogs.update_changelogs_for_release()
     changed_files = changelogs.get_changed_files()

exasol/toolbox/templates/github/workflows/checks.yml

@@ -166,8 +166,6 @@ jobs:
     runs-on: ubuntu-24.04
     permissions:
       contents: read
-    env:
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }}

exasol/toolbox/util/dependencies/licenses.py

@@ -2,13 +2,18 @@
 
 import subprocess
 import tempfile
+from collections import OrderedDict
+from dataclasses import dataclass
 from inspect import cleandoc
 from json import loads
 from typing import Optional
 
 from pydantic import field_validator
 
-from exasol.toolbox.util.dependencies.shared_models import Package
+from exasol.toolbox.util.dependencies.shared_models import (
+    NormalizedPackageStr,
+    Package,
+)
 
 LICENSE_MAPPING_TO_ABBREVIATION = {
     "BSD License": "BSD",
@@ -88,20 +93,23 @@ def select_most_restrictive(licenses: list[str]) -> str:
     return LICENSE_MAPPING_TO_ABBREVIATION.get(_license, _license)
 
 
-def _packages_from_json(json: str) -> list[PackageLicense]:
+def _packages_from_json(json: str) -> dict[NormalizedPackageStr, PackageLicense]:
     packages = loads(json)
-    return [
-        PackageLicense(
-            name=package["Name"],
-            package_link=package["URL"],
-            version=package["Version"],
-            license=package["License"],
-        )
+    return {
+        package_license.normalized_name: package_license
         for package in packages
-    ]
+        if (
+            package_license := PackageLicense(
+                name=package["Name"],
+                package_link=package["URL"],
+                version=package["Version"],
+                license=package["License"],
+            )
+        )
+    }
 
 
-def licenses() -> list[PackageLicense]:
+def get_licenses() -> dict[NormalizedPackageStr, PackageLicense]:
     with tempfile.NamedTemporaryFile() as file:
         subprocess.run(
             [
@@ -117,62 +125,51 @@ def licenses() -> list[PackageLicense]:
         return _packages_from_json(file.read().decode())
 
 
-def packages_to_markdown(
-    dependencies: dict[str, list], packages: list[PackageLicense]
-) -> str:
-    def heading():
-        return "# Dependencies\n"
+@dataclass(frozen=True)
+class PackageLicenseReport:
+    dependencies: OrderedDict[str, dict[NormalizedPackageStr, Package]]
+    licenses: dict[NormalizedPackageStr, PackageLicense]
 
-    def dependency(
-        group: str,
-        group_packages: list[Package],
-        packages: list[PackageLicense],
-    ) -> str:
-        def _header(_group: str):
-            _group = "".join([word.capitalize() for word in _group.strip().split()])
-            text = f"## {_group} Dependencies\n"
-            text += "|Package|Version|License|\n"
-            text += "|---|---|---|\n"
-            return text
-
-        def _rows(
-            _group_packages: list[Package], _packages: list[PackageLicense]
-        ) -> str:
-            text = ""
-            for package in _group_packages:
-                consistent = filter(
-                    lambda elem: elem.normalized_name == package.normalized_name,
-                    _packages,
-                )
-                for content in consistent:
-                    if content.package_link:
-                        text += f"|[{content.name}]({content.package_link})"
-                    else:
-                        text += f"|{content.name}"
-                    text += f"|{content.version}"
-                    if content.license_link:
-                        text += f"|[{content.license}]({content.license_link})|\n"
-                    else:
-                        text += f"|{content.license}|\n"
-            text += "\n"
-            return text
-
-        _template = cleandoc(
+    @staticmethod
+    def _format_group_table_header(group: str) -> str:
+        return cleandoc(
+            f"""## `{group}` Dependencies
+            |Package|Version|License|
+            |---|---|---|
             """
-            {header}{rows}
-        """
         )
-        return _template.format(
-            header=_header(group), rows=_rows(group_packages, packages)
-        )
-
-    template = cleandoc(
-        """
-        {heading}{rows}
-    """
-    )
 
-    rows = ""
-    for group in dependencies:
-        rows += dependency(group, dependencies[group], packages)
-    return template.format(heading=heading(), rows=rows)
+    def _format_group_table(
+        self, group: str, group_package_names: set[NormalizedPackageStr]
+    ) -> str:
+        group_header = self._format_group_table_header(group=group)
+
+        rows = []
+        for package_name in sorted(group_package_names):
+            if license_info := self.licenses.get(package_name):
+                rows.append(self._format_table_row(license_info=license_info))
+
+        return f"""{group_header}\n{''.join(rows)}\n"""
+
+    @staticmethod
+    def _format_table_row(license_info: PackageLicense) -> str:
+        row_package = f"{license_info.name}"
+        if license_info.package_link:
+            row_package = f"[{license_info.name}]({license_info.package_link})"
+
+        row_license = f"{license_info.license}"
+        if license_info.license_link:
+            row_license = f"[{license_info.license}]({license_info.license_link})"
+
+        return f"|{row_package}|{license_info.version}|{row_license}|\n"
+
+    def to_markdown(self) -> str:
+        rows = []
+        for group in self.dependencies:
+            group_package_names = set(self.dependencies[group].keys())
+            rows.append(
+                self._format_group_table(
+                    group=group, group_package_names=group_package_names
+                )
+            )
+        return cleandoc(f"""# Dependencies\n\n{''.join(rows)}""")

exasol/toolbox/util/dependencies/poetry_dependencies.py

@@ -2,6 +2,7 @@
 
 import subprocess
 import tempfile
+from collections import OrderedDict
 from pathlib import Path
 from typing import Optional
 
@@ -12,7 +13,10 @@
 )
 from tomlkit import TOMLDocument
 
-from exasol.toolbox.util.dependencies.shared_models import Package
+from exasol.toolbox.util.dependencies.shared_models import (
+    NormalizedPackageStr,
+    Package,
+)
 from exasol.toolbox.util.git import Git
 
 
@@ -92,16 +96,20 @@ def _extract_from_line(line: str) -> Optional[Package]:
             return None
         return Package(name=split_line[0], version=split_line[1])
 
-    def _extract_from_poetry_show(self, output_text: str) -> list[Package]:
-        return [
-            package
+    def _extract_from_poetry_show(
+        self, output_text: str
+    ) -> dict[NormalizedPackageStr, Package]:
+        return {
+            package.normalized_name: package
             for line in output_text.splitlines()
             if (package := self._extract_from_line(line))
-        ]
+        }
 
     @property
-    def direct_dependencies(self) -> dict[str, list[Package]]:
-        dependencies = {}
+    def direct_dependencies(
+        self,
+    ) -> OrderedDict[str, dict[NormalizedPackageStr, Package]]:
+        dependencies = OrderedDict()
         for group in self.groups:
             command = (
                 "poetry",
@@ -122,7 +130,7 @@ def direct_dependencies(self) -> dict[str, list[Package]]:
         return dependencies
 
     @property
-    def all_dependencies(self) -> dict[str, list[Package]]:
+    def all_dependencies(self) -> OrderedDict[str, dict[NormalizedPackageStr, Package]]:
         command = ("poetry", "show", "--no-truncate")
         output = subprocess.run(
             command,
@@ -133,28 +141,33 @@ def all_dependencies(self) -> dict[str, list[Package]]:
         )
 
         direct_dependencies = self.direct_dependencies.copy()
-        transitive_dependencies = []
+
+        transitive_dependencies = {}
         names_direct_dependencies = {
-            dep.name
-            for group_list in direct_dependencies.values()
-            for dep in group_list
+            package_name
+            for group_list in direct_dependencies
+            for package_name in group_list
         }
         for line in output.stdout.splitlines():
             dep = self._extract_from_line(line=line)
             if dep and dep.name not in names_direct_dependencies:
-                transitive_dependencies.append(dep)
+                transitive_dependencies[dep.normalized_name] = dep
 
         return direct_dependencies | {TRANSITIVE_GROUP.name: transitive_dependencies}
 
 
-def get_dependencies(working_directory: Path) -> dict[str, list[Package]]:
+def get_dependencies(
+    working_directory: Path,
+) -> OrderedDict[str, dict[NormalizedPackageStr, Package]]:
     poetry_dep = PoetryToml.load_from_toml(working_directory=working_directory)
     return PoetryDependencies(
         groups=poetry_dep.groups, working_directory=working_directory
     ).direct_dependencies
 
 
-def get_dependencies_from_latest_tag() -> dict[str, list[Package]]:
+def get_dependencies_from_latest_tag() -> (
+    OrderedDict[str, dict[NormalizedPackageStr, Package]]
+):
     latest_tag = Git.get_latest_tag()
     with tempfile.TemporaryDirectory() as path:
         tmpdir = Path(path)

exasol/toolbox/util/dependencies/shared_models.py

@@ -1,6 +1,9 @@
 from __future__ import annotations
 
-from typing import Annotated
+from typing import (
+    Annotated,
+    NewType,
+)
 
 from packaging.version import Version
 from pydantic import (
@@ -9,15 +12,21 @@
     ConfigDict,
 )
 
+NormalizedPackageStr = NewType("NormalizedPackageStr", str)
+
 VERSION_TYPE = Annotated[str, AfterValidator(lambda v: Version(v))]
 
 
+def normalize_package_name(package_name: str) -> NormalizedPackageStr:
+    return NormalizedPackageStr(package_name.lower().replace("_", "-"))
+
+
 class Package(BaseModel):
     model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True)
 
     name: str
     version: VERSION_TYPE
 
     @property
-    def normalized_name(self) -> str:
-        return self.name.lower().replace("_", "-")
+    def normalized_name(self) -> NormalizedPackageStr:
+        return normalize_package_name(self.name)