Support different versions per instance of a component (#559)

This PR changes Commodore to

* Create separate worktrees for each component alias
* Create class symlinks for each component alias
* Create each alias target with only the defaults and component class
symlinked from the alias worktree
* Read instance versions from `parameters.components.<instance-name>`

Note that per-instance versions only work correctly only for components
which use `${_base_directory}` in their config when specifying Jsonnet
files or Helm chart/YAML locations in `kapitan.compile` and
`kapitan.dependencies`. Note, that components should use
`${_base_directory}` anyway, and new components created from the
template use `${_base_directory}` out of the box.

Components must signal that they support per-instance versions by setting
component parameter `_metadata.multi_version=true`.

Resolves #563 

Co-authored-by: Aline Abler <aline.abler@vshn.ch>
Co-authored-by: Aline Abler <alinea@riseup.net>

Author: simu

README.md

@@ -115,9 +115,9 @@ Commodore also supports additional processing on the output of Kapitan, such as
 
 1. Run linting and tests
 
-   Auto format with autopep8
+   Automatically apply Black formatting
    ```console
-   poetry run autopep
+   poetry run black .
    ```
 
    List all Tox targets
@@ -132,7 +132,7 @@ Commodore also supports additional processing on the output of Kapitan, such as
 
    Run just a specific target
    ```console
-   poetry run tox -e py38
+   poetry run tox -e py312
    ```
 
 

commodore/cluster.py

@@ -169,7 +169,9 @@ def generate_target(
         "_instance": target,
     }
     if not bootstrap:
-        parameters["_base_directory"] = str(components[component].target_directory)
+        parameters["_base_directory"] = str(
+            components[component].alias_directory(target)
+        )
         parameters["_kustomize_wrapper"] = str(__kustomize_wrapper__)
         parameters["kapitan"] = {
             "vars": {
@@ -206,24 +208,28 @@ def render_target(
     classes = [f"params.{inv.bootstrap_target}"]
 
     for c in sorted(components):
-        if inv.defaults_file(c).is_file():
-            classes.append(f"defaults.{c}")
+        defaults_file = inv.defaults_file(c)
+        if c == component and target != component:
+            # Special case alias defaults symlink
+            defaults_file = inv.defaults_file(target)
+
+        if defaults_file.is_file():
+            classes.append(f"defaults.{defaults_file.stem}")
         else:
             click.secho(f" > Default file for class {c} missing", fg="yellow")
 
     classes.append("global.commodore")
 
     if not bootstrap:
-        if not inv.component_file(component).is_file():
+        if not inv.component_file(target).is_file():
             raise click.ClickException(
                 f"Target rendering failed for {target}: component class is missing"
             )
-        classes.append(f"components.{component}")
+        classes.append(f"components.{target}")
 
     return generate_target(inv, target, components, classes, component)
 
 
-# pylint: disable=unsubscriptable-object
 def update_target(cfg: Config, target: str, component: Optional[str] = None):
     click.secho(f"Updating Kapitan target for {target}...", bold=True)
     file = cfg.inventory.target_file(target)

commodore/compile.py

@@ -240,7 +240,7 @@ def setup_compile_environment(config: Config) -> tuple[dict[str, Any], Iterable[
     config.register_component_deprecations(cluster_parameters)
     # Raise exception if component version override without URL is present in the
     # hierarchy.
-    verify_version_overrides(cluster_parameters)
+    verify_version_overrides(cluster_parameters, config.get_component_aliases())
 
     for component in config.get_components().values():
         ckey = component.parameters_key

commodore/component/__init__.py

@@ -19,6 +19,8 @@ class Component:
     _version: Optional[str] = None
     _dir: P
     _sub_path: str
+    _aliases: dict[str, tuple[str, str]]
+    _work_dir: Optional[P]
 
     @classmethod
     def clone(cls, cfg, clone_url: str, name: str, version: str = "master"):
@@ -57,6 +59,8 @@ def __init__(
         self.version = version
         self._sub_path = sub_path
         self._repo = None
+        self._aliases = {self.name: (self.version or "", self.sub_path or "")}
+        self._work_dir = work_dir
 
     @property
     def name(self) -> str:
@@ -67,8 +71,12 @@ def repo(self) -> GitRepo:
         if not self._repo:
             if self._dependency:
                 dep_repo = self._dependency.bare_repo
-                author_name = dep_repo.author.name
-                author_email = dep_repo.author.email
+                author_name = (
+                    dep_repo.author.name if hasattr(dep_repo, "author") else None
+                )
+                author_email = (
+                    dep_repo.author.email if hasattr(dep_repo, "author") else None
+                )
             else:
                 # Fall back to author detection if we don't have a dependency
                 author_name = None
@@ -126,21 +134,46 @@ def sub_path(self) -> str:
     def repo_directory(self) -> P:
         return self._dir
 
+    @property
+    def work_directory(self) -> Optional[P]:
+        return self._work_dir
+
     @property
     def target_directory(self) -> P:
-        return self._dir / self._sub_path
+        return self.alias_directory(self.name)
 
     @property
     def target_dir(self) -> P:
         return self.target_directory
 
     @property
     def class_file(self) -> P:
-        return self.target_directory / "class" / f"{self.name}.yml"
+        return self.alias_class_file(self.name)
 
     @property
     def defaults_file(self) -> P:
-        return self.target_directory / "class" / "defaults.yml"
+        return self.alias_defaults_file(self.name)
+
+    def alias_directory(self, alias: str) -> P:
+        if not self._dependency:
+            return self._dir / self._sub_path
+        apath = self._dependency.get_component(alias)
+        if not apath:
+            raise ValueError(f"unknown alias {alias} for component {self.name}")
+        if alias not in self._aliases:
+            raise ValueError(
+                f"alias {alias} for component {self.name} has not been registered"
+            )
+        return apath / self._aliases[alias][1]
+
+    def alias_class_file(self, alias: str) -> P:
+        return self.alias_directory(alias) / "class" / f"{self.name}.yml"
+
+    def alias_defaults_file(self, alias: str) -> P:
+        return self.alias_directory(alias) / "class" / "defaults.yml"
+
+    def has_alias(self, alias: str):
+        return alias in self._aliases
 
     @property
     def lib_files(self) -> Iterable[P]:
@@ -177,6 +210,35 @@ def checkout(self):
             )
         self._dependency.checkout_component(self.name, self.version)
 
+    def register_alias(self, alias: str, version: str, sub_path: str = ""):
+        if not self._work_dir:
+            raise ValueError(
+                f"Can't register alias on component {self.name} "
+                + "which isn't configured with a working directory"
+            )
+        if alias in self._aliases:
+            raise ValueError(
+                f"alias {alias} already registered on component {self.name}"
+            )
+        self._aliases[alias] = (version, sub_path)
+        if self._dependency:
+            self._dependency.register_component(
+                alias, component_dir(self._work_dir, alias)
+            )
+
+    def checkout_alias(
+        self, alias: str, alias_dependency: Optional[MultiDependency] = None
+    ):
+        if alias not in self._aliases:
+            raise ValueError(
+                f"alias {alias} is not registered on component {self.name}"
+            )
+
+        if alias_dependency:
+            alias_dependency.checkout_component(alias, self._aliases[alias][0])
+        elif self._dependency:
+            self._dependency.checkout_component(alias, self._aliases[alias][0])
+
     def is_checked_out(self) -> bool:
         return self.target_dir is not None and self.target_dir.is_dir()
 

commodore/config.py

@@ -377,14 +377,26 @@ def get_component_aliases(self):
         return self._component_aliases
 
     def register_component_aliases(self, aliases: dict[str, str]):
-        self._component_aliases = aliases
+        self._component_aliases.update(aliases)
 
     def verify_component_aliases(self, cluster_parameters: dict):
         for alias, cn in self._component_aliases.items():
-            if alias != cn and not _component_is_aliasable(cluster_parameters, cn):
-                raise click.ClickException(
-                    f"Component {cn} with alias {alias} does not support instantiation."
+            if alias != cn:
+                if not _component_is_aliasable(cluster_parameters, cn):
+                    raise click.ClickException(
+                        f"Component {cn} with alias {alias} does not support instantiation."
+                    )
+
+                cv = cluster_parameters.get("components", {}).get(alias, {})
+                alias_has_version = (
+                    cv.get("url") is not None or cv.get("version") is not None
                 )
+                if alias_has_version and not _component_supports_alias_version(
+                    cluster_parameters, cn, alias
+                ):
+                    raise click.ClickException(
+                        f"Component {cn} with alias {alias} does not support overriding instance version."
+                    )
 
     def get_component_alias_versioninfos(self) -> dict[str, InstanceVersionInfo]:
         return {
@@ -453,6 +465,18 @@ def _component_is_aliasable(cluster_parameters: dict, component_name: str):
     return cmeta.get("multi_instance", False)
 
 
+def _component_supports_alias_version(
+    cluster_parameters: dict,
+    component_name: str,
+    alias: str,
+):
+    ckey = component_parameters_key(component_name)
+    cmeta = cluster_parameters[ckey].get("_metadata", {})
+    akey = component_parameters_key(alias)
+    ameta = cluster_parameters.get(akey, {}).get("_metadata", {})
+    return cmeta.get("multi_version", False) and ameta.get("multi_version", False)
+
+
 def set_fact_value(facts: dict[str, Any], raw_key: str, value: Any) -> None:
     """Set value for nested fact at `raw_key` (expected form `path.to.key`) to `value`.
 

commodore/dependency_mgmt/__init__.py

@@ -40,6 +40,24 @@ def create_component_symlinks(cfg, component: Component):
         )
 
 
+def create_alias_symlinks(cfg, component: Component, alias: str):
+    if not component.has_alias(alias):
+        raise ValueError(
+            f"component {component.name} doesn't have alias {alias} registered"
+        )
+    relsymlink(
+        component.alias_class_file(alias),
+        cfg.inventory.components_dir,
+        dest_name=f"{alias}.yml",
+    )
+    inventory_default = cfg.inventory.defaults_file(alias)
+    relsymlink(
+        component.alias_defaults_file(alias),
+        inventory_default.parent,
+        dest_name=inventory_default.name,
+    )
+
+
 def create_package_symlink(cfg, pname: str, package: Package):
     """
     Create package symlink in the inventory.
@@ -69,7 +87,7 @@ def fetch_components(cfg: Config):
     component_names, component_aliases = _discover_components(cfg)
     click.secho("Registering component aliases...", bold=True)
     cfg.register_component_aliases(component_aliases)
-    cspecs = _read_components(cfg, component_names)
+    cspecs = _read_components(cfg, component_aliases)
     click.secho("Fetching components...", bold=True)
 
     deps: dict[str, list] = {}
@@ -93,6 +111,25 @@ def fetch_components(cfg: Config):
         deps.setdefault(cdep.url, []).append(c)
     fetch_parallel(fetch_component, cfg, deps.values())
 
+    components = cfg.get_components()
+
+    for alias, component in component_aliases.items():
+        if alias == component:
+            # Nothing to setup for identity alias
+            continue
+
+        c = components[component]
+        aspec = cspecs[alias]
+        adep = None
+        if aspec.url != c.repo_url:
+            adep = cfg.register_dependency_repo(aspec.url)
+            adep.register_component(alias, component_dir(cfg.work_dir, alias))
+
+        c.register_alias(alias, aspec.version, aspec.path)
+        c.checkout_alias(alias, adep)
+
+        create_alias_symlinks(cfg, c, alias)
+
 
 def fetch_component(cfg, dependencies):
     """
@@ -126,7 +163,7 @@ def register_components(cfg: Config):
     click.secho("Discovering included components...", bold=True)
     try:
         components, component_aliases = _discover_components(cfg)
-        cspecs = _read_components(cfg, components)
+        cspecs = _read_components(cfg, component_aliases)
     except KeyError as e:
         raise click.ClickException(f"While discovering components: {e}")
     click.secho("Registering components and aliases...", bold=True)
@@ -152,9 +189,9 @@ def register_components(cfg: Config):
         cfg.register_component(component)
         create_component_symlinks(cfg, component)
 
-    registered_components = cfg.get_components().keys()
+    registered_components = cfg.get_components()
     pruned_aliases = {
-        a: c for a, c in component_aliases.items() if c in registered_components
+        a: c for a, c in component_aliases.items() if c in registered_components.keys()
     }
     pruned = sorted(set(component_aliases.keys()) - set(pruned_aliases.keys()))
     if len(pruned) > 0:
@@ -163,6 +200,24 @@ def register_components(cfg: Config):
         )
     cfg.register_component_aliases(pruned_aliases)
 
+    for alias, cn in pruned_aliases.items():
+        if alias == cn:
+            # Nothing to setup for identity alias
+            continue
+
+        c = registered_components[cn]
+        aspec = cspecs[alias]
+
+        if aspec.url != c.repo_url:
+            adep = cfg.register_dependency_repo(aspec.url)
+            adep.register_component(alias, component_dir(cfg.work_dir, alias))
+        c.register_alias(alias, aspec.version, aspec.path)
+
+        if not component_dir(cfg.work_dir, alias).is_dir():
+            raise click.ClickException(f"Missing alias checkout for '{alias} as {cn}'")
+
+        create_alias_symlinks(cfg, c, alias)
+
 
 def fetch_packages(cfg: Config):
     """
@@ -235,10 +290,18 @@ def register_packages(cfg: Config):
         create_package_symlink(cfg, p, pkg)
 
 
-def verify_version_overrides(cluster_parameters):
+def verify_version_overrides(cluster_parameters, component_aliases: dict[str, str]):
     errors = []
+    aliases = set(component_aliases.keys()) - set(component_aliases.values())
     for cname, cspec in cluster_parameters["components"].items():
-        if "url" not in cspec:
+        if cname in aliases:
+            # We don't require an url in component alias version configs
+            # but we do require the base component to have one
+            if component_aliases[cname] not in cluster_parameters["components"]:
+                errors.append(
+                    f"component '{component_aliases[cname]}' (imported as {cname})"
+                )
+        elif "url" not in cspec:
             errors.append(f"component '{cname}'")
 
     for pname, pspec in cluster_parameters.get("packages", {}).items():