feat: add support for building linux/arm64 images from code (#1093)

Details:
* Update the session launcher API to support specifying target platforms. This means the API can support multi-platform builds.
* At the moment, images can be built with only ONE target platform, so validation only accepts a single target platform.
* Image building can be configured to support the `linux/arm64` platform with env variables

See chart changes here: SwissDataScienceCenter/renku#4231

---------

Co-authored-by: Tasko Olevski <olevski90@gmail.com>

Author: leafty

components/renku_data_services/migrations/versions/df2c0e65612a_add_platforms_field_to_build_parameters.py

@@ -0,0 +1,40 @@
+"""add platforms field to build_parameters
+
+Revision ID: df2c0e65612a
+Revises: 21fd67938150
+Create Date: 2025-11-03 08:59:27.001063
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+from renku_data_services.utils.sqlalchemy import ULIDType
+
+# revision identifiers, used by Alembic.
+revision = "df2c0e65612a"
+down_revision = "21fd67938150"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "build_platforms",
+        sa.Column("id", sa.Integer(), sa.Identity(always=True), nullable=False),
+        sa.Column("platform", sa.Enum("linux_amd64", "linux_arm64", name="build_platform"), nullable=False),
+        sa.Column("build_parameters_id", ULIDType(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["build_parameters_id"],
+            ["sessions.build_parameters.id"],
+            name="build_platform_build_parameters_id_fk",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        schema="sessions",
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("build_platforms", schema="sessions")
+    op.execute("DROP type build_platform")

components/renku_data_services/session/api.spec.yaml

@@ -648,6 +648,15 @@ components:
       minLength: 1
       maxLength: 99
       example: My Renku Session :)
+    BuildPlatforms:
+      description: The target runtime platforms of a session environment.
+      type: array
+      items:
+        $ref: "#/components/schemas/BuildPlatform"
+    BuildPlatform:
+      description: A runtime platform, e.g. "linux/amd64".
+      type: string
+      enum: ["linux/amd64", "linux/arm64"]
     BuilderVariant:
       description: Type of virtual environment manager when building custom environments.
       type: string
@@ -728,6 +737,8 @@ components:
       properties:
         repository:
           $ref: "#/components/schemas/Repository"
+        platforms:
+          $ref: "#/components/schemas/BuildPlatforms"
         builder_variant:
           $ref: "#/components/schemas/BuilderVariant"
         frontend_variant:
@@ -755,6 +766,8 @@ components:
       properties:
         repository:
           $ref: "#/components/schemas/Repository"
+        platforms:
+          $ref: "#/components/schemas/BuildPlatforms"
         builder_variant:
           $ref: "#/components/schemas/BuilderVariant"
         frontend_variant:

components/renku_data_services/session/apispec.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api.spec.yaml
-#   timestamp: 2025-10-23T07:30:34+00:00
+#   timestamp: 2025-11-13T12:27:39+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,11 @@
 from renku_data_services.session.apispec_base import BaseAPISpec
 
 
+class BuildPlatform(Enum):
+    linux_amd64 = "linux/amd64"
+    linux_arm64 = "linux/arm64"
+
+
 class EnvironmentKind(Enum):
     GLOBAL = "GLOBAL"
     CUSTOM = "CUSTOM"
@@ -363,6 +368,9 @@ class BuildParameters(BaseAPISpec):
         extra="forbid",
     )
     repository: str = Field(..., description="A git repository URL")
+    platforms: Optional[List[BuildPlatform]] = Field(
+        None, description="The target runtime platforms of a session environment."
+    )
     builder_variant: str = Field(
         ...,
         description="Type of virtual environment manager when building custom environments.",
@@ -386,6 +394,9 @@ class BuildParametersPost(BuildParameters):
 
 class BuildParametersPatch(BaseAPISpec):
     repository: Optional[str] = Field(None, description="A git repository URL")
+    platforms: Optional[List[BuildPlatform]] = Field(
+        None, description="The target runtime platforms of a session environment."
+    )
     builder_variant: Optional[str] = Field(
         None,
         description="Type of virtual environment manager when building custom environments.",

components/renku_data_services/session/config.py

@@ -8,10 +8,22 @@
 
 from renku_data_services.app_config import logging
 from renku_data_services.session import crs as session_crs
+from renku_data_services.session import models
 
 logger = logging.getLogger(__name__)
 
 
+@dataclass
+class BuildPlatformOverrides:
+    """Configuration overrides for a given target platform."""
+
+    builder_image: str | None = None
+    run_image: str | None = None
+    strategy_name: str | None = None
+    node_selector: dict[str, str] | None = None
+    tolerations: list[session_crs.Toleration] | None = None
+
+
 @dataclass
 class BuildsConfig:
     """Configuration for container image builds."""
@@ -21,6 +33,7 @@ class BuildsConfig:
     build_builder_image: str | None = None
     build_run_image: str | None = None
     build_strategy_name: str | None = None
+    build_platform_overrides: dict[str, BuildPlatformOverrides] | None = None
     push_secret_name: str | None = None
     buildrun_retention_after_failed: timedelta | None = None
     buildrun_retention_after_succeeded: timedelta | None = None
@@ -75,12 +88,38 @@ def from_env(cls) -> "BuildsConfig":
             except PydanticValidationError:
                 logger.error("Could not validate BUILD_NODE_TOLERATIONS. Will not use tolerations for image builds.")
 
+        build_platform_overrides: dict[str, BuildPlatformOverrides] | None = None
+        build_platform_overrides_str = os.environ.get("BUILD_PLATFORM_OVERRIDES")
+        if build_platform_overrides_str:
+            try:
+                parsed = session_crs.BuildPlatformOverridesDict.model_validate_json(build_platform_overrides_str).root
+                if parsed:
+                    for platform, data in parsed.items():
+                        if platform not in models.Platform:
+                            logger.error(f"Ignoring unknown platform {platform}.")
+                            continue
+                        if build_platform_overrides is None:
+                            build_platform_overrides = dict()
+                        build_platform_overrides[platform] = BuildPlatformOverrides(
+                            builder_image=data.builderImage,
+                            run_image=data.runImage,
+                            strategy_name=data.strategyName,
+                            node_selector=data.nodeSelector,
+                            tolerations=data.tolerations,
+                        )
+            except PydanticValidationError:
+                logger.error(
+                    "Could not validate BUILD_PLATFORM_OVERRIDES. "
+                    "Will not use platform-specific overrides for image builds."
+                )
+
         return cls(
             enabled=enabled or False,
             build_output_image_prefix=build_output_image_prefix or None,
             build_builder_image=build_builder_image,
             build_run_image=build_run_image,
             build_strategy_name=build_strategy_name or None,
+            build_platform_overrides=build_platform_overrides,
             push_secret_name=push_secret_name or None,
             buildrun_retention_after_failed=buildrun_retention_after_failed,
             buildrun_retention_after_succeeded=buildrun_retention_after_succeeded,

components/renku_data_services/session/core.py

@@ -64,8 +64,11 @@ def validate_unsaved_build_parameters(
             )
         )
 
+    platforms = __validate_build_parameters_platforms(environment.platforms)
+
     return models.UnsavedBuildParameters(
         repository=environment.repository,
+        platforms=platforms,
         builder_variant=environment.builder_variant,
         frontend_variant=environment.frontend_variant,
         repository_revision=environment.repository_revision if environment.repository_revision else None,
@@ -90,8 +93,13 @@ def validate_build_parameters_patch(environment: apispec.BuildParametersPatch) -
             )
         )
 
+    platforms: list[models.Platform] | None = None
+    if environment.platforms is not None:
+        platforms = __validate_build_parameters_platforms(environment.platforms)
+
     return models.BuildParametersPatch(
         repository=environment.repository,
+        platforms=platforms,
         builder_variant=environment.builder_variant,
         frontend_variant=environment.frontend_variant,
         repository_revision=environment.repository_revision,
@@ -344,3 +352,25 @@ def validate_build_patch(patch: apispec.BuildPatch) -> models.BuildPatch:
     """Validate the update to a session launcher."""
     status = models.BuildStatus(patch.status.value) if patch.status else None
     return models.BuildPatch(status=status)
+
+
+def __validate_build_parameters_platforms(platforms: list[apispec.BuildPlatform] | None) -> list[models.Platform]:
+    """Validate the platforms field for build parameters."""
+    platforms_str_list: list[str] = [models.Platform.linux_amd64]
+    if platforms:
+        platforms_str_list = [item.value for item in platforms]
+    platforms_str_list = sorted(set(platforms_str_list))
+    if len(platforms_str_list) != 1:
+        raise errors.ValidationError(
+            message=f"Invalid value for the field 'platforms': {platforms}: "
+            "only one platform at a time is supported at the moment."
+        )
+    for platform in platforms_str_list:
+        if platform not in models.Platform:
+            raise errors.ValidationError(
+                message=(
+                    f"Invalid value for the field 'platforms': {platforms}: "
+                    f"Valid values are {[e.value for e in models.Platform]}"
+                )
+            )
+    return [models.Platform(item) for item in platforms_str_list]

components/renku_data_services/session/crs.py

@@ -63,3 +63,25 @@ class Tolerations(RootModel[list[Toleration] | None]):
     """A list of k8s tolerations."""
 
     root: list[Toleration] | None = None
+
+
+class BuildPlatformOverrides(BaseModel):
+    """Configuration overrides for a given target platform.
+
+    Used to validate the builds configuration.
+    """
+
+    builderImage: str | None = None
+    runImage: str | None = None
+    strategyName: str | None = None
+    nodeSelector: dict[str, str] | None = None
+    tolerations: list[Toleration] | None = None
+
+
+class BuildPlatformOverridesDict(RootModel[dict[str, BuildPlatformOverrides] | None]):
+    """A map of target platforms to configuration overrides.
+
+    Used to validate the builds configuration.
+    """
+
+    root: dict[str, BuildPlatformOverrides] | None = None

components/renku_data_services/session/db.py

@@ -138,7 +138,11 @@ def __copy_environment(
         if environment.environment_image_source == models.EnvironmentImageSource.build:
             if not environment.build_parameters:
                 raise errors.ProgrammingError(message="Environment has no build parameters.")
+            platforms = [
+                schemas.BuildPlatformORM(platform=platform) for platform in environment.build_parameters.platforms
+            ]
             new_build_parameters = schemas.BuildParametersORM(
+                platforms=platforms,
                 builder_variant=environment.build_parameters.builder_variant,
                 frontend_variant=environment.build_parameters.frontend_variant,
                 repository=environment.build_parameters.repository,
@@ -164,7 +168,11 @@ def __insert_build_parameters_environment(
             raise errors.UnauthorizedError(
                 message="You have to be authenticated to insert an environment in the DB.", quiet=True
             )
+        platforms = [
+            schemas.BuildPlatformORM(platform=platform) for platform in new_build_parameters_environment.platforms
+        ]
         build_parameters_orm = schemas.BuildParametersORM(
+            platforms=platforms,
             builder_variant=new_build_parameters_environment.builder_variant,
             frontend_variant=new_build_parameters_environment.frontend_variant,
             repository=new_build_parameters_environment.repository,
@@ -265,6 +273,10 @@ async def __update_environment_build_parameters(
 
         if build_parameters.repository is not None:
             environment.build_parameters.repository = build_parameters.repository
+        if build_parameters.platforms is not None:
+            environment.build_parameters.platforms = [
+                schemas.BuildPlatformORM(platform=platform) for platform in build_parameters.platforms
+            ]
         if build_parameters.builder_variant is not None:
             environment.build_parameters.builder_variant = build_parameters.builder_variant
         if build_parameters.frontend_variant is not None:
@@ -418,7 +430,9 @@ async def insert_launcher(
                 )
                 session.add(environment_orm)
             elif isinstance(launcher.environment, models.UnsavedBuildParameters):
+                platforms = [schemas.BuildPlatformORM(platform=platform) for platform in launcher.environment.platforms]
                 build_parameters_orm = schemas.BuildParametersORM(
+                    platforms=platforms,
                     builder_variant=launcher.environment.builder_variant,
                     frontend_variant=launcher.environment.frontend_variant,
                     repository=launcher.environment.repository,
@@ -718,7 +732,11 @@ async def __update_launcher_environment(
                 await session.flush()
             case models.UnsavedBuildParameters() as new_custom_built_environment, models.EnvironmentKind.CUSTOM:
                 # Custom environment with image is replaced by a custom environment with build
+                platforms = [
+                    schemas.BuildPlatformORM(platform=platform) for platform in new_custom_built_environment.platforms
+                ]
                 build_parameters_orm = schemas.BuildParametersORM(
+                    platforms=platforms,
                     builder_variant=new_custom_built_environment.builder_variant,
                     frontend_variant=new_custom_built_environment.frontend_variant,
                     repository=new_custom_built_environment.repository,
@@ -884,6 +902,15 @@ async def start_build(self, user: base_models.APIUser, build: models.UnsavedBuil
                         message=f"Session environment with id '{build.environment_id}' already has a build in progress."
                     )
 
+            # We check that we build for a single target platform
+            if len(build_parameters.platforms) > 1:
+                raise errors.CannotStartBuildError(
+                    message=(
+                        f"Building images can target only one platform at a time. "
+                        f"Current value: {build_parameters.platforms}"
+                    )
+                )
+
             build_orm = schemas.BuildORM(
                 environment_id=build.environment_id,
                 status=models.BuildStatus.in_progress,
@@ -1052,6 +1079,9 @@ def _get_buildrun_params(
         git_repository_revision = build_parameters.repository_revision
         context_dir = build_parameters.context_dir
 
+        builder_image = self.builds_config.build_builder_image or constants.BUILD_DEFAULT_BUILDER_IMAGE
+        run_image = self.builds_config.build_run_image or constants.BUILD_DEFAULT_RUN_IMAGE
+
         output_image_prefix = (
             self.builds_config.build_output_image_prefix or constants.BUILD_DEFAULT_OUTPUT_IMAGE_PREFIX
         )
@@ -1063,6 +1093,9 @@ def _get_buildrun_params(
         build_strategy_name = self.builds_config.build_strategy_name or constants.BUILD_DEFAULT_BUILD_STRATEGY_NAME
         push_secret_name = self.builds_config.push_secret_name or constants.BUILD_DEFAULT_PUSH_SECRET_NAME
 
+        node_selector = self.builds_config.node_selector
+        tolerations = self.builds_config.tolerations
+
         retention_after_failed = (
             self.builds_config.buildrun_retention_after_failed or constants.BUILD_RUN_DEFAULT_RETENTION_AFTER_FAILED
         )
@@ -1083,26 +1116,36 @@ def _get_buildrun_params(
             annotations["renku.io/launcher_id"] = str(launcher.id)
             annotations["renku.io/project_id"] = str(launcher.project_id)
 
-        return models.ShipwrightBuildRunParams(
+        params = models.ShipwrightBuildRunParams(
             name=build.k8s_name,
             git_repository=git_repository,
-            build_image=self.builds_config.build_builder_image or constants.BUILD_DEFAULT_BUILDER_IMAGE,
-            run_image=self.builds_config.build_run_image or constants.BUILD_DEFAULT_RUN_IMAGE,
+            builder_image=builder_image,
+            run_image=run_image,
             output_image=output_image,
             build_strategy_name=build_strategy_name,
             push_secret_name=push_secret_name,
             retention_after_failed=retention_after_failed,
             retention_after_succeeded=retention_after_succeeded,
             build_timeout=build_timeout,
-            node_selector=self.builds_config.node_selector,
-            tolerations=self.builds_config.tolerations,
+            node_selector=node_selector,
+            tolerations=tolerations,
             labels=labels,
             annotations=annotations,
             frontend=build_parameters.frontend_variant,
             git_repository_revision=git_repository_revision,
             context_dir=context_dir,
         )
 
+        platform = models.Platform.linux_amd64
+        if build_parameters.platforms:
+            platform = build_parameters.platforms[0]
+        overrides = None
+        if self.builds_config.build_platform_overrides:
+            overrides = self.builds_config.build_platform_overrides.get(platform.value)
+        params = params.with_overrides(overrides=overrides)
+
+        return params
+
     async def _get_environment_authorization(
         self, session: AsyncSession, user: base_models.APIUser, environment: schemas.EnvironmentORM, scope: Scope
     ) -> bool: