Speed up invalidated packaging image rebuilds

TODO.md

@@ -1,5 +1,15 @@
 # Production Readiness TODO
 
+- [x] Persist package-manager downloads independently of Docker layers.
+  Final result: the native package stage uses locked, stable BuildKit cache IDs
+  separated by distro and architecture. The runtime stage additionally
+  separates cache IDs by backend and backend version.
+  QA: validate the image matrix, run Dockerfile checks for Ubuntu, Alpine, and
+  Arch inputs, and benchmark invalidated Ubuntu package-manager steps before
+  and after the cache mounts. The ARM64 Ubuntu package target measured 17s and
+  14s before versus 13s and 13s after. Comparing the second invalidated rebuild
+  in each set, the warm cache-mount build was 7.1% faster (14s to 13s).
+
 - [x] Replace widening workflow filters with a typed, fail-closed release plan.
   Final result: manual runs select native, Homebrew, and npm validation through
   typed booleans plus exact checked-in IDs or `all`; publication implications

docker/Dockerfile.mesh-llm

@@ -12,7 +12,12 @@ ARG BACKEND_VERSION=
 ARG TARGET_ARCH=amd64
 ARG MESH_LLM_VERSION
 COPY docker/install-package-build-deps.sh /usr/local/bin/install-package-build-deps.sh
-RUN sh /usr/local/bin/install-package-build-deps.sh "${DISTRO}"
+RUN --mount=type=cache,id=mesh-packaging-${DISTRO}-${TARGET_ARCH}-package-manager-packages,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,id=mesh-packaging-${DISTRO}-${TARGET_ARCH}-package-manager-index,target=/var/lib/apt/lists,sharing=locked \
+    --mount=type=cache,id=mesh-packaging-${DISTRO}-${TARGET_ARCH}-apk-packages,target=/var/cache/apk,sharing=locked \
+    --mount=type=cache,id=mesh-packaging-${DISTRO}-${TARGET_ARCH}-pacman-packages,target=/var/cache/pacman/pkg,sharing=locked \
+    --mount=type=cache,id=mesh-packaging-${DISTRO}-${TARGET_ARCH}-pacman-index,target=/var/lib/pacman/sync,sharing=locked \
+    sh /usr/local/bin/install-package-build-deps.sh "${DISTRO}"
 WORKDIR /work
 COPY packaging/native/build-package.sh ./build-package.sh
 COPY artifacts/upstream ./mesh-bundle
@@ -34,6 +39,7 @@ FROM ${RUNTIME_BASE_IMAGE} AS runtime
 ARG DISTRO=ubuntu
 ARG BACKEND=cpu
 ARG BACKEND_VERSION=
+ARG TARGETARCH
 ARG MESH_LLM_VERSION
 ARG MESH_LLM_REF
 ARG MESH_LLM_SOURCE_SHA
@@ -57,7 +63,12 @@ LABEL org.opencontainers.image.title="mesh-llm" \
 COPY docker/install-runtime-deps.sh /usr/local/bin/install-runtime-deps.sh
 COPY docker/install-native-package.sh /usr/local/bin/install-native-package.sh
 COPY artifacts/native-package /packages
-RUN sh /usr/local/bin/install-runtime-deps.sh "${DISTRO}" "${BACKEND}" && \
+RUN --mount=type=cache,id=mesh-packaging-${DISTRO}-${TARGETARCH}-${BACKEND}-${BACKEND_VERSION}-package-manager-packages,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,id=mesh-packaging-${DISTRO}-${TARGETARCH}-${BACKEND}-${BACKEND_VERSION}-package-manager-index,target=/var/lib/apt/lists,sharing=locked \
+    --mount=type=cache,id=mesh-packaging-${DISTRO}-${TARGETARCH}-${BACKEND}-${BACKEND_VERSION}-apk-packages,target=/var/cache/apk,sharing=locked \
+    --mount=type=cache,id=mesh-packaging-${DISTRO}-${TARGETARCH}-${BACKEND}-${BACKEND_VERSION}-pacman-packages,target=/var/cache/pacman/pkg,sharing=locked \
+    --mount=type=cache,id=mesh-packaging-${DISTRO}-${TARGETARCH}-${BACKEND}-${BACKEND_VERSION}-pacman-index,target=/var/lib/pacman/sync,sharing=locked \
+    sh /usr/local/bin/install-runtime-deps.sh "${DISTRO}" "${BACKEND}" && \
     sh /usr/local/bin/install-native-package.sh "${DISTRO}" /packages && \
     rm -rf /packages
 COPY docker/entrypoint.sh /usr/local/bin/mesh-llm-entrypoint

docker/install-native-package.sh

@@ -32,17 +32,17 @@ case "$distro" in
   ubuntu)
     package="$(find_one_package '*.deb')"
     export DEBIAN_FRONTEND=noninteractive
+    rm -f /etc/apt/apt.conf.d/docker-clean
     apt-get update
     apt-get install -y --no-install-recommends "$package"
     # Vendor packages such as ROCm install shared objects outside the default
     # library directories. Refresh explicitly because container build layers
     # do not reliably leave deferred libc triggers reflected in ld.so.cache.
     ldconfig
-    rm -rf /var/lib/apt/lists/*
     ;;
   alpine)
     package="$(find_one_package '*.apk')"
-    apk add --no-cache --allow-untrusted "$package"
+    apk add --cache-dir /var/cache/apk --allow-untrusted "$package"
     ;;
   arch)
     package="$(find_one_package '*.pkg.tar.zst')"

docker/install-package-build-deps.sh

@@ -21,12 +21,12 @@ refresh_pacman() {
 case "$distro" in
   ubuntu)
     export DEBIAN_FRONTEND=noninteractive
+    rm -f /etc/apt/apt.conf.d/docker-clean
     apt-get update
     apt-get install -y --no-install-recommends ca-certificates coreutils dpkg tar
-    rm -rf /var/lib/apt/lists/*
     ;;
   alpine)
-    apk add --no-cache abuild ca-certificates coreutils gzip tar
+    apk add --cache-dir /var/cache/apk abuild ca-certificates coreutils gzip tar
     ;;
   arch)
     refresh_pacman

docker/install-runtime-deps.sh

@@ -22,17 +22,17 @@ refresh_pacman() {
 case "$distro" in
   ubuntu)
     export DEBIAN_FRONTEND=noninteractive
+    rm -f /etc/apt/apt.conf.d/docker-clean
     apt-get update
     apt-get install -y --no-install-recommends ca-certificates libdbus-1-3 libgomp1
     if [ "$backend" = "vulkan" ]; then
       apt-get install -y --no-install-recommends libvulkan1
     fi
-    rm -rf /var/lib/apt/lists/*
     ;;
   alpine)
-    apk add --no-cache ca-certificates dbus-libs libatomic libgcc libgomp libstdc++
+    apk add --cache-dir /var/cache/apk ca-certificates dbus-libs libatomic libgcc libgomp libstdc++
     if [ "$backend" = "vulkan" ]; then
-      apk add --no-cache vulkan-loader
+      apk add --cache-dir /var/cache/apk vulkan-loader
     fi
     ;;
   arch)
@@ -47,7 +47,6 @@ case "$distro" in
     if [ "$backend" = "rocm" ]; then
       pacman -S --noconfirm --needed hip-runtime-amd rocm-core
     fi
-    pacman -Scc --noconfirm
     ;;
   *)
     echo "unsupported distro: $distro" >&2