Use Sonic VPP image

.github/workflows/base-image.yaml

@@ -16,6 +16,7 @@ jobs:
           - name: 202311
           - name: 202411
           - name: 202505
+          - name: 202511-vpp
 
     steps:
     - name: Log in to the container registry

Makefile

@@ -10,8 +10,8 @@ KINDCONFIG := $(or $(KINDCONFIG),control-plane/kind.yaml)
 KUBECONFIG := $(shell pwd)/.kubeconfig
 
 METALCTL_HMAC := $(or $(METALCTL_HMAC),metal-admin)
-METALCTL_API_URL := $(or $(METALCTL_API_URL),http://api.172.17.0.1.nip.io:8080/metal)
-METAL_APIV2_URL := $(or $(METAL_APIV2_URL),http://v2.172.17.0.1.nip.io:8080)
+METALCTL_API_URL := $(or $(METALCTL_API_URL),http://api.172.42.0.42.nip.io:8080/metal)
+METAL_APIV2_URL := $(or $(METAL_APIV2_URL),http://v2.172.42.0.42.nip.io:8080)
 
 MKE2FS_CONFIG := $(shell pwd)/mke2fs.conf
 # Default values
@@ -24,9 +24,14 @@ ANSIBLE_DISPLAY_SKIPPED_HOSTS=false
 
 MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic)
 MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:latest)
-MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest)
+MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:202511-vpp)
 MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1)
 
+MINI_LAB_INTERNAL_NETWORK=mini_lab_internal
+# define this here as well so that kind picks up the network on a clean checkout,
+# where .env does not exist yet at make parse time (-include .env above)
+KIND_EXPERIMENTAL_DOCKER_NETWORK := $(or $(KIND_EXPERIMENTAL_DOCKER_NETWORK),$(MINI_LAB_INTERNAL_NETWORK))
+
 MACHINE_OS=debian-13.0
 MAX_RETRIES := 30
 
@@ -118,13 +123,16 @@ create-proxy-registries:
 
 .PHONY: control-plane-bake
 control-plane-bake:
+
+	@if ! docker network ls | grep -q mini_lab_internal; then docker network create mini_lab_internal --gateway 172.42.0.1 --ip-range=172.42.0.0/16 --subnet=172.42.0.0/16 --ipv6=false ; fi
 	@if ! which kind > /dev/null; then echo "kind needs to be installed"; exit 1; fi
 	@if ! kind get clusters | grep metal-control-plane > /dev/null; then \
 		kind create cluster $(KIND_ARGS) \
 			--name metal-control-plane \
 			--config $(KINDCONFIG) \
 			--kubeconfig $(KUBECONFIG); fi
 	$(MAKE) create-proxy-registries
+	docker compose up -d --force-recreate cloud-provider-kind
 
 .PHONY: partition
 partition: partition-bake
@@ -176,6 +184,7 @@ env:
 
 .PHONY: cleanup
 cleanup: cleanup-control-plane cleanup-partition
+	docker network rm --force mini_lab_internal
 
 .PHONY: cleanup-control-plane
 cleanup-control-plane:
@@ -422,6 +431,7 @@ build-sonic-base:
 	docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202311 images/sonic/base-202311
 	docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202411 images/sonic/base-202411
 	docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202505 images/sonic/base-202505
+	docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202511-vpp images/sonic/base-202511-vpp
 
 ## DEV TARGETS ##
 
@@ -451,7 +461,7 @@ build-dell-sonic:
 fetch-virtual-kubeconfig:
 	# TODO: it's hard to get the latest issued generic kubeconfig secret... just take the first result for now
 	kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden $(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden -l managed-by=secrets-manager,manager-identity=gardener-operator,name=generic-token-kubeconfig --no-headers | awk '{ print $$1 }') -o jsonpath='{.data.kubeconfig}' | base64 -d > .virtual-kubeconfig
-	@kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.17.0.1.nip.io:4443
+	@kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.42.0.1.nip.io:4443
 	@kubectl --kubeconfig=.virtual-kubeconfig config set-credentials garden --token=$(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden shoot-access-virtual-garden -o jsonpath='{.data.token}' | base64 -d)
 	@kubectl --kubeconfig=$(KUBECONFIG) config unset users.garden
 	@kubectl --kubeconfig=$(KUBECONFIG) config unset contexts.garden

compose.yaml

@@ -126,6 +126,20 @@ services:
       - REGISTRY_PROXY_TTL=168h
       - REGISTRY_STORAGE_DELETE_ENABLED=true
       - OTEL_TRACES_EXPORTER=none
+  cloud-provider-kind:
+    image: registry.k8s.io/cloud-provider-kind/cloud-controller-manager:v0.10.0
+    restart: always
+    networks:
+      - kind
+    environment:
+      - KIND_EXPERIMENTAL_DOCKER_NETWORK=${KIND_EXPERIMENTAL_DOCKER_NETWORK:-kind}
+    command:
+    # v0.10.0 of cloud controller does not support tcproutes, since it does not support the experimental gateway api channel
+    # using envoy-gateway deployed via roles/gateway instead
+      - --gateway-channel
+      - disabled
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
 volumes:
   proxy-docker:
   proxy-gcr:

control-plane/kind.yaml

@@ -2,7 +2,7 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 networking:
   apiServerPort: 6443
-  apiServerAddress: 0.0.0.0
+  apiServerAddress: 172.42.0.1
 nodes:
   - role: control-plane
     extraMounts:
@@ -13,10 +13,6 @@ nodes:
         hostPort: 4443
       - containerPort: 8080
         hostPort: 8080
-      - containerPort: 4150
-        hostPort: 4150
-      - containerPort: 50051
-        hostPort: 50051
     # if you want to run gardener operator + metal-stack, you need more pods
     kubeadmConfigPatches:
       - |

deploy_control_plane.yaml

@@ -6,10 +6,12 @@
   roles:
     - name: ansible-common
       tags: always
-    - name: ingress-controller
-      tags: ingress-controller
     - name: metal-roles/control-plane/roles/prepare
       tags: prepare
+    - name: gateway
+      tags: gateway
+    - name: ingress-controller
+      tags: ingress-controller
     - name: metal-roles/control-plane/roles/nsq
       tags: nsq
     - name: metal-roles/control-plane/roles/metal-db

deploy_gardener.yaml

@@ -13,7 +13,7 @@
             name: shoot-info
             namespace: kube-system
           data:
-            nodeNetwork: 172.18.0.0/16
+            nodeNetwork: 172.42.0.0/16
             podNetwork: 10.244.0.0/24
             serviceNetwork: 10.96.0.0/16
       tags: gardener
@@ -81,7 +81,7 @@
             status:
               loadBalancer:
                 ingress:
-                  - ip: "172.17.0.1"
+                  - ip: "172.42.0.1"
         tags: gardener
 
       - name: Expose istio gateway through ingress-nginx (for local environments)

deploy_partition.yaml

@@ -7,7 +7,7 @@
     - name: Wait for system to become reachable
       ansible.builtin.wait_for_connection:
         delay: 10
-        timeout: 50
+        timeout: 120
   roles:
     - name: ansible-common
       tags: always
@@ -161,10 +161,6 @@
   hosts: dell_sonic
   any_errors_fatal: true
   become: true
-  pre_tasks:
-    - name: Wait some time
-      pause:
-        seconds: 120
   roles:
     - name: ansible-common
       tags: always
@@ -203,6 +199,35 @@
     - name: Wait until no route entries have "queued"
       include_tasks: tasks/check_queued.yaml
 
+- name: Work around isc-dhcp/Vlan4000 startup race (Community SONiC)
+  hosts: leaves:!dell_sonic
+  any_errors_fatal: true
+  gather_facts: false
+  tasks:
+    # dhcpd binds Vlan4000 at boot, before the SVI has carrier. Its LPF receive socket then dies
+    # ("receive_packet failed on Vlan4000: Network is down") while dhcpd stays up holding
+    # 0.0.0.0:67 -- so it answers nothing and PXE clients loop DISCOVER forever, never getting a
+    # lease. Restarting it once the underlay is up rebinds the socket, hence the placement after
+    # the underlay wait above.
+    #
+    # The probe solicits a real broadcast DISCOVER rather than poking 10.0.1.1:67, because the UDP
+    # socket stays healthy even when the LPF path is dead -- a unicast check would report a false
+    # pass. Restart and probe are deliberately one task: a retry must re-do the restart, not just
+    # re-probe, and `until` only retries a single task.
+    - name: Restart isc-dhcp-server until it serves an offer on Vlan4000
+      ansible.builtin.shell: |
+        systemctl restart isc-dhcp-server
+        sleep 2
+        dhcp-server-detector Vlan4000 --exit-on-first-offer --duration 10
+      args:
+        executable: /bin/bash
+      when: dhcp_subnets is defined
+      register: dhcp_offer_check
+      until: dhcp_offer_check.rc == 0
+      retries: 10
+      delay: 5
+      changed_when: true
+
 - name: Configure IPv6 and LLDP ports (Enterprise SONiC)
   hosts: dell_sonic
   any_errors_fatal: true

env.sh

@@ -24,4 +24,5 @@ DEPLOYMENT_BASE_IMAGE_TAG=${DEPLOYMENT_BASE_IMAGE_TAG}
 CI=${CI:=false}
 DOCKER_HUB_USER=${DOCKER_HUB_USER:=}
 DOCKER_HUB_TOKEN=${DOCKER_HUB_TOKEN:=}
+KIND_EXPERIMENTAL_DOCKER_NETWORK=${MINI_LAB_INTERNAL_NETWORK:=mini_lab_internal}
 EOF

files/certs/grpc/server.json

@@ -1,7 +1,7 @@
 {
   "CN": "metal-api",
   "hosts": [
-    "172.17.0.1",
+    "172.42.0.42",
     "203.0.113.1"
   ],
   "key": {

files/certs/metal-api/server.json

@@ -0,0 +1,20 @@
+{
+  "CN": "metal-api",
+  "hosts": [
+    "api.172.42.0.42.nip.io",
+    "v2.172.42.0.42.nip.io"
+  ],
+  "key": {
+    "algo": "rsa",
+    "size": 4096
+  },
+  "names": [
+    {
+      "C":  "DE",
+      "L":  "Munich",
+      "O":  "metal-stack",
+      "OU": "DevOps",
+      "ST": "Bavaria"
+    }
+  ]
+}

files/dev_images.yaml

@@ -1,6 +1,6 @@
 ---
 # Do not change these values
 metal_api_image_tag: dev
-metal_core_image_name: 172.17.0.1:5000/metalstack/metal-core
+metal_core_image_name: 172.42.0.42:5000/metalstack/metal-core
 metal_core_image_tag: dev
-metal_hammer_image_url: http://172.17.0.1:20015/metal-hammer-initrd.img.lz4
+metal_hammer_image_url: http://172.42.0.42:20015/metal-hammer-initrd.img.lz4

files/exit/frr.conf

@@ -35,6 +35,11 @@ router bgp 4200000021
  !
  address-family ipv4 unicast
   redistribute connected route-map LOOPBACKS
+  ! PXE egress: hand the leaves a default so machine traffic routes over the fabric to us, and we
+  ! NAT it out eth0 (mgmt). NB this default is also installed into each leaf's KERNEL, which
+  ! redirects the leaf's OWN egress (pixiecore -> github/DNS) through us sourced from its loopback
+  ! -- that only works because network.sh MASQUERADEs everything leaving eth0. Don't drop that NAT.
+  neighbor FABRIC default-originate
  exit-address-family
  !
  address-family ipv6 unicast

files/exit/network.sh

@@ -27,3 +27,19 @@ bridge vlan add vid 1000 untagged pvid dev vniInternet
 ip link set up dev vniInternet
 
 sysctl -w net.ipv6.conf.all.forwarding=1
+
+# PXE egress return-path NAT. Traffic reaching us over the fabric keeps its original source --
+# machines are 10.0.1.0/24, and the leaves' own traffic is sourced from their 10.0.0.0/24
+# loopbacks. The mgmt docker bridge only NATs 172.42.0.0/16, so anything we forward out eth0 must
+# be SNAT'd or the replies have no path back. Masquerade the whole interface rather than a single
+# prefix, so both the machine and leaf-originated ranges are covered.
+# The frr image ships no iptables; bounded + non-fatal on purpose, since this script runs under
+# `set -o errexit` and must never block exit bring-up if the package mirror is unreachable.
+for _ in 1 2 3 4 5; do
+	if apk add --no-cache iptables; then
+		iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
+		break
+	fi
+	echo "apk could not reach the mirror (iptables); retrying ..."
+	sleep 2
+done || true

files/sonic_frr_with_fpm.tpl

@@ -3,6 +3,12 @@ frr defaults datacenter
 # This is the only line changed from upstream: https://github.com/metal-stack/metal-core/blob/master/cmd/internal/switcher/templates/tpl/sonic_frr.tpl
 # Follow-up issue: https://github.com/metal-stack/metal-core/issues/199
 fpm address 127.0.0.1
+# SONiC default (upstream ships this in its zebra config; the metal-core template dropped it).
+# With next-hop-groups enabled (FRR's default under FPM), zebra hands fpmsyncd an NHG id instead
+# of inline nexthops, and the EVPN encap attrs (vni_label/router_mac, carried as RTA_ENCAP) are
+# lost -> ROUTE_TABLE loses vni_label/router_mac -> RouteOrch builds a plain IP nexthop instead of
+# TUNNEL_ENCAP.
+no fpm use-next-hop-groups
 hostname {{ .Name }}
 password zebra
 enable password zebra

files/startup-config/leaf01_4.4.3.json

@@ -11,7 +11,7 @@
     }
   },
   "DNS_SERVER": {
-    "172.17.0.1": {},
+    "172.42.0.1": {},
     "1.1.1.1": {},
     "1.0.0.1": {}
   },

files/startup-config/leaf01_4.5.1.json

@@ -11,7 +11,7 @@
     }
   },
   "DNS_SERVER": {
-    "172.17.0.1": {},
+    "172.42.0.1": {},
     "1.1.1.1": {},
     "1.0.0.1": {}
   },

files/startup-config/leaf02_4.4.3.json

@@ -11,7 +11,7 @@
     }
   },
   "DNS_SERVER": {
-    "172.17.0.1": {},
+    "172.42.0.1": {},
     "1.1.1.1": {},
     "1.0.0.1": {}
   },

files/startup-config/leaf02_4.5.1.json

@@ -11,7 +11,7 @@
     }
   },
   "DNS_SERVER": {
-    "172.17.0.1": {},
+    "172.42.0.1": {},
     "1.1.1.1": {},
     "1.0.0.1": {}
   },

images/sonic/Dockerfile

@@ -9,13 +9,17 @@ RUN apt-get update && \
         iproute2 \
         linux-image-cloud-amd64 \
         python3 \
+        python3-pip \
         python3-guestfs \
         python3-scapy \
         qemu-system-x86 \
         telnet
 
-COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:202505 /sonic-vs.img /sonic-vs.img
-COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:202505 /frr-pythontools.deb /frr-pythontools.deb
+COPY requirements.txt /
+RUN pip install --break-system-packages -r requirements.txt
+
+COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:master-vpp /sonic-vs.img /sonic-vs.img
+COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:master-vpp /frr-pythontools.deb /frr-pythontools.deb
 
 ENTRYPOINT ["/launch.py"]
 

images/sonic/README.md

@@ -0,0 +1,12 @@
+# Virtual Sonic Images
+
+We use sonic-vpp to emulate SONiC switches. It is running in kvm inside a containerlab container. To provide better emulation accuracy we use sonic-vpp, which used the Vector Package Processor to emulate somthing like a switch ASIC, like the Broadcom Tomahawk 3 used in our Edgecore Accton AS7726-X32 workhorse we use in production. We migrated to sonic-vpp because the sonic-vs image used mostly netlink primitives, which behaved differently than an ASIC driven through SONiCs SAI layer.  It's slower but still sane.
+
+
+# Configuration knobs
+
+You can edit the port_config.ini to add more ports.
+
+
+# Boot process
+The switch will boot with a default first-boot configuration. This is required since first boot will generate some required configuration for VPP. After a short while the configuration that is generated in launch.py is injected and the sonic is reloaded. After the new configuration is loaded the container will be marked ready. Check the docker logs for errors if bootup takes more than a minute.