From dfbbe11052648f46dc36ee8154087894f1a9c8aa Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 7 Jul 2026 12:32:13 +0200 Subject: [PATCH 01/11] feat: move mini-lab to a dedicated docker network To enable fixed IPs for the Gateway API, mini-lab needs a docker network that is not the docker default bridge. Create a dedicated `mini_lab_internal` network (172.42.0.0/16) and relocate everything off the old default network bridge (172.17.0.1 / 172.18.0.0). This shifts the control-plane ingress DNS from 172.17.0.1.nip.io to 172.42.0.1.nip.io. No Gateway API migration yet Signed-off-by: Benjamin Ritter --- Makefile | 10 +++++++++- control-plane/kind.yaml | 2 +- deploy_gardener.yaml | 4 ++-- docs/overview-kamaji.drawio.svg | 4 ++-- docs/overview.drawio.svg | 4 ++-- env.sh | 1 + files/startup-config/leaf01_4.4.3.json | 2 +- files/startup-config/leaf01_4.5.1.json | 2 +- files/startup-config/leaf02_4.4.3.json | 2 +- files/startup-config/leaf02_4.5.1.json | 2 +- inventories/group_vars/all/control_plane.yaml | 2 +- .../group_vars/control_plane/gardener/gardenlet.yaml | 2 +- .../group_vars/control_plane/gardener/operator.yaml | 4 ++-- inventories/group_vars/control_plane/minio.yaml | 2 +- inventories/group_vars/control_plane/powerdns.yaml | 4 ++-- inventories/group_vars/leaves/main.yaml | 2 +- inventories/group_vars/partition/router.yaml | 2 +- mini-lab.dell_sonic.yaml | 2 +- mini-lab.sonic.yaml | 2 +- roles/kamaji/templates/metallb-kind-ip-pool.yaml | 2 +- 20 files changed, 33 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 837e44bf..81018bf1 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,11 @@ MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms: MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest) MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1) +MINI_LAB_INTERNAL_NETWORK=mini_lab_internal +# define this here as well so that kind picks up the network on a clean checkout, +# where .env does not exist yet at make parse time (-include .env above) +KIND_EXPERIMENTAL_DOCKER_NETWORK := $(or $(KIND_EXPERIMENTAL_DOCKER_NETWORK),$(MINI_LAB_INTERNAL_NETWORK)) + MACHINE_OS=debian-13.0 MAX_RETRIES := 30 @@ -118,6 +123,8 @@ create-proxy-registries: .PHONY: control-plane-bake control-plane-bake: + + @if ! docker network ls | grep -q mini_lab_internal; then docker network create mini_lab_internal --gateway 172.42.0.1 --ip-range=172.42.0.0/16 --subnet=172.42.0.0/16 --ipv6=false ; fi @if ! which kind > /dev/null; then echo "kind needs to be installed"; exit 1; fi @if ! kind get clusters | grep metal-control-plane > /dev/null; then \ kind create cluster $(KIND_ARGS) \ @@ -176,6 +183,7 @@ env: .PHONY: cleanup cleanup: cleanup-control-plane cleanup-partition + docker network rm --force mini_lab_internal .PHONY: cleanup-control-plane cleanup-control-plane: @@ -451,7 +459,7 @@ build-dell-sonic: fetch-virtual-kubeconfig: # TODO: it's hard to get the latest issued generic kubeconfig secret... just take the first result for now kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden $(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden -l managed-by=secrets-manager,manager-identity=gardener-operator,name=generic-token-kubeconfig --no-headers | awk '{ print $$1 }') -o jsonpath='{.data.kubeconfig}' | base64 -d > .virtual-kubeconfig - @kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.17.0.1.nip.io:4443 + @kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.42.0.1.nip.io:4443 @kubectl --kubeconfig=.virtual-kubeconfig config set-credentials garden --token=$(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden shoot-access-virtual-garden -o jsonpath='{.data.token}' | base64 -d) @kubectl --kubeconfig=$(KUBECONFIG) config unset users.garden @kubectl --kubeconfig=$(KUBECONFIG) config unset contexts.garden diff --git a/control-plane/kind.yaml b/control-plane/kind.yaml index a55b4aaf..c0af5d47 100644 --- a/control-plane/kind.yaml +++ b/control-plane/kind.yaml @@ -2,7 +2,7 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 networking: apiServerPort: 6443 - apiServerAddress: 0.0.0.0 + apiServerAddress: 172.42.0.1 nodes: - role: control-plane extraMounts: diff --git a/deploy_gardener.yaml b/deploy_gardener.yaml index 43eb72d4..9cfad385 100644 --- a/deploy_gardener.yaml +++ b/deploy_gardener.yaml @@ -13,7 +13,7 @@ name: shoot-info namespace: kube-system data: - nodeNetwork: 172.18.0.0/16 + nodeNetwork: 172.42.0.0/16 podNetwork: 10.244.0.0/24 serviceNetwork: 10.96.0.0/16 tags: gardener @@ -81,7 +81,7 @@ status: loadBalancer: ingress: - - ip: "172.17.0.1" + - ip: "172.42.0.1" tags: gardener - name: Expose istio gateway through ingress-nginx (for local environments) diff --git a/docs/overview-kamaji.drawio.svg b/docs/overview-kamaji.drawio.svg index f6e083c1..e9b2b33e 100644 --- a/docs/overview-kamaji.drawio.svg +++ b/docs/overview-kamaji.drawio.svg @@ -470,13 +470,13 @@
- docker network 172.17.0.1/16 + docker network 172.42.0.1/16
- docker network 172.17.0.1/16 + docker network 172.42.0.1/16 diff --git a/docs/overview.drawio.svg b/docs/overview.drawio.svg index 1267de68..fdb8f9b9 100644 --- a/docs/overview.drawio.svg +++ b/docs/overview.drawio.svg @@ -470,13 +470,13 @@
- docker network 172.17.0.1/16 + docker network 172.42.0.1/16
- docker network 172.17.0.1/16 + docker network 172.42.0.1/16 diff --git a/env.sh b/env.sh index a01de375..bf233635 100755 --- a/env.sh +++ b/env.sh @@ -24,4 +24,5 @@ DEPLOYMENT_BASE_IMAGE_TAG=${DEPLOYMENT_BASE_IMAGE_TAG} CI=${CI:=false} DOCKER_HUB_USER=${DOCKER_HUB_USER:=} DOCKER_HUB_TOKEN=${DOCKER_HUB_TOKEN:=} +KIND_EXPERIMENTAL_DOCKER_NETWORK=${MINI_LAB_INTERNAL_NETWORK:=mini_lab_internal} EOF diff --git a/files/startup-config/leaf01_4.4.3.json b/files/startup-config/leaf01_4.4.3.json index 622b5df9..60393ec3 100644 --- a/files/startup-config/leaf01_4.4.3.json +++ b/files/startup-config/leaf01_4.4.3.json @@ -11,7 +11,7 @@ } }, "DNS_SERVER": { - "172.17.0.1": {}, + "172.42.0.1": {}, "1.1.1.1": {}, "1.0.0.1": {} }, diff --git a/files/startup-config/leaf01_4.5.1.json b/files/startup-config/leaf01_4.5.1.json index cd4224ac..78106fe8 100644 --- a/files/startup-config/leaf01_4.5.1.json +++ b/files/startup-config/leaf01_4.5.1.json @@ -11,7 +11,7 @@ } }, "DNS_SERVER": { - "172.17.0.1": {}, + "172.42.0.1": {}, "1.1.1.1": {}, "1.0.0.1": {} }, diff --git a/files/startup-config/leaf02_4.4.3.json b/files/startup-config/leaf02_4.4.3.json index 5a8d4a78..7359f574 100644 --- a/files/startup-config/leaf02_4.4.3.json +++ b/files/startup-config/leaf02_4.4.3.json @@ -11,7 +11,7 @@ } }, "DNS_SERVER": { - "172.17.0.1": {}, + "172.42.0.1": {}, "1.1.1.1": {}, "1.0.0.1": {} }, diff --git a/files/startup-config/leaf02_4.5.1.json b/files/startup-config/leaf02_4.5.1.json index 41c8f118..335ef382 100644 --- a/files/startup-config/leaf02_4.5.1.json +++ b/files/startup-config/leaf02_4.5.1.json @@ -11,7 +11,7 @@ } }, "DNS_SERVER": { - "172.17.0.1": {}, + "172.42.0.1": {}, "1.1.1.1": {}, "1.0.0.1": {} }, diff --git a/inventories/group_vars/all/control_plane.yaml b/inventories/group_vars/all/control_plane.yaml index 1a93b331..a1beeeb7 100644 --- a/inventories/group_vars/all/control_plane.yaml +++ b/inventories/group_vars/all/control_plane.yaml @@ -1,6 +1,6 @@ --- metal_control_plane_provider_tenant: metal-stack -metal_control_plane_ingress_dns: 172.17.0.1.nip.io +metal_control_plane_ingress_dns: 172.42.0.1.nip.io metal_control_plane_stage_name: test metal_control_plane_namespace: metal-control-plane metal_control_plane_image_pull_policy: Always diff --git a/inventories/group_vars/control_plane/gardener/gardenlet.yaml b/inventories/group_vars/control_plane/gardener/gardenlet.yaml index 8fed1d0f..fe02bb70 100644 --- a/inventories/group_vars/control_plane/gardener/gardenlet.yaml +++ b/inventories/group_vars/control_plane/gardener/gardenlet.yaml @@ -1,5 +1,5 @@ --- -gardener_gardenlet_default_dns_domain: "gardener.172.17.0.1.nip.io" +gardener_gardenlet_default_dns_domain: "gardener.172.42.0.1.nip.io" gardener_gardenlet_default_dns_provider: powerdns gardener_gardenlet_default_dns_credentials: apiKey: "{{ powerdns_api_key | b64encode }}" diff --git a/inventories/group_vars/control_plane/gardener/operator.yaml b/inventories/group_vars/control_plane/gardener/operator.yaml index 5cddad2e..06f6d525 100644 --- a/inventories/group_vars/control_plane/gardener/operator.yaml +++ b/inventories/group_vars/control_plane/gardener/operator.yaml @@ -1,5 +1,5 @@ --- -gardener_operator_ingress_dns_domain: "gardener.172.17.0.1.nip.io" +gardener_operator_ingress_dns_domain: "gardener.{{ metal_control_plane_ingress_dns }}" gardener_operator_backup_infrastructure: provider: S3 @@ -17,7 +17,7 @@ gardener_operator_backup_infrastructure_secret: s3ForcePathStyle: "{{ 'true' | b64encode }}" # enable mini-lab patches -gardener_operator_patch_istio_ingress_gateway_service_ip: 172.17.0.1 +gardener_operator_patch_istio_ingress_gateway_service_ip: 172.42.0.1 gardener_operator_expose_virtual_garden_through_ingress_nginx: true # for local setups this should be sufficient diff --git a/inventories/group_vars/control_plane/minio.yaml b/inventories/group_vars/control_plane/minio.yaml index 73801c7c..0e7566ee 100644 --- a/inventories/group_vars/control_plane/minio.yaml +++ b/inventories/group_vars/control_plane/minio.yaml @@ -2,4 +2,4 @@ minio_root_user: mini-lab minio_root_password: change-me -minio_dns_name: minio.172.17.0.1.nip.io +minio_dns_name: minio.172.42.0.1.nip.io diff --git a/inventories/group_vars/control_plane/powerdns.yaml b/inventories/group_vars/control_plane/powerdns.yaml index bde65e72..d001532f 100644 --- a/inventories/group_vars/control_plane/powerdns.yaml +++ b/inventories/group_vars/control_plane/powerdns.yaml @@ -10,11 +10,11 @@ powerdns_load_balancer_dns_name: "ns.{{ metal_control_plane_ingress_dns }}" powerdns_api_dns_name: "powerdns-api.{{ metal_control_plane_ingress_dns }}" powerdns_zones: - - name: "gardener.172.17.0.1.nip.io." + - name: "gardener.172.42.0.1.nip.io." kind: Master nameservers: - "{{ powerdns_load_balancer_dns_name }}." - - name: "gardener-kube-apiserver.172.17.0.1.nip.io." + - name: "gardener-kube-apiserver.172.42.0.1.nip.io." kind: Master nameservers: - "{{ powerdns_load_balancer_dns_name }}." diff --git a/inventories/group_vars/leaves/main.yaml b/inventories/group_vars/leaves/main.yaml index fbc83ac4..e1d1e596 100644 --- a/inventories/group_vars/leaves/main.yaml +++ b/inventories/group_vars/leaves/main.yaml @@ -12,7 +12,7 @@ sonic_config_frr_render: false sonic_config_loopback_address: "{{ lo }}" sonic_config_mgmt_interface: ip: "{{ ansible_host }}/16`" - gateway_address: "172.17.0.1" + gateway_address: "172.42.0.1" sonic_config_mgmt_vrf: false sonic_config_nameservers: "{{ router_nameservers }}" diff --git a/inventories/group_vars/partition/router.yaml b/inventories/group_vars/partition/router.yaml index bdcbc460..a85d2998 100644 --- a/inventories/group_vars/partition/router.yaml +++ b/inventories/group_vars/partition/router.yaml @@ -1,5 +1,5 @@ --- router_nameservers: - - 172.17.0.1 + - 172.42.0.1 - 1.1.1.1 - 1.0.0.1 diff --git a/mini-lab.dell_sonic.yaml b/mini-lab.dell_sonic.yaml index 91bf0990..d1342437 100644 --- a/mini-lab.dell_sonic.yaml +++ b/mini-lab.dell_sonic.yaml @@ -2,7 +2,7 @@ name: mini-lab prefix: "" mgmt: - network: bridge + network: mini_lab_internal topology: defaults: diff --git a/mini-lab.sonic.yaml b/mini-lab.sonic.yaml index 5d46a128..a14ad7e0 100644 --- a/mini-lab.sonic.yaml +++ b/mini-lab.sonic.yaml @@ -2,7 +2,7 @@ name: mini-lab prefix: "" mgmt: - network: bridge + network: mini_lab_internal topology: defaults: diff --git a/roles/kamaji/templates/metallb-kind-ip-pool.yaml b/roles/kamaji/templates/metallb-kind-ip-pool.yaml index 45034660..eeea201c 100644 --- a/roles/kamaji/templates/metallb-kind-ip-pool.yaml +++ b/roles/kamaji/templates/metallb-kind-ip-pool.yaml @@ -6,7 +6,7 @@ metadata: namespace: metallb-system spec: addresses: - - 172.18.255.200-172.18.255.250 + - 172.42.255.200-172.42.255.250 --- apiVersion: metallb.io/v1beta1 kind: L2Advertisement From f1f016a7c739cf09b3eb26b14df4538dc625a22c Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 7 Jul 2026 12:32:21 +0200 Subject: [PATCH 02/11] feat: add metal-api server certificate Add a metal-api server cert (api./v2. nip.io SANs) for the Gateway to terminate TLS with, add it to roll_certs.sh, and update the grpc cert host to the Gateway LoadBalancer IP. Signed-off-by: Benjamin Ritter --- files/certs/grpc/server.json | 2 +- files/certs/metal-api/server.json | 20 ++++++++++++++++++++ scripts/roll_certs.sh | 3 ++- 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 files/certs/metal-api/server.json diff --git a/files/certs/grpc/server.json b/files/certs/grpc/server.json index 17ba2a53..e1b8bc35 100644 --- a/files/certs/grpc/server.json +++ b/files/certs/grpc/server.json @@ -1,7 +1,7 @@ { "CN": "metal-api", "hosts": [ - "172.17.0.1", + "172.42.0.42", "203.0.113.1" ], "key": { diff --git a/files/certs/metal-api/server.json b/files/certs/metal-api/server.json new file mode 100644 index 00000000..da41eb2a --- /dev/null +++ b/files/certs/metal-api/server.json @@ -0,0 +1,20 @@ +{ + "CN": "metal-api", + "hosts": [ + "api.172.42.0.42.nip.io", + "v2.172.42.0.42.nip.io" + ], + "key": { + "algo": "rsa", + "size": 4096 + }, + "names": [ + { + "C": "DE", + "L": "Munich", + "O": "metal-stack", + "OU": "DevOps", + "ST": "Bavaria" + } + ] +} diff --git a/scripts/roll_certs.sh b/scripts/roll_certs.sh index 7fc34acd..69bcdcda 100755 --- a/scripts/roll_certs.sh +++ b/scripts/roll_certs.sh @@ -13,7 +13,8 @@ rm *.csr for component in \ grpc \ - masterdata-api; do + masterdata-api \ + metal-api; do pushd $component echo "generating $component certs" From 215ddca82b944615b42059f1e444460744f4083a Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 7 Jul 2026 12:32:56 +0200 Subject: [PATCH 03/11] feat: deploy cloud-provider-kind and Envoy Gateway Add the Gateway API plumbing without wiring any application to it yet: Deploy cloud-provider-kind to provide LoadBalancer IPs. Add a `gateway` role deploying the Envoy Gateway controller, a GatewayClass, an EnvoyProxy pinned to the fixed LB IP 172.42.0.42, and the metal-control-plane Gateway. Register the role in the control-plane playbook and expose metal_control_plane_gateway_dns Signed-off-by: Benjamin Ritter --- Makefile | 1 + compose.yaml | 14 ++++++++ deploy_control_plane.yaml | 6 ++-- inventories/group_vars/all/control_plane.yaml | 1 + .../group_vars/control_plane/ingress.yaml | 3 ++ roles/gateway/defaults/main.yml | 8 +++++ roles/gateway/files/gatewayclass.yaml | 6 ++++ roles/gateway/tasks/main.yml | 34 ++++++++++++++++++ roles/gateway/templates/envoyproxy.yaml | 14 ++++++++ roles/gateway/templates/gateway.yaml | 35 +++++++++++++++++++ 10 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 roles/gateway/defaults/main.yml create mode 100644 roles/gateway/files/gatewayclass.yaml create mode 100644 roles/gateway/tasks/main.yml create mode 100644 roles/gateway/templates/envoyproxy.yaml create mode 100644 roles/gateway/templates/gateway.yaml diff --git a/Makefile b/Makefile index 81018bf1..3f46dd28 100644 --- a/Makefile +++ b/Makefile @@ -132,6 +132,7 @@ control-plane-bake: --config $(KINDCONFIG) \ --kubeconfig $(KUBECONFIG); fi $(MAKE) create-proxy-registries + docker compose up -d --force-recreate cloud-provider-kind .PHONY: partition partition: partition-bake diff --git a/compose.yaml b/compose.yaml index 8fd3e0d9..82e87843 100644 --- a/compose.yaml +++ b/compose.yaml @@ -126,6 +126,20 @@ services: - REGISTRY_PROXY_TTL=168h - REGISTRY_STORAGE_DELETE_ENABLED=true - OTEL_TRACES_EXPORTER=none + cloud-provider-kind: + image: registry.k8s.io/cloud-provider-kind/cloud-controller-manager:v0.10.0 + restart: always + networks: + - kind + environment: + - KIND_EXPERIMENTAL_DOCKER_NETWORK=${KIND_EXPERIMENTAL_DOCKER_NETWORK:-kind} + command: + # v0.10.0 of cloud controller does not support tcproutes, since it does not support the experimental gateway api channel + # using envoy-gateway deployed via roles/gateway instead + - --gateway-channel + - disabled + volumes: + - /var/run/docker.sock:/var/run/docker.sock volumes: proxy-docker: proxy-gcr: diff --git a/deploy_control_plane.yaml b/deploy_control_plane.yaml index bd0f0241..7336cf70 100644 --- a/deploy_control_plane.yaml +++ b/deploy_control_plane.yaml @@ -6,10 +6,12 @@ roles: - name: ansible-common tags: always - - name: ingress-controller - tags: ingress-controller - name: metal-roles/control-plane/roles/prepare tags: prepare + - name: gateway + tags: gateway + - name: ingress-controller + tags: ingress-controller - name: metal-roles/control-plane/roles/nsq tags: nsq - name: metal-roles/control-plane/roles/metal-db diff --git a/inventories/group_vars/all/control_plane.yaml b/inventories/group_vars/all/control_plane.yaml index a1beeeb7..a0f045fe 100644 --- a/inventories/group_vars/all/control_plane.yaml +++ b/inventories/group_vars/all/control_plane.yaml @@ -1,6 +1,7 @@ --- metal_control_plane_provider_tenant: metal-stack metal_control_plane_ingress_dns: 172.42.0.1.nip.io +metal_control_plane_gateway_dns: 172.42.0.42.nip.io metal_control_plane_stage_name: test metal_control_plane_namespace: metal-control-plane metal_control_plane_image_pull_policy: Always diff --git a/inventories/group_vars/control_plane/ingress.yaml b/inventories/group_vars/control_plane/ingress.yaml index 0ef8644b..fe68a6b6 100644 --- a/inventories/group_vars/control_plane/ingress.yaml +++ b/inventories/group_vars/control_plane/ingress.yaml @@ -2,3 +2,6 @@ ingress_tcp_service_exposals: "4150": "{{ metal_control_plane_namespace }}/nsqd:4150" "50051": "{{ metal_control_plane_namespace }}/metal-api:50051" +gateway_tcp_listeners: + nsq: 4150 + metal-api-grpc: 50051 \ No newline at end of file diff --git a/roles/gateway/defaults/main.yml b/roles/gateway/defaults/main.yml new file mode 100644 index 00000000..5bb8dff6 --- /dev/null +++ b/roles/gateway/defaults/main.yml @@ -0,0 +1,8 @@ +--- +gateway_namespace: "{{ metal_control_plane_namespace }}" +gateway_http_port: 8080 +gateway_https_port: 4443 + +gateway_tcp_listeners: {} + # nsq: 4150 + # metal-api: 50051 diff --git a/roles/gateway/files/gatewayclass.yaml b/roles/gateway/files/gatewayclass.yaml new file mode 100644 index 00000000..a750b9fd --- /dev/null +++ b/roles/gateway/files/gatewayclass.yaml @@ -0,0 +1,6 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller \ No newline at end of file diff --git a/roles/gateway/tasks/main.yml b/roles/gateway/tasks/main.yml new file mode 100644 index 00000000..d0f1a4d3 --- /dev/null +++ b/roles/gateway/tasks/main.yml @@ -0,0 +1,34 @@ +- name: Deploy envoy-gateway + kubernetes.core.helm: + release_name: envoy-gateway + release_namespace: envoy-gateway-system + create_namespace: true + chart_ref: "oci://docker.io/envoyproxy/gateway-helm" + chart_version: "{{ envoy_gateway_chart_version | default(omit) }}" + # values: "{{ lookup('template', 'values.yaml') | from_yaml }}" + wait: true + +- name: Deploy Envoy GatewayClass + kubernetes.core.k8s: + definition: "{{ lookup('file', 'gatewayclass.yaml') | from_yaml }}" + +- name: Deploy metal-control-plane EnvoyProxy + kubernetes.core.k8s: + definition: "{{ lookup('template', 'envoyproxy.yaml') | from_yaml }}" + +- name: Deploy metal-api certificate secret + kubernetes.core.k8s: + definition: + apiVersion: v1 + kind: Secret + type: kubernetes.io/tls + metadata: + name: metal-api-tls + namespace: "{{ gateway_namespace }}" + data: + tls.crt: "{{ lookup('file', 'certs/metal-api/server.pem') | b64encode }}" + tls.key: "{{ lookup('file', 'certs/metal-api/server-key.pem') | b64encode }}" + +- name: Deploy metal-control-plane Gateway + kubernetes.core.k8s: + definition: "{{ lookup('template', 'gateway.yaml') | from_yaml }}" diff --git a/roles/gateway/templates/envoyproxy.yaml b/roles/gateway/templates/envoyproxy.yaml new file mode 100644 index 00000000..edb7c0fe --- /dev/null +++ b/roles/gateway/templates/envoyproxy.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: metal-control-plane + namespace: "{{ gateway_namespace }}" +spec: + provider: + type: Kubernetes + kubernetes: + envoyService: + externalTrafficPolicy: Local + type: LoadBalancer + loadBalancerIP: 172.42.0.42 \ No newline at end of file diff --git a/roles/gateway/templates/gateway.yaml b/roles/gateway/templates/gateway.yaml new file mode 100644 index 00000000..49b0ec4e --- /dev/null +++ b/roles/gateway/templates/gateway.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: metal-control-plane + namespace: "{{ gateway_namespace }}" +spec: + gatewayClassName: eg + addresses: + - value: 172.42.0.42 + infrastructure: + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: metal-control-plane + listeners: + - protocol: HTTP + port: {{ gateway_http_port }} + name: http + hostname: "*.{{ metal_control_plane_gateway_dns }}" + - protocol: HTTPS + port: {{ gateway_https_port }} + name: https + hostname: "*.{{ metal_control_plane_gateway_dns }}" + tls: + mode: Terminate + certificateRefs: + - group: "" + kind: Secret + name: metal-api-tls +{% for name, port in gateway_tcp_listeners.items() %} + - protocol: TCP + name: {{ name }} + port: {{ port }} +{% endfor %} From fc3caf2d89cdf6e4d09b1e78e0554fa1d426fed6 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 7 Jul 2026 12:33:25 +0200 Subject: [PATCH 04/11] feat: switch control-plane services to Gateway API Migrate zitadel, nsq, metal-api and metal-apiserver off ingress-nginx to Gateway API. Enable http/tcp routes, point their external URLs at the Gateway LoadBalancer (172.42.0.42 / gateway_dns). Signed-off-by: Benjamin Ritter --- Makefile | 4 +-- files/dev_images.yaml | 4 +-- .../group_vars/control_plane/metal.yml | 33 +++++++++++++++---- inventories/group_vars/control_plane/nsq.yaml | 5 +++ .../group_vars/control_plane/zitadel.yaml | 9 +++-- inventories/group_vars/partition/common.yaml | 4 +-- 6 files changed, 45 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 3f46dd28..fb2368e0 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,8 @@ KINDCONFIG := $(or $(KINDCONFIG),control-plane/kind.yaml) KUBECONFIG := $(shell pwd)/.kubeconfig METALCTL_HMAC := $(or $(METALCTL_HMAC),metal-admin) -METALCTL_API_URL := $(or $(METALCTL_API_URL),http://api.172.17.0.1.nip.io:8080/metal) -METAL_APIV2_URL := $(or $(METAL_APIV2_URL),http://v2.172.17.0.1.nip.io:8080) +METALCTL_API_URL := $(or $(METALCTL_API_URL),http://api.172.42.0.42.nip.io:8080/metal) +METAL_APIV2_URL := $(or $(METAL_APIV2_URL),http://v2.172.42.0.42.nip.io:8080) MKE2FS_CONFIG := $(shell pwd)/mke2fs.conf # Default values diff --git a/files/dev_images.yaml b/files/dev_images.yaml index e0699518..91b8285c 100644 --- a/files/dev_images.yaml +++ b/files/dev_images.yaml @@ -1,6 +1,6 @@ --- # Do not change these values metal_api_image_tag: dev -metal_core_image_name: 172.17.0.1:5000/metalstack/metal-core +metal_core_image_name: 172.42.0.42:5000/metalstack/metal-core metal_core_image_tag: dev -metal_hammer_image_url: http://172.17.0.1:20015/metal-hammer-initrd.img.lz4 +metal_hammer_image_url: http://172.42.0.42:20015/metal-hammer-initrd.img.lz4 diff --git a/inventories/group_vars/control_plane/metal.yml b/inventories/group_vars/control_plane/metal.yml index 40db02a6..814a55a0 100644 --- a/inventories/group_vars/control_plane/metal.yml +++ b/inventories/group_vars/control_plane/metal.yml @@ -1,9 +1,21 @@ --- metal_set_resource_limits: no -metal_check_api_health_endpoint: http://api.{{ metal_control_plane_ingress_dns }}:8080/metal/v1/health -metal_api_headscale_control_plane_address: "http://headscale.{{ metal_control_plane_ingress_dns }}:8080" +metal_check_api_health_endpoint: http://api.{{ metal_control_plane_gateway_dns }}:8080/metal/v1/health +metal_api_headscale_control_plane_address: "http://headscale.{{ metal_control_plane_gateway_dns }}:8080" -# metal_helm_chart_local_path: /helm-charts/charts/metal-control-plane +metal_deploy_ingress: false + +metal_api_httproute_enabled: true +metal_api_httproute_parent_refs: +- name: metal-control-plane + namespace: "{{ metal_control_plane_namespace }}" + sectionName: http + +metal_api_tcproute_enabled: true +metal_api_tcproute_parent_refs: +- name: metal-control-plane + namespace: "{{ metal_control_plane_namespace }}" + sectionName: metal-api-grpc metal_api_pdb_min_available: 1 metal_api_replicas: 1 @@ -16,12 +28,21 @@ metal_api_nsq_tcp_address: nsqd:4150 metal_apiserver_pdb_min_available: 1 metal_apiserver_enabled: true -metal_apiserver_url: http://v2.{{ metal_control_plane_ingress_dns }}:8080 +metal_apiserver_url: http://v2.{{ metal_control_plane_gateway_dns }}:8080 -metal_apiserver_oidc_discovery_url: https://auth.{{ metal_control_plane_ingress_dns }}:4443/.well-known/openid-configuration -metal_apiserver_oidc_end_session_url: "https://auth.{{ metal_control_plane_ingress_dns }}:4443/oidc/v1/end_session" +metal_apiserver_oidc_secret_name: zitadel-client-credentials +metal_apiserver_oidc_discovery_url: https://zitadel.{{ metal_control_plane_gateway_dns }}:4443/.well-known/openid-configuration +metal_apiserver_oidc_end_session_url: "https://zitadel.{{ metal_control_plane_gateway_dns }}:4443/oidc/v1/end_session" +metal_apiserver_redis_password: change-me-soon metal_apiserver_secure_cookie: false +metal_apiserver_admin_subjects: "admin@metal-stack.zitadel.{{ metal_control_plane_gateway_dns }}@openid-connect" + +metal_apiserver_httproute_enabled: true +metal_apiserver_httproute_parent_refs: +- name: metal-control-plane + namespace: "{{ metal_control_plane_namespace }}" + sectionName: http metal_api_images: - id: firewall-ubuntu-3.0 diff --git a/inventories/group_vars/control_plane/nsq.yaml b/inventories/group_vars/control_plane/nsq.yaml index b8983676..c7a1f802 100644 --- a/inventories/group_vars/control_plane/nsq.yaml +++ b/inventories/group_vars/control_plane/nsq.yaml @@ -9,3 +9,8 @@ nsq_certs_client_cert: "{{ lookup('file', 'certs/nsq/client.crt') }}" nsq_certs_ca_cert: "{{ lookup('file', 'certs/ca.pem') }}" nsq_broadcast_address: nsqd + +nsq_tcproute_enabled: true +nsq_tcproute_parent_refs: +- name: metal-control-plane + sectionName: nsq diff --git a/inventories/group_vars/control_plane/zitadel.yaml b/inventories/group_vars/control_plane/zitadel.yaml index 4a2675be..b316798c 100644 --- a/inventories/group_vars/control_plane/zitadel.yaml +++ b/inventories/group_vars/control_plane/zitadel.yaml @@ -1,11 +1,16 @@ --- zitadel_endpoint: zitadel.{{ metal_control_plane_namespace }}.svc.cluster.local -zitadel_external_domain: auth.{{ metal_control_plane_ingress_dns }} +zitadel_external_domain: zitadel.{{ metal_control_plane_gateway_dns }} zitadel_ingress_dns: https://{{ zitadel_external_domain }}:4443 zitadel_port: 8080 zitadel_skip_verify_tls: true zitadel_insecure: true +zitadel_httproute_enabled: true +zitadel_httproute_parent_refs: +- name: metal-control-plane + sectionName: https + zitadel_init_config: static_users: - first_name: Olli @@ -23,4 +28,4 @@ zitadel_init_config: # later id will be added but currently not possible with zitadel id: metal-stack name: metal-stack - redirect_uri: http://v2.172.17.0.1.nip.io:8080/auth/openid-connect/callback + redirect_uri: http://v2.172.42.0.42.nip.io:8080/auth/openid-connect/callback diff --git a/inventories/group_vars/partition/common.yaml b/inventories/group_vars/partition/common.yaml index 61da08a0..57761405 100644 --- a/inventories/group_vars/partition/common.yaml +++ b/inventories/group_vars/partition/common.yaml @@ -3,7 +3,7 @@ metal_partition_timezone: Europe/Berlin metal_partition_id: mini-lab metal_partition_metal_api_protocol: http -metal_partition_metal_api_addr: api.{{ metal_control_plane_ingress_dns }} +metal_partition_metal_api_addr: api.{{ metal_control_plane_gateway_dns }} metal_partition_metal_api_port: 8080 metal_partition_metal_api_basepath: /metal/ metal_partition_metal_api_hmac_edit_key: metal-edit @@ -14,4 +14,4 @@ metal_partition_metal_api_grpc_ca_cert: "{{ lookup('file', 'certs/ca.pem') }}" metal_partition_metal_api_grpc_client_cert: "{{ lookup('file', 'certs/grpc/client.pem') }}" metal_partition_metal_api_grpc_client_key: "{{ lookup('file', 'certs/grpc/client-key.pem') }}" -metal_partition_mgmt_gateway: 172.17.0.1 +metal_partition_mgmt_gateway: 172.42.0.42 From 4e3bdebeaa4cd6f23c399dfd894fbe74e74a1748 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Fri, 8 May 2026 16:48:01 +0200 Subject: [PATCH 05/11] feat: remove ingress-nginx exposed ports Signed-off-by: Benjamin Ritter --- control-plane/kind.yaml | 4 ---- inventories/group_vars/control_plane/ingress.yaml | 3 --- 2 files changed, 7 deletions(-) diff --git a/control-plane/kind.yaml b/control-plane/kind.yaml index c0af5d47..018c5454 100644 --- a/control-plane/kind.yaml +++ b/control-plane/kind.yaml @@ -13,10 +13,6 @@ nodes: hostPort: 4443 - containerPort: 8080 hostPort: 8080 - - containerPort: 4150 - hostPort: 4150 - - containerPort: 50051 - hostPort: 50051 # if you want to run gardener operator + metal-stack, you need more pods kubeadmConfigPatches: - | diff --git a/inventories/group_vars/control_plane/ingress.yaml b/inventories/group_vars/control_plane/ingress.yaml index fe68a6b6..d7a9fd25 100644 --- a/inventories/group_vars/control_plane/ingress.yaml +++ b/inventories/group_vars/control_plane/ingress.yaml @@ -1,7 +1,4 @@ --- -ingress_tcp_service_exposals: - "4150": "{{ metal_control_plane_namespace }}/nsqd:4150" - "50051": "{{ metal_control_plane_namespace }}/metal-api:50051" gateway_tcp_listeners: nsq: 4150 metal-api-grpc: 50051 \ No newline at end of file From 4b9e3922499d3083c8ef14b1c57b2493c17299d5 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Wed, 20 May 2026 21:25:33 +0200 Subject: [PATCH 06/11] feat: add sonic-vpp image --- .github/workflows/base-image.yaml | 1 + Makefile | 1 + images/sonic/Dockerfile | 4 ++-- images/sonic/base-vpp/Dockerfile | 26 ++++++++++++++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 images/sonic/base-vpp/Dockerfile diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index 039e6f01..83ca6040 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -16,6 +16,7 @@ jobs: - name: 202311 - name: 202411 - name: 202505 + - name: 202511-vpp steps: - name: Log in to the container registry diff --git a/Makefile b/Makefile index fb2368e0..93fff875 100644 --- a/Makefile +++ b/Makefile @@ -431,6 +431,7 @@ build-sonic-base: docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202311 images/sonic/base-202311 docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202411 images/sonic/base-202411 docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202505 images/sonic/base-202505 + docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202511 images/sonic/base-202511-vpp ## DEV TARGETS ## diff --git a/images/sonic/Dockerfile b/images/sonic/Dockerfile index 9565c738..09b4ebc0 100644 --- a/images/sonic/Dockerfile +++ b/images/sonic/Dockerfile @@ -14,8 +14,8 @@ RUN apt-get update && \ qemu-system-x86 \ telnet -COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:202505 /sonic-vs.img /sonic-vs.img -COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:202505 /frr-pythontools.deb /frr-pythontools.deb +COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:master-vpp /sonic-vs.img /sonic-vs.img +COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:master-vpp /frr-pythontools.deb /frr-pythontools.deb ENTRYPOINT ["/launch.py"] diff --git a/images/sonic/base-vpp/Dockerfile b/images/sonic/base-vpp/Dockerfile new file mode 100644 index 00000000..277cda85 --- /dev/null +++ b/images/sonic/base-vpp/Dockerfile @@ -0,0 +1,26 @@ +# Check: https://sonic-build.azurewebsites.net/ui/sonic/pipelines +ARG SONIC_BRANCH=master +ARG SONIC_BASE_URL=https://sonic-build.azurewebsites.net/api/sonic/artifacts?branchName=${SONIC_BRANCH}&definitionId=2818&artifactName=sonic-buildimage.vpp +ARG SONIC_IMG_URL=${SONIC_BASE_URL}&target=target%2Fsonic-vpp.img.gz +ARG FRR_RELOAD_URL=${SONIC_BASE_URL}&target=target%2Fdebs%2Fbookworm%2Ffrr-pythontools_10.5.4-sonic-0_all.deb + +FROM docker.io/library/busybox:stable AS download + +ARG SONIC_IMG_URL +ARG FRR_RELOAD_URL + +ADD "${SONIC_IMG_URL}" /sonic-vs.img.gz +ADD "${FRR_RELOAD_URL}" /frr-pythontools.deb + +RUN gunzip /sonic-vs.img.gz + +FROM scratch + +ARG SONIC_IMG_URL +ARG FRR_RELOAD_URL + +LABEL sonic-img-url=${SONIC_IMG_URL} \ + frr-reload-url=${FRR_RELOAD_URL} + +COPY --from=download /frr-pythontools.deb /frr-pythontools.deb +COPY --from=download /sonic-vs.img /sonic-vs.img From 88d7e85373943590abfcaf109b08ad511c608370 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Sat, 23 May 2026 10:59:08 +0200 Subject: [PATCH 07/11] feat: fix sonic-vpp boot by cleaning up launch.py Removed hacks, that just about worked in sonic-vs but broke in sonic-vpp - sonic-vpp requires first time init to generate VPP config files from hwsku lanemap and port config. If skipped the syncd container, in which vpp runs, will crash immediately - /etc/sonic/sonic-environment is generated on first boot from /usr/share/sonic/device/x86_64-kvm_x86_64-r0/default_sku, so writing it serves no purpose as we reenabled firstboot - switch to telnetlib3, due to telnetlib being deprecated Signed-off-by: Benjamin Ritter --- Makefile | 4 +- deploy_partition.yaml | 6 +- images/sonic/Dockerfile | 4 + images/sonic/README.md | 8 ++ images/sonic/port_config.ini | 125 +----------------------- images/sonic/requirements.txt | 1 + inventories/group_vars/leaves/main.yaml | 2 +- mini-lab.sonic.yaml | 8 +- roles/sonic/tasks/main.yaml | 22 ++--- 9 files changed, 35 insertions(+), 145 deletions(-) create mode 100644 images/sonic/README.md create mode 100644 images/sonic/requirements.txt diff --git a/Makefile b/Makefile index 93fff875..5d17f0a9 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ ANSIBLE_DISPLAY_SKIPPED_HOSTS=false MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic) MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:latest) -MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest) +MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:202511-vpp) MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1) MINI_LAB_INTERNAL_NETWORK=mini_lab_internal @@ -431,7 +431,7 @@ build-sonic-base: docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202311 images/sonic/base-202311 docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202411 images/sonic/base-202411 docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202505 images/sonic/base-202505 - docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202511 images/sonic/base-202511-vpp + docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202511-vpp images/sonic/base-202511-vpp ## DEV TARGETS ## diff --git a/deploy_partition.yaml b/deploy_partition.yaml index 2ad4de4d..295355e8 100644 --- a/deploy_partition.yaml +++ b/deploy_partition.yaml @@ -7,7 +7,7 @@ - name: Wait for system to become reachable ansible.builtin.wait_for_connection: delay: 10 - timeout: 50 + timeout: 120 roles: - name: ansible-common tags: always @@ -161,10 +161,6 @@ hosts: dell_sonic any_errors_fatal: true become: true - pre_tasks: - - name: Wait some time - pause: - seconds: 120 roles: - name: ansible-common tags: always diff --git a/images/sonic/Dockerfile b/images/sonic/Dockerfile index 09b4ebc0..c4339242 100644 --- a/images/sonic/Dockerfile +++ b/images/sonic/Dockerfile @@ -9,11 +9,15 @@ RUN apt-get update && \ iproute2 \ linux-image-cloud-amd64 \ python3 \ + python3-pip \ python3-guestfs \ python3-scapy \ qemu-system-x86 \ telnet +COPY requirements.txt / +RUN pip install --break-system-packages -r requirements.txt + COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:master-vpp /sonic-vs.img /sonic-vs.img COPY --from=ghcr.io/metal-stack/mini-lab-sonic-base:master-vpp /frr-pythontools.deb /frr-pythontools.deb diff --git a/images/sonic/README.md b/images/sonic/README.md new file mode 100644 index 00000000..bdec0df1 --- /dev/null +++ b/images/sonic/README.md @@ -0,0 +1,8 @@ +# Virtual Sonic Images + +We use sonic-vpp to emulate SONiC switches. It is running in kvm inside a containerlab container. To provide better emulation accuracy we use sonic-vpp, which used the Vector Package Processor to emulate somthing like a switch ASIC, like the Broadcom Tomahawk 3 used in our Edgecore Accton AS7726-X32 workhorse we use in production. We migrated to sonic-vpp because the sonic-vs image used mostly netlink primitives, which behaved differently than an ASIC driven through SONiCs SAI layer. It's slower but still sane. + + +# Configuration knobs + +You can edit the port_config.ini to add more ports. Keep the number as low as possible. It will put less strain on your system because it will spawn fewer VPP worker threads. You will have to set up the switch from scratch afterwards, since VPP will generate some configuration on first startup. \ No newline at end of file diff --git a/images/sonic/port_config.ini b/images/sonic/port_config.ini index acc1f3d2..2dfa3d87 100644 --- a/images/sonic/port_config.ini +++ b/images/sonic/port_config.ini @@ -1,123 +1,4 @@ # name lanes alias index speed -Ethernet0 1 Eth1/1 1 25000 -Ethernet1 2 Eth1/2 1 25000 -Ethernet2 3 Eth1/3 1 25000 -Ethernet3 4 Eth1/4 1 25000 -Ethernet4 5 Eth2/1 2 25000 -Ethernet5 6 Eth2/2 2 25000 -Ethernet6 7 Eth2/3 2 25000 -Ethernet7 8 Eth2/4 2 25000 -Ethernet8 9 Eth3/1 3 25000 -Ethernet9 10 Eth3/2 3 25000 -Ethernet10 11 Eth3/3 3 25000 -Ethernet11 12 Eth3/4 3 25000 -Ethernet12 13 Eth4/1 4 25000 -Ethernet13 14 Eth4/2 4 25000 -Ethernet14 15 Eth4/3 4 25000 -Ethernet15 16 Eth4/4 4 25000 -Ethernet16 17 Eth5/1 5 25000 -Ethernet17 18 Eth5/2 5 25000 -Ethernet18 19 Eth5/3 5 25000 -Ethernet19 20 Eth5/4 5 25000 -Ethernet20 21 Eth6/1 6 25000 -Ethernet21 22 Eth6/2 6 25000 -Ethernet22 23 Eth6/3 6 25000 -Ethernet23 24 Eth6/4 6 25000 -Ethernet24 25 Eth7/1 7 25000 -Ethernet25 26 Eth7/2 7 25000 -Ethernet26 27 Eth7/3 7 25000 -Ethernet27 28 Eth7/4 7 25000 -Ethernet28 29 Eth8/1 8 25000 -Ethernet29 30 Eth8/2 8 25000 -Ethernet30 31 Eth8/3 8 25000 -Ethernet31 32 Eth8/4 8 25000 -Ethernet32 33 Eth9/1 9 25000 -Ethernet33 34 Eth9/2 9 25000 -Ethernet34 35 Eth9/3 9 25000 -Ethernet35 36 Eth9/4 9 25000 -Ethernet36 37 Eth10/1 10 25000 -Ethernet37 38 Eth10/2 10 25000 -Ethernet38 39 Eth10/3 10 25000 -Ethernet39 40 Eth10/4 10 25000 -Ethernet40 41 Eth11/1 11 25000 -Ethernet41 42 Eth11/2 11 25000 -Ethernet42 43 Eth11/3 11 25000 -Ethernet43 44 Eth11/4 11 25000 -Ethernet44 45 Eth12/1 12 25000 -Ethernet45 46 Eth12/2 12 25000 -Ethernet46 47 Eth12/3 12 25000 -Ethernet47 48 Eth12/4 12 25000 -Ethernet48 49 Eth13/1 13 25000 -Ethernet49 50 Eth13/2 13 25000 -Ethernet50 51 Eth13/3 13 25000 -Ethernet51 52 Eth13/4 13 25000 -Ethernet52 53 Eth14/1 14 25000 -Ethernet53 54 Eth14/2 14 25000 -Ethernet54 55 Eth14/3 14 25000 -Ethernet55 56 Eth14/4 14 25000 -Ethernet56 57 Eth15/1 15 25000 -Ethernet57 58 Eth15/2 15 25000 -Ethernet58 59 Eth15/3 15 25000 -Ethernet59 60 Eth15/4 15 25000 -Ethernet60 61 Eth16/1 16 25000 -Ethernet61 62 Eth16/2 16 25000 -Ethernet62 63 Eth16/3 16 25000 -Ethernet63 64 Eth16/4 16 25000 -Ethernet64 65 Eth17/1 17 25000 -Ethernet65 66 Eth17/2 17 25000 -Ethernet66 67 Eth17/3 17 25000 -Ethernet67 68 Eth17/4 17 25000 -Ethernet68 69 Eth18/1 18 25000 -Ethernet69 70 Eth18/2 18 25000 -Ethernet70 71 Eth18/3 18 25000 -Ethernet71 72 Eth18/4 18 25000 -Ethernet72 73 Eth19/1 19 25000 -Ethernet73 74 Eth19/2 19 25000 -Ethernet74 75 Eth19/3 19 25000 -Ethernet75 76 Eth19/4 19 25000 -Ethernet76 77 Eth20/1 20 25000 -Ethernet77 78 Eth20/2 20 25000 -Ethernet78 79 Eth20/3 20 25000 -Ethernet79 80 Eth20/4 20 25000 -Ethernet80 81 Eth21/1 21 25000 -Ethernet81 82 Eth21/2 21 25000 -Ethernet82 83 Eth21/3 21 25000 -Ethernet83 84 Eth21/4 21 25000 -Ethernet84 85 Eth22/1 22 25000 -Ethernet85 86 Eth22/2 22 25000 -Ethernet86 87 Eth22/3 22 25000 -Ethernet87 88 Eth22/4 22 25000 -Ethernet88 89 Eth23/1 23 25000 -Ethernet89 90 Eth23/2 23 25000 -Ethernet90 91 Eth23/3 23 25000 -Ethernet91 92 Eth23/4 23 25000 -Ethernet92 93 Eth24/1 24 25000 -Ethernet93 94 Eth24/2 24 25000 -Ethernet94 95 Eth24/3 24 25000 -Ethernet95 96 Eth24/4 24 25000 -Ethernet96 97 Eth25/1 25 25000 -Ethernet97 98 Eth25/2 25 25000 -Ethernet98 99 Eth25/3 25 25000 -Ethernet99 100 Eth25/4 25 25000 -Ethernet100 101 Eth26/1 26 25000 -Ethernet101 102 Eth26/2 26 25000 -Ethernet102 103 Eth26/3 26 25000 -Ethernet103 104 Eth26/4 26 25000 -Ethernet104 105 Eth27/1 27 25000 -Ethernet105 106 Eth27/2 27 25000 -Ethernet106 107 Eth27/3 27 25000 -Ethernet107 108 Eth27/4 27 25000 -Ethernet108 109 Eth28/1 28 25000 -Ethernet109 110 Eth28/2 28 25000 -Ethernet110 111 Eth28/3 28 25000 -Ethernet111 112 Eth28/4 28 25000 -Ethernet112 113 Eth29/1 29 25000 -Ethernet113 114 Eth29/2 29 25000 -Ethernet114 115 Eth29/3 29 25000 -Ethernet115 116 Eth29/4 29 25000 -Ethernet116 117 Eth30/1 30 25000 -Ethernet117 118 Eth30/2 30 25000 -Ethernet118 119 Eth30/3 30 25000 -Ethernet119 120 Eth30/4 30 25000 -Ethernet120 121,122,123,124 Eth31 31 100000 -Ethernet124 125,126,127,128 Eth32 32 100000 +Ethernet0 1,2,3,4 Eth1 1 100000 +Ethernet4 5,6,7,8 Eth2 2 100000 +Ethernet8 121,122,123,124 Eth3 3 100000 \ No newline at end of file diff --git a/images/sonic/requirements.txt b/images/sonic/requirements.txt new file mode 100644 index 00000000..4973e51b --- /dev/null +++ b/images/sonic/requirements.txt @@ -0,0 +1 @@ +telnetlib3~=4.0.4 \ No newline at end of file diff --git a/inventories/group_vars/leaves/main.yaml b/inventories/group_vars/leaves/main.yaml index e1d1e596..765c3bec 100644 --- a/inventories/group_vars/leaves/main.yaml +++ b/inventories/group_vars/leaves/main.yaml @@ -4,7 +4,7 @@ dhcp_listening_interfaces: metal_core_cidr_mask: 25 metal_core_spine_uplinks: - - Ethernet120 + - Ethernet8 sonic_config_docker_routing_config_mode: split-unified sonic_config_frr_render: false diff --git a/mini-lab.sonic.yaml b/mini-lab.sonic.yaml index a14ad7e0..c29fd11a 100644 --- a/mini-lab.sonic.yaml +++ b/mini-lab.sonic.yaml @@ -58,7 +58,7 @@ topology: mtu: 9000 - endpoints: ["leaf01:Ethernet0", "machine01:lan0"] - endpoints: ["leaf02:Ethernet0", "machine01:lan1"] - - endpoints: ["leaf01:Ethernet1", "machine02:lan0"] - - endpoints: ["leaf02:Ethernet1", "machine02:lan1"] - - endpoints: ["leaf01:Ethernet120", "exit:eth1"] - - endpoints: ["leaf02:Ethernet120", "exit:eth2"] + - endpoints: ["leaf01:Ethernet4", "machine02:lan0"] + - endpoints: ["leaf02:Ethernet4", "machine02:lan1"] + - endpoints: ["leaf01:Ethernet8", "exit:eth1"] + - endpoints: ["leaf02:Ethernet8", "exit:eth2"] diff --git a/roles/sonic/tasks/main.yaml b/roles/sonic/tasks/main.yaml index c8ee8460..444c7367 100644 --- a/roles/sonic/tasks/main.yaml +++ b/roles/sonic/tasks/main.yaml @@ -2,15 +2,15 @@ - name: Install frr-pythontools ansible.builtin.import_tasks: frr-reload.yaml -- name: Fix Network Performance - ansible.builtin.import_tasks: fix-network-performance.yaml +# - name: Fix Network Performance +# ansible.builtin.import_tasks: fix-network-performance.yaml -- name: Set lldp tx-interval to 10 - ansible.builtin.command: lldpcli configure lldp tx-interval 10 - retries: 10 - delay: 3 - register: result - until: result.rc == 0 +# - name: Set lldp tx-interval to 10 +# ansible.builtin.command: lldpcli configure lldp tx-interval 10 +# retries: 10 +# delay: 3 +# register: result +# until: result.rc == 0 - name: Activate IP MASQUERADE on eth0 ansible.builtin.iptables: @@ -26,9 +26,9 @@ sysctl_set: yes value: "1" -# We need to fill some values for the sonic-exporter (uses the STATE_DB) -- name: Mock sonic platform for kvm - ansible.builtin.import_tasks: mock-platform.yaml +# # We need to fill some values for the sonic-exporter (uses the STATE_DB) +# - name: Mock sonic platform for kvm +# ansible.builtin.import_tasks: mock-platform.yaml # ntp restarting for monitoring -> otherwise some NodeTimeOutOfSync error - name: restart chrony From 63d87fcaaa75c60e52b656a988c802eace5ecd80 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Fri, 29 May 2026 15:30:54 +0200 Subject: [PATCH 08/11] fix: add more documentation Signed-off-by: Benjamin Ritter --- images/sonic/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/images/sonic/README.md b/images/sonic/README.md index bdec0df1..ecc5d770 100644 --- a/images/sonic/README.md +++ b/images/sonic/README.md @@ -5,4 +5,8 @@ We use sonic-vpp to emulate SONiC switches. It is running in kvm inside a contai # Configuration knobs -You can edit the port_config.ini to add more ports. Keep the number as low as possible. It will put less strain on your system because it will spawn fewer VPP worker threads. You will have to set up the switch from scratch afterwards, since VPP will generate some configuration on first startup. \ No newline at end of file +You can edit the port_config.ini to add more ports. + + +# Boot process +The switch will boot with a default first-boot configuration. This is required since first boot will generate some required configuration for VPP. After a short while the configuration that is generated in launch.py is injected and the sonic is reloaded. After the new configuration is loaded the container will be marked ready. Check the docker logs for errors if bootup takes more than a minute. \ No newline at end of file From 6314bab5d5d3e0dd7bf9fd71ffc56b7d6215c2b8 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Sun, 19 Jul 2026 16:15:36 +0200 Subject: [PATCH 09/11] feat: implement internet egress via exit node --- files/exit/frr.conf | 5 +++++ files/exit/network.sh | 16 ++++++++++++++++ mini-lab.sonic.yaml | 1 - 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/files/exit/frr.conf b/files/exit/frr.conf index 5a94dc6b..bf16e1dd 100644 --- a/files/exit/frr.conf +++ b/files/exit/frr.conf @@ -35,6 +35,11 @@ router bgp 4200000021 ! address-family ipv4 unicast redistribute connected route-map LOOPBACKS + ! PXE egress: hand the leaves a default so machine traffic routes over the fabric to us, and we + ! NAT it out eth0 (mgmt). NB this default is also installed into each leaf's KERNEL, which + ! redirects the leaf's OWN egress (pixiecore -> github/DNS) through us sourced from its loopback + ! -- that only works because network.sh MASQUERADEs everything leaving eth0. Don't drop that NAT. + neighbor FABRIC default-originate exit-address-family ! address-family ipv6 unicast diff --git a/files/exit/network.sh b/files/exit/network.sh index 6ad64477..ca5800db 100644 --- a/files/exit/network.sh +++ b/files/exit/network.sh @@ -27,3 +27,19 @@ bridge vlan add vid 1000 untagged pvid dev vniInternet ip link set up dev vniInternet sysctl -w net.ipv6.conf.all.forwarding=1 + +# PXE egress return-path NAT. Traffic reaching us over the fabric keeps its original source -- +# machines are 10.0.1.0/24, and the leaves' own traffic is sourced from their 10.0.0.0/24 +# loopbacks. The mgmt docker bridge only NATs 172.42.0.0/16, so anything we forward out eth0 must +# be SNAT'd or the replies have no path back. Masquerade the whole interface rather than a single +# prefix, so both the machine and leaf-originated ranges are covered. +# The frr image ships no iptables; bounded + non-fatal on purpose, since this script runs under +# `set -o errexit` and must never block exit bring-up if the package mirror is unreachable. +for _ in 1 2 3 4 5; do + if apk add --no-cache iptables; then + iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE + break + fi + echo "apk could not reach the mirror (iptables); retrying ..." + sleep 2 +done || true diff --git a/mini-lab.sonic.yaml b/mini-lab.sonic.yaml index c29fd11a..44d7e8fd 100644 --- a/mini-lab.sonic.yaml +++ b/mini-lab.sonic.yaml @@ -10,7 +10,6 @@ topology: nodes: exit: image: quay.io/frrouting/frr:10.3.0 - network-mode: none binds: - files/exit/daemons:/etc/frr/daemons - files/exit/frr.conf:/etc/frr/frr.conf From bc01512821272ca0e5990e303adcf9cd1baefa2d Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Sun, 19 Jul 2026 16:15:44 +0200 Subject: [PATCH 10/11] fix: add workaround for isc not binding to Vlan4000 Signed-off-by: Benjamin Ritter --- deploy_partition.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/deploy_partition.yaml b/deploy_partition.yaml index 295355e8..5ae172d7 100644 --- a/deploy_partition.yaml +++ b/deploy_partition.yaml @@ -199,6 +199,35 @@ - name: Wait until no route entries have "queued" include_tasks: tasks/check_queued.yaml +- name: Work around isc-dhcp/Vlan4000 startup race (Community SONiC) + hosts: leaves:!dell_sonic + any_errors_fatal: true + gather_facts: false + tasks: + # dhcpd binds Vlan4000 at boot, before the SVI has carrier. Its LPF receive socket then dies + # ("receive_packet failed on Vlan4000: Network is down") while dhcpd stays up holding + # 0.0.0.0:67 -- so it answers nothing and PXE clients loop DISCOVER forever, never getting a + # lease. Restarting it once the underlay is up rebinds the socket, hence the placement after + # the underlay wait above. + # + # The probe solicits a real broadcast DISCOVER rather than poking 10.0.1.1:67, because the UDP + # socket stays healthy even when the LPF path is dead -- a unicast check would report a false + # pass. Restart and probe are deliberately one task: a retry must re-do the restart, not just + # re-probe, and `until` only retries a single task. + - name: Restart isc-dhcp-server until it serves an offer on Vlan4000 + ansible.builtin.shell: | + systemctl restart isc-dhcp-server + sleep 2 + dhcp-server-detector Vlan4000 --exit-on-first-offer --duration 10 + args: + executable: /bin/bash + when: dhcp_subnets is defined + register: dhcp_offer_check + until: dhcp_offer_check.rc == 0 + retries: 10 + delay: 5 + changed_when: true + - name: Configure IPv6 and LLDP ports (Enterprise SONiC) hosts: dell_sonic any_errors_fatal: true From 71f821f0d5079667a70a393449efc4d5a0a6776a Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Sun, 19 Jul 2026 16:17:25 +0200 Subject: [PATCH 11/11] fix: fix zebra handing over invalid next-hop groups to fpmsyncd Signed-off-by: Benjamin Ritter --- files/sonic_frr_with_fpm.tpl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/files/sonic_frr_with_fpm.tpl b/files/sonic_frr_with_fpm.tpl index 7019b34d..cab4f8a6 100644 --- a/files/sonic_frr_with_fpm.tpl +++ b/files/sonic_frr_with_fpm.tpl @@ -3,6 +3,12 @@ frr defaults datacenter # This is the only line changed from upstream: https://github.com/metal-stack/metal-core/blob/master/cmd/internal/switcher/templates/tpl/sonic_frr.tpl # Follow-up issue: https://github.com/metal-stack/metal-core/issues/199 fpm address 127.0.0.1 +# SONiC default (upstream ships this in its zebra config; the metal-core template dropped it). +# With next-hop-groups enabled (FRR's default under FPM), zebra hands fpmsyncd an NHG id instead +# of inline nexthops, and the EVPN encap attrs (vni_label/router_mac, carried as RTA_ENCAP) are +# lost -> ROUTE_TABLE loses vni_label/router_mac -> RouteOrch builds a plain IP nexthop instead of +# TUNNEL_ENCAP. +no fpm use-next-hop-groups hostname {{ .Name }} password zebra enable password zebra