From 37a7993dd4cd3e0dfc1c540d39d97d6b9f5a29c3 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 7 Jul 2026 12:32:13 +0200 Subject: [PATCH 1/9] feat: move mini-lab to a dedicated docker network To enable fixed IPs for the Gateway API, mini-lab needs a docker network that is not the docker default bridge. Create a dedicated `mini_lab_internal` network (172.42.0.0/16) and relocate everything off the old default network bridge (172.17.0.1 / 172.18.0.0). This shifts the control-plane ingress DNS from 172.17.0.1.nip.io to 172.42.0.1.nip.io. No Gateway API migration yet Signed-off-by: Benjamin Ritter --- Makefile | 10 +++++++++- control-plane/kind.yaml | 2 +- deploy_gardener.yaml | 4 ++-- docs/overview-kamaji.drawio.svg | 4 ++-- docs/overview.drawio.svg | 4 ++-- env.sh | 1 + files/startup-config/leaf01_4.4.3.json | 2 +- files/startup-config/leaf01_4.5.1.json | 2 +- files/startup-config/leaf02_4.4.3.json | 2 +- files/startup-config/leaf02_4.5.1.json | 2 +- inventories/group_vars/all/control_plane.yaml | 2 +- .../group_vars/control_plane/gardener/gardenlet.yaml | 2 +- .../group_vars/control_plane/gardener/operator.yaml | 4 ++-- inventories/group_vars/control_plane/minio.yaml | 2 +- inventories/group_vars/control_plane/powerdns.yaml | 4 ++-- inventories/group_vars/leaves/main.yaml | 2 +- inventories/group_vars/partition/router.yaml | 2 +- mini-lab.dell_sonic.yaml | 2 +- mini-lab.sonic.yaml | 2 +- roles/kamaji/templates/metallb-kind-ip-pool.yaml | 2 +- 20 files changed, 33 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 837e44bf..81018bf1 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,11 @@ MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms: MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest) MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1) +MINI_LAB_INTERNAL_NETWORK=mini_lab_internal +# define this here as well so that kind picks up the network on a clean checkout, +# where .env does not exist yet at make parse time (-include .env above) +KIND_EXPERIMENTAL_DOCKER_NETWORK := $(or $(KIND_EXPERIMENTAL_DOCKER_NETWORK),$(MINI_LAB_INTERNAL_NETWORK)) + MACHINE_OS=debian-13.0 MAX_RETRIES := 30 @@ -118,6 +123,8 @@ create-proxy-registries: .PHONY: control-plane-bake control-plane-bake: + + @if ! docker network ls | grep -q mini_lab_internal; then docker network create mini_lab_internal --gateway 172.42.0.1 --ip-range=172.42.0.0/16 --subnet=172.42.0.0/16 --ipv6=false ; fi @if ! which kind > /dev/null; then echo "kind needs to be installed"; exit 1; fi @if ! kind get clusters | grep metal-control-plane > /dev/null; then \ kind create cluster $(KIND_ARGS) \ @@ -176,6 +183,7 @@ env: .PHONY: cleanup cleanup: cleanup-control-plane cleanup-partition + docker network rm --force mini_lab_internal .PHONY: cleanup-control-plane cleanup-control-plane: @@ -451,7 +459,7 @@ build-dell-sonic: fetch-virtual-kubeconfig: # TODO: it's hard to get the latest issued generic kubeconfig secret... just take the first result for now kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden $(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden -l managed-by=secrets-manager,manager-identity=gardener-operator,name=generic-token-kubeconfig --no-headers | awk '{ print $$1 }') -o jsonpath='{.data.kubeconfig}' | base64 -d > .virtual-kubeconfig - @kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.17.0.1.nip.io:4443 + @kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.42.0.1.nip.io:4443 @kubectl --kubeconfig=.virtual-kubeconfig config set-credentials garden --token=$(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden shoot-access-virtual-garden -o jsonpath='{.data.token}' | base64 -d) @kubectl --kubeconfig=$(KUBECONFIG) config unset users.garden @kubectl --kubeconfig=$(KUBECONFIG) config unset contexts.garden diff --git a/control-plane/kind.yaml b/control-plane/kind.yaml index a55b4aaf..c0af5d47 100644 --- a/control-plane/kind.yaml +++ b/control-plane/kind.yaml @@ -2,7 +2,7 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 networking: apiServerPort: 6443 - apiServerAddress: 0.0.0.0 + apiServerAddress: 172.42.0.1 nodes: - role: control-plane extraMounts: diff --git a/deploy_gardener.yaml b/deploy_gardener.yaml index 43eb72d4..9cfad385 100644 --- a/deploy_gardener.yaml +++ b/deploy_gardener.yaml @@ -13,7 +13,7 @@ name: shoot-info namespace: kube-system data: - nodeNetwork: 172.18.0.0/16 + nodeNetwork: 172.42.0.0/16 podNetwork: 10.244.0.0/24 serviceNetwork: 10.96.0.0/16 tags: gardener @@ -81,7 +81,7 @@ status: loadBalancer: ingress: - - ip: "172.17.0.1" + - ip: "172.42.0.1" tags: gardener - name: Expose istio gateway through ingress-nginx (for local environments) diff --git a/docs/overview-kamaji.drawio.svg b/docs/overview-kamaji.drawio.svg index f6e083c1..e9b2b33e 100644 --- a/docs/overview-kamaji.drawio.svg +++ b/docs/overview-kamaji.drawio.svg @@ -470,13 +470,13 @@
- docker network 172.17.0.1/16 + docker network 172.42.0.1/16
- docker network 172.17.0.1/16 + docker network 172.42.0.1/16 diff --git a/docs/overview.drawio.svg b/docs/overview.drawio.svg index 1267de68..fdb8f9b9 100644 --- a/docs/overview.drawio.svg +++ b/docs/overview.drawio.svg @@ -470,13 +470,13 @@
- docker network 172.17.0.1/16 + docker network 172.42.0.1/16
- docker network 172.17.0.1/16 + docker network 172.42.0.1/16 diff --git a/env.sh b/env.sh index a01de375..bf233635 100755 --- a/env.sh +++ b/env.sh @@ -24,4 +24,5 @@ DEPLOYMENT_BASE_IMAGE_TAG=${DEPLOYMENT_BASE_IMAGE_TAG} CI=${CI:=false} DOCKER_HUB_USER=${DOCKER_HUB_USER:=} DOCKER_HUB_TOKEN=${DOCKER_HUB_TOKEN:=} +KIND_EXPERIMENTAL_DOCKER_NETWORK=${MINI_LAB_INTERNAL_NETWORK:=mini_lab_internal} EOF diff --git a/files/startup-config/leaf01_4.4.3.json b/files/startup-config/leaf01_4.4.3.json index 622b5df9..60393ec3 100644 --- a/files/startup-config/leaf01_4.4.3.json +++ b/files/startup-config/leaf01_4.4.3.json @@ -11,7 +11,7 @@ } }, "DNS_SERVER": { - "172.17.0.1": {}, + "172.42.0.1": {}, "1.1.1.1": {}, "1.0.0.1": {} }, diff --git a/files/startup-config/leaf01_4.5.1.json b/files/startup-config/leaf01_4.5.1.json index cd4224ac..78106fe8 100644 --- a/files/startup-config/leaf01_4.5.1.json +++ b/files/startup-config/leaf01_4.5.1.json @@ -11,7 +11,7 @@ } }, "DNS_SERVER": { - "172.17.0.1": {}, + "172.42.0.1": {}, "1.1.1.1": {}, "1.0.0.1": {} }, diff --git a/files/startup-config/leaf02_4.4.3.json b/files/startup-config/leaf02_4.4.3.json index 5a8d4a78..7359f574 100644 --- a/files/startup-config/leaf02_4.4.3.json +++ b/files/startup-config/leaf02_4.4.3.json @@ -11,7 +11,7 @@ } }, "DNS_SERVER": { - "172.17.0.1": {}, + "172.42.0.1": {}, "1.1.1.1": {}, "1.0.0.1": {} }, diff --git a/files/startup-config/leaf02_4.5.1.json b/files/startup-config/leaf02_4.5.1.json index 41c8f118..335ef382 100644 --- a/files/startup-config/leaf02_4.5.1.json +++ b/files/startup-config/leaf02_4.5.1.json @@ -11,7 +11,7 @@ } }, "DNS_SERVER": { - "172.17.0.1": {}, + "172.42.0.1": {}, "1.1.1.1": {}, "1.0.0.1": {} }, diff --git a/inventories/group_vars/all/control_plane.yaml b/inventories/group_vars/all/control_plane.yaml index 1a93b331..a1beeeb7 100644 --- a/inventories/group_vars/all/control_plane.yaml +++ b/inventories/group_vars/all/control_plane.yaml @@ -1,6 +1,6 @@ --- metal_control_plane_provider_tenant: metal-stack -metal_control_plane_ingress_dns: 172.17.0.1.nip.io +metal_control_plane_ingress_dns: 172.42.0.1.nip.io metal_control_plane_stage_name: test metal_control_plane_namespace: metal-control-plane metal_control_plane_image_pull_policy: Always diff --git a/inventories/group_vars/control_plane/gardener/gardenlet.yaml b/inventories/group_vars/control_plane/gardener/gardenlet.yaml index 8fed1d0f..fe02bb70 100644 --- a/inventories/group_vars/control_plane/gardener/gardenlet.yaml +++ b/inventories/group_vars/control_plane/gardener/gardenlet.yaml @@ -1,5 +1,5 @@ --- -gardener_gardenlet_default_dns_domain: "gardener.172.17.0.1.nip.io" +gardener_gardenlet_default_dns_domain: "gardener.172.42.0.1.nip.io" gardener_gardenlet_default_dns_provider: powerdns gardener_gardenlet_default_dns_credentials: apiKey: "{{ powerdns_api_key | b64encode }}" diff --git a/inventories/group_vars/control_plane/gardener/operator.yaml b/inventories/group_vars/control_plane/gardener/operator.yaml index 5cddad2e..06f6d525 100644 --- a/inventories/group_vars/control_plane/gardener/operator.yaml +++ b/inventories/group_vars/control_plane/gardener/operator.yaml @@ -1,5 +1,5 @@ --- -gardener_operator_ingress_dns_domain: "gardener.172.17.0.1.nip.io" +gardener_operator_ingress_dns_domain: "gardener.{{ metal_control_plane_ingress_dns }}" gardener_operator_backup_infrastructure: provider: S3 @@ -17,7 +17,7 @@ gardener_operator_backup_infrastructure_secret: s3ForcePathStyle: "{{ 'true' | b64encode }}" # enable mini-lab patches -gardener_operator_patch_istio_ingress_gateway_service_ip: 172.17.0.1 +gardener_operator_patch_istio_ingress_gateway_service_ip: 172.42.0.1 gardener_operator_expose_virtual_garden_through_ingress_nginx: true # for local setups this should be sufficient diff --git a/inventories/group_vars/control_plane/minio.yaml b/inventories/group_vars/control_plane/minio.yaml index 73801c7c..0e7566ee 100644 --- a/inventories/group_vars/control_plane/minio.yaml +++ b/inventories/group_vars/control_plane/minio.yaml @@ -2,4 +2,4 @@ minio_root_user: mini-lab minio_root_password: change-me -minio_dns_name: minio.172.17.0.1.nip.io +minio_dns_name: minio.172.42.0.1.nip.io diff --git a/inventories/group_vars/control_plane/powerdns.yaml b/inventories/group_vars/control_plane/powerdns.yaml index bde65e72..d001532f 100644 --- a/inventories/group_vars/control_plane/powerdns.yaml +++ b/inventories/group_vars/control_plane/powerdns.yaml @@ -10,11 +10,11 @@ powerdns_load_balancer_dns_name: "ns.{{ metal_control_plane_ingress_dns }}" powerdns_api_dns_name: "powerdns-api.{{ metal_control_plane_ingress_dns }}" powerdns_zones: - - name: "gardener.172.17.0.1.nip.io." + - name: "gardener.172.42.0.1.nip.io." kind: Master nameservers: - "{{ powerdns_load_balancer_dns_name }}." - - name: "gardener-kube-apiserver.172.17.0.1.nip.io." + - name: "gardener-kube-apiserver.172.42.0.1.nip.io." kind: Master nameservers: - "{{ powerdns_load_balancer_dns_name }}." diff --git a/inventories/group_vars/leaves/main.yaml b/inventories/group_vars/leaves/main.yaml index fbc83ac4..e1d1e596 100644 --- a/inventories/group_vars/leaves/main.yaml +++ b/inventories/group_vars/leaves/main.yaml @@ -12,7 +12,7 @@ sonic_config_frr_render: false sonic_config_loopback_address: "{{ lo }}" sonic_config_mgmt_interface: ip: "{{ ansible_host }}/16`" - gateway_address: "172.17.0.1" + gateway_address: "172.42.0.1" sonic_config_mgmt_vrf: false sonic_config_nameservers: "{{ router_nameservers }}" diff --git a/inventories/group_vars/partition/router.yaml b/inventories/group_vars/partition/router.yaml index bdcbc460..a85d2998 100644 --- a/inventories/group_vars/partition/router.yaml +++ b/inventories/group_vars/partition/router.yaml @@ -1,5 +1,5 @@ --- router_nameservers: - - 172.17.0.1 + - 172.42.0.1 - 1.1.1.1 - 1.0.0.1 diff --git a/mini-lab.dell_sonic.yaml b/mini-lab.dell_sonic.yaml index 91bf0990..d1342437 100644 --- a/mini-lab.dell_sonic.yaml +++ b/mini-lab.dell_sonic.yaml @@ -2,7 +2,7 @@ name: mini-lab prefix: "" mgmt: - network: bridge + network: mini_lab_internal topology: defaults: diff --git a/mini-lab.sonic.yaml b/mini-lab.sonic.yaml index 5d46a128..a14ad7e0 100644 --- a/mini-lab.sonic.yaml +++ b/mini-lab.sonic.yaml @@ -2,7 +2,7 @@ name: mini-lab prefix: "" mgmt: - network: bridge + network: mini_lab_internal topology: defaults: diff --git a/roles/kamaji/templates/metallb-kind-ip-pool.yaml b/roles/kamaji/templates/metallb-kind-ip-pool.yaml index 45034660..eeea201c 100644 --- a/roles/kamaji/templates/metallb-kind-ip-pool.yaml +++ b/roles/kamaji/templates/metallb-kind-ip-pool.yaml @@ -6,7 +6,7 @@ metadata: namespace: metallb-system spec: addresses: - - 172.18.255.200-172.18.255.250 + - 172.42.255.200-172.42.255.250 --- apiVersion: metallb.io/v1beta1 kind: L2Advertisement From b033476dab1ee8b972533160527e261927b3297f Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 7 Jul 2026 12:32:21 +0200 Subject: [PATCH 2/9] feat: add metal-api server certificate Add a metal-api server cert (api./v2. nip.io SANs) for the Gateway to terminate TLS with, add it to roll_certs.sh, and update the grpc cert host to the Gateway LoadBalancer IP. Signed-off-by: Benjamin Ritter --- files/certs/grpc/server.json | 2 +- files/certs/metal-api/server.json | 20 ++++++++++++++++++++ scripts/roll_certs.sh | 3 ++- 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 files/certs/metal-api/server.json diff --git a/files/certs/grpc/server.json b/files/certs/grpc/server.json index 17ba2a53..e1b8bc35 100644 --- a/files/certs/grpc/server.json +++ b/files/certs/grpc/server.json @@ -1,7 +1,7 @@ { "CN": "metal-api", "hosts": [ - "172.17.0.1", + "172.42.0.42", "203.0.113.1" ], "key": { diff --git a/files/certs/metal-api/server.json b/files/certs/metal-api/server.json new file mode 100644 index 00000000..da41eb2a --- /dev/null +++ b/files/certs/metal-api/server.json @@ -0,0 +1,20 @@ +{ + "CN": "metal-api", + "hosts": [ + "api.172.42.0.42.nip.io", + "v2.172.42.0.42.nip.io" + ], + "key": { + "algo": "rsa", + "size": 4096 + }, + "names": [ + { + "C": "DE", + "L": "Munich", + "O": "metal-stack", + "OU": "DevOps", + "ST": "Bavaria" + } + ] +} diff --git a/scripts/roll_certs.sh b/scripts/roll_certs.sh index 7fc34acd..69bcdcda 100755 --- a/scripts/roll_certs.sh +++ b/scripts/roll_certs.sh @@ -13,7 +13,8 @@ rm *.csr for component in \ grpc \ - masterdata-api; do + masterdata-api \ + metal-api; do pushd $component echo "generating $component certs" From 6bfbb859bbe0eca1f942bef95657386378c244db Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 7 Jul 2026 12:32:56 +0200 Subject: [PATCH 3/9] feat: deploy cloud-provider-kind and Envoy Gateway Add the Gateway API plumbing without wiring any application to it yet: Deploy cloud-provider-kind to provide LoadBalancer IPs. Add a `gateway` role deploying the Envoy Gateway controller, a GatewayClass, an EnvoyProxy pinned to the fixed LB IP 172.42.0.42, and the metal-control-plane Gateway. Register the role in the control-plane playbook and expose metal_control_plane_gateway_dns Signed-off-by: Benjamin Ritter --- Makefile | 1 + compose.yaml | 14 ++++++++ deploy_control_plane.yaml | 6 ++-- inventories/group_vars/all/control_plane.yaml | 1 + .../group_vars/control_plane/ingress.yaml | 3 ++ roles/gateway/defaults/main.yml | 8 +++++ roles/gateway/files/gatewayclass.yaml | 6 ++++ roles/gateway/tasks/main.yml | 34 ++++++++++++++++++ roles/gateway/templates/envoyproxy.yaml | 14 ++++++++ roles/gateway/templates/gateway.yaml | 35 +++++++++++++++++++ 10 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 roles/gateway/defaults/main.yml create mode 100644 roles/gateway/files/gatewayclass.yaml create mode 100644 roles/gateway/tasks/main.yml create mode 100644 roles/gateway/templates/envoyproxy.yaml create mode 100644 roles/gateway/templates/gateway.yaml diff --git a/Makefile b/Makefile index 81018bf1..3f46dd28 100644 --- a/Makefile +++ b/Makefile @@ -132,6 +132,7 @@ control-plane-bake: --config $(KINDCONFIG) \ --kubeconfig $(KUBECONFIG); fi $(MAKE) create-proxy-registries + docker compose up -d --force-recreate cloud-provider-kind .PHONY: partition partition: partition-bake diff --git a/compose.yaml b/compose.yaml index 8fd3e0d9..82e87843 100644 --- a/compose.yaml +++ b/compose.yaml @@ -126,6 +126,20 @@ services: - REGISTRY_PROXY_TTL=168h - REGISTRY_STORAGE_DELETE_ENABLED=true - OTEL_TRACES_EXPORTER=none + cloud-provider-kind: + image: registry.k8s.io/cloud-provider-kind/cloud-controller-manager:v0.10.0 + restart: always + networks: + - kind + environment: + - KIND_EXPERIMENTAL_DOCKER_NETWORK=${KIND_EXPERIMENTAL_DOCKER_NETWORK:-kind} + command: + # v0.10.0 of cloud controller does not support tcproutes, since it does not support the experimental gateway api channel + # using envoy-gateway deployed via roles/gateway instead + - --gateway-channel + - disabled + volumes: + - /var/run/docker.sock:/var/run/docker.sock volumes: proxy-docker: proxy-gcr: diff --git a/deploy_control_plane.yaml b/deploy_control_plane.yaml index bd0f0241..7336cf70 100644 --- a/deploy_control_plane.yaml +++ b/deploy_control_plane.yaml @@ -6,10 +6,12 @@ roles: - name: ansible-common tags: always - - name: ingress-controller - tags: ingress-controller - name: metal-roles/control-plane/roles/prepare tags: prepare + - name: gateway + tags: gateway + - name: ingress-controller + tags: ingress-controller - name: metal-roles/control-plane/roles/nsq tags: nsq - name: metal-roles/control-plane/roles/metal-db diff --git a/inventories/group_vars/all/control_plane.yaml b/inventories/group_vars/all/control_plane.yaml index a1beeeb7..a0f045fe 100644 --- a/inventories/group_vars/all/control_plane.yaml +++ b/inventories/group_vars/all/control_plane.yaml @@ -1,6 +1,7 @@ --- metal_control_plane_provider_tenant: metal-stack metal_control_plane_ingress_dns: 172.42.0.1.nip.io +metal_control_plane_gateway_dns: 172.42.0.42.nip.io metal_control_plane_stage_name: test metal_control_plane_namespace: metal-control-plane metal_control_plane_image_pull_policy: Always diff --git a/inventories/group_vars/control_plane/ingress.yaml b/inventories/group_vars/control_plane/ingress.yaml index 0ef8644b..fe68a6b6 100644 --- a/inventories/group_vars/control_plane/ingress.yaml +++ b/inventories/group_vars/control_plane/ingress.yaml @@ -2,3 +2,6 @@ ingress_tcp_service_exposals: "4150": "{{ metal_control_plane_namespace }}/nsqd:4150" "50051": "{{ metal_control_plane_namespace }}/metal-api:50051" +gateway_tcp_listeners: + nsq: 4150 + metal-api-grpc: 50051 \ No newline at end of file diff --git a/roles/gateway/defaults/main.yml b/roles/gateway/defaults/main.yml new file mode 100644 index 00000000..5bb8dff6 --- /dev/null +++ b/roles/gateway/defaults/main.yml @@ -0,0 +1,8 @@ +--- +gateway_namespace: "{{ metal_control_plane_namespace }}" +gateway_http_port: 8080 +gateway_https_port: 4443 + +gateway_tcp_listeners: {} + # nsq: 4150 + # metal-api: 50051 diff --git a/roles/gateway/files/gatewayclass.yaml b/roles/gateway/files/gatewayclass.yaml new file mode 100644 index 00000000..a750b9fd --- /dev/null +++ b/roles/gateway/files/gatewayclass.yaml @@ -0,0 +1,6 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller \ No newline at end of file diff --git a/roles/gateway/tasks/main.yml b/roles/gateway/tasks/main.yml new file mode 100644 index 00000000..d0f1a4d3 --- /dev/null +++ b/roles/gateway/tasks/main.yml @@ -0,0 +1,34 @@ +- name: Deploy envoy-gateway + kubernetes.core.helm: + release_name: envoy-gateway + release_namespace: envoy-gateway-system + create_namespace: true + chart_ref: "oci://docker.io/envoyproxy/gateway-helm" + chart_version: "{{ envoy_gateway_chart_version | default(omit) }}" + # values: "{{ lookup('template', 'values.yaml') | from_yaml }}" + wait: true + +- name: Deploy Envoy GatewayClass + kubernetes.core.k8s: + definition: "{{ lookup('file', 'gatewayclass.yaml') | from_yaml }}" + +- name: Deploy metal-control-plane EnvoyProxy + kubernetes.core.k8s: + definition: "{{ lookup('template', 'envoyproxy.yaml') | from_yaml }}" + +- name: Deploy metal-api certificate secret + kubernetes.core.k8s: + definition: + apiVersion: v1 + kind: Secret + type: kubernetes.io/tls + metadata: + name: metal-api-tls + namespace: "{{ gateway_namespace }}" + data: + tls.crt: "{{ lookup('file', 'certs/metal-api/server.pem') | b64encode }}" + tls.key: "{{ lookup('file', 'certs/metal-api/server-key.pem') | b64encode }}" + +- name: Deploy metal-control-plane Gateway + kubernetes.core.k8s: + definition: "{{ lookup('template', 'gateway.yaml') | from_yaml }}" diff --git a/roles/gateway/templates/envoyproxy.yaml b/roles/gateway/templates/envoyproxy.yaml new file mode 100644 index 00000000..edb7c0fe --- /dev/null +++ b/roles/gateway/templates/envoyproxy.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: metal-control-plane + namespace: "{{ gateway_namespace }}" +spec: + provider: + type: Kubernetes + kubernetes: + envoyService: + externalTrafficPolicy: Local + type: LoadBalancer + loadBalancerIP: 172.42.0.42 \ No newline at end of file diff --git a/roles/gateway/templates/gateway.yaml b/roles/gateway/templates/gateway.yaml new file mode 100644 index 00000000..49b0ec4e --- /dev/null +++ b/roles/gateway/templates/gateway.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: metal-control-plane + namespace: "{{ gateway_namespace }}" +spec: + gatewayClassName: eg + addresses: + - value: 172.42.0.42 + infrastructure: + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: metal-control-plane + listeners: + - protocol: HTTP + port: {{ gateway_http_port }} + name: http + hostname: "*.{{ metal_control_plane_gateway_dns }}" + - protocol: HTTPS + port: {{ gateway_https_port }} + name: https + hostname: "*.{{ metal_control_plane_gateway_dns }}" + tls: + mode: Terminate + certificateRefs: + - group: "" + kind: Secret + name: metal-api-tls +{% for name, port in gateway_tcp_listeners.items() %} + - protocol: TCP + name: {{ name }} + port: {{ port }} +{% endfor %} From 849de3a2bb2a7058e78f5be47b4dd4acc2e63954 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Tue, 7 Jul 2026 12:33:25 +0200 Subject: [PATCH 4/9] feat: switch control-plane services to Gateway API Migrate zitadel, nsq, metal-api and metal-apiserver off ingress-nginx to Gateway API. Enable http/tcp routes, point their external URLs at the Gateway LoadBalancer (172.42.0.42 / gateway_dns). Signed-off-by: Benjamin Ritter --- Makefile | 4 +-- files/dev_images.yaml | 4 +-- .../group_vars/control_plane/metal.yml | 30 +++++++++++++++---- inventories/group_vars/control_plane/nsq.yaml | 5 ++++ .../group_vars/control_plane/zitadel.yaml | 9 ++++-- inventories/group_vars/partition/common.yaml | 4 +-- 6 files changed, 42 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 3f46dd28..fb2368e0 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,8 @@ KINDCONFIG := $(or $(KINDCONFIG),control-plane/kind.yaml) KUBECONFIG := $(shell pwd)/.kubeconfig METALCTL_HMAC := $(or $(METALCTL_HMAC),metal-admin) -METALCTL_API_URL := $(or $(METALCTL_API_URL),http://api.172.17.0.1.nip.io:8080/metal) -METAL_APIV2_URL := $(or $(METAL_APIV2_URL),http://v2.172.17.0.1.nip.io:8080) +METALCTL_API_URL := $(or $(METALCTL_API_URL),http://api.172.42.0.42.nip.io:8080/metal) +METAL_APIV2_URL := $(or $(METAL_APIV2_URL),http://v2.172.42.0.42.nip.io:8080) MKE2FS_CONFIG := $(shell pwd)/mke2fs.conf # Default values diff --git a/files/dev_images.yaml b/files/dev_images.yaml index e0699518..91b8285c 100644 --- a/files/dev_images.yaml +++ b/files/dev_images.yaml @@ -1,6 +1,6 @@ --- # Do not change these values metal_api_image_tag: dev -metal_core_image_name: 172.17.0.1:5000/metalstack/metal-core +metal_core_image_name: 172.42.0.42:5000/metalstack/metal-core metal_core_image_tag: dev -metal_hammer_image_url: http://172.17.0.1:20015/metal-hammer-initrd.img.lz4 +metal_hammer_image_url: http://172.42.0.42:20015/metal-hammer-initrd.img.lz4 diff --git a/inventories/group_vars/control_plane/metal.yml b/inventories/group_vars/control_plane/metal.yml index 40db02a6..af68119b 100644 --- a/inventories/group_vars/control_plane/metal.yml +++ b/inventories/group_vars/control_plane/metal.yml @@ -1,9 +1,21 @@ --- metal_set_resource_limits: no -metal_check_api_health_endpoint: http://api.{{ metal_control_plane_ingress_dns }}:8080/metal/v1/health -metal_api_headscale_control_plane_address: "http://headscale.{{ metal_control_plane_ingress_dns }}:8080" +metal_check_api_health_endpoint: http://api.{{ metal_control_plane_gateway_dns }}:8080/metal/v1/health +metal_api_headscale_control_plane_address: "http://headscale.{{ metal_control_plane_gateway_dns }}:8080" -# metal_helm_chart_local_path: /helm-charts/charts/metal-control-plane +metal_deploy_ingress: false + +metal_api_httproute_enabled: true +metal_api_httproute_parent_refs: +- name: metal-control-plane + namespace: "{{ metal_control_plane_namespace }}" + sectionName: http + +metal_api_tcproute_enabled: true +metal_api_tcproute_parent_refs: +- name: metal-control-plane + namespace: "{{ metal_control_plane_namespace }}" + sectionName: metal-api-grpc metal_api_pdb_min_available: 1 metal_api_replicas: 1 @@ -16,13 +28,19 @@ metal_api_nsq_tcp_address: nsqd:4150 metal_apiserver_pdb_min_available: 1 metal_apiserver_enabled: true -metal_apiserver_url: http://v2.{{ metal_control_plane_ingress_dns }}:8080 +metal_apiserver_url: http://v2.{{ metal_control_plane_gateway_dns }}:8080 -metal_apiserver_oidc_discovery_url: https://auth.{{ metal_control_plane_ingress_dns }}:4443/.well-known/openid-configuration -metal_apiserver_oidc_end_session_url: "https://auth.{{ metal_control_plane_ingress_dns }}:4443/oidc/v1/end_session" +metal_apiserver_oidc_discovery_url: https://zitadel.{{ metal_control_plane_gateway_dns }}/.well-known/openid-configuration +metal_apiserver_oidc_end_session_url: "https://zitadel.{{ metal_control_plane_gateway_dns }}/oidc/v1/end_session" metal_apiserver_secure_cookie: false +metal_apiserver_httproute_enabled: true +metal_apiserver_httproute_parent_refs: +- name: metal-control-plane + namespace: "{{ metal_control_plane_namespace }}" + sectionName: http + metal_api_images: - id: firewall-ubuntu-3.0 name: Firewall 3 Ubuntu diff --git a/inventories/group_vars/control_plane/nsq.yaml b/inventories/group_vars/control_plane/nsq.yaml index b8983676..c7a1f802 100644 --- a/inventories/group_vars/control_plane/nsq.yaml +++ b/inventories/group_vars/control_plane/nsq.yaml @@ -9,3 +9,8 @@ nsq_certs_client_cert: "{{ lookup('file', 'certs/nsq/client.crt') }}" nsq_certs_ca_cert: "{{ lookup('file', 'certs/ca.pem') }}" nsq_broadcast_address: nsqd + +nsq_tcproute_enabled: true +nsq_tcproute_parent_refs: +- name: metal-control-plane + sectionName: nsq diff --git a/inventories/group_vars/control_plane/zitadel.yaml b/inventories/group_vars/control_plane/zitadel.yaml index 4a2675be..b316798c 100644 --- a/inventories/group_vars/control_plane/zitadel.yaml +++ b/inventories/group_vars/control_plane/zitadel.yaml @@ -1,11 +1,16 @@ --- zitadel_endpoint: zitadel.{{ metal_control_plane_namespace }}.svc.cluster.local -zitadel_external_domain: auth.{{ metal_control_plane_ingress_dns }} +zitadel_external_domain: zitadel.{{ metal_control_plane_gateway_dns }} zitadel_ingress_dns: https://{{ zitadel_external_domain }}:4443 zitadel_port: 8080 zitadel_skip_verify_tls: true zitadel_insecure: true +zitadel_httproute_enabled: true +zitadel_httproute_parent_refs: +- name: metal-control-plane + sectionName: https + zitadel_init_config: static_users: - first_name: Olli @@ -23,4 +28,4 @@ zitadel_init_config: # later id will be added but currently not possible with zitadel id: metal-stack name: metal-stack - redirect_uri: http://v2.172.17.0.1.nip.io:8080/auth/openid-connect/callback + redirect_uri: http://v2.172.42.0.42.nip.io:8080/auth/openid-connect/callback diff --git a/inventories/group_vars/partition/common.yaml b/inventories/group_vars/partition/common.yaml index 61da08a0..57761405 100644 --- a/inventories/group_vars/partition/common.yaml +++ b/inventories/group_vars/partition/common.yaml @@ -3,7 +3,7 @@ metal_partition_timezone: Europe/Berlin metal_partition_id: mini-lab metal_partition_metal_api_protocol: http -metal_partition_metal_api_addr: api.{{ metal_control_plane_ingress_dns }} +metal_partition_metal_api_addr: api.{{ metal_control_plane_gateway_dns }} metal_partition_metal_api_port: 8080 metal_partition_metal_api_basepath: /metal/ metal_partition_metal_api_hmac_edit_key: metal-edit @@ -14,4 +14,4 @@ metal_partition_metal_api_grpc_ca_cert: "{{ lookup('file', 'certs/ca.pem') }}" metal_partition_metal_api_grpc_client_cert: "{{ lookup('file', 'certs/grpc/client.pem') }}" metal_partition_metal_api_grpc_client_key: "{{ lookup('file', 'certs/grpc/client-key.pem') }}" -metal_partition_mgmt_gateway: 172.17.0.1 +metal_partition_mgmt_gateway: 172.42.0.42 From 71941ea0101f277b3fb66ad881eb42ac5ae10ad5 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Fri, 8 May 2026 16:48:01 +0200 Subject: [PATCH 5/9] feat: remove ingress-nginx exposed ports Signed-off-by: Benjamin Ritter --- control-plane/kind.yaml | 4 ---- inventories/group_vars/control_plane/ingress.yaml | 3 --- 2 files changed, 7 deletions(-) diff --git a/control-plane/kind.yaml b/control-plane/kind.yaml index c0af5d47..018c5454 100644 --- a/control-plane/kind.yaml +++ b/control-plane/kind.yaml @@ -13,10 +13,6 @@ nodes: hostPort: 4443 - containerPort: 8080 hostPort: 8080 - - containerPort: 4150 - hostPort: 4150 - - containerPort: 50051 - hostPort: 50051 # if you want to run gardener operator + metal-stack, you need more pods kubeadmConfigPatches: - | diff --git a/inventories/group_vars/control_plane/ingress.yaml b/inventories/group_vars/control_plane/ingress.yaml index fe68a6b6..d7a9fd25 100644 --- a/inventories/group_vars/control_plane/ingress.yaml +++ b/inventories/group_vars/control_plane/ingress.yaml @@ -1,7 +1,4 @@ --- -ingress_tcp_service_exposals: - "4150": "{{ metal_control_plane_namespace }}/nsqd:4150" - "50051": "{{ metal_control_plane_namespace }}/metal-api:50051" gateway_tcp_listeners: nsq: 4150 metal-api-grpc: 50051 \ No newline at end of file From 1bf6c8cc721931237b0231e899ba0f2f8e7e9543 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Mon, 13 Jul 2026 11:47:09 +0200 Subject: [PATCH 6/9] feat: use a single certificate for metal-api and metal-apiserver Signed-off-by: Benjamin Ritter --- files/certs/grpc/server.json | 20 -------------------- files/certs/{grpc => metal-api}/client.json | 0 files/certs/metal-api/server.json | 4 +++- inventories/group_vars/partition/common.yaml | 4 ++-- scripts/roll_certs.sh | 1 - 5 files changed, 5 insertions(+), 24 deletions(-) delete mode 100644 files/certs/grpc/server.json rename files/certs/{grpc => metal-api}/client.json (100%) diff --git a/files/certs/grpc/server.json b/files/certs/grpc/server.json deleted file mode 100644 index e1b8bc35..00000000 --- a/files/certs/grpc/server.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "CN": "metal-api", - "hosts": [ - "172.42.0.42", - "203.0.113.1" - ], - "key": { - "algo": "rsa", - "size": 4096 - }, - "names": [ - { - "C": "DE", - "L": "Munich", - "O": "metal-stack", - "OU": "DevOps", - "ST": "Bavaria" - } - ] -} diff --git a/files/certs/grpc/client.json b/files/certs/metal-api/client.json similarity index 100% rename from files/certs/grpc/client.json rename to files/certs/metal-api/client.json diff --git a/files/certs/metal-api/server.json b/files/certs/metal-api/server.json index da41eb2a..6f271cf2 100644 --- a/files/certs/metal-api/server.json +++ b/files/certs/metal-api/server.json @@ -2,7 +2,9 @@ "CN": "metal-api", "hosts": [ "api.172.42.0.42.nip.io", - "v2.172.42.0.42.nip.io" + "v2.172.42.0.42.nip.io", + "172.42.0.42", + "203.0.113.1" ], "key": { "algo": "rsa", diff --git a/inventories/group_vars/partition/common.yaml b/inventories/group_vars/partition/common.yaml index 57761405..381ed8b6 100644 --- a/inventories/group_vars/partition/common.yaml +++ b/inventories/group_vars/partition/common.yaml @@ -11,7 +11,7 @@ metal_partition_metal_api_hmac_view_key: metal-view metal_partition_metal_api_grpc_address: "{{ metal_partition_mgmt_gateway }}:50051" metal_partition_metal_api_grpc_ca_cert: "{{ lookup('file', 'certs/ca.pem') }}" -metal_partition_metal_api_grpc_client_cert: "{{ lookup('file', 'certs/grpc/client.pem') }}" -metal_partition_metal_api_grpc_client_key: "{{ lookup('file', 'certs/grpc/client-key.pem') }}" +metal_partition_metal_api_grpc_client_cert: "{{ lookup('file', 'certs/metal-api/client.pem') }}" +metal_partition_metal_api_grpc_client_key: "{{ lookup('file', 'certs/metal-api/client-key.pem') }}" metal_partition_mgmt_gateway: 172.42.0.42 diff --git a/scripts/roll_certs.sh b/scripts/roll_certs.sh index 69bcdcda..947a8fd4 100755 --- a/scripts/roll_certs.sh +++ b/scripts/roll_certs.sh @@ -12,7 +12,6 @@ cfssl genkey -initca ca-csr.json | cfssljson -bare ca rm *.csr for component in \ - grpc \ masterdata-api \ metal-api; do pushd $component From 3e53230480378108176a24c4068b158899b2b52c Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Mon, 13 Jul 2026 13:05:44 +0200 Subject: [PATCH 7/9] feat: add https listeners for metal-api and metal-apiserver Signed-off-by: Benjamin Ritter --- inventories/group_vars/control_plane/metal.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/inventories/group_vars/control_plane/metal.yml b/inventories/group_vars/control_plane/metal.yml index af68119b..ffae00aa 100644 --- a/inventories/group_vars/control_plane/metal.yml +++ b/inventories/group_vars/control_plane/metal.yml @@ -10,7 +10,9 @@ metal_api_httproute_parent_refs: - name: metal-control-plane namespace: "{{ metal_control_plane_namespace }}" sectionName: http - +- name: metal-control-plane + namespace: "{{ metal_control_plane_namespace }}" + sectionName: https metal_api_tcproute_enabled: true metal_api_tcproute_parent_refs: - name: metal-control-plane @@ -40,6 +42,9 @@ metal_apiserver_httproute_parent_refs: - name: metal-control-plane namespace: "{{ metal_control_plane_namespace }}" sectionName: http +- name: metal-control-plane + namespace: "{{ metal_control_plane_namespace }}" + sectionName: https metal_api_images: - id: firewall-ubuntu-3.0 @@ -173,10 +178,10 @@ metal_masterdata_api_projects: metal_console_enabled: false -metal_api_grpc_certs_server_key: "{{ lookup('file', 'certs/grpc/server-key.pem') }}" -metal_api_grpc_certs_server_cert: "{{ lookup('file', 'certs/grpc/server.pem') }}" -metal_api_grpc_certs_client_key: "{{ lookup('file', 'certs/grpc/client-key.pem') }}" -metal_api_grpc_certs_client_cert: "{{ lookup('file', 'certs/grpc/client.pem') }}" +metal_api_grpc_certs_server_key: "{{ lookup('file', 'certs/metal-api/server-key.pem') }}" +metal_api_grpc_certs_server_cert: "{{ lookup('file', 'certs/metal-api/server.pem') }}" +metal_api_grpc_certs_client_key: "{{ lookup('file', 'certs/metal-api/client-key.pem') }}" +metal_api_grpc_certs_client_cert: "{{ lookup('file', 'certs/metal-api/client.pem') }}" metal_api_grpc_certs_ca_cert: "{{ lookup('file', 'certs/ca.pem') }}" # auditing From 9bba781c07162d7b21ca09b02949f6ed2378a296 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Mon, 13 Jul 2026 13:10:40 +0200 Subject: [PATCH 8/9] feat: use 80 and 443 on gateway listeners as we are not listening on the host we are free to use any port we like. So we switch to the default ports for https and http Signed-off-by: Benjamin Ritter --- inventories/group_vars/control_plane/metal.yml | 6 +++--- inventories/group_vars/control_plane/zitadel.yaml | 4 ++-- inventories/group_vars/partition/common.yaml | 2 +- roles/gateway/defaults/main.yml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/inventories/group_vars/control_plane/metal.yml b/inventories/group_vars/control_plane/metal.yml index ffae00aa..91ae43d7 100644 --- a/inventories/group_vars/control_plane/metal.yml +++ b/inventories/group_vars/control_plane/metal.yml @@ -1,7 +1,7 @@ --- metal_set_resource_limits: no -metal_check_api_health_endpoint: http://api.{{ metal_control_plane_gateway_dns }}:8080/metal/v1/health -metal_api_headscale_control_plane_address: "http://headscale.{{ metal_control_plane_gateway_dns }}:8080" +metal_check_api_health_endpoint: http://api.{{ metal_control_plane_gateway_dns }}/metal/v1/health +metal_api_headscale_control_plane_address: "http://headscale.{{ metal_control_plane_gateway_dns }}" metal_deploy_ingress: false @@ -30,7 +30,7 @@ metal_api_nsq_tcp_address: nsqd:4150 metal_apiserver_pdb_min_available: 1 metal_apiserver_enabled: true -metal_apiserver_url: http://v2.{{ metal_control_plane_gateway_dns }}:8080 +metal_apiserver_url: http://v2.{{ metal_control_plane_gateway_dns }} metal_apiserver_oidc_discovery_url: https://zitadel.{{ metal_control_plane_gateway_dns }}/.well-known/openid-configuration metal_apiserver_oidc_end_session_url: "https://zitadel.{{ metal_control_plane_gateway_dns }}/oidc/v1/end_session" diff --git a/inventories/group_vars/control_plane/zitadel.yaml b/inventories/group_vars/control_plane/zitadel.yaml index b316798c..ebc8efb0 100644 --- a/inventories/group_vars/control_plane/zitadel.yaml +++ b/inventories/group_vars/control_plane/zitadel.yaml @@ -1,7 +1,7 @@ --- zitadel_endpoint: zitadel.{{ metal_control_plane_namespace }}.svc.cluster.local zitadel_external_domain: zitadel.{{ metal_control_plane_gateway_dns }} -zitadel_ingress_dns: https://{{ zitadel_external_domain }}:4443 +zitadel_ingress_dns: https://{{ zitadel_external_domain }} zitadel_port: 8080 zitadel_skip_verify_tls: true zitadel_insecure: true @@ -28,4 +28,4 @@ zitadel_init_config: # later id will be added but currently not possible with zitadel id: metal-stack name: metal-stack - redirect_uri: http://v2.172.42.0.42.nip.io:8080/auth/openid-connect/callback + redirect_uri: http://v2.172.42.0.42.nip.io/auth/openid-connect/callback diff --git a/inventories/group_vars/partition/common.yaml b/inventories/group_vars/partition/common.yaml index 381ed8b6..bb3791fe 100644 --- a/inventories/group_vars/partition/common.yaml +++ b/inventories/group_vars/partition/common.yaml @@ -4,7 +4,7 @@ metal_partition_id: mini-lab metal_partition_metal_api_protocol: http metal_partition_metal_api_addr: api.{{ metal_control_plane_gateway_dns }} -metal_partition_metal_api_port: 8080 +metal_partition_metal_api_port: 80 metal_partition_metal_api_basepath: /metal/ metal_partition_metal_api_hmac_edit_key: metal-edit metal_partition_metal_api_hmac_view_key: metal-view diff --git a/roles/gateway/defaults/main.yml b/roles/gateway/defaults/main.yml index 5bb8dff6..b97b0625 100644 --- a/roles/gateway/defaults/main.yml +++ b/roles/gateway/defaults/main.yml @@ -1,7 +1,7 @@ --- gateway_namespace: "{{ metal_control_plane_namespace }}" -gateway_http_port: 8080 -gateway_https_port: 4443 +gateway_http_port: 80 +gateway_https_port: 443 gateway_tcp_listeners: {} # nsq: 4150 From 0bd111aacf2241cec84a67ec068af06c634e18a3 Mon Sep 17 00:00:00 2001 From: Benjamin Ritter Date: Mon, 13 Jul 2026 13:11:42 +0200 Subject: [PATCH 9/9] feat: use HTTPS endpoints for metalctl we need to put the CA data into an env var to later access metal-api. So we run cert generation on before calling the requested make target if certs are not generated yet. Signed-off-by: Benjamin Ritter --- Makefile | 51 ++++++++++++++++++++++---------------------- compose.yaml | 1 + env.sh | 6 ++++++ scripts/gen-certs.sh | 18 ++++++++++++++++ 4 files changed, 50 insertions(+), 26 deletions(-) create mode 100755 scripts/gen-certs.sh diff --git a/Makefile b/Makefile index fb2368e0..5bd2ddd5 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,15 @@ +# runs at parse time on every invocation: ensure certs exist, then (re)generate .env +$(shell [ -f files/certs/ca.pem ] || ./scripts/gen-certs.sh) +$(shell ./env.sh 1>&2) +-include .env + .DEFAULT_GOAL := up .EXPORT_ALL_VARIABLES: --include .env +# disable built-in implicit rules (e.g. the "%: %.sh" rule that silently turns a +# stray/misspelled prerequisite into a junk executable instead of erroring out) +MAKEFLAGS += --no-builtin-rules +.SUFFIXES: # Commands YQ=docker run --rm -i -v $(shell pwd):/workdir mikefarah/yq:4 @@ -9,10 +17,6 @@ YQ=docker run --rm -i -v $(shell pwd):/workdir mikefarah/yq:4 KINDCONFIG := $(or $(KINDCONFIG),control-plane/kind.yaml) KUBECONFIG := $(shell pwd)/.kubeconfig -METALCTL_HMAC := $(or $(METALCTL_HMAC),metal-admin) -METALCTL_API_URL := $(or $(METALCTL_API_URL),http://api.172.42.0.42.nip.io:8080/metal) -METAL_APIV2_URL := $(or $(METAL_APIV2_URL),http://v2.172.42.0.42.nip.io:8080) - MKE2FS_CONFIG := $(shell pwd)/mke2fs.conf # Default values CONTAINERLAB=$(shell which containerlab) @@ -80,7 +84,7 @@ COMPOSE_ARGS += $(if $(MINI_LAB_HELM_CHARTS),-f compose.dev/helm-charts.yaml) endif .PHONY: up -up: env gen-certs verify-deployment-image control-plane-bake partition-bake +up: gen-certs verify-deployment-image control-plane-bake partition-bake @chmod 600 files/ssh/id_ed25519 docker compose $(COMPOSE_ARGS) up --pull=always --abort-on-container-failure --remove-orphans --force-recreate control-plane partition @$(MAKE) --no-print-directory start-machines @@ -98,23 +102,12 @@ restart: down up .PHONY: down down: cleanup -.PHONY: gen-certs -gen-certs: - @if ! [ -f "files/certs/ca.pem" ]; then \ - echo "certificate generation required, running cfssl container"; \ - docker run --rm \ - --user $$(id -u):$$(id -g) \ - --entrypoint bash \ - -v ${PWD}:/work \ - cfssl/cfssl /work/scripts/roll_certs.sh; fi - .PHONY: roll-certs roll-certs: - rm files/certs/ca.pem - $(MAKE) gen-certs + ./scripts/gen-certs.sh .PHONY: control-plane -control-plane: control-plane-bake env +control-plane: control-plane-bake docker compose $(COMPOSE_ARGS) up --remove-orphans --force-recreate control-plane .PHONY: create-proxy-registries @@ -152,7 +145,7 @@ endif ./scripts/deactivate_offloading.sh; fi .PHONY: verify-deployment-image -verify-deployment-image: env +verify-deployment-image: @if which cosign 1> /dev/null 2> /dev/null; then \ echo -e "\033[0;32mcosign is installed, verifying deployment base image\033[0m" && \ . ./.env && cosign verify --key files/cosign.pub ghcr.io/metal-stack/metal-deployment-base:$$DEPLOYMENT_BASE_IMAGE_TAG; \ @@ -178,9 +171,14 @@ external_network: sudo ip -6 route add 2001:db8:0:113::/64 via 2001:db8:0:1::1 dev mini_lab_ext; \ fi -.PHONY: env -env: - @./env.sh +files/certs/ca.pem: + @./scripts/gen-certs.sh + +.env: files/certs/ca.pem + @./env.sh 1>&2 + +.PHONY: gen-certs # keep as a convenience alias +gen-certs: files/certs/ca.pem .PHONY: cleanup cleanup: cleanup-control-plane cleanup-partition @@ -202,7 +200,7 @@ cleanup-partition: docker network rm --force mini_lab_ext .PHONY: _privatenet -_privatenet: env +_privatenet: docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network list --name user-private-network | grep user-private-network || docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network allocate --partition mini-lab --project 00000000-0000-0000-0000-000000000001 --name user-private-network .PHONY: update-userdata @@ -245,7 +243,7 @@ public-ipv6: @docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network ip create --name test --network internet-mini-lab --project 00000000-0000-0000-0000-000000000001 --addressfamily IPv6 -o template --template "{{ .ipaddress }}" .PHONY: ls -ls: env +ls: docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl machine ls ## SWITCH MANAGEMENT ## @@ -351,7 +349,7 @@ console-machine04: @$(MAKE) --no-print-directory _console VM=machine04 .PHONY: _password -_password: env +_password: docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl machine consolepassword $(MACHINE_UUID) .PHONY: password-machine01 @@ -442,6 +440,7 @@ dev-env: @echo "export METALCTL_HMAC=${METALCTL_HMAC}" @echo "export METAL_APIV2_URL=${METAL_APIV2_URL}" @echo "export KUBECONFIG=$(KUBECONFIG)" + @echo "export METALCTL_CERTIFICATE_AUTHORITY_DATA=$(METALCTL_CERTIFICATE_AUTHORITY_DATA)" build-dell-sonic: if [ ! -f "sonic-vs.img" ]; then \ diff --git a/compose.yaml b/compose.yaml index 82e87843..05c4bc5e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -53,6 +53,7 @@ services: environment: - METALCTL_HMAC=${METALCTL_HMAC} - METALCTL_API_URL=${METALCTL_API_URL} + - METALCTL_CERTIFICATE_AUTHORITY_DATA=${METALCTL_CERTIFICATE_AUTHORITY_DATA} - METALCTL_HMAC_AUTH_TYPE=${METALCTL_HMAC_AUTH_TYPE:-Metal-Admin} volumes: - ./files/ssh:/root/.ssh:ro diff --git a/env.sh b/env.sh index bf233635..1ed00eb6 100755 --- a/env.sh +++ b/env.sh @@ -25,4 +25,10 @@ CI=${CI:=false} DOCKER_HUB_USER=${DOCKER_HUB_USER:=} DOCKER_HUB_TOKEN=${DOCKER_HUB_TOKEN:=} KIND_EXPERIMENTAL_DOCKER_NETWORK=${MINI_LAB_INTERNAL_NETWORK:=mini_lab_internal} + +METALCTL_HMAC=${METALCTL_HMAC:=metal-admin} +METALCTL_API_URL=${METALCTL_API_URL:=https://api.172.42.0.42.nip.io/metal} +METALCTL_CERTIFICATE_AUTHORITY_DATA=${METALCTL_CERTIFICATE_AUTHORITY_DATA:=$(cat files/certs/ca.pem | base64 -w0)} + +METAL_APIV2_URL=${METAL_APIV2_URL:=http://v2.172.42.0.42.nip.io} EOF diff --git a/scripts/gen-certs.sh b/scripts/gen-certs.sh new file mode 100755 index 00000000..7e1755e9 --- /dev/null +++ b/scripts/gen-certs.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Generates the mini-lab TLS certificates by running the cfssl container, which +# in turn executes scripts/roll_certs.sh inside it. All output goes to stderr so +# this can be called safely from $(shell ...) / recipes that capture stdout. + +set -eo pipefail + +# resolve the repo root (parent of this script's dir) so we work from any cwd +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +echo "certificate generation required, running cfssl container" >&2 + +docker run --rm \ + --user "$(id -u):$(id -g)" \ + --entrypoint bash \ + -v "${repo_root}:/work" \ + cfssl/cfssl /work/scripts/roll_certs.sh 1>&2