diff --git a/Makefile b/Makefile index 837e44bf..5bd2ddd5 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,15 @@ +# runs at parse time on every invocation: ensure certs exist, then (re)generate .env +$(shell [ -f files/certs/ca.pem ] || ./scripts/gen-certs.sh) +$(shell ./env.sh 1>&2) +-include .env + .DEFAULT_GOAL := up .EXPORT_ALL_VARIABLES: --include .env +# disable built-in implicit rules (e.g. the "%: %.sh" rule that silently turns a +# stray/misspelled prerequisite into a junk executable instead of erroring out) +MAKEFLAGS += --no-builtin-rules +.SUFFIXES: # Commands YQ=docker run --rm -i -v $(shell pwd):/workdir mikefarah/yq:4 @@ -9,10 +17,6 @@ YQ=docker run --rm -i -v $(shell pwd):/workdir mikefarah/yq:4 KINDCONFIG := $(or $(KINDCONFIG),control-plane/kind.yaml) KUBECONFIG := $(shell pwd)/.kubeconfig -METALCTL_HMAC := $(or $(METALCTL_HMAC),metal-admin) -METALCTL_API_URL := $(or $(METALCTL_API_URL),http://api.172.17.0.1.nip.io:8080/metal) -METAL_APIV2_URL := $(or $(METAL_APIV2_URL),http://v2.172.17.0.1.nip.io:8080) - MKE2FS_CONFIG := $(shell pwd)/mke2fs.conf # Default values CONTAINERLAB=$(shell which containerlab) @@ -27,6 +31,11 @@ MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms: MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest) MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1) +MINI_LAB_INTERNAL_NETWORK=mini_lab_internal +# define this here as well so that kind picks up the network on a clean checkout, +# where .env does not exist yet at make parse time (-include .env above) +KIND_EXPERIMENTAL_DOCKER_NETWORK := $(or $(KIND_EXPERIMENTAL_DOCKER_NETWORK),$(MINI_LAB_INTERNAL_NETWORK)) + MACHINE_OS=debian-13.0 MAX_RETRIES := 30 @@ -75,7 +84,7 @@ COMPOSE_ARGS += $(if $(MINI_LAB_HELM_CHARTS),-f compose.dev/helm-charts.yaml) endif .PHONY: up -up: env gen-certs verify-deployment-image control-plane-bake partition-bake +up: gen-certs verify-deployment-image control-plane-bake partition-bake @chmod 600 files/ssh/id_ed25519 docker compose $(COMPOSE_ARGS) up --pull=always --abort-on-container-failure --remove-orphans --force-recreate control-plane partition @$(MAKE) --no-print-directory start-machines @@ -93,23 +102,12 @@ restart: down up .PHONY: down down: cleanup -.PHONY: gen-certs -gen-certs: - @if ! [ -f "files/certs/ca.pem" ]; then \ - echo "certificate generation required, running cfssl container"; \ - docker run --rm \ - --user $$(id -u):$$(id -g) \ - --entrypoint bash \ - -v ${PWD}:/work \ - cfssl/cfssl /work/scripts/roll_certs.sh; fi - .PHONY: roll-certs roll-certs: - rm files/certs/ca.pem - $(MAKE) gen-certs + ./scripts/gen-certs.sh .PHONY: control-plane -control-plane: control-plane-bake env +control-plane: control-plane-bake docker compose $(COMPOSE_ARGS) up --remove-orphans --force-recreate control-plane .PHONY: create-proxy-registries @@ -118,6 +116,8 @@ create-proxy-registries: .PHONY: control-plane-bake control-plane-bake: + + @if ! docker network ls | grep -q mini_lab_internal; then docker network create mini_lab_internal --gateway 172.42.0.1 --ip-range=172.42.0.0/16 --subnet=172.42.0.0/16 --ipv6=false ; fi @if ! which kind > /dev/null; then echo "kind needs to be installed"; exit 1; fi @if ! kind get clusters | grep metal-control-plane > /dev/null; then \ kind create cluster $(KIND_ARGS) \ @@ -125,6 +125,7 @@ control-plane-bake: --config $(KINDCONFIG) \ --kubeconfig $(KUBECONFIG); fi $(MAKE) create-proxy-registries + docker compose up -d --force-recreate cloud-provider-kind .PHONY: partition partition: partition-bake @@ -144,7 +145,7 @@ endif ./scripts/deactivate_offloading.sh; fi .PHONY: verify-deployment-image -verify-deployment-image: env +verify-deployment-image: @if which cosign 1> /dev/null 2> /dev/null; then \ echo -e "\033[0;32mcosign is installed, verifying deployment base image\033[0m" && \ . ./.env && cosign verify --key files/cosign.pub ghcr.io/metal-stack/metal-deployment-base:$$DEPLOYMENT_BASE_IMAGE_TAG; \ @@ -170,12 +171,18 @@ external_network: sudo ip -6 route add 2001:db8:0:113::/64 via 2001:db8:0:1::1 dev mini_lab_ext; \ fi -.PHONY: env -env: - @./env.sh +files/certs/ca.pem: + @./scripts/gen-certs.sh + +.env: files/certs/ca.pem + @./env.sh 1>&2 + +.PHONY: gen-certs # keep as a convenience alias +gen-certs: files/certs/ca.pem .PHONY: cleanup cleanup: cleanup-control-plane cleanup-partition + docker network rm --force mini_lab_internal .PHONY: cleanup-control-plane cleanup-control-plane: @@ -193,7 +200,7 @@ cleanup-partition: docker network rm --force mini_lab_ext .PHONY: _privatenet -_privatenet: env +_privatenet: docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network list --name user-private-network | grep user-private-network || docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network allocate --partition mini-lab --project 00000000-0000-0000-0000-000000000001 --name user-private-network .PHONY: update-userdata @@ -236,7 +243,7 @@ public-ipv6: @docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network ip create --name test --network internet-mini-lab --project 00000000-0000-0000-0000-000000000001 --addressfamily IPv6 -o template --template "{{ .ipaddress }}" .PHONY: ls -ls: env +ls: docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl machine ls ## SWITCH MANAGEMENT ## @@ -342,7 +349,7 @@ console-machine04: @$(MAKE) --no-print-directory _console VM=machine04 .PHONY: _password -_password: env +_password: docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl machine consolepassword $(MACHINE_UUID) .PHONY: password-machine01 @@ -433,6 +440,7 @@ dev-env: @echo "export METALCTL_HMAC=${METALCTL_HMAC}" @echo "export METAL_APIV2_URL=${METAL_APIV2_URL}" @echo "export KUBECONFIG=$(KUBECONFIG)" + @echo "export METALCTL_CERTIFICATE_AUTHORITY_DATA=$(METALCTL_CERTIFICATE_AUTHORITY_DATA)" build-dell-sonic: if [ ! -f "sonic-vs.img" ]; then \ @@ -451,7 +459,7 @@ build-dell-sonic: fetch-virtual-kubeconfig: # TODO: it's hard to get the latest issued generic kubeconfig secret... just take the first result for now kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden $(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden -l managed-by=secrets-manager,manager-identity=gardener-operator,name=generic-token-kubeconfig --no-headers | awk '{ print $$1 }') -o jsonpath='{.data.kubeconfig}' | base64 -d > .virtual-kubeconfig - @kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.17.0.1.nip.io:4443 + @kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.42.0.1.nip.io:4443 @kubectl --kubeconfig=.virtual-kubeconfig config set-credentials garden --token=$(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden shoot-access-virtual-garden -o jsonpath='{.data.token}' | base64 -d) @kubectl --kubeconfig=$(KUBECONFIG) config unset users.garden @kubectl --kubeconfig=$(KUBECONFIG) config unset contexts.garden diff --git a/compose.yaml b/compose.yaml index 8fd3e0d9..05c4bc5e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -53,6 +53,7 @@ services: environment: - METALCTL_HMAC=${METALCTL_HMAC} - METALCTL_API_URL=${METALCTL_API_URL} + - METALCTL_CERTIFICATE_AUTHORITY_DATA=${METALCTL_CERTIFICATE_AUTHORITY_DATA} - METALCTL_HMAC_AUTH_TYPE=${METALCTL_HMAC_AUTH_TYPE:-Metal-Admin} volumes: - ./files/ssh:/root/.ssh:ro @@ -126,6 +127,20 @@ services: - REGISTRY_PROXY_TTL=168h - REGISTRY_STORAGE_DELETE_ENABLED=true - OTEL_TRACES_EXPORTER=none + cloud-provider-kind: + image: registry.k8s.io/cloud-provider-kind/cloud-controller-manager:v0.10.0 + restart: always + networks: + - kind + environment: + - KIND_EXPERIMENTAL_DOCKER_NETWORK=${KIND_EXPERIMENTAL_DOCKER_NETWORK:-kind} + command: + # v0.10.0 of cloud controller does not support tcproutes, since it does not support the experimental gateway api channel + # using envoy-gateway deployed via roles/gateway instead + - --gateway-channel + - disabled + volumes: + - /var/run/docker.sock:/var/run/docker.sock volumes: proxy-docker: proxy-gcr: diff --git a/control-plane/kind.yaml b/control-plane/kind.yaml index a55b4aaf..018c5454 100644 --- a/control-plane/kind.yaml +++ b/control-plane/kind.yaml @@ -2,7 +2,7 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 networking: apiServerPort: 6443 - apiServerAddress: 0.0.0.0 + apiServerAddress: 172.42.0.1 nodes: - role: control-plane extraMounts: @@ -13,10 +13,6 @@ nodes: hostPort: 4443 - containerPort: 8080 hostPort: 8080 - - containerPort: 4150 - hostPort: 4150 - - containerPort: 50051 - hostPort: 50051 # if you want to run gardener operator + metal-stack, you need more pods kubeadmConfigPatches: - | diff --git a/deploy_control_plane.yaml b/deploy_control_plane.yaml index bd0f0241..7336cf70 100644 --- a/deploy_control_plane.yaml +++ b/deploy_control_plane.yaml @@ -6,10 +6,12 @@ roles: - name: ansible-common tags: always - - name: ingress-controller - tags: ingress-controller - name: metal-roles/control-plane/roles/prepare tags: prepare + - name: gateway + tags: gateway + - name: ingress-controller + tags: ingress-controller - name: metal-roles/control-plane/roles/nsq tags: nsq - name: metal-roles/control-plane/roles/metal-db diff --git a/deploy_gardener.yaml b/deploy_gardener.yaml index 43eb72d4..9cfad385 100644 --- a/deploy_gardener.yaml +++ b/deploy_gardener.yaml @@ -13,7 +13,7 @@ name: shoot-info namespace: kube-system data: - nodeNetwork: 172.18.0.0/16 + nodeNetwork: 172.42.0.0/16 podNetwork: 10.244.0.0/24 serviceNetwork: 10.96.0.0/16 tags: gardener @@ -81,7 +81,7 @@ status: loadBalancer: ingress: - - ip: "172.17.0.1" + - ip: "172.42.0.1" tags: gardener - name: Expose istio gateway through ingress-nginx (for local environments) diff --git a/docs/overview-kamaji.drawio.svg b/docs/overview-kamaji.drawio.svg index f6e083c1..e9b2b33e 100644 --- a/docs/overview-kamaji.drawio.svg +++ b/docs/overview-kamaji.drawio.svg @@ -470,13 +470,13 @@
- docker network 172.17.0.1/16 + docker network 172.42.0.1/16
- docker network 172.17.0.1/16 + docker network 172.42.0.1/16 diff --git a/docs/overview.drawio.svg b/docs/overview.drawio.svg index 1267de68..fdb8f9b9 100644 --- a/docs/overview.drawio.svg +++ b/docs/overview.drawio.svg @@ -470,13 +470,13 @@
- docker network 172.17.0.1/16 + docker network 172.42.0.1/16
- docker network 172.17.0.1/16 + docker network 172.42.0.1/16 diff --git a/env.sh b/env.sh index a01de375..1ed00eb6 100755 --- a/env.sh +++ b/env.sh @@ -24,4 +24,11 @@ DEPLOYMENT_BASE_IMAGE_TAG=${DEPLOYMENT_BASE_IMAGE_TAG} CI=${CI:=false} DOCKER_HUB_USER=${DOCKER_HUB_USER:=} DOCKER_HUB_TOKEN=${DOCKER_HUB_TOKEN:=} +KIND_EXPERIMENTAL_DOCKER_NETWORK=${MINI_LAB_INTERNAL_NETWORK:=mini_lab_internal} + +METALCTL_HMAC=${METALCTL_HMAC:=metal-admin} +METALCTL_API_URL=${METALCTL_API_URL:=https://api.172.42.0.42.nip.io/metal} +METALCTL_CERTIFICATE_AUTHORITY_DATA=${METALCTL_CERTIFICATE_AUTHORITY_DATA:=$(cat files/certs/ca.pem | base64 -w0)} + +METAL_APIV2_URL=${METAL_APIV2_URL:=http://v2.172.42.0.42.nip.io} EOF diff --git a/files/certs/grpc/client.json b/files/certs/metal-api/client.json similarity index 100% rename from files/certs/grpc/client.json rename to files/certs/metal-api/client.json diff --git a/files/certs/grpc/server.json b/files/certs/metal-api/server.json similarity index 76% rename from files/certs/grpc/server.json rename to files/certs/metal-api/server.json index 17ba2a53..6f271cf2 100644 --- a/files/certs/grpc/server.json +++ b/files/certs/metal-api/server.json @@ -1,7 +1,9 @@ { "CN": "metal-api", "hosts": [ - "172.17.0.1", + "api.172.42.0.42.nip.io", + "v2.172.42.0.42.nip.io", + "172.42.0.42", "203.0.113.1" ], "key": { diff --git a/files/dev_images.yaml b/files/dev_images.yaml index e0699518..91b8285c 100644 --- a/files/dev_images.yaml +++ b/files/dev_images.yaml @@ -1,6 +1,6 @@ --- # Do not change these values metal_api_image_tag: dev -metal_core_image_name: 172.17.0.1:5000/metalstack/metal-core +metal_core_image_name: 172.42.0.42:5000/metalstack/metal-core metal_core_image_tag: dev -metal_hammer_image_url: http://172.17.0.1:20015/metal-hammer-initrd.img.lz4 +metal_hammer_image_url: http://172.42.0.42:20015/metal-hammer-initrd.img.lz4 diff --git a/files/startup-config/leaf01_4.4.3.json b/files/startup-config/leaf01_4.4.3.json index 622b5df9..60393ec3 100644 --- a/files/startup-config/leaf01_4.4.3.json +++ b/files/startup-config/leaf01_4.4.3.json @@ -11,7 +11,7 @@ } }, "DNS_SERVER": { - "172.17.0.1": {}, + "172.42.0.1": {}, "1.1.1.1": {}, "1.0.0.1": {} }, diff --git a/files/startup-config/leaf01_4.5.1.json b/files/startup-config/leaf01_4.5.1.json index cd4224ac..78106fe8 100644 --- a/files/startup-config/leaf01_4.5.1.json +++ b/files/startup-config/leaf01_4.5.1.json @@ -11,7 +11,7 @@ } }, "DNS_SERVER": { - "172.17.0.1": {}, + "172.42.0.1": {}, "1.1.1.1": {}, "1.0.0.1": {} }, diff --git a/files/startup-config/leaf02_4.4.3.json b/files/startup-config/leaf02_4.4.3.json index 5a8d4a78..7359f574 100644 --- a/files/startup-config/leaf02_4.4.3.json +++ b/files/startup-config/leaf02_4.4.3.json @@ -11,7 +11,7 @@ } }, "DNS_SERVER": { - "172.17.0.1": {}, + "172.42.0.1": {}, "1.1.1.1": {}, "1.0.0.1": {} }, diff --git a/files/startup-config/leaf02_4.5.1.json b/files/startup-config/leaf02_4.5.1.json index 41c8f118..335ef382 100644 --- a/files/startup-config/leaf02_4.5.1.json +++ b/files/startup-config/leaf02_4.5.1.json @@ -11,7 +11,7 @@ } }, "DNS_SERVER": { - "172.17.0.1": {}, + "172.42.0.1": {}, "1.1.1.1": {}, "1.0.0.1": {} }, diff --git a/inventories/group_vars/all/control_plane.yaml b/inventories/group_vars/all/control_plane.yaml index 1a93b331..a0f045fe 100644 --- a/inventories/group_vars/all/control_plane.yaml +++ b/inventories/group_vars/all/control_plane.yaml @@ -1,6 +1,7 @@ --- metal_control_plane_provider_tenant: metal-stack -metal_control_plane_ingress_dns: 172.17.0.1.nip.io +metal_control_plane_ingress_dns: 172.42.0.1.nip.io +metal_control_plane_gateway_dns: 172.42.0.42.nip.io metal_control_plane_stage_name: test metal_control_plane_namespace: metal-control-plane metal_control_plane_image_pull_policy: Always diff --git a/inventories/group_vars/control_plane/gardener/gardenlet.yaml b/inventories/group_vars/control_plane/gardener/gardenlet.yaml index 8fed1d0f..fe02bb70 100644 --- a/inventories/group_vars/control_plane/gardener/gardenlet.yaml +++ b/inventories/group_vars/control_plane/gardener/gardenlet.yaml @@ -1,5 +1,5 @@ --- -gardener_gardenlet_default_dns_domain: "gardener.172.17.0.1.nip.io" +gardener_gardenlet_default_dns_domain: "gardener.172.42.0.1.nip.io" gardener_gardenlet_default_dns_provider: powerdns gardener_gardenlet_default_dns_credentials: apiKey: "{{ powerdns_api_key | b64encode }}" diff --git a/inventories/group_vars/control_plane/gardener/operator.yaml b/inventories/group_vars/control_plane/gardener/operator.yaml index 5cddad2e..06f6d525 100644 --- a/inventories/group_vars/control_plane/gardener/operator.yaml +++ b/inventories/group_vars/control_plane/gardener/operator.yaml @@ -1,5 +1,5 @@ --- -gardener_operator_ingress_dns_domain: "gardener.172.17.0.1.nip.io" +gardener_operator_ingress_dns_domain: "gardener.{{ metal_control_plane_ingress_dns }}" gardener_operator_backup_infrastructure: provider: S3 @@ -17,7 +17,7 @@ gardener_operator_backup_infrastructure_secret: s3ForcePathStyle: "{{ 'true' | b64encode }}" # enable mini-lab patches -gardener_operator_patch_istio_ingress_gateway_service_ip: 172.17.0.1 +gardener_operator_patch_istio_ingress_gateway_service_ip: 172.42.0.1 gardener_operator_expose_virtual_garden_through_ingress_nginx: true # for local setups this should be sufficient diff --git a/inventories/group_vars/control_plane/ingress.yaml b/inventories/group_vars/control_plane/ingress.yaml index 0ef8644b..d7a9fd25 100644 --- a/inventories/group_vars/control_plane/ingress.yaml +++ b/inventories/group_vars/control_plane/ingress.yaml @@ -1,4 +1,4 @@ --- -ingress_tcp_service_exposals: - "4150": "{{ metal_control_plane_namespace }}/nsqd:4150" - "50051": "{{ metal_control_plane_namespace }}/metal-api:50051" +gateway_tcp_listeners: + nsq: 4150 + metal-api-grpc: 50051 \ No newline at end of file diff --git a/inventories/group_vars/control_plane/metal.yml b/inventories/group_vars/control_plane/metal.yml index 40db02a6..91ae43d7 100644 --- a/inventories/group_vars/control_plane/metal.yml +++ b/inventories/group_vars/control_plane/metal.yml @@ -1,9 +1,23 @@ --- metal_set_resource_limits: no -metal_check_api_health_endpoint: http://api.{{ metal_control_plane_ingress_dns }}:8080/metal/v1/health -metal_api_headscale_control_plane_address: "http://headscale.{{ metal_control_plane_ingress_dns }}:8080" - -# metal_helm_chart_local_path: /helm-charts/charts/metal-control-plane +metal_check_api_health_endpoint: http://api.{{ metal_control_plane_gateway_dns }}/metal/v1/health +metal_api_headscale_control_plane_address: "http://headscale.{{ metal_control_plane_gateway_dns }}" + +metal_deploy_ingress: false + +metal_api_httproute_enabled: true +metal_api_httproute_parent_refs: +- name: metal-control-plane + namespace: "{{ metal_control_plane_namespace }}" + sectionName: http +- name: metal-control-plane + namespace: "{{ metal_control_plane_namespace }}" + sectionName: https +metal_api_tcproute_enabled: true +metal_api_tcproute_parent_refs: +- name: metal-control-plane + namespace: "{{ metal_control_plane_namespace }}" + sectionName: metal-api-grpc metal_api_pdb_min_available: 1 metal_api_replicas: 1 @@ -16,13 +30,22 @@ metal_api_nsq_tcp_address: nsqd:4150 metal_apiserver_pdb_min_available: 1 metal_apiserver_enabled: true -metal_apiserver_url: http://v2.{{ metal_control_plane_ingress_dns }}:8080 +metal_apiserver_url: http://v2.{{ metal_control_plane_gateway_dns }} -metal_apiserver_oidc_discovery_url: https://auth.{{ metal_control_plane_ingress_dns }}:4443/.well-known/openid-configuration -metal_apiserver_oidc_end_session_url: "https://auth.{{ metal_control_plane_ingress_dns }}:4443/oidc/v1/end_session" +metal_apiserver_oidc_discovery_url: https://zitadel.{{ metal_control_plane_gateway_dns }}/.well-known/openid-configuration +metal_apiserver_oidc_end_session_url: "https://zitadel.{{ metal_control_plane_gateway_dns }}/oidc/v1/end_session" metal_apiserver_secure_cookie: false +metal_apiserver_httproute_enabled: true +metal_apiserver_httproute_parent_refs: +- name: metal-control-plane + namespace: "{{ metal_control_plane_namespace }}" + sectionName: http +- name: metal-control-plane + namespace: "{{ metal_control_plane_namespace }}" + sectionName: https + metal_api_images: - id: firewall-ubuntu-3.0 name: Firewall 3 Ubuntu @@ -155,10 +178,10 @@ metal_masterdata_api_projects: metal_console_enabled: false -metal_api_grpc_certs_server_key: "{{ lookup('file', 'certs/grpc/server-key.pem') }}" -metal_api_grpc_certs_server_cert: "{{ lookup('file', 'certs/grpc/server.pem') }}" -metal_api_grpc_certs_client_key: "{{ lookup('file', 'certs/grpc/client-key.pem') }}" -metal_api_grpc_certs_client_cert: "{{ lookup('file', 'certs/grpc/client.pem') }}" +metal_api_grpc_certs_server_key: "{{ lookup('file', 'certs/metal-api/server-key.pem') }}" +metal_api_grpc_certs_server_cert: "{{ lookup('file', 'certs/metal-api/server.pem') }}" +metal_api_grpc_certs_client_key: "{{ lookup('file', 'certs/metal-api/client-key.pem') }}" +metal_api_grpc_certs_client_cert: "{{ lookup('file', 'certs/metal-api/client.pem') }}" metal_api_grpc_certs_ca_cert: "{{ lookup('file', 'certs/ca.pem') }}" # auditing diff --git a/inventories/group_vars/control_plane/minio.yaml b/inventories/group_vars/control_plane/minio.yaml index 73801c7c..0e7566ee 100644 --- a/inventories/group_vars/control_plane/minio.yaml +++ b/inventories/group_vars/control_plane/minio.yaml @@ -2,4 +2,4 @@ minio_root_user: mini-lab minio_root_password: change-me -minio_dns_name: minio.172.17.0.1.nip.io +minio_dns_name: minio.172.42.0.1.nip.io diff --git a/inventories/group_vars/control_plane/nsq.yaml b/inventories/group_vars/control_plane/nsq.yaml index b8983676..c7a1f802 100644 --- a/inventories/group_vars/control_plane/nsq.yaml +++ b/inventories/group_vars/control_plane/nsq.yaml @@ -9,3 +9,8 @@ nsq_certs_client_cert: "{{ lookup('file', 'certs/nsq/client.crt') }}" nsq_certs_ca_cert: "{{ lookup('file', 'certs/ca.pem') }}" nsq_broadcast_address: nsqd + +nsq_tcproute_enabled: true +nsq_tcproute_parent_refs: +- name: metal-control-plane + sectionName: nsq diff --git a/inventories/group_vars/control_plane/powerdns.yaml b/inventories/group_vars/control_plane/powerdns.yaml index bde65e72..d001532f 100644 --- a/inventories/group_vars/control_plane/powerdns.yaml +++ b/inventories/group_vars/control_plane/powerdns.yaml @@ -10,11 +10,11 @@ powerdns_load_balancer_dns_name: "ns.{{ metal_control_plane_ingress_dns }}" powerdns_api_dns_name: "powerdns-api.{{ metal_control_plane_ingress_dns }}" powerdns_zones: - - name: "gardener.172.17.0.1.nip.io." + - name: "gardener.172.42.0.1.nip.io." kind: Master nameservers: - "{{ powerdns_load_balancer_dns_name }}." - - name: "gardener-kube-apiserver.172.17.0.1.nip.io." + - name: "gardener-kube-apiserver.172.42.0.1.nip.io." kind: Master nameservers: - "{{ powerdns_load_balancer_dns_name }}." diff --git a/inventories/group_vars/control_plane/zitadel.yaml b/inventories/group_vars/control_plane/zitadel.yaml index 4a2675be..ebc8efb0 100644 --- a/inventories/group_vars/control_plane/zitadel.yaml +++ b/inventories/group_vars/control_plane/zitadel.yaml @@ -1,11 +1,16 @@ --- zitadel_endpoint: zitadel.{{ metal_control_plane_namespace }}.svc.cluster.local -zitadel_external_domain: auth.{{ metal_control_plane_ingress_dns }} -zitadel_ingress_dns: https://{{ zitadel_external_domain }}:4443 +zitadel_external_domain: zitadel.{{ metal_control_plane_gateway_dns }} +zitadel_ingress_dns: https://{{ zitadel_external_domain }} zitadel_port: 8080 zitadel_skip_verify_tls: true zitadel_insecure: true +zitadel_httproute_enabled: true +zitadel_httproute_parent_refs: +- name: metal-control-plane + sectionName: https + zitadel_init_config: static_users: - first_name: Olli @@ -23,4 +28,4 @@ zitadel_init_config: # later id will be added but currently not possible with zitadel id: metal-stack name: metal-stack - redirect_uri: http://v2.172.17.0.1.nip.io:8080/auth/openid-connect/callback + redirect_uri: http://v2.172.42.0.42.nip.io/auth/openid-connect/callback diff --git a/inventories/group_vars/leaves/main.yaml b/inventories/group_vars/leaves/main.yaml index fbc83ac4..e1d1e596 100644 --- a/inventories/group_vars/leaves/main.yaml +++ b/inventories/group_vars/leaves/main.yaml @@ -12,7 +12,7 @@ sonic_config_frr_render: false sonic_config_loopback_address: "{{ lo }}" sonic_config_mgmt_interface: ip: "{{ ansible_host }}/16`" - gateway_address: "172.17.0.1" + gateway_address: "172.42.0.1" sonic_config_mgmt_vrf: false sonic_config_nameservers: "{{ router_nameservers }}" diff --git a/inventories/group_vars/partition/common.yaml b/inventories/group_vars/partition/common.yaml index 61da08a0..bb3791fe 100644 --- a/inventories/group_vars/partition/common.yaml +++ b/inventories/group_vars/partition/common.yaml @@ -3,15 +3,15 @@ metal_partition_timezone: Europe/Berlin metal_partition_id: mini-lab metal_partition_metal_api_protocol: http -metal_partition_metal_api_addr: api.{{ metal_control_plane_ingress_dns }} -metal_partition_metal_api_port: 8080 +metal_partition_metal_api_addr: api.{{ metal_control_plane_gateway_dns }} +metal_partition_metal_api_port: 80 metal_partition_metal_api_basepath: /metal/ metal_partition_metal_api_hmac_edit_key: metal-edit metal_partition_metal_api_hmac_view_key: metal-view metal_partition_metal_api_grpc_address: "{{ metal_partition_mgmt_gateway }}:50051" metal_partition_metal_api_grpc_ca_cert: "{{ lookup('file', 'certs/ca.pem') }}" -metal_partition_metal_api_grpc_client_cert: "{{ lookup('file', 'certs/grpc/client.pem') }}" -metal_partition_metal_api_grpc_client_key: "{{ lookup('file', 'certs/grpc/client-key.pem') }}" +metal_partition_metal_api_grpc_client_cert: "{{ lookup('file', 'certs/metal-api/client.pem') }}" +metal_partition_metal_api_grpc_client_key: "{{ lookup('file', 'certs/metal-api/client-key.pem') }}" -metal_partition_mgmt_gateway: 172.17.0.1 +metal_partition_mgmt_gateway: 172.42.0.42 diff --git a/inventories/group_vars/partition/router.yaml b/inventories/group_vars/partition/router.yaml index bdcbc460..a85d2998 100644 --- a/inventories/group_vars/partition/router.yaml +++ b/inventories/group_vars/partition/router.yaml @@ -1,5 +1,5 @@ --- router_nameservers: - - 172.17.0.1 + - 172.42.0.1 - 1.1.1.1 - 1.0.0.1 diff --git a/mini-lab.dell_sonic.yaml b/mini-lab.dell_sonic.yaml index 91bf0990..d1342437 100644 --- a/mini-lab.dell_sonic.yaml +++ b/mini-lab.dell_sonic.yaml @@ -2,7 +2,7 @@ name: mini-lab prefix: "" mgmt: - network: bridge + network: mini_lab_internal topology: defaults: diff --git a/mini-lab.sonic.yaml b/mini-lab.sonic.yaml index 5d46a128..a14ad7e0 100644 --- a/mini-lab.sonic.yaml +++ b/mini-lab.sonic.yaml @@ -2,7 +2,7 @@ name: mini-lab prefix: "" mgmt: - network: bridge + network: mini_lab_internal topology: defaults: diff --git a/roles/gateway/defaults/main.yml b/roles/gateway/defaults/main.yml new file mode 100644 index 00000000..b97b0625 --- /dev/null +++ b/roles/gateway/defaults/main.yml @@ -0,0 +1,8 @@ +--- +gateway_namespace: "{{ metal_control_plane_namespace }}" +gateway_http_port: 80 +gateway_https_port: 443 + +gateway_tcp_listeners: {} + # nsq: 4150 + # metal-api: 50051 diff --git a/roles/gateway/files/gatewayclass.yaml b/roles/gateway/files/gatewayclass.yaml new file mode 100644 index 00000000..a750b9fd --- /dev/null +++ b/roles/gateway/files/gatewayclass.yaml @@ -0,0 +1,6 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller \ No newline at end of file diff --git a/roles/gateway/tasks/main.yml b/roles/gateway/tasks/main.yml new file mode 100644 index 00000000..d0f1a4d3 --- /dev/null +++ b/roles/gateway/tasks/main.yml @@ -0,0 +1,34 @@ +- name: Deploy envoy-gateway + kubernetes.core.helm: + release_name: envoy-gateway + release_namespace: envoy-gateway-system + create_namespace: true + chart_ref: "oci://docker.io/envoyproxy/gateway-helm" + chart_version: "{{ envoy_gateway_chart_version | default(omit) }}" + # values: "{{ lookup('template', 'values.yaml') | from_yaml }}" + wait: true + +- name: Deploy Envoy GatewayClass + kubernetes.core.k8s: + definition: "{{ lookup('file', 'gatewayclass.yaml') | from_yaml }}" + +- name: Deploy metal-control-plane EnvoyProxy + kubernetes.core.k8s: + definition: "{{ lookup('template', 'envoyproxy.yaml') | from_yaml }}" + +- name: Deploy metal-api certificate secret + kubernetes.core.k8s: + definition: + apiVersion: v1 + kind: Secret + type: kubernetes.io/tls + metadata: + name: metal-api-tls + namespace: "{{ gateway_namespace }}" + data: + tls.crt: "{{ lookup('file', 'certs/metal-api/server.pem') | b64encode }}" + tls.key: "{{ lookup('file', 'certs/metal-api/server-key.pem') | b64encode }}" + +- name: Deploy metal-control-plane Gateway + kubernetes.core.k8s: + definition: "{{ lookup('template', 'gateway.yaml') | from_yaml }}" diff --git a/roles/gateway/templates/envoyproxy.yaml b/roles/gateway/templates/envoyproxy.yaml new file mode 100644 index 00000000..edb7c0fe --- /dev/null +++ b/roles/gateway/templates/envoyproxy.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: metal-control-plane + namespace: "{{ gateway_namespace }}" +spec: + provider: + type: Kubernetes + kubernetes: + envoyService: + externalTrafficPolicy: Local + type: LoadBalancer + loadBalancerIP: 172.42.0.42 \ No newline at end of file diff --git a/roles/gateway/templates/gateway.yaml b/roles/gateway/templates/gateway.yaml new file mode 100644 index 00000000..49b0ec4e --- /dev/null +++ b/roles/gateway/templates/gateway.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: metal-control-plane + namespace: "{{ gateway_namespace }}" +spec: + gatewayClassName: eg + addresses: + - value: 172.42.0.42 + infrastructure: + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: metal-control-plane + listeners: + - protocol: HTTP + port: {{ gateway_http_port }} + name: http + hostname: "*.{{ metal_control_plane_gateway_dns }}" + - protocol: HTTPS + port: {{ gateway_https_port }} + name: https + hostname: "*.{{ metal_control_plane_gateway_dns }}" + tls: + mode: Terminate + certificateRefs: + - group: "" + kind: Secret + name: metal-api-tls +{% for name, port in gateway_tcp_listeners.items() %} + - protocol: TCP + name: {{ name }} + port: {{ port }} +{% endfor %} diff --git a/roles/kamaji/templates/metallb-kind-ip-pool.yaml b/roles/kamaji/templates/metallb-kind-ip-pool.yaml index 45034660..eeea201c 100644 --- a/roles/kamaji/templates/metallb-kind-ip-pool.yaml +++ b/roles/kamaji/templates/metallb-kind-ip-pool.yaml @@ -6,7 +6,7 @@ metadata: namespace: metallb-system spec: addresses: - - 172.18.255.200-172.18.255.250 + - 172.42.255.200-172.42.255.250 --- apiVersion: metallb.io/v1beta1 kind: L2Advertisement diff --git a/scripts/gen-certs.sh b/scripts/gen-certs.sh new file mode 100755 index 00000000..7e1755e9 --- /dev/null +++ b/scripts/gen-certs.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Generates the mini-lab TLS certificates by running the cfssl container, which +# in turn executes scripts/roll_certs.sh inside it. All output goes to stderr so +# this can be called safely from $(shell ...) / recipes that capture stdout. + +set -eo pipefail + +# resolve the repo root (parent of this script's dir) so we work from any cwd +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +echo "certificate generation required, running cfssl container" >&2 + +docker run --rm \ + --user "$(id -u):$(id -g)" \ + --entrypoint bash \ + -v "${repo_root}:/work" \ + cfssl/cfssl /work/scripts/roll_certs.sh 1>&2 diff --git a/scripts/roll_certs.sh b/scripts/roll_certs.sh index 7fc34acd..947a8fd4 100755 --- a/scripts/roll_certs.sh +++ b/scripts/roll_certs.sh @@ -12,8 +12,8 @@ cfssl genkey -initca ca-csr.json | cfssljson -bare ca rm *.csr for component in \ - grpc \ - masterdata-api; do + masterdata-api \ + metal-api; do pushd $component echo "generating $component certs"