diff --git a/README.md b/README.md index a86f327..a04656c 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,19 @@ A comprehensive test‑infrastructure repository designed to support software te ## Port Configuration -The cluster exposes several ports for easy access without requiring port-forwarding: +NodePorts for the Envoy Gateway and Grafana are allocated **dynamically** by Kubernetes rather than pinned to fixed host ports. This means multiple `test-infra` clusters (different `CLUSTER_NAME`s) can run concurrently on the same Docker host without port collisions. -- **30443**: HTTPS Gateway (Envoy Gateway) -- **30000**: Grafana dashboard (after installing observability) +To reach these services, use the built-in port-forward tasks (recommended): -All ports use non-privileged ranges (>1024) to avoid requiring administrative privileges. +- `task gateway-port-forward` – forwards the Envoy Gateway to `localhost:8443` (HTTPS) and `localhost:8080` (HTTP) +- `task grafana-port-forward` – forwards Grafana to `localhost:3000` (after `task install-observability`) + +Or find the assigned NodePort directly and reach it via the cluster's Docker network: + +```bash +kubectl get svc -n envoy-gateway-system -l app.kubernetes.io/name=envoy -o jsonpath='{.items[0].spec.ports[*].nodePort}' +kubectl get svc -n telemetry-system grafana-service -o jsonpath='{.spec.ports[0].nodePort}' +``` --- @@ -145,7 +152,7 @@ task install-observability # Add telemetry stack - **Victoria Metrics** - Time-series metrics collection and storage - **Loki** - Log aggregation with container log collection via Promtail - **Tempo** - Distributed tracing storage -- **Grafana** - Unified dashboard (accessible at http://localhost:30000, admin/datum123) +- **Grafana** - Unified dashboard (run `task grafana-port-forward`, then visit http://localhost:3000, admin/datum123) - **Prometheus CRDs** - Custom resources for advanced metrics scraping and alerting (servicemonitors, podmonitors, etc.) The observability stack is designed for development and testing environments with appropriate resource limits and simplified configurations. @@ -212,7 +219,7 @@ task test-infra:k9s # Launch k9s terminal UI **Versions** – run `task ensure-tools` regularly; it will upgrade outdated binaries. -**Docker conflicts** – if port collisions occur, delete the cluster and recreate with a different name: `task cluster-up CLUSTER_NAME=my‑test`. +**Docker conflicts** – NodePorts are no longer bound to fixed host ports, so running several clusters concurrently (each with a distinct `CLUSTER_NAME`) should not collide on ports. If you still hit a Docker naming/network conflict, delete the cluster and recreate with a different name: `task cluster-up CLUSTER_NAME=my‑test`. **Permissions** – tools are installed to system directories and may require sudo privileges. diff --git a/Taskfile.yml b/Taskfile.yml index 08f011f..13eefde 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -142,9 +142,11 @@ tasks: - echo "" - echo "🎉 Test infrastructure deployment complete!" - echo "" - - echo "🌐 Available endpoints:" - - 'echo " • HTTPS Gateway: https://localhost:8443"' - - 'echo " • Grafana: http://localhost:30000 (after installing observability)"' + - echo "🌐 Reaching the gateway/Grafana:" + - echo " NodePorts are allocated dynamically (not bound to fixed host ports), so" + - echo " multiple clusters can run concurrently on this host. Use:" + - echo " • task gateway-port-forward # forward the Envoy Gateway to localhost:8443/8080" + - echo " • task grafana-port-forward # forward Grafana to localhost:3000 (after installing observability)" - echo "" - echo "📦 Optional add-ons:" - echo " • task install-observability" # Deploy full telemetry stack @@ -153,7 +155,7 @@ tasks: - echo " • export KUBECONFIG={{.KUBECONFIG_FILE}}" # Connect to test-infra cluster - echo " • task cluster-status" # Check cluster health - echo " • kubectl get pods --all-namespaces" # List all pods - - echo " • curl -k https://localhost:8443" # Test gateway + - echo " • task gateway-port-forward # then: curl -k https://localhost:8443" - echo " • task help" # See all commands cluster-down: @@ -210,6 +212,43 @@ tasks: echo "📝 Using kubeconfig: {{.KUBECONFIG_FILE}}" k9s --kubeconfig {{.KUBECONFIG_FILE}} + gateway-port-forward: + desc: "Port-forward the Envoy Gateway to localhost:8443 (HTTPS) and :8080 (HTTP)" + silent: true + cmds: + - | + if [ ! -f "{{.KUBECONFIG_FILE}}" ]; then + echo "❌ Kubeconfig not found at {{.KUBECONFIG_FILE}}" + echo "💡 Run 'task cluster-up' first to create the cluster" + exit 1 + fi + SVC=$(kubectl --kubeconfig {{.KUBECONFIG_FILE}} get svc -n envoy-gateway-system \ + -l app.kubernetes.io/name=envoy \ + -o jsonpath='{.items[0].metadata.name}') + if [ -z "$SVC" ]; then + echo "❌ Could not find the Envoy Gateway service. Is the cluster/gateway installed?" + exit 1 + fi + echo "🌐 Forwarding svc/$SVC -> localhost:8443 (https), localhost:8080 (http)" + kubectl --kubeconfig {{.KUBECONFIG_FILE}} port-forward -n envoy-gateway-system svc/$SVC 8443:8443 8080:8080 + + grafana-port-forward: + desc: "Port-forward Grafana to localhost:3000 (after installing observability)" + silent: true + cmds: + - | + if [ ! -f "{{.KUBECONFIG_FILE}}" ]; then + echo "❌ Kubeconfig not found at {{.KUBECONFIG_FILE}}" + echo "💡 Run 'task cluster-up' first to create the cluster" + exit 1 + fi + if ! kubectl --kubeconfig {{.KUBECONFIG_FILE}} get svc grafana-service -n telemetry-system >/dev/null 2>&1; then + echo "❌ Grafana service not found. Run 'task install-observability' first." + exit 1 + fi + echo "📊 Forwarding svc/grafana-service -> localhost:3000 (admin/datum123)" + kubectl --kubeconfig {{.KUBECONFIG_FILE}} port-forward -n telemetry-system svc/grafana-service 3000:3000 + cluster-status: desc: "Show cluster status and health" silent: true @@ -402,9 +441,9 @@ tasks: - kubectl wait --for=condition=Programmed gateway/default-gateway -n envoy-gateway-system --timeout={{.WAIT_TIMEOUT}} || true - echo "✅ Envoy Gateway Operator and merged gateway are ready" - echo "" - - echo "🌐 Gateway exposed ports:" - - 'echo " HTTPS Gateway: https://localhost:8443 (self-signed cert - use -k flag)"' - - echo " To find the actual NodePort assignments, run:" + - echo "🌐 Gateway access:" + - echo " NodePort is allocated dynamically. Use 'task gateway-port-forward' to reach" + - echo " it at https://localhost:8443, or find the assigned NodePort directly:" - echo " kubectl get svc -n envoy-gateway-system -l app.kubernetes.io/name=envoy" install-observability: diff --git a/cluster/kind-config.yaml b/cluster/kind-config.yaml index 6eb76e0..2c64d30 100644 --- a/cluster/kind-config.yaml +++ b/cluster/kind-config.yaml @@ -2,19 +2,11 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - extraPortMappings: - # Grafana (Observability) - - containerPort: 30000 - hostPort: 30000 - protocol: TCP - # Envoy Gateway HTTPS - - containerPort: 30443 - hostPort: 30443 - protocol: TCP - # Envoy Gateway HTTP - - containerPort: 30080 - hostPort: 30080 - protocol: TCP + # No extraPortMappings: NodePorts for Envoy Gateway/Grafana are allocated + # dynamically (see components/envoy-gateway-operator and + # components/observability), so multiple clusters can run concurrently on + # the same Docker host without colliding on fixed host ports. Reach + # services via `kubectl port-forward` or the kind Docker network. # Add audit log configurations kubeadmConfigPatches: diff --git a/components/envoy-gateway-operator/gateway-resources/gateway-config.yaml b/components/envoy-gateway-operator/gateway-resources/gateway-config.yaml index e3d8e3f..4411ca5 100644 --- a/components/envoy-gateway-operator/gateway-resources/gateway-config.yaml +++ b/components/envoy-gateway-operator/gateway-resources/gateway-config.yaml @@ -12,6 +12,12 @@ spec: provider: type: Kubernetes kubernetes: + # NodePort without a fixed nodePort: Kubernetes allocates an ephemeral + # port per cluster, so concurrent clusters on the same host never + # collide. Find the assigned port with: + # kubectl get svc -n envoy-gateway-system -l app.kubernetes.io/name=envoy \ + # -o jsonpath='{.items[0].spec.ports[*].nodePort}' + # or use `task gateway-port-forward` for a stable localhost:8443/8080. envoyService: type: NodePort patch: @@ -22,12 +28,10 @@ spec: - name: https port: 8443 targetPort: 8443 - nodePort: 30443 protocol: TCP - name: http port: 8080 targetPort: 8080 - nodePort: 30080 protocol: TCP envoyDeployment: container: diff --git a/components/observability/grafana-instance.yaml b/components/observability/grafana-instance.yaml index 852559f..fc9104a 100644 --- a/components/observability/grafana-instance.yaml +++ b/components/observability/grafana-instance.yaml @@ -19,6 +19,11 @@ spec: alertmanager_config_from_datasources: "true" alerting: enabled: "false" # Disable legacy alerting in favor of unified alerting + # NodePort without a fixed nodePort: Kubernetes allocates an ephemeral port + # per cluster so concurrent clusters on the same host never collide. Find + # the assigned port with: + # kubectl get svc -n telemetry-system grafana-service \ + # -o jsonpath='{.spec.ports[0].nodePort}' service: spec: type: NodePort @@ -26,7 +31,6 @@ spec: - name: grafana-http port: 3000 targetPort: 3000 - nodePort: 30000 protocol: TCP deployment: spec: