diff --git a/charts/integration/templates/gateway-federator.yaml b/charts/integration/templates/gateway-federator.yaml new file mode 100644 index 00000000000..6cd4aa9a3a9 --- /dev/null +++ b/charts/integration/templates/gateway-federator.yaml @@ -0,0 +1,233 @@ +{{- if .Values.gatewayAPI.enabled }} +# Gateway API resources for integration-test dynamic federation backends. +# Replaces the Ingress loop in ingress.yaml when gatewayAPI.enabled=true. +# +# Each dynamicBackend gets its own HTTPRoute under a single shared Gateway. +# The Gateway uses a wildcard listener hostname to cover all dynamic-backend-N +# hostnames without requiring one listener per backend. +# +# Relationship to nginx-ingress-services: +# The nginx-ingress-services release in this namespace handles the main +# federation endpoint (federation-test-helper → federator service) and must +# also have federator.gatewayAPI.enabled=true. The Gateway here is separate +# so that its ClientTrafficPolicy and routing rules are scoped to integration +# backends only, and the two Gateways can have independent proxy configurations. +# +# Self-signed CA: +# The ClientTrafficPolicy references federator-ca-secret. In integration tests +# this secret lives in {{ or .Values.tls.caNamespace .Release.Namespace }}. +# If tls.caNamespace differs from the release namespace a ReferenceGrant is +# rendered below — see the nginx-ingress-services gateway-federator.yaml for +# the ReferenceGrant that covers the main federation endpoint; a separate one +# is needed here because this ClientTrafficPolicy is a distinct resource. +--- +# EnvoyProxy: use ClusterIP so in-cluster SRV resolution works without an +# external load balancer. The federation-test-helper Service in the +# nginx-ingress-services release selects the federator-gateway proxy pods; +# the integration-federator-gateway proxy pods are accessed directly via the +# dynamic-backend-N..svc.cluster.local hostnames resolved by DNS. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: integration-federator-proxy-config +spec: + provider: + type: Kubernetes + kubernetes: + envoyService: + type: ClusterIP +--- +# Gateway: single wildcard HTTPS listener covering all dynamic-backend-N hostnames, +# plus an HTTP listener for the redirect rule. +# +# The wildcard hostname *.{{ .Release.Namespace }}.svc.cluster.local matches: +# dynamic-backend-1.{{ .Release.Namespace }}.svc.cluster.local +# dynamic-backend-2.{{ .Release.Namespace }}.svc.cluster.local +# ... +# +# The federator-certificate-secret must be a wildcard cert for this domain +# (certificateDomain: *.{{ .Release.Namespace }}.svc.cluster.local). +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: integration-federator-gateway +spec: + gatewayClassName: {{ .Values.gatewayAPI.gatewayClassName | quote }} + infrastructure: + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: integration-federator-proxy-config + listeners: + - name: https + protocol: HTTPS + port: 443 + hostname: {{ printf "*.%s.svc.cluster.local" .Release.Namespace | quote }} + tls: + mode: Terminate + certificateRefs: + - kind: Secret + group: "" + name: federator-certificate-secret + allowedRoutes: + namespaces: + from: Same + - name: http + protocol: HTTP + port: 80 + hostname: {{ printf "*.%s.svc.cluster.local" .Release.Namespace | quote }} + allowedRoutes: + namespaces: + from: Same +--- +# ClientTrafficPolicy: enforce mTLS on all dynamic-backend routes. +# Targets the Gateway (not a specific sectionName) so it applies to the https +# listener; the http listener carries no traffic in practice (redirect only). +# +# NOTE: auth-tls-verify-depth has no equivalent in ClientTrafficPolicy. +# See the commented EnvoyPatchPolicy at the bottom of this file. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: ClientTrafficPolicy +metadata: + name: integration-federator-mtls +spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: integration-federator-gateway + sectionName: https + tls: + clientValidation: + optional: false + caCertificateRefs: + - kind: Secret + group: "" + name: federator-ca-secret + {{- if .Values.tls.caNamespace }} + namespace: {{ .Values.tls.caNamespace | quote }} + {{- end }} +--- +# HTTPRoute: 301 redirect from HTTP to HTTPS for all dynamic backends. +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: integration-federator-http-redirect +spec: + parentRefs: + - name: integration-federator-gateway + sectionName: http + hostnames: + - {{ printf "*.%s.svc.cluster.local" .Release.Namespace | quote }} + rules: + - filters: + - type: RequestRedirect + requestRedirect: + scheme: https + statusCode: 301 +--- +# EnvoyPatchPolicy: inject the mTLS peer certificate as X-SSL-Certificate on +# all routes served by this Gateway. +# +# Adding at the RouteConfiguration level (path "/request_headers_to_add") rather +# than per virtual_host means the header is set once and inherited by all routes, +# equivalent to having the configuration-snippet on every individual Ingress. +# +# Verify the RouteConfiguration name before deploying — see the comment in +# nginx-ingress-services/templates/gateway-federator.yaml for instructions. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyPatchPolicy +metadata: + name: integration-federator-cert-header +spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: integration-federator-gateway + type: JSONPatch + jsonPatches: + - type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" + name: {{ printf "%s/integration-federator-gateway/https" .Release.Namespace | quote }} + operation: + op: add + path: "/request_headers_to_add" + value: + - header: + key: "X-SSL-Certificate" + value: "%DOWNSTREAM_PEER_CERT%" + keep_empty_value: false +{{- if .Values.tls.caNamespace }} +--- +# ReferenceGrant: authorise this ClientTrafficPolicy to read federator-ca-secret +# from {{ .Values.tls.caNamespace }}. +# +# This is a second ReferenceGrant distinct from the one in nginx-ingress-services +# because each ClientTrafficPolicy is a separate resource that requires its own +# grant — a single grant covers all ClientTrafficPolicies in the listed namespace, +# but we need one grant entry per consuming namespace. +# +# Name includes both the release namespace and a suffix to avoid colliding with +# the grant created by nginx-ingress-services in the same CA namespace. +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: ReferenceGrant +metadata: + name: {{ printf "integration-federator-ca-access-%s" .Release.Namespace | quote }} + namespace: {{ .Values.tls.caNamespace | quote }} +spec: + from: + - group: gateway.envoyproxy.io + kind: ClientTrafficPolicy + namespace: {{ .Release.Namespace | quote }} + to: + - group: "" + kind: Secret + name: federator-ca-secret +{{- end }} +{{- range $name, $dynamicBackend := .Values.config.dynamicBackends }} +--- +# HTTPRoute for dynamic backend {{ $name }}. +# Routes federation traffic arriving at +# {{ $dynamicBackend.federatorExternalHostPrefix }}.{{ $.Release.Namespace }}.svc.cluster.local +# to the integration service on port {{ $dynamicBackend.federatorExternalPort }}. +# Replaces: the Ingress resource integration-federator-{{ $name }} in ingress.yaml. +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: integration-federator-{{ $name }} +spec: + parentRefs: + - name: integration-federator-gateway + sectionName: https + hostnames: + - {{ printf "%s.%s.svc.cluster.local" $dynamicBackend.federatorExternalHostPrefix $.Release.Namespace | quote }} + rules: + - backendRefs: + - name: integration + port: {{ $dynamicBackend.federatorExternalPort }} +{{- end }} +{{- end }} + +{{- /* +-------------------------------------------------------------------------------- +VERIFY_DEPTH NOTE +-------------------------------------------------------------------------------- +To enforce tls.verify_depth on this Gateway, add: + +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyPatchPolicy +metadata: + name: integration-federator-verify-depth +spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: integration-federator-gateway + type: JSONPatch + jsonPatches: + - type: "type.googleapis.com/envoy.config.listener.v3.Listener" + name: "/integration-federator-gateway/https" + operation: + op: add + path: "/filter_chains/0/transport_socket/typed_config/common_tls_context/validation_context/max_verify_depth" + value: + value: {{ .Values.tls.verify_depth }} +-------------------------------------------------------------------------------- */}} diff --git a/charts/integration/templates/ingress.yaml b/charts/integration/templates/ingress.yaml index 362b7b0d8f9..e12602cda8c 100644 --- a/charts/integration/templates/ingress.yaml +++ b/charts/integration/templates/ingress.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.gatewayAPI.enabled }} {{- range $name, $dynamicBackend := .Values.config.dynamicBackends }} --- apiVersion: networking.k8s.io/v1 @@ -30,3 +31,4 @@ spec: port: number: {{ $dynamicBackend.federatorExternalPort }} {{- end }} +{{- end }} diff --git a/charts/integration/values.yaml b/charts/integration/values.yaml index 36305b2be75..0b010e1f117 100644 --- a/charts/integration/values.yaml +++ b/charts/integration/values.yaml @@ -129,4 +129,16 @@ tls: ingress: class: nginx +gatewayAPI: + # Set to true to use Envoy Gateway instead of ingress-nginx for the dynamic + # backend federation ingresses. Requires federator.gatewayAPI.enabled=true in + # the nginx-ingress-services release deployed in the same namespace (which + # handles the main federation endpoint and the federation-test-helper service). + enabled: false + # Name of the GatewayClass created by Envoy Gateway. + gatewayClassName: envoy-gateway + # Container port on which the Envoy proxy pod listens for HTTPS. + # See nginx-ingress-services values for details. + envoyContainerPort: 443 + secrets: {} diff --git a/charts/nginx-ingress-services/templates/federation-test-helper.yaml b/charts/nginx-ingress-services/templates/federation-test-helper.yaml index 0fb621e34d2..b96830f822f 100644 --- a/charts/nginx-ingress-services/templates/federation-test-helper.yaml +++ b/charts/nginx-ingress-services/templates/federation-test-helper.yaml @@ -6,15 +6,27 @@ apiVersion: v1 kind: Service metadata: name: federation-test-helper - namespace: {{ .Release.namespace }} + namespace: {{ .Release.Namespace }} spec: ports: - name: wire-server-federator port: 443 protocol: TCP + {{- if .Values.federator.gatewayAPI.enabled }} + # Envoy Gateway: proxy pod container port for the HTTPS listener. + # Adjust federator.gatewayAPI.envoyContainerPort if your Envoy Gateway + # version uses a different port (e.g. 10443 for pre-v1.1 deployments). + targetPort: {{ .Values.federator.gatewayAPI.envoyContainerPort }} + {{- else }} targetPort: https + {{- end }} selector: - {{- if $newLabels }} + {{- if .Values.federator.gatewayAPI.enabled }} + # Envoy Gateway proxy pod labels set by the Gateway controller. + # The controller deploys one proxy Deployment per Gateway into this namespace. + gateway.envoyproxy.io/owning-gateway-name: federator-gateway + gateway.envoyproxy.io/owning-gateway-namespace: {{ .Release.Namespace }} + {{- else if $newLabels }} app.kubernetes.io/component: controller app.kubernetes.io/name: ingress-nginx {{- else }} diff --git a/charts/nginx-ingress-services/templates/gateway-federator.yaml b/charts/nginx-ingress-services/templates/gateway-federator.yaml new file mode 100644 index 00000000000..2ff7a597935 --- /dev/null +++ b/charts/nginx-ingress-services/templates/gateway-federator.yaml @@ -0,0 +1,246 @@ +{{- if and .Values.federator.enabled .Values.federator.gatewayAPI.enabled }} +{{- if .Values.config.isAdditionalIngress -}} + {{ fail "Federation and multi-backend-domain (multi-ingress) cannot be configured together." }} +{{- end -}} +# Gateway API resources for federation mTLS — replaces federator-ingress when +# federator.gatewayAPI.enabled=true. +# +# Prerequisites: +# Envoy Gateway installed and a GatewayClass named {{ .Values.federator.gatewayAPI.gatewayClassName }} +# https://gateway.envoyproxy.io/docs/tasks/quickstart/ +# +# Replacing these ingress-nginx annotations: +# ssl-redirect → HTTPRoute with RequestRedirect filter +# auth-tls-verify-client → ClientTrafficPolicy tls.clientValidation.optional=false +# auth-tls-secret → ClientTrafficPolicy tls.clientValidation.caCertificateRefs +# auth-tls-verify-depth → EnvoyPatchPolicy (not in ClientTrafficPolicy API; see note below) +# configuration-snippet → EnvoyPatchPolicy with %DOWNSTREAM_PEER_CERT% +--- +# EnvoyProxy configures the proxy Deployment and Service Envoy Gateway generates +# for this Gateway. Setting an explicit serviceType prevents Envoy Gateway from +# defaulting to LoadBalancer in environments where that is unwanted (e.g. +# integration tests that rely on ClusterIP + federation-test-helper for +# in-cluster SRV discovery). +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: federator-proxy-config +spec: + provider: + type: Kubernetes + kubernetes: + envoyService: + type: {{ .Values.federator.gatewayAPI.serviceType }} +--- +# Gateway: TLS-terminating HTTPS listener + plain HTTP listener for redirect. +# Envoy Gateway creates one proxy Deployment and Service in this namespace +# for each Gateway (the proxy pods are labelled with owning-gateway-name=federator-gateway). +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: federator-gateway +spec: + gatewayClassName: {{ .Values.federator.gatewayAPI.gatewayClassName | quote }} + infrastructure: + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: federator-proxy-config + listeners: + - name: https + protocol: HTTPS + port: 443 + hostname: {{ .Values.config.dns.federator | quote }} + tls: + mode: Terminate + certificateRefs: + - kind: Secret + group: "" + name: federator-certificate-secret + allowedRoutes: + namespaces: + from: Same + - name: http + protocol: HTTP + port: 80 + hostname: {{ .Values.config.dns.federator | quote }} + allowedRoutes: + namespaces: + from: Same +--- +# ClientTrafficPolicy: enforce mTLS on the HTTPS listener only. +# sectionName scopes this policy to the https listener; the http listener +# (used only for redirect) is intentionally left without client cert validation. +# +# NOTE: auth-tls-verify-depth has no direct equivalent in ClientTrafficPolicy. +# For verify_depth > 1 (intermediate CA chains) add an EnvoyPatchPolicy that +# patches max_verify_depth into the DownstreamTlsContext — see the commented +# example at the bottom of this file. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: ClientTrafficPolicy +metadata: + name: federator-mtls +spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: federator-gateway + sectionName: https + tls: + clientValidation: + optional: false + caCertificateRefs: + - kind: Secret + group: "" + name: federator-ca-secret + {{- if .Values.tls.caNamespace }} + namespace: {{ .Values.tls.caNamespace | quote }} + {{- end }} +--- +# HTTPRoute: forward HTTPS traffic to the federator backend. +# The X-SSL-Certificate header carrying the peer certificate is added by the +# EnvoyPatchPolicy below — HTTPRoute filters only support static header values. +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: federator-https +spec: + parentRefs: + - name: federator-gateway + sectionName: https + hostnames: + - {{ .Values.config.dns.federator | quote }} + rules: + - backendRefs: + - name: federator + port: {{ .Values.federator.gatewayAPI.servicePort }} +--- +# HTTPRoute: 301 redirect from HTTP to HTTPS. +# Replaces: nginx.ingress.kubernetes.io/ssl-redirect: "true" +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: federator-http-redirect +spec: + parentRefs: + - name: federator-gateway + sectionName: http + hostnames: + - {{ .Values.config.dns.federator | quote }} + rules: + - filters: + - type: RequestRedirect + requestRedirect: + scheme: https + statusCode: 301 +--- +# EnvoyPatchPolicy: inject the mTLS peer certificate as a request header. +# Replaces: configuration-snippet: proxy_set_header "X-SSL-Certificate" $ssl_client_escaped_cert; +# +# %DOWNSTREAM_PEER_CERT% is Envoy's equivalent of nginx's $ssl_client_escaped_cert: +# the URI-encoded PEM of the downstream mTLS client certificate. +# +# The RouteConfiguration name is generated by Envoy Gateway as: +# "//" +# Verify the exact name before deploying (the patch is silently a no-op if wrong): +# +# egctl x translate --from gateway-api --to xds \ +# --namespace {{ .Release.Namespace }} -f <(helm template ...) \ +# | grep -B2 '"type.googleapis.com/envoy.config.route.v3.RouteConfiguration"' +# +# Alternatively via the Envoy admin endpoint: +# kubectl -n {{ .Release.Namespace }} port-forward \ +# $(kubectl -n {{ .Release.Namespace }} get pod \ +# -l gateway.envoyproxy.io/owning-gateway-name=federator-gateway \ +# -o name | head -1) 19000 \ +# && curl -s localhost:19000/config_dump \ +# | jq '[.. | objects | select(."@type"? | contains("RoutesConfigDump")) +# | .dynamic_route_configs[].route_config.name]' +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyPatchPolicy +metadata: + name: federator-cert-header +spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: federator-gateway + type: JSONPatch + jsonPatches: + - type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" + name: {{ printf "%s/federator-gateway/https" .Release.Namespace | quote }} + operation: + op: add + path: "/request_headers_to_add" + value: + - header: + key: "X-SSL-Certificate" + value: "%DOWNSTREAM_PEER_CERT%" + keep_empty_value: false +{{- if .Values.tls.caNamespace }} +--- +# ReferenceGrant: authorise the ClientTrafficPolicy in {{ .Release.Namespace }} +# to read the CA Secret from {{ .Values.tls.caNamespace }}. +# Replaces: the implicit cross-namespace reference in the auth-tls-secret annotation. +# +# Gateway API enforces namespace isolation: a ClientTrafficPolicy cannot read a +# Secret from another namespace without a ReferenceGrant in that namespace. +# +# This resource carries metadata.namespace={{ .Values.tls.caNamespace }}, so Helm +# will create it there. The Helm service account must have create/update rights on +# ReferenceGrant in {{ .Values.tls.caNamespace }}. +# +# If you run separate Helm releases per environment, ensure this release's +# service account has the required RBAC, or extract and apply this resource +# independently with kubectl. +# +# Name includes the release namespace so that two test backends (each running +# this chart in a different namespace) can coexist without name collision. +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: ReferenceGrant +metadata: + name: {{ printf "federator-ca-access-%s" .Release.Namespace | quote }} + namespace: {{ .Values.tls.caNamespace | quote }} +spec: + from: + - group: gateway.envoyproxy.io + kind: ClientTrafficPolicy + namespace: {{ .Release.Namespace | quote }} + to: + - group: "" + kind: Secret + name: federator-ca-secret +{{- end }} +{{- end }} + +{{- /* +-------------------------------------------------------------------------------- +VERIFY_DEPTH NOTE +-------------------------------------------------------------------------------- +ClientTrafficPolicy does not expose max_verify_depth. The default Envoy +behaviour is no depth limit. If tls.verify_depth must be enforced (e.g. to +reject certificates signed by an intermediate CA deeper than expected), add +this EnvoyPatchPolicy. + +Replace the listener name with the actual generated xDS Listener name; see the +instructions on the EnvoyPatchPolicy above for how to discover it. + +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyPatchPolicy +metadata: + name: federator-verify-depth +spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: federator-gateway + type: JSONPatch + jsonPatches: + - type: "type.googleapis.com/envoy.config.listener.v3.Listener" + name: "/federator-gateway/https" + operation: + op: add + path: "/filter_chains/0/transport_socket/typed_config/common_tls_context/validation_context/max_verify_depth" + value: + value: {{ .Values.tls.verify_depth }} +-------------------------------------------------------------------------------- */}} diff --git a/charts/nginx-ingress-services/templates/ingress_federator.yaml b/charts/nginx-ingress-services/templates/ingress_federator.yaml index 4602fe98112..c6a8a5960ef 100644 --- a/charts/nginx-ingress-services/templates/ingress_federator.yaml +++ b/charts/nginx-ingress-services/templates/ingress_federator.yaml @@ -1,4 +1,4 @@ -{{- if .Values.federator.enabled }} +{{- if and .Values.federator.enabled (not .Values.federator.gatewayAPI.enabled) }} {{- if .Values.config.isAdditionalIngress -}} {{ fail "Federation and multi-backend-domain (multi-ingress) cannot be configured together." }} {{- end -}} diff --git a/charts/nginx-ingress-services/values.yaml b/charts/nginx-ingress-services/values.yaml index e77de580fb9..fdfc52057bc 100644 --- a/charts/nginx-ingress-services/values.yaml +++ b/charts/nginx-ingress-services/values.yaml @@ -15,6 +15,25 @@ fakeS3: federator: enabled: false integrationTestHelper: false + gatewayAPI: + # Set to true to use Envoy Gateway (Gateway API) instead of ingress-nginx. + # Requires Envoy Gateway to be installed cluster-wide. + # https://gateway.envoyproxy.io/ + enabled: false + # Name of the GatewayClass created by Envoy Gateway (default: "envoy-gateway"). + gatewayClassName: envoy-gateway + # Port number for the federator-ext service port. + # Must match federator chart value: service.externalFederatorPort (default 8081). + servicePort: 8081 + # Kubernetes Service type for the Envoy proxy Service created alongside the Gateway. + # Use LoadBalancer for production; ClusterIP for in-cluster integration tests + # where federation-test-helper provides the SRV-discoverable endpoint. + serviceType: LoadBalancer + # Container port on which the Envoy proxy pod listens for HTTPS. + # Envoy Gateway (v1.1+) binds directly to the Gateway listener port (443) + # using NET_BIND_SERVICE. Override to 10443 for older versions that use + # a +10000 port offset for privileged ports. + envoyContainerPort: 443 tls: duration: 2160h # 90d Letsencrypt default; NOTE: changes are ignored if using Letsencrypt renewBefore: 360h # 15d