OPNET-783: network: Add BGP VIP management support

bindata/network/frr-k8s/003-static-pod-rbac.yaml

@@ -0,0 +1,100 @@
+{{ if .BGPVIPManagement }}
+# RBAC for the frr-k8s static pod on control plane nodes (BGP VIP
+# management). The static pod authenticates with the node kubeconfig, whose
+# identity is the MCO node-bootstrapper ServiceAccount.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: frr-k8s-static-pod
+rules:
+- apiGroups:
+  - frrk8s.metallb.io
+  resources:
+  - frrconfigurations
+  - frrk8sconfigurations
+  verbs:
+  - get
+  - list
+  - watch
+# No delete: stale state objects are removed by frr-k8s's status cleaner.
+- apiGroups:
+  - frrk8s.metallb.io
+  resources:
+  - frrnodestates
+  - bgpsessionstates
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - frrk8s.metallb.io
+  resources:
+  - frrnodestates/status
+  - bgpsessionstates/status
+  verbs:
+  - get
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: frr-k8s-static-pod
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: frr-k8s-static-pod
+subjects:
+- kind: ServiceAccount
+  name: node-bootstrapper
+  namespace: openshift-machine-config-operator
+---
+# Read-only subset of the DaemonSet SA's Role in 002-rbac.yaml.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: frr-k8s-static-pod
+  namespace: openshift-frr-k8s
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: frr-k8s-static-pod
+  namespace: openshift-frr-k8s
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: frr-k8s-static-pod
+subjects:
+- kind: ServiceAccount
+  name: node-bootstrapper
+  namespace: openshift-machine-config-operator
+{{ end }}

bindata/network/frr-k8s/frr-k8s.yaml

@@ -294,6 +294,17 @@ spec:
           requests:
             cpu: 10m
             memory: 20Mi
+{{ if .BGPVIPManagement }}
+      # Control plane nodes run the MCO-rendered frr-k8s static pod instead;
+      # on compact/SNO this DaemonSet correctly matches zero nodes.
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: DoesNotExist
+{{ end }}
       nodeSelector:
         kubernetes.io/os: linux
       tolerations:

bindata/network/frr-k8s/master-monitor.yaml

@@ -0,0 +1,202 @@
+{{ if .BGPVIPManagement }}
+# Metrics companion for the frr-k8s static pods on control plane nodes.
+# Static pods cannot mount Secrets, so the FRR metrics exporter runs here
+# instead, reading the static pod's FRR through hostPath sockets. Only the
+# dataplane needs static pod survivability; metrics do not.
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: frr-k8s-master-monitor
+  namespace: openshift-frr-k8s
+  labels:
+    app: frr-k8s
+    component: frr-k8s-master-monitor
+  annotations:
+    release.openshift.io/version: "{{.ReleaseVersion}}"
+spec:
+  selector:
+    matchLabels:
+      app: frr-k8s
+      component: frr-k8s-master-monitor
+  template:
+    metadata:
+      labels:
+        app: frr-k8s
+        component: frr-k8s-master-monitor
+    spec:
+      serviceAccountName: frr-k8s-daemon
+      nodeSelector:
+        kubernetes.io/os: linux
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - operator: "Exists"
+      initContainers:
+      - name: cp-metrics
+        image: {{.FRRK8sImage}}
+        command: ["/bin/sh", "-c", "cp -f /frr-metrics /etc/frr_metrics/"]
+        resources:
+          requests:
+            cpu: 10m
+            memory: 20Mi
+          limits:
+            cpu: 100m
+            memory: 128Mi
+        securityContext:
+          runAsNonRoot: true
+          runAsUser: 65534
+          runAsGroup: 65534
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop: ["ALL"]
+          seccompProfile:
+            type: RuntimeDefault
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - name: metrics
+          mountPath: /etc/frr_metrics
+      containers:
+      - name: frr-metrics
+        image: {{.FRRK8sImage}}
+        command: ["/etc/frr_metrics/frr-metrics"]
+        args:
+        - --metrics-port=9141
+        - "--metrics-bind-address=[::]"
+        - --tls-cert-file=/etc/metrics/tls.crt
+        - --tls-private-key-file=/etc/metrics/tls.key
+        ports:
+        - containerPort: 9141
+          name: frrmetricshttps
+        # Root without extra capabilities except DAC_OVERRIDE: vtysh requires
+        # root, and the FRR vty sockets are owned by the frr user (100).
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop: ["ALL"]
+            add: ["DAC_OVERRIDE"]
+          seccompProfile:
+            type: RuntimeDefault
+        readinessProbe:
+          tcpSocket:
+            port: 9141
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        livenessProbe:
+          tcpSocket:
+            port: 9141
+          initialDelaySeconds: 10
+          periodSeconds: 30
+        resources:
+          requests:
+            cpu: 10m
+            memory: 20Mi
+          limits:
+            cpu: 500m
+            memory: 128Mi
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - name: frr-sockets
+          mountPath: /var/run/frr
+        - name: frr-conf
+          mountPath: /etc/frr
+        - name: metrics
+          mountPath: /etc/frr_metrics
+        - name: metrics-certs
+          mountPath: /etc/metrics
+          readOnly: true
+      volumes:
+      - name: frr-sockets
+        hostPath:
+          path: /run/frr-k8s/sockets
+          type: DirectoryOrCreate
+      - name: frr-conf
+        hostPath:
+          path: /run/frr-k8s/conf
+          type: DirectoryOrCreate
+      - name: metrics
+        emptyDir: {}
+      - name: metrics-certs
+        secret:
+          secretName: frr-k8s-master-monitor-certs
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: frr-k8s-master-monitor
+  namespace: openshift-frr-k8s
+  labels:
+    name: frr-k8s-master-monitor
+  annotations:
+    prometheus.io/scrape: "true"
+    service.beta.openshift.io/serving-cert-secret-name: frr-k8s-master-monitor-certs
+spec:
+  selector:
+    app: frr-k8s
+    component: frr-k8s-master-monitor
+  clusterIP: None
+  ports:
+  - name: frrmetricshttps
+    port: 9141
+    targetPort: 9141
+  sessionAffinity: None
+  type: ClusterIP
+---
+# The namespace carries a default-deny NetworkPolicy; the worker DaemonSet
+# is exempt as hostNetwork, the pod-network companion is not.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: frr-k8s-master-monitor
+  namespace: openshift-frr-k8s
+spec:
+  podSelector:
+    matchLabels:
+      component: frr-k8s-master-monitor
+  policyTypes:
+  - Ingress
+  - Egress
+  ingress:
+  - from:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: openshift-monitoring
+    ports:
+    - protocol: TCP
+      port: 9141
+  # TokenReview/SubjectAccessReview for scrape authentication. The API
+  # server endpoints are node IPs, not selectable by pod selectors.
+  egress:
+  - ports:
+    - protocol: TCP
+      port: 443
+    - protocol: TCP
+      port: 6443
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: frr-k8s-master-monitor
+  namespace: openshift-frr-k8s
+  labels:
+    app: frr-k8s
+    component: frr-k8s-master-monitor
+  annotations:
+    networkoperator.openshift.io/ignore-errors: ""
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    port: frrmetricshttps
+    scheme: https
+    tlsConfig:
+      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      serverName: frr-k8s-master-monitor.openshift-frr-k8s.svc
+  jobLabel: app
+  namespaceSelector:
+    matchNames:
+    - openshift-frr-k8s
+  selector:
+    matchLabels:
+      name: frr-k8s-master-monitor
+{{ end }}

docs/bgp_vip_management.md

@@ -0,0 +1,63 @@
+# BGP-based VIP management
+
+Feature gate: `BGPBasedVIPManagement` (DevPreviewNoUpgrade). BareMetal
+platform only, and only when the Infrastructure CR reports
+`status.platformStatus.baremetal.vipManagement: BGP`. Without all three,
+everything below is inert and frr-k8s behaves exactly as before.
+When all three hold, the FRR additional routing capability provider
+(`network.operator.openshift.io/cluster`:
+`spec.additionalRoutingCapabilities.providers: [FRR]`) is additionally
+required - it ships the FRRConfiguration CRD; without it, rendering fails
+explicitly rather than staying inert.
+
+Enhancement: openshift/enhancements#1982.
+
+## What CNO does
+
+When active, CNO renders a single cluster-wide `FRRConfiguration`
+(`openshift-frr-k8s/bgp-vip`) from the installer-generated `bgp-vip-config`
+ConfigMap:
+
+- The CR spec carries the BGP **sessions** (neighbors, optional BFD).
+- VIP **advertisement** is in `rawConfig`: `redistribute table-direct 198`
+  filtered to exactly the API/ingress VIP prefixes, plus per-neighbor egress
+  permits. kube-vip (rendered by MCO) installs a VIP route into kernel table
+  198 only while that node's backend health check passes, so each node
+  advertises a VIP only while it can serve it; withdrawal is automatic.
+  Advertisement cannot use the CRD's `prefixes`/`toAdvertise` surface: it
+  renders unconditional `network` statements and cannot express redistributed
+  routes (native support proposed in metallb/frr-k8s#469).
+- Configured `communities` are attached to the VIP routes where they enter
+  the BGP table (the redistribution route-maps), so every peer receives them.
+
+## Known DevPreview limitations
+
+- `hostOverrides` (per-host peer lists) reach control plane nodes through the
+  MCO-rendered per-node peers file — runtimecfg resolves them by short
+  hostname at render time. The cluster-wide `FRRConfiguration` intentionally
+  carries only `defaultPeers`: expressing per-host sessions as per-node CRs
+  would require hostname/label semantics shared across installer, runtimecfg
+  and node selectors, which the planned TechPreview structured API addresses.
+  CNO logs a warning when overrides are present.
+- BGP peer passwords travel from the installer through the `bgp-vip-config`
+  ConfigMap into `FRRConfiguration.spec` in plaintext. Moving to
+  `passwordSecret` (`kubernetes.io/basic-auth`) requires the installer to
+  generate Secrets and is planned alongside the TechPreview structured API.
+
+## Placement
+
+Control plane nodes run an MCO-rendered frr-k8s **static pod** (needed at
+bootstrap, before any workload can schedule). The frr-k8s DaemonSet therefore
+avoids masters by role under BGP VIP management; on compact/SNO topologies it
+correctly matches zero nodes. Workers keep the DaemonSet and advertise the
+ingress VIP when they host healthy routers.
+
+## RBAC
+
+The static pod authenticates with the node kubeconfig (the MCO
+node-bootstrapper ServiceAccount). `003-static-pod-rbac.yaml` grants the
+static pod the read permissions the frr-k8s controller's informers require,
+and write access to `FRRNodeState`/`BGPSessionState`. Rendered only under
+BGP VIP management.
+Per-node write scoping is not expressible in RBAC; a ValidatingAdmissionPolicy
+is planned follow-up.