diff --git a/Taskfile.yaml b/Taskfile.yaml index 5898bdf..088fd21 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -99,6 +99,7 @@ tasks: cmds: - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-cni ./cmd/galactic-cni - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-router ./cmd/galactic-router + - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-webhook ./cmd/galactic-webhook - go build -ldflags "{{.LDFLAGS}}" -o bin/vmtap-cni ./cmd/vmtap-cni - GOBIN={{.LOCALBIN}} go install github.com/containernetworking/plugins/plugins/main/host-device@v1.9.1 diff --git a/cmd/galactic-webhook/main.go b/cmd/galactic-webhook/main.go new file mode 100644 index 0000000..5c306ed --- /dev/null +++ b/cmd/galactic-webhook/main.go @@ -0,0 +1,18 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Command galactic-webhook is the mutating admission webhook that provisions +// NAD + VPCAttachment ID allocation for pods requesting VPC attachment. See +// internal/webhook for the handler logic and this repo's design plan +// (.local/plan-vpc-nad-webhook-plan.md) for the full rationale. +package main + +import "os" + +func main() { + cmd := newRootCommand() + if err := cmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/cmd/galactic-webhook/root.go b/cmd/galactic-webhook/root.go new file mode 100644 index 0000000..c402e1b --- /dev/null +++ b/cmd/galactic-webhook/root.go @@ -0,0 +1,127 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "errors" + "fmt" + "net/http" + "strings" + + "github.com/spf13/cobra" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "go.datum.net/galactic/internal/config" + "go.datum.net/galactic/internal/metadata" + galacticwebhook "go.datum.net/galactic/internal/webhook" +) + +const ( + appName = "galactic-webhook" + + appDesc = `Galactic VPC-attachment mutating admission webhook + + Find more information at: https://www.datum.net/docs` + + // mutatePodPath is the path the MutatingWebhookConfiguration in + // config/webhook/ targets. + mutatePodPath = "/mutate-v1-pod-vpc-attachment" +) + +// runCmd contains the application startup logic. +func runCmd(cfg *config.WebhookConfig) error { + ctrl.SetLogger(zap.New(zap.UseDevMode(true))) + + scheme := galacticwebhook.NewScheme() + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{ + BindAddress: fmt.Sprintf(":%d", cfg.MetricsPort), + }, + HealthProbeBindAddress: fmt.Sprintf(":%d", cfg.HealthPort), + WebhookServer: webhook.NewServer(webhook.Options{ + Port: cfg.Port, + CertDir: cfg.CertDir, + }), + }) + if err != nil { + return fmt.Errorf("create manager: %w", err) + } + + // Cache-sync-gated readiness: a cold cache means the ID allocator can't + // see existing NADs and would double-allocate IDs — see this repo's + // design plan, "Client: cache-backed reads, direct writes." + if err := mgr.AddReadyzCheck("informer-sync", func(req *http.Request) error { + if !mgr.GetCache().WaitForCacheSync(req.Context()) { + return errors.New("informer cache not yet synced") + } + return nil + }); err != nil { + return fmt.Errorf("add readyz check: %w", err) + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return fmt.Errorf("add healthz check: %w", err) + } + + mutator := &galacticwebhook.PodMutator{ + Client: mgr.GetClient(), + Decoder: admission.NewDecoder(scheme), + NADDefaults: galacticwebhook.NADDefaults{ + MTU: cfg.MTU, + InterfaceType: cfg.InterfaceType, + }, + } + mgr.GetWebhookServer().Register(mutatePodPath, &webhook.Admission{Handler: mutator}) + + ctx := ctrl.SetupSignalHandler() + if err := mgr.Start(ctx); err != nil { + return fmt.Errorf("manager exited: %w", err) + } + return nil +} + +// newRootCommand builds the root cobra command with all flags and the +// application startup logic. +func newRootCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: appName, + Short: strings.Split(appDesc, "\n")[0], + Long: appDesc, + RunE: func(cmd *cobra.Command, _ []string) error { + if ok, _ := cmd.Flags().GetBool("build-info"); ok { + fmt.Println(metadata.BuildInfo(appName)) + return nil + } + if ok, _ := cmd.Flags().GetBool("version"); ok { + fmt.Printf("galactic-webhook version %s\n", metadata.Version) + return nil + } + + cfg := config.NewWebhookConfig() + cfg.BindFlags(cmd.Flags()) + if err := cfg.Validate(); err != nil { + return err + } + return runCmd(cfg) + }, + } + + cmd.Flags().IntP("port", "p", config.DefaultWebhookPort, "Webhook server listen port") + cmd.Flags().IntP("metrics-port", "", config.DefaultWebhookMetricsPort, "Metrics listen port") + cmd.Flags().IntP("health-port", "", config.DefaultWebhookHealthPort, "Health/readiness probe listen port") + cmd.Flags().StringP("cert-dir", "", config.DefaultWebhookCertDir, "Directory containing tls.crt/tls.key") + cmd.Flags().IntP("mtu", "", config.DefaultWebhookMTU, "MTU baked into every NAD's CNI conflist") + cmd.Flags().StringP("interface-type", "", config.DefaultWebhookInterfaceType, + "Interface type baked into every NAD's CNI conflist (\"veth\" or \"tap\")") + cmd.Flags().Bool("build-info", false, "Print build information and exit") + cmd.Flags().BoolP("version", "V", false, "Print version and exit") + return cmd +} diff --git a/config/cni/rbac.yaml b/config/cni/rbac.yaml index 71cf5f8..9e67770 100644 --- a/config/cni/rbac.yaml +++ b/config/cni/rbac.yaml @@ -16,6 +16,17 @@ rules: resources: - network-attachment-definitions verbs: ["patch"] + # vpcattachments: galactic-cni creates the VPCAttachment CR and writes its + # status at ADD time (internal/cni's applyVPCAttachment), at the same + # point it creates BGPVRFInstance/BGPAdvertisement above. + - apiGroups: ["cloud.datumapis.com"] + resources: + - vpcattachments + verbs: ["get", "list", "create", "update", "patch"] + - apiGroups: ["cloud.datumapis.com"] + resources: + - vpcattachments/status + verbs: ["get", "update", "patch"] - apiGroups: [""] resources: - nodes diff --git a/config/kustomization.yaml b/config/kustomization.yaml index 503df7d..2e35930 100644 --- a/config/kustomization.yaml +++ b/config/kustomization.yaml @@ -3,3 +3,4 @@ resources: - cni - router - vmtap + - webhook diff --git a/config/router/rbac.yaml b/config/router/rbac.yaml index 6e939e5..79120c0 100644 --- a/config/router/rbac.yaml +++ b/config/router/rbac.yaml @@ -33,6 +33,30 @@ rules: resources: - nodes verbs: ["get", "list", "watch"] + # pods (cluster-wide, all namespaces): the GC reconciler checks pod + # existence to reclaim orphaned VPCAttachments (Status.PodName) and NADs + # (referenced via the Multus networks annotation) — see internal/gc's + # CollectOrphanedVPCAttachments/CollectOrphanedNADs. + - apiGroups: [""] + resources: + - pods + verbs: ["get", "list", "watch"] + # vpcattachments (cluster-wide): galactic-cni (a different ServiceAccount, + # see config/cni/rbac.yaml) creates these and writes their status at ADD + # time. galactic-router's GC reconciler only reads and deletes orphaned + # ones — no create/update/status verbs needed here. + - apiGroups: ["cloud.datumapis.com"] + resources: + - vpcattachments + verbs: ["get", "list", "watch", "delete"] + # network-attachment-definitions (cluster-wide): galactic-webhook (a + # different ServiceAccount, see config/webhook/rbac.yaml) creates these; + # galactic-cni patches the host-interface annotation onto them. GC only + # reads and deletes orphaned ones. + - apiGroups: ["k8s.cni.cncf.io"] + resources: + - network-attachment-definitions + verbs: ["get", "list", "watch", "delete"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] diff --git a/config/webhook/certificate.yaml b/config/webhook/certificate.yaml new file mode 100644 index 0000000..eb12120 --- /dev/null +++ b/config/webhook/certificate.yaml @@ -0,0 +1,28 @@ +# Self-signed CA + serving certificate for the webhook's TLS listener, +# mirroring the minimal cert-manager setup common for controller-runtime +# webhooks: a self-signed Issuer bootstraps a CA, which then issues the +# actual serving Certificate. The MutatingWebhookConfiguration's +# cert-manager.io/inject-ca-from annotation (see +# mutatingwebhookconfiguration.yaml) has cert-manager auto-populate +# caBundle from this Certificate — no manual CA distribution needed. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: galactic-webhook-selfsigned + namespace: galactic-system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: galactic-webhook-serving-cert + namespace: galactic-system +spec: + secretName: galactic-webhook-serving-certs + issuerRef: + name: galactic-webhook-selfsigned + kind: Issuer + dnsNames: + - galactic-webhook.galactic-system.svc + - galactic-webhook.galactic-system.svc.cluster.local diff --git a/config/webhook/deployment.yaml b/config/webhook/deployment.yaml new file mode 100644 index 0000000..023ae4e --- /dev/null +++ b/config/webhook/deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: galactic-webhook + namespace: galactic-system + labels: + app.kubernetes.io/name: galactic-webhook +spec: + # 2+ replicas: failurePolicy: Fail on the MutatingWebhookConfiguration + # makes webhook unavailability a pod-admission outage — see this repo's + # design plan, Component placement. + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: galactic-webhook + template: + metadata: + labels: + app.kubernetes.io/name: galactic-webhook + spec: + serviceAccountName: galactic-webhook + containers: + - name: galactic-webhook + # ":latest" is a placeholder only: no such tag is ever pushed to + # GHCR. CI resolves and stamps a real version tag here when it + # publishes the ghcr.io/datum-cloud/galactic-kustomize OCI bundle + # (see .github/workflows/publish.yaml); applying this manifest + # directly from a git checkout will fail to pull the image unless + # you override the tag first — see the README's "Production + # Deployment" section. + image: ghcr.io/datum-cloud/galactic-webhook:latest + command: + - /galactic-webhook + ports: + - name: webhook + containerPort: 9443 + protocol: TCP + - name: metrics + containerPort: 8080 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: health + initialDelaySeconds: 5 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 5 + periodSeconds: 10 + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + volumeMounts: + - name: serving-certs + mountPath: /etc/webhook/certs + readOnly: true + volumes: + - name: serving-certs + secret: + secretName: galactic-webhook-serving-certs diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 0000000..ea94665 --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,7 @@ +resources: + - serviceaccount.yaml + - rbac.yaml + - deployment.yaml + - service.yaml + - certificate.yaml + - mutatingwebhookconfiguration.yaml diff --git a/config/webhook/mutatingwebhookconfiguration.yaml b/config/webhook/mutatingwebhookconfiguration.yaml new file mode 100644 index 0000000..3c8c117 --- /dev/null +++ b/config/webhook/mutatingwebhookconfiguration.yaml @@ -0,0 +1,35 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: galactic-webhook + annotations: + # Has cert-manager auto-populate every webhook's clientConfig.caBundle + # below from this Certificate's CA — see certificate.yaml. + cert-manager.io/inject-ca-from: galactic-system/galactic-webhook-serving-cert +webhooks: + - name: vpc-attachment-injector.galactic.datumapis.com + clientConfig: + service: + name: galactic-webhook + namespace: galactic-system + path: /mutate-v1-pod-vpc-attachment + port: 443 + rules: + - apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + operations: ["CREATE"] + failurePolicy: Fail + sideEffects: NoneOnDryRun + matchPolicy: Equivalent + reinvocationPolicy: IfNeeded + admissionReviewVersions: ["v1"] + timeoutSeconds: 10 + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kube-node-lease + - galactic-system diff --git a/config/webhook/rbac.yaml b/config/webhook/rbac.yaml new file mode 100644 index 0000000..a96825f --- /dev/null +++ b/config/webhook/rbac.yaml @@ -0,0 +1,32 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: galactic-webhook +rules: + # vpcs (cluster-wide): a pod's vpc-attachment annotation names a VPC in + # the pod's own namespace, which can be any namespace — see this repo's + # design plan, Handler logic step 5. + - apiGroups: ["cloud.datumapis.com"] + resources: + - vpcs + verbs: ["get", "list", "watch"] + # network-attachment-definitions (cluster-wide): the webhook creates one + # NAD per pod attachment and lists existing ones (labelVPC) to allocate + # a free VPCAttachment ID — see internal/webhook/allocate.go. + - apiGroups: ["k8s.cni.cncf.io"] + resources: + - network-attachment-definitions + verbs: ["get", "list", "watch", "create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: galactic-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: galactic-webhook +subjects: + - kind: ServiceAccount + name: galactic-webhook + namespace: galactic-system diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 0000000..41ecf8a --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: galactic-webhook + namespace: galactic-system +spec: + selector: + app.kubernetes.io/name: galactic-webhook + ports: + - name: webhook + port: 443 + targetPort: webhook diff --git a/config/webhook/serviceaccount.yaml b/config/webhook/serviceaccount.yaml new file mode 100644 index 0000000..98ce712 --- /dev/null +++ b/config/webhook/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: galactic-webhook + namespace: galactic-system diff --git a/containers/galactic-webhook/Dockerfile b/containers/galactic-webhook/Dockerfile new file mode 100644 index 0000000..07b0e4d --- /dev/null +++ b/containers/galactic-webhook/Dockerfile @@ -0,0 +1,42 @@ +# Build the galactic-webhook mutating admission webhook +FROM --platform=$BUILDPLATFORM golang:1.26 AS builder +ARG TARGETOS +ARG TARGETARCH +ARG VERSION=dev +ARG GIT_COMMIT=unknown +ARG GIT_TREE_STATE=unknown +ARG BUILD_DATE=unknown +ARG SPDX_LICENSE=AGPL-3.0-or-later +ARG GIT_URL=https://github.com/datum-cloud/galactic + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY cmd/ cmd/ +COPY internal/ internal/ + +# Build galactic-webhook +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \ + -ldflags "-s -w \ + -X go.datum.net/galactic/internal/metadata.Version=${VERSION} \ + -X go.datum.net/galactic/internal/metadata.GitCommit=${GIT_COMMIT} \ + -X go.datum.net/galactic/internal/metadata.GitTreeState=${GIT_TREE_STATE} \ + -X go.datum.net/galactic/internal/metadata.BuildDate=${BUILD_DATE} \ + -X go.datum.net/galactic/internal/metadata.SPDXLicense=${SPDX_LICENSE} \ + -X go.datum.net/galactic/internal/metadata.GitURL=${GIT_URL}" \ + -o galactic-webhook cmd/galactic-webhook/main.go + +# Use distroless as minimal base image to package the binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot + +COPY --from=builder /workspace/galactic-webhook /galactic-webhook + +USER 65532:65532 +ENTRYPOINT ["/galactic-webhook"] diff --git a/docs/agents/ARCHITECTURE.md b/docs/agents/ARCHITECTURE.md index f1351a7..727dfe2 100644 --- a/docs/agents/ARCHITECTURE.md +++ b/docs/agents/ARCHITECTURE.md @@ -1,26 +1,38 @@ # Architecture -> Galactic is the SRv6 data plane for multi-cloud VPC networking, deployed as two -> binaries on each Kubernetes node: a CNI plugin that attaches containers to VPC -> networks, and a router that reconciles BGP CRDs and drives an embedded -> GoBGP server to distribute EVPN (L2VPN/EVPN AFI/SAFI) paths between nodes. +> Galactic is the SRv6 data plane for multi-cloud VPC networking, deployed as three +> binaries: a per-node CNI plugin that attaches containers to VPC networks, a +> per-node router that reconciles BGP CRDs and drives an embedded GoBGP server to +> distribute EVPN (L2VPN/EVPN AFI/SAFI) paths between nodes, and a cluster-wide +> mutating admission webhook that provisions VPC attachments for pods on CREATE. -_Last updated: 2026-07-14_ +_Last updated: 2026-07-30_ --- ## Overview Galactic implements VPC isolation and cross-cluster reachability using Linux SRv6. -When a pod is attached to a VPC, the CNI plugin creates the required kernel state -(VRF, veth pair, SRv6 ingress route) and writes a `BGPAdvertisement` CRD. -`galactic-router` watches that CRD and injects the EVPN path into the node-local -GoBGP server. GoBGP distributes the path to a BGP route reflector, enabling pods -on different nodes or clusters to reach each other via SRv6-encapsulated traffic. - -VPC and VPCAttachment CRDs are owned by a separate companion operator -(`go.datum.net/cloud`). Galactic receives pre-populated identifiers through the -CNI config and acts on them. `galactic-router` reconciles BGP CRDs +When a pod carrying the `galactic.datumapis.com/vpc=` annotation is +created, `galactic-webhook` allocates a VPCAttachment ID for it, creates a +`NetworkAttachmentDefinition` (NAD) pointing `galactic-cni` at that VPC + +VPCAttachment pair, and patches the pod to attach via that NAD (Multus). When +the pod is scheduled, `galactic-cni` creates the required kernel state (VRF, +veth pair, SRv6 ingress route), creates the `VPCAttachment` CR itself (Spec and +Status), and writes a `BGPAdvertisement` CRD. `galactic-router` watches that CRD +and injects the EVPN path into the node-local GoBGP server. GoBGP distributes +the path to a BGP route reflector, enabling pods on different nodes or clusters +to reach each other via SRv6-encapsulated traffic. + +The `VPC` CRD is owned by a separate companion operator (`go.datum.net/cloud`, +types-only — no controller exists there). `VPCAttachment` and NAD provisioning, +previously assumed to belong to that companion operator, are now owned by +Galactic itself: `galactic-webhook` creates the NAD (and allocates the +VPCAttachment ID baked into it); `galactic-cni` creates the `VPCAttachment` CR +and writes its status, at the same point it already creates +`BGPVRFInstance`/`BGPAdvertisement`. See `.local/plan-vpc-nad-webhook-plan.md` +for the full design rationale, including why the webhook creates only the NAD +and not the VPCAttachment CR itself. `galactic-router` reconciles BGP CRDs (`go.datum.net/network`) directly — no gRPC sidecar, no provider CRD lifecycle. ### SRv6 SID encoding @@ -54,8 +66,13 @@ Distinguisher and import/export Route Target. galactic/ ├── cmd/ │ ├── galactic-cni/ # CNI binary -│ └── galactic-router/ # Router binary (controller-runtime reconciler) +│ ├── galactic-router/ # Router binary (controller-runtime reconciler) +│ └── galactic-webhook/ # Mutating admission webhook binary ├── internal/ +│ ├── webhook/ # PodMutator admission.Handler: VPCAttachment ID +│ │ # allocation (via NAD labels) + NAD creation + +│ │ # pod patch. Creates NADs only — see internal/cni's +│ │ # applyVPCAttachment for VPCAttachment CR creation. │ ├── controller/ # controller-runtime reconcilers (BGPRouter, BGPPeer, │ │ # BGPAdvertisement, BGPVRFInstance, BGPPolicy, Secret, │ │ # Node, GC); field index registration; status helpers @@ -67,10 +84,12 @@ galactic/ │ ├── model/ # DesiredRouter and family; re-exports BGP API enums │ ├── hash/ # SHA-256 change detection over DesiredRouter │ ├── metadata/ # Build-time version info (Version, GitCommit, etc.) -│ ├── gc/ # Orphaned BGPAdvertisement/BGPVRFInstance CRD and -│ │ # stale kernel VRF cleanup, driven by the GC controller +│ ├── gc/ # Orphaned BGPAdvertisement/BGPVRFInstance/VPCAttachment +│ │ # CRD, orphaned NAD, and stale kernel VRF cleanup, +│ │ # driven by the GC controller │ ├── cni/ # CNI cmdAdd / cmdDel / cmdCheck, PluginConf parsing, -│ │ # BGP CRD publish, built-in IPAM wiring +│ │ # BGP CRD publish, VPCAttachment CR create+status +│ │ # (vpcattachment.go), built-in IPAM wiring │ │ ├── ipam/ # Built-in IPv6 pool + static IP allocators │ │ ├── route/ # Host-side static routes via netlink │ │ ├── tap/ # Tap interface management (VM workloads) @@ -91,19 +110,26 @@ galactic/ │ │ │ # and tenant-control nodes │ │ └── tenant-control/ # route-reflector role: base + GALACTIC_ROUTER_REFLECTOR=true, │ │ # opt-in via the galactic.datumapis.com/node=control node label -│ └── cni/ # hostNetwork DaemonSet: `init` container stages -│ # galactic-cni/host-device into /opt/cni/bin -│ # and writes the conflist + kubeconfig; `run` -│ # container refreshes credentials and serves -│ # gRPC health checks +│ ├── cni/ # hostNetwork DaemonSet: `init` container stages +│ │ # galactic-cni/host-device into /opt/cni/bin +│ │ # and writes the conflist + kubeconfig; `run` +│ │ # container refreshes credentials and serves +│ │ # gRPC health checks +│ └── webhook/ # HA Deployment (2 replicas) + Service + +│ # MutatingWebhookConfiguration + cert-manager +│ # self-signed Issuer/Certificate for TLS ├── deploy/ │ └── containerlab/ # ContainerLab lab topology and scripts └── containers/ ├── galactic-cni/ # galactic-cni + host-device image (e2e test and production publish) - └── galactic-router/ # galactic-router production image + ├── galactic-router/ # galactic-router production image + └── galactic-webhook/ # galactic-webhook image (not yet wired into task test:e2e or publish.yaml) ``` -Production images are published by `.github/workflows/publish.yaml` — see CI/CD below. +Production images for `galactic-cni`/`galactic-router` are published by +`.github/workflows/publish.yaml` — see CI/CD below. `galactic-webhook`'s +Dockerfile exists but isn't yet wired into either `task test:e2e` or the +publish pipeline — see Known Constraints. --- @@ -113,6 +139,28 @@ See [docs/cni-cmd-sequence.md](../cni-cmd-sequence.md) for the full CNI ADD/DEL See [docs/agent-startup.md](../agent-startup.md) for the router startup sequence diagram. +**Pod → VPC attachment, end to end:** + +1. Pod CREATE with `galactic.datumapis.com/vpc=` reaches + `galactic-webhook`. It validates the VPC exists, allocates a free 16-bit + VPCAttachment ID (scanning existing NADs labeled for that VPC — not the + `VPCAttachment` CRD, which doesn't exist yet at this point), creates a + deterministically-named NAD (`-`) with + the CNI conflist embedded, and patches the pod's + `k8s.v1.cni.cncf.io/networks` annotation plus a reinvocation-guard + annotation. +2. Once scheduled, Multus resolves the NAD and invokes `galactic-cni`'s ADD + with that conflist. `galactic-cni` creates the VRF/veth/SRv6 kernel state, + then — at the same point it creates `BGPVRFInstance`/`BGPAdvertisement` — + creates the `VPCAttachment` CR (Spec from real IPAM-allocated addresses) + and writes its Status (`Node`, `ContainerID`, `PodName`, interface names, + `PodSubnet`). +3. `galactic-router` watches `BGPAdvertisement`/`BGPVRFInstance` and injects + the EVPN path into GoBGP, same as before this feature existed. +4. Cleanup: the GC controller reclaims NADs no live pod still references, and + `VPCAttachment` CRs whose `Status.PodName` no longer exists — see + `internal/gc`'s `CollectOrphanedNADs`/`CollectOrphanedVPCAttachments`. + --- ## Components @@ -127,7 +175,8 @@ See [docs/agent-startup.md](../agent-startup.md) for the router startup sequence | `internal/hash` | `galactic-router` | Change detection | | `internal/metadata` | both | Build-time version info stamped via `-ldflags` | | `internal/gc` | `galactic-router` | Orphaned CRD/VRF cleanup, driven by the GC controller's ticker | -| `internal/cni` | `galactic-cni` | CNI cmdAdd / cmdDel / cmdCheck; BGP CRD publish | +| `internal/cni` | `galactic-cni` | CNI cmdAdd / cmdDel / cmdCheck; BGP CRD publish; VPCAttachment CR create+status (`vpcattachment.go`) | +| `internal/webhook` | `galactic-webhook` | Mutating admission webhook: VPCAttachment ID allocation, NAD creation, pod patch | | `internal/cni/ipam` | `galactic-cni` | Built-in IPv6 pool + static allocators | | `internal/cni/tap` | `galactic-cni` | Tap interface create/delete (VM workloads) | | `internal/installer` | `galactic-cni` | DaemonSet `init`/`run` logic: binary staging, conflist/kubeconfig templating, credential refresh, gRPC health server | @@ -199,6 +248,47 @@ lives in `root.go`'s `runCmd`: that waits for cache sync, then runs on `--gc-interval`, default 5m). 8. `mgr.Start(ctx)` — blocks until the signal-handler context is cancelled. +### `cmd/galactic-webhook/main.go` / `root.go` — Mutating admission webhook + +`main.go` is a 3-line wrapper around `newRootCommand().Execute()`, mirroring +`galactic-router`'s split; all startup logic lives in `root.go`'s `runCmd`: + +1. Build a controller-runtime manager with `webhook.NewServer(webhook.Options{ + Port: cfg.Port, CertDir: cfg.CertDir})` (default port `9443`, cert dir + `/etc/webhook/certs` — populated by the cert-manager-issued Secret, see + `config/webhook/certificate.yaml`). +2. Register a cache-sync-gated readyz check (`mgr.GetCache().WaitForCacheSync`) + — a cold cache means the ID allocator can't see existing NADs and would + double-allocate — plus a plain `healthz.Ping` healthz check, both served on + `--health-port` (default `8081`). +3. Construct `internal/webhook.PodMutator{Client: mgr.GetClient(), Decoder: + admission.NewDecoder(scheme), NADDefaults: {MTU: cfg.MTU, InterfaceType: + cfg.InterfaceType}}` and register it at `/mutate-v1-pod-vpc-attachment`. +4. `mgr.Start(ctx)` — blocks until the signal-handler context is cancelled. + +Env vars: `GALACTIC_WEBHOOK_PORT`, `GALACTIC_WEBHOOK_METRICS_PORT`, +`GALACTIC_WEBHOOK_HEALTH_PORT`, `GALACTIC_WEBHOOK_CERT_DIR`, +`GALACTIC_WEBHOOK_MTU`, `GALACTIC_WEBHOOK_INTERFACE_TYPE` (`internal/config/webhook.go`). + +`PodMutator.Handle` (`internal/webhook/pod_mutator.go`) is the pure `Pod in → +Pod/patch out or Deny` logic: + +1. Decode the admitted Pod. +2. Reinvocation guard: if `galactic.datumapis.com/vpc-attachment-ref` is + already set, allow unpatched (already processed). +3. If `galactic.datumapis.com/vpc` is unset, allow unpatched (silent no-op). +4. If `pod.Spec.HostNetwork`, allow unpatched (VPC attach doesn't apply). +5. `Get` the named `VPC`; deny if not found or `Status.VPC` is still empty. +6. On dry-run, allow unpatched — never create real objects. +7. `createNAD` (`internal/webhook/pod_mutator.go`): allocate a free + VPCAttachment ID (`AllocateVPCAttachmentID`, `allocate.go` — scans NADs + labeled for this VPC, not the `VPCAttachment` CRD) and create the NAD + (`buildNAD`, `nad.go`), retrying up to 5 times on an `AlreadyExists` + collision. +8. Patch the pod: parse-merge `k8s.v1.cni.cncf.io/networks` (preserving any + existing entries) and set the reinvocation-guard annotation, then return + `admission.PatchResponseFromRaw`. + --- ## Configuration @@ -224,6 +314,7 @@ See [docs/router/configuration.md](../router/configuration.md) for the full refe | Field | Type | Description | |-----------------|----------|-------------------------------------------------------------------------| | `vpc` | string | Base62-encoded 48-bit VPC identifier | +| `vpc_name` | string | VPC CR's Kubernetes object name (distinct from `vpc` above); optional — when empty, `galactic-cni` skips VPCAttachment CR creation. Set by `galactic-webhook` when it builds the NAD. | | `vpcattachment` | string | Base62-encoded 16-bit VPCAttachment identifier | | `interface_type`| string | `veth` (default) or `tap`; tap mode omits guest-side/host-device config but still runs IPAM and SRv6/BGP publish (see the ADD result section below) | | `namespace` | string | Kubernetes namespace for BGP CRDs; resolution order is this field → `GALACTIC_CNI_NAMESPACE` → `HostConf.Namespace` (from the conflist) → `DefaultNamespace` (`galactic-system`) | @@ -319,8 +410,9 @@ pod's IPAM bookkeeping and does not attempt to unwind kernel/CRD state — see t | `internal/model` | both | `DesiredRouter`, `DesiredPeer`, `DesiredAdvertisement`, `DesiredPolicy`, `DesiredVRFInstance`, `RuntimeStatus`; re-exports BGP API enums | No | | `internal/hash` | galactic-router | SHA-256 fingerprint of `DesiredRouter` for no-op suppression | No | | `internal/metadata` | both | Build-time vars (`Version`, `GitCommit`, `GitTreeState`, `BuildDate`) stamped via `-ldflags` | No | -| `internal/gc` | galactic-router | Collects orphaned `BGPAdvertisement`/`BGPVRFInstance` CRDs and stale kernel VRFs; invoked by the GC controller's ticker | No | -| `internal/cni` | galactic-cni | `cmdAdd` / `cmdDel` / `cmdCheck`; CNI PluginConf parsing; BGPVRFInstance/BGPAdvertisement lifecycle; delegates kernel work to plumbing | No | +| `internal/gc` | galactic-router | Collects orphaned `BGPAdvertisement`/`BGPVRFInstance`/`VPCAttachment` CRDs, orphaned NADs (reference-counted via the Multus networks annotation), and stale kernel VRFs; invoked by the GC controller's ticker | No | +| `internal/cni` | galactic-cni | `cmdAdd` / `cmdDel` / `cmdCheck`; CNI PluginConf parsing; BGPVRFInstance/BGPAdvertisement lifecycle; VPCAttachment CR create+status (`vpcattachment.go`); delegates kernel work to plumbing | No | +| `internal/webhook` | galactic-webhook| `PodMutator` mutating admission handler: VPCAttachment ID allocation (`allocate.go`, scans NAD labels), NAD construction (`nad.go`), pod patch (`pod_mutator.go`) | No | | `internal/cni/ipam` | galactic-cni | Built-in IPv6 pool allocator (in-memory, ephemeral) and static IP allocator | Yes (pool allocations) | | `internal/cni/route` | galactic-cni | Host-side static route add/delete via netlink | No | | `internal/cni/tap` | galactic-cni | Tap interface create/delete for VM workloads (Kata, Firecracker, QEMU) | No | @@ -339,9 +431,10 @@ pod's IPAM bookkeeping and does not attempt to unwind kernel/CRD state — see t |-----------------------------------------|----------|----------------------------------------------------------| | `github.com/osrg/gobgp/v4` | v4.7.0 | Embedded BGP server (tenant mode) | | `go.datum.net/network` | bumped frequently | BGP CRD API types (BGPRouter, BGPPeer, BGPAdvertisement, BGPPolicy, BGPVRFInstance) | -| `sigs.k8s.io/controller-runtime` | v0.24.1 | Reconciler framework, manager, field indexes | -| `github.com/spf13/cobra` | v1.10.2 | CLI command/flag handling for both binaries | -| `github.com/spf13/viper` | v1.21.0 | Config resolution (flags/env/defaults) for `galactic-router` only; `galactic-cni` resolves config itself (conflist/env/API auto-detect in `internal/cni/config.go`) and does not import viper | +| `go.datum.net/cloud` | pinned pseudo-version | `VPC`/`VPCAttachment` API types; types-only, no controller — imported by both `internal/cni` (creates/updates `VPCAttachment`) and `internal/webhook` (reads `VPC`) | +| `sigs.k8s.io/controller-runtime` | v0.24.1 | Reconciler framework, manager, field indexes, webhook server/admission | +| `github.com/spf13/cobra` | v1.10.2 | CLI command/flag handling for all three binaries | +| `github.com/spf13/viper` | v1.21.0 | Config resolution (flags/env/defaults) for `galactic-router`/`galactic-webhook`; `galactic-cni` resolves config itself (conflist/env/API auto-detect in `internal/cni/config.go`) and does not import viper | | `github.com/containernetworking/cni` | v1.3.0 | CNI plugin spec, skel, invoke | | `github.com/containernetworking/plugins` | v1.9.1 | `host-device` plugin, delegated to for moving the guest veth into the pod netns | | `github.com/vishvananda/netlink` | pinned pseudo-version | Linux netlink: VRF, veth, SRv6 routes | @@ -372,8 +465,8 @@ pod's IPAM bookkeeping and does not attempt to unwind kernel/CRD state — see t | Layer | Command | Framework | Scope | |------------|------------------|---------------------|------------------------------------------------------------------------| -| Unit | `task test:unit` | `go test -race` | `internal/cni` (`cni_test.go`, `bgp_test.go`, `netns_test.go` — `buildResult`, `parseConf`, `routeTarget`, `lookupBGPRouter`), `internal/cni/{ipam,tap,veth}`, `internal/installer` (`installer_test.go` — `Bootstrap`/`Run` with mocked k8s client and netlink/host paths), `internal/plumbing/srv6`, `internal/gc`, `internal/reconcile`, `internal/controller`, `internal/plumbing/intf`, `internal/metadata`, `internal/runtime/gobgp` (partial), `internal/runtime/frr` | -| E2E | `task test:e2e` | Kind + `go test` | Full BGPRouter lifecycle in a Kind cluster; builds and loads image | +| Unit | `task test:unit` | `go test -race` | `internal/cni` (`cni_test.go`, `bgp_test.go`, `netns_test.go`, `vpcattachment_test.go` — `buildResult`, `parseConf`, `routeTarget`, `lookupBGPRouter`, `applyVPCAttachment`), `internal/cni/{ipam,tap,veth}`, `internal/installer` (`installer_test.go` — `Bootstrap`/`Run` with mocked k8s client and netlink/host paths), `internal/plumbing/srv6`, `internal/gc` (incl. `orphans_test.go` — NAD reference-counting, VPCAttachment `Status.PodName` checks), `internal/webhook` (`allocate_test.go`, `nad_test.go`, `pod_mutator_test.go` — fake-client `PodMutator.Handle` coverage), `internal/reconcile`, `internal/controller`, `internal/plumbing/intf`, `internal/metadata`, `internal/runtime/gobgp` (partial), `internal/runtime/frr` | +| E2E | `task test:e2e` | Kind + `go test` | Full BGPRouter lifecycle in a Kind cluster; builds and loads image; `TestCNIVPCAttachmentCreation` (`tests/e2e/e2e_test.go`) exercises `applyVPCAttachment` end-to-end against a real `VPCAttachment` CRD (installed from `datum-cloud/cloud`, same pattern as the BGP CRDs) and a VPC fixture (`scripts/ci.sh`). Does **not** yet cover `galactic-webhook` — see Known Constraints. | | CI full | `task ci` | all of the above | lint → build → test:unit → test:e2e | `internal/plumbing/vrf` has no unit tests — it requires `CAP_NET_ADMIN` and a real kernel. `internal/cni` and `internal/plumbing/srv6` now have unit coverage for their pure-logic paths (this used to not be the case). `internal/plumbing/intf` is pure-function and fully unit-testable. @@ -406,6 +499,9 @@ Runs on every PR and push to `main`. Two tiers: - **`cmdDel` does not tear down shared kernel/CRD state.** By design (see Key Design Decisions above) — cleanup of VRF, veth/tap, routes, SRv6 ingress, and BGP CRDs is deferred to `galactic-router`'s asynchronous GC controller, not performed synchronously in `cmdDel`. - **`internal/plumbing/vrf` has no unit tests.** It requires `CAP_NET_ADMIN` and a real kernel. `internal/cni` and `internal/plumbing/srv6` do now have unit coverage for their pure-logic paths. `internal/plumbing/intf` is fully unit-testable (pure functions only). Kernel-path coverage otherwise comes from the e2e suite (`task test:e2e`). - **`--mode=transit` is unimplemented.** Accepted by CLI/env validation, but `runCmd` returns an error at startup ("mode=transit is not yet supported"). +- **`galactic-webhook` is not yet wired into `task test:e2e` or `publish.yaml`.** `containers/galactic-webhook/Dockerfile` and `config/webhook/` exist and `task build`/`task lint`/`task test:unit` all cover it, but there is no chainsaw e2e coverage exercising a real admission flow (would need cert-manager installed in the Kind cluster) and no CI job publishing its image. Unit tests use `sigs.k8s.io/controller-runtime/pkg/client/fake` throughout — see `internal/webhook/*_test.go`. +- **`galactic-cni`'s VPCAttachment integration degrades gracefully, not strictly.** `applyVPCAttachment` (`internal/cni/vpcattachment.go`) silently skips VPCAttachment CR creation when `PluginConf.VPCName` is empty (a NAD not built by `galactic-webhook`) or when there's no IPAM allocation to populate the CRD's required `Spec.Interface.Addresses`/`Status.PodSubnet` fields — existing CNI configs/e2e fixtures that predate this feature keep working unchanged. +- **`VPCAttachmentStatus.ContainerID` requires exactly 46 hex characters; real container IDs are 64.** `truncateContainerID` (`internal/cni/vpcattachment.go`) truncates to fit, mirroring the same accommodation this repo already makes for annotation-key length limits (`annotationContainerIDLen`). - **`galactic-cni`'s install DaemonSet is a Go installer, not a shell script.** `config/cni/configmap.yaml`/`install.sh` were deleted; `config/cni/daemonset.yaml` now runs `hostNetwork: true` with an `install-cni` init container (`command: ["/galactic-cni", "init"]`, calling `installer.Bootstrap`) and a `credential-refresh` main container (`command: ["/galactic-cni", "run"]`, calling `installer.Run`), both on the same image (see CI/CD above). `Bootstrap` writes the CNI binaries to `/opt/cni/bin`, the static conflist to `/etc/cni/net.d/10-galactic.conflist`, and `ca.crt`/kubeconfig to `/var/lib/galactic` (chosen over `/etc/galactic` specifically so it lands under `/var`, the one path immutable-root distros like Talos allow hostPath writes to without a host-level `extraMounts` entry); `Run` refreshes the kubeconfig token every 300s and rotates the CNI log once it exceeds 10MB. `/opt/cni/bin` is fixed by the CNI/kubelet plugin-discovery convention and can't be relocated by this DaemonSet alone — on Talos it needs its own `extraMounts` entry in the machine config if it isn't writable by default. The `run` container also serves gRPC health checks on port `5180` (`livenessProbe`/`readinessProbe` in the DaemonSet spec), and `config/cni/rbac.yaml` grants `get` on `nodes` for `Bootstrap`'s node-identity check. --- @@ -425,8 +521,11 @@ Runs on every PR and push to `main`. Two tiers: | BGP peer / VRF / advertisement / policy CRUD | `internal/runtime/gobgp/peers.go`, `runtime.go` (`applyVRFs`), `paths.go`, `policies.go` | | Controller watch graph | `internal/controller/bgprouter_controller.go:SetupWithManager` | | CRD status update logic | `internal/controller/status.go`, `bgprouter_controller.go:updateRouterStatus` | -| Orphaned CRD/VRF garbage collection | `internal/controller/gc_controller.go`, `internal/gc/gc.go` | +| Orphaned CRD/VRF/VPCAttachment/NAD garbage collection | `internal/controller/gc_controller.go`, `internal/gc/gc.go`, `internal/gc/orphans_test.go` | | RBAC pre-flight self-check | `cmd/galactic-router/main.go:checkWatchPermissions` | +| Pod → VPC attachment webhook handler | `internal/webhook/pod_mutator.go:PodMutator.Handle` | +| VPCAttachment ID allocation | `internal/webhook/allocate.go:AllocateVPCAttachmentID` | +| VPCAttachment CR creation + status (CNI side) | `internal/cni/vpcattachment.go:applyVPCAttachment` | | Interface naming / base62 encoding | `internal/plumbing/intf/intf.go` | | Hash-based no-op suppression | `internal/hash/hash.go`; annotation `galactic.datum.net/config-hash` on BGPRouter | | GoBGP server lifecycle (start/reconfigure) | `internal/runtime/gobgp/server.go` | diff --git a/go.mod b/go.mod index 3b796d6..6550281 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/containernetworking/cni v1.3.0 github.com/containernetworking/plugins v1.9.1 github.com/coreos/go-iptables v0.8.0 + github.com/evanphx/json-patch/v5 v5.9.11 github.com/kenshaw/baseconv v0.1.1 github.com/lorenzosaino/go-sysctl v0.3.1 github.com/osrg/gobgp/v4 v4.7.0 @@ -13,6 +14,7 @@ require ( github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.21.0 github.com/vishvananda/netlink v1.3.2-0.20260629151558-4e35dc940f49 + go.datum.net/cloud v0.0.0-20260719203343-b2deee16a43a go.datum.net/network v0.0.0-20260719211723-8caa91ab0b37 golang.org/x/sys v0.47.0 golang.org/x/term v0.45.0 @@ -32,7 +34,6 @@ require ( github.com/eapache/channels v1.1.0 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect - github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/gaissmai/bart v0.26.1 // indirect diff --git a/go.sum b/go.sum index c6e3758..f825814 100644 --- a/go.sum +++ b/go.sum @@ -166,6 +166,8 @@ github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zd github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.datum.net/cloud v0.0.0-20260719203343-b2deee16a43a h1:/FmH15DHxI1++oIFUb6UEjDz4I+rmbC9ftFF+aps7Tw= +go.datum.net/cloud v0.0.0-20260719203343-b2deee16a43a/go.mod h1:GXxlcB60RJUXIdJNKbmmAxdZDm/FTff/s8ekHY28s34= go.datum.net/network v0.0.0-20260719211723-8caa91ab0b37 h1:KldPQuababwYJyw3KToFIPaIfKyjMYbBLE9kTHVtBNM= go.datum.net/network v0.0.0-20260719211723-8caa91ab0b37/go.mod h1:dqzM8WZczbiZ9bCvsxjkoI10GJqQ24NVWnc9boXgOkE= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -212,8 +214,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= -golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/cni/bgp_test.go b/internal/cni/bgp_test.go index 1bc3796..2e1cdb8 100644 --- a/internal/cni/bgp_test.go +++ b/internal/cni/bgp_test.go @@ -277,8 +277,8 @@ func TestIPAMAdvertisementPrefixesIPv4Only(t *testing.T) { if ipv4Addr != "10.128.0.5" { t.Errorf("ipv4Addr = %q, want 10.128.0.5", ipv4Addr) } - if len(prefixes) != 1 || prefixes[0] != "10.128.0.5/32" { - t.Errorf("prefixes = %v, want exactly [\"10.128.0.5/32\"] (no panic, no empty-prefixes case)", prefixes) + if len(prefixes) != 1 || prefixes[0] != testIPv4Prefix { + t.Errorf("prefixes = %v, want exactly [%q] (no panic, no empty-prefixes case)", prefixes, testIPv4Prefix) } } diff --git a/internal/cni/cni_test.go b/internal/cni/cni_test.go index 5d316f0..3b47318 100644 --- a/internal/cni/cni_test.go +++ b/internal/cni/cni_test.go @@ -26,6 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" "go.datum.net/galactic/internal/config" bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) @@ -50,6 +51,10 @@ const ( testSID128 = "2001:db8::1/128" testCNIVersion = "1.0.0" testIPv4Subnet = "10.128.0.0/20" + testIPv4Prefix = "10.128.0.5/32" + testNamespace = "default" + testPodName = "my-pod" + testVPCName = "my-vpc" // testPrevResult is a valid CNI v1.0.0 result used in prevResult tests. testPrevResult = `{"cniVersion":"1.0.0",` + @@ -59,7 +64,15 @@ const ( ) func fakeClient(objs ...client.Object) client.Client { - return fake.NewClientBuilder().WithScheme(cniScheme).WithObjects(objs...).Build() + // VPCAttachment declares a status subresource (+kubebuilder:subresource:status); + // the fake client requires WithStatusSubresource for Status().Update() calls + // against it to work (a fake-client-only requirement — a real API server needs + // no equivalent client-side declaration). + return fake.NewClientBuilder(). + WithScheme(cniScheme). + WithStatusSubresource(&cloudv1alpha1.VPCAttachment{}). + WithObjects(objs...). + Build() } // routerForNode builds a BGPRouter with spec.targetRef.name set to nodeName. @@ -1462,7 +1475,7 @@ func TestResourceTrackerCleanupPartialState(t *testing.T) { vpc: testVPC, vpcAttachment: testAttachment, ifaceType: interfaceTypeVeth, - namespace: "default", + namespace: testNamespace, } ctx := context.Background() tracker.cleanup(ctx) // should not panic; vrf.Delete will fail but is logged diff --git a/internal/cni/nad_test.go b/internal/cni/nad_test.go index 7bff756..edaf927 100644 --- a/internal/cni/nad_test.go +++ b/internal/cni/nad_test.go @@ -26,8 +26,8 @@ func TestParsePodNamespace(t *testing.T) { }, { name: "namespace only", - cniArgs: "K8S_POD_NAMESPACE=default", - expected: "default", + cniArgs: "K8S_POD_NAMESPACE=" + testNamespace, + expected: testNamespace, }, { name: "full multus args", diff --git a/internal/cni/ops_add.go b/internal/cni/ops_add.go index 551e7e4..044cde9 100644 --- a/internal/cni/ops_add.go +++ b/internal/cni/ops_add.go @@ -115,6 +115,7 @@ func cmdAdd(args *skel.CmdArgs) (err error) { } tracker.k8s = k8sClient podNamespace := parsePodNamespace(args.Args) + tracker.podNamespace = podNamespace if err := annotateNAD(rollbackCtx, k8sClient, pluginConf.Name, podNamespace, hostName); err != nil { return fmt.Errorf("annotate NAD: %w", err) } @@ -133,9 +134,10 @@ func cmdAdd(args *skel.CmdArgs) (err error) { // Host-device delegation and IPAM are veth-only. // In tap mode the guest VM manages its own networking. var ipamResult *ipamResult + var guestName string // empty for tap: the guest VM manages its own interface, there is no galactic-owned guest device switch pluginConf.InterfaceType { case interfaceTypeVeth: - guestName := intf.GenerateInterfaceNameGuest(pluginConf.VPC, pluginConf.VPCAttachment) + guestName = intf.GenerateInterfaceNameGuest(pluginConf.VPC, pluginConf.VPCAttachment) ipamResult, err = buildVethResult(args, pluginConf, hostName, guestName, hostMac, hostMTU) if err != nil { return err @@ -146,50 +148,89 @@ func cmdAdd(args *skel.CmdArgs) (err error) { "ipv4Address", ipamResult.ipv4Address, "ipv4Gateway", ipamResult.ipv4Gateway) } case interfaceTypeTap: - // Allocate IPAM for the tap interface (same as veth). - // The VM manages its own guest interface; the CNI only configures the host side. - ipamResult, err = allocateIPAM(args, pluginConf) - if err != nil { - return fmt.Errorf("allocate IPAM: %w", err) - } - if ipamResult != nil { - slog.Debug("ADD: IPAM allocated", "containerID", args.ContainerID, - "ipv6Subnet", ipamResult.ipv6Subnet, "ipv6Gateway", ipamResult.ipv6Gateway, - "ipv4Address", ipamResult.ipv4Address, "ipv4Gateway", ipamResult.ipv4Gateway) - } + return cmdAddTap(args, pluginConf, tracker, nodeName, namespace, podNamespace, hostName, hostMac, hostMTU) + } - // Configure the gateway address on the host tap and install the VRF route. - if err := configureHostGateway(pluginConf.VPC, pluginConf.VPCAttachment, ipamResult); err != nil { - return err - } - if ipamResult != nil && ipamResult.ipv6Gateway != nil { - slog.Debug("ADD: host gateway configured", "name", hostName, "gateway", ipamResult.ipv6Gateway) - } + err = applyVPCAttachmentRetrying(args, pluginConf, podNamespace, nodeName, hostName, guestName, ipamResult, tracker) + if err != nil { + return err + } + slog.Debug("ADD: publishing BGP state", "containerID", args.ContainerID, "interfaceType", pluginConf.InterfaceType) + return publishBGPState(args, pluginConf, nodeName, namespace, ipamResult, tracker) +} - // Print the CNI result with IP info. - result := buildTapResult(pluginConf, ipamResult, hostName, hostMac, hostMTU) - if err := types.PrintResult(result, pluginConf.CNIVersion); err != nil { - return fmt.Errorf("print CNI result: %w", err) - } +// cmdAddTap finishes cmdAdd's tap-mode path: IPAM allocation, host gateway +// configuration, the CNI result, and BGP/VPCAttachment state publishing. +// Split out of cmdAdd to keep its cyclomatic complexity in check — this is +// pure extraction, not a behavior change. +func cmdAddTap( + args *skel.CmdArgs, pluginConf *PluginConf, tracker *resourceTracker, + nodeName, namespace, podNamespace, hostName, hostMac string, hostMTU int, +) error { + // Allocate IPAM for the tap interface (same as veth). + // The VM manages its own guest interface; the CNI only configures the host side. + ipamResult, err := allocateIPAM(args, pluginConf) + if err != nil { + return fmt.Errorf("allocate IPAM: %w", err) + } + if ipamResult != nil { + slog.Debug("ADD: IPAM allocated", "containerID", args.ContainerID, + "ipv6Subnet", ipamResult.ipv6Subnet, "ipv6Gateway", ipamResult.ipv6Gateway, + "ipv4Address", ipamResult.ipv4Address, "ipv4Gateway", ipamResult.ipv4Gateway) + } - // Decode VPC/VRFID for BGP state publish. - vpcHex, err := intf.Base62ToHex(pluginConf.VPC) - if err != nil { - return fmt.Errorf("decode VPC: %w", err) - } - vrfID, err := vrfIDFromAttachment(pluginConf.VPCAttachment) - if err != nil { - return fmt.Errorf("decode VPCAttachment: %w", err) - } + // Configure the gateway address on the host tap and install the VRF route. + if err := configureHostGateway(pluginConf.VPC, pluginConf.VPCAttachment, ipamResult); err != nil { + return err + } + if ipamResult != nil && ipamResult.ipv6Gateway != nil { + slog.Debug("ADD: host gateway configured", "name", hostName, "gateway", ipamResult.ipv6Gateway) + } - // Publish BGP state (SRv6 ingress + BGP CRDs). - if tracker.k8s == nil { - return errors.New("k8s client not set in tracker") - } - slog.Debug("ADD: publishing BGP state", "containerID", args.ContainerID, "interfaceType", interfaceTypeTap) - return publishBGPStateK8s(args, pluginConf, nodeName, namespace, ipamResult, vpcHex, vrfID, tracker.k8s, tracker) + // Print the CNI result with IP info. + result := buildTapResult(pluginConf, ipamResult, hostName, hostMac, hostMTU) + if err := types.PrintResult(result, pluginConf.CNIVersion); err != nil { + return fmt.Errorf("print CNI result: %w", err) } - slog.Debug("ADD: publishing BGP state", "containerID", args.ContainerID, "interfaceType", pluginConf.InterfaceType) - return publishBGPState(args, pluginConf, nodeName, namespace, ipamResult, tracker) + // Decode VPC/VRFID for BGP state publish. + vpcHex, err := intf.Base62ToHex(pluginConf.VPC) + if err != nil { + return fmt.Errorf("decode VPC: %w", err) + } + vrfID, err := vrfIDFromAttachment(pluginConf.VPCAttachment) + if err != nil { + return fmt.Errorf("decode VPCAttachment: %w", err) + } + + // Publish BGP state (SRv6 ingress + BGP CRDs). + if tracker.k8s == nil { + return errors.New("k8s client not set in tracker") + } + // guestName is always empty in tap mode: the guest VM manages its own + // interface, there is no galactic-owned guest device. + if err := applyVPCAttachmentRetrying( + args, pluginConf, podNamespace, nodeName, hostName, "", ipamResult, tracker, + ); err != nil { + return err + } + slog.Debug("ADD: publishing BGP state", "containerID", args.ContainerID, "interfaceType", interfaceTypeTap) + return publishBGPStateK8s(args, pluginConf, nodeName, namespace, ipamResult, vpcHex, vrfID, tracker.k8s, tracker) +} + +// applyVPCAttachmentRetrying wraps applyVPCAttachment in retryK8sOps, shared +// by both cmdAdd's veth tail and cmdAddTap. +func applyVPCAttachmentRetrying( + args *skel.CmdArgs, pluginConf *PluginConf, podNamespace, nodeName, hostName, guestName string, + ipamResult *ipamResult, tracker *resourceTracker, +) error { + err := retryK8sOps(cniTimeout, func(ctx context.Context) error { + return applyVPCAttachment( + ctx, tracker.k8s, args, pluginConf, podNamespace, nodeName, hostName, guestName, ipamResult, tracker, + ) + }) + if err != nil { + return fmt.Errorf("apply VPCAttachment: %w", err) + } + return nil } diff --git a/internal/cni/resource.go b/internal/cni/resource.go index c8c8c2b..476cb82 100644 --- a/internal/cni/resource.go +++ b/internal/cni/resource.go @@ -14,6 +14,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" "go.datum.net/galactic/internal/cni/tap" "go.datum.net/galactic/internal/cni/veth" "go.datum.net/galactic/internal/plumbing/srv6" @@ -35,19 +36,22 @@ func SetEnableLocalIPAM(v bool) { func init() { utilruntime.Must(clientgoscheme.AddToScheme(cniScheme)) utilruntime.Must(bgpv1alpha1.AddToScheme(cniScheme)) + utilruntime.Must(cloudv1alpha1.AddToScheme(cniScheme)) } // resourceTracker tracks resources created during cmdAdd for selective rollback. type resourceTracker struct { - vpc, vpcAttachment string - ifaceType string - vrfCreated bool - routesCreated int - srv6SID string - vrfInstanceCreated bool - advCreated bool - k8s client.Client - namespace string + vpc, vpcAttachment string + ifaceType string + vrfCreated bool + routesCreated int + srv6SID string + vrfInstanceCreated bool + advCreated bool + vpcAttachmentCreated bool + podNamespace string + k8s client.Client + namespace string } // cleanup rolls back all tracked resources in reverse creation order. @@ -56,7 +60,18 @@ func (rt *resourceTracker) cleanup(ctx context.Context) { slog.Info("Selective rollback: cleaning up resources created during failed ADD", "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) - // 1. Delete BGPAdvertisement (withdraws prefixes) + // 1. Delete VPCAttachment (lives in the pod's namespace, not rt.namespace) + if rt.vpcAttachmentCreated && rt.k8s != nil { + if err := deleteVPCAttachment(ctx, rt.k8s, rt.vpc, rt.vpcAttachment, rt.podNamespace); err != nil { + slog.Error("Rollback: failed to delete VPCAttachment", "err", err, + "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment, "namespace", rt.podNamespace) + } else { + slog.Debug("Rollback: deleted VPCAttachment", "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment, + "namespace", rt.podNamespace) + } + } + + // 2. Delete BGPAdvertisement (withdraws prefixes) if rt.advCreated && rt.k8s != nil { adv := &bgpv1alpha1.BGPAdvertisement{ ObjectMeta: metav1.ObjectMeta{ @@ -72,7 +87,7 @@ func (rt *resourceTracker) cleanup(ctx context.Context) { } } - // 2. Delete BGPVRFInstance + // 3. Delete BGPVRFInstance if rt.vrfInstanceCreated && rt.k8s != nil { vrfInst := &bgpv1alpha1.BGPVRFInstance{ ObjectMeta: metav1.ObjectMeta{ @@ -88,7 +103,7 @@ func (rt *resourceTracker) cleanup(ctx context.Context) { } } - // 3. Delete SRv6 ingress route (only if we got a SID) + // 4. Delete SRv6 ingress route (only if we got a SID) if rt.srv6SID != "" { if err := srv6.RouteIngressDel(rt.srv6SID, rt.vpc, rt.vpcAttachment); err != nil { slog.Error("Rollback: failed to delete SRv6 ingress route", "err", err, @@ -98,7 +113,7 @@ func (rt *resourceTracker) cleanup(ctx context.Context) { } } - // 4. Delete host veth (veth mode only) + // 5. Delete host veth (veth mode only) if rt.ifaceType == interfaceTypeVeth { if err := veth.Delete(rt.vpc, rt.vpcAttachment); err != nil { slog.Error("Rollback: failed to delete veth", "err", err, @@ -108,7 +123,7 @@ func (rt *resourceTracker) cleanup(ctx context.Context) { } } - // 5. Delete tap (tap mode only) + // 6. Delete tap (tap mode only) if rt.ifaceType == interfaceTypeTap { if err := tap.Delete(rt.vpc, rt.vpcAttachment); err != nil { slog.Error("Rollback: failed to delete tap", "err", err, @@ -118,7 +133,7 @@ func (rt *resourceTracker) cleanup(ctx context.Context) { } } - // 6. Delete VRF (flushes all routes, removes VRF interface) + // 7. Delete VRF (flushes all routes, removes VRF interface) if err := vrf.Delete(rt.vpc, rt.vpcAttachment); err != nil { slog.Error("Rollback: failed to delete VRF", "err", err, "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) diff --git a/internal/cni/types.go b/internal/cni/types.go index 6398f6a..f6139cb 100644 --- a/internal/cni/types.go +++ b/internal/cni/types.go @@ -46,7 +46,10 @@ type Address struct { // validation in parseConf lands in a later phase. type PluginConf struct { types.PluginConf - VPC string `json:"vpc"` + VPC string `json:"vpc"` + // VPCName is the VPC CR's Kubernetes object name, distinct from the + // base62 VPC identifier above — see applyVPCAttachment (vpcattachment.go). + VPCName string `json:"vpc_name,omitempty"` VPCAttachment string `json:"vpcattachment"` MTU int `json:"mtu,omitempty"` InterfaceType string `json:"interface_type,omitempty"` // interfaceTypeVeth or interfaceTypeTap diff --git a/internal/cni/vpcattachment.go b/internal/cni/vpcattachment.go new file mode 100644 index 0000000..fc066e2 --- /dev/null +++ b/internal/cni/vpcattachment.go @@ -0,0 +1,182 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cni + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + + "github.com/containernetworking/cni/pkg/skel" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + "go.datum.net/galactic/internal/plumbing/intf" +) + +// containerIDStatusLen is the exact length VPCAttachmentStatus.ContainerID +// requires (go.datum.net/cloud's CEL/length validation: MinLength=MaxLength=46). +// Real container IDs in this codebase are 64 hex characters (see +// annotationContainerIDLen and cni_test.go's fullContainerID), so the value +// is truncated to fit — the same accommodation already made for annotation +// keys elsewhere in this package, not a new pattern. +const containerIDStatusLen = 46 + +// vpcAttachmentName returns the deterministic name for a VPCAttachment CR. +// Mirrors bgpVRFInstanceName/bgpAdvertisementName (bgp.go): the (vpc, +// vpcAttachment) pair is a reliable 1:1 key, so NAD and VPCAttachment end up +// with the same name without either looking the other up. +func vpcAttachmentName(vpc, vpcAttachment string) string { + return fmt.Sprintf("%s-%s", vpc, vpcAttachment) +} + +// parsePodName extracts the K8S_POD_NAME value from the CNI_ARGS environment +// variable string passed as args.Args by Multus. Mirrors parsePodNamespace +// (nad.go). Returns an empty string when the value is not present. +func parsePodName(cniArgs string) string { + for _, part := range strings.Split(cniArgs, ";") { + key, value, ok := strings.Cut(part, "=") + if ok && key == "K8S_POD_NAME" { + return value + } + } + return "" +} + +// vpcAttachmentAddresses converts ipamResult's allocated address(es) into the +// CIDR-notation strings VPCAttachmentSpec.Interface.Addresses requires. +// Mirrors ipamAdvertisementPrefixes (bgp.go:300-317): the IPv4 address needs +// an explicit /32 appended, the IPv6 subnet is already CIDR-shaped. +func vpcAttachmentAddresses(ipamResult *ipamResult) []cloudv1alpha1.IPAddress { + if ipamResult == nil { + return nil + } + var addrs []cloudv1alpha1.IPAddress + if ipamResult.ipv6Subnet != nil { + addrs = append(addrs, cloudv1alpha1.IPAddress(ipamResult.ipv6Subnet.String())) + } + if ipamResult.ipv4Address != nil { + addrs = append(addrs, cloudv1alpha1.IPAddress(ipamResult.ipv4Address.String()+"/32")) + } + return addrs +} + +// vpcAttachmentPodSubnet returns the value for VPCAttachmentStatus.PodSubnet, +// derived from the same ipamResult used for Spec.Interface.Addresses above — +// kept consistent by construction, not by convention. +func vpcAttachmentPodSubnet(ipamResult *ipamResult) string { + if ipamResult == nil || ipamResult.ipv6Subnet == nil { + return "" + } + return ipamResult.ipv6Subnet.String() +} + +// truncateContainerID shortens id to containerIDStatusLen characters. See +// containerIDStatusLen's doc comment for why this truncation is necessary. +func truncateContainerID(id string) string { + if len(id) > containerIDStatusLen { + return id[:containerIDStatusLen] + } + return id +} + +// applyVPCAttachment creates (or updates) the VPCAttachment CR for this +// attachment and populates its Status with the attach-time facts galactic-cni +// has just gathered — Node, ContainerID, PodName, interface names, and +// PodSubnet. This mirrors how publishBGPStateK8s already creates +// BGPVRFInstance/BGPAdvertisement via controllerutil.CreateOrUpdate: the +// caller is expected to run this inside retryK8sOps so transient k8s errors +// retry, and to track success via tracker.vpcAttachmentCreated so a later ADD +// failure rolls this back the same way vrfInstanceCreated/advCreated do. +// +// Skipped (logged at Debug, not an error) when: +// - pluginConf.VPCName is empty — VPCAttachment provisioning is additive; +// CNI configs not built by galactic-webhook (e.g. existing e2e fixtures) +// keep working exactly as before. +// - ipamResult has no usable addresses — VPCAttachmentSpec.Interface.Addresses +// (MinItems=1) and VPCAttachmentStatus.PodSubnet (MinLength=1, no +// omitempty) are both required by the CRD schema; an attachment with no +// IPAM allocation (e.g. a tap workload managing its own addressing — +// see ipamAdvertisementPrefixes's doc comment) cannot satisfy either, so +// there is nothing valid to create. +func applyVPCAttachment( + ctx context.Context, k8s client.Client, args *skel.CmdArgs, pluginConf *PluginConf, + podNamespace, nodeName, hostName, guestName string, ipamResult *ipamResult, + tracker *resourceTracker, +) error { + if pluginConf.VPCName == "" { + slog.Debug("ADD: skipping VPCAttachment (no vpc_name in CNI config)", + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + return nil + } + if podNamespace == "" { + return errors.New("K8S_POD_NAMESPACE not present in CNI_ARGS: cannot create VPCAttachment") + } + + addresses := vpcAttachmentAddresses(ipamResult) + podSubnet := vpcAttachmentPodSubnet(ipamResult) + if len(addresses) == 0 || podSubnet == "" { + slog.Debug("ADD: skipping VPCAttachment (no IPAM allocation for this attachment)", + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + return nil + } + + vrfName := intf.GenerateInterfaceNameVRF(pluginConf.VPC, pluginConf.VPCAttachment) + name := vpcAttachmentName(pluginConf.VPC, pluginConf.VPCAttachment) + + attachment := &cloudv1alpha1.VPCAttachment{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: podNamespace}, + } + _, err := controllerutil.CreateOrUpdate(ctx, k8s, attachment, func() error { + attachment.Spec = cloudv1alpha1.VPCAttachmentSpec{ + VPC: cloudv1alpha1.VPCRef{Name: pluginConf.VPCName}, + Interface: cloudv1alpha1.VPCAttachmentInterface{ + Name: args.IfName, + Addresses: addresses, + }, + } + return nil + }) + if err != nil { + return fmt.Errorf("apply VPCAttachment: %w", err) + } + tracker.vpcAttachmentCreated = true + slog.Debug("ADD: VPCAttachment applied", "name", name, "namespace", podNamespace) + + attachment.Status = cloudv1alpha1.VPCAttachmentStatus{ + VPC: pluginConf.VPC, + VPCAttachment: pluginConf.VPCAttachment, + Node: nodeName, + ContainerID: truncateContainerID(args.ContainerID), + PodName: parsePodName(args.Args), + HostInterface: hostName, + VRFInterface: vrfName, + GuestInterface: guestName, + PodSubnet: podSubnet, + } + if err := k8s.Status().Update(ctx, attachment); err != nil { + return fmt.Errorf("update VPCAttachment status: %w", err) + } + slog.Debug("ADD: VPCAttachment status updated", "name", name, "namespace", podNamespace, + "node", nodeName, "containerID", attachment.Status.ContainerID, "podName", attachment.Status.PodName) + return nil +} + +// deleteVPCAttachment rolls back the VPCAttachment CR created during a +// failed ADD. Called from resourceTracker.cleanup(), mirroring how +// BGPAdvertisement/BGPVRFInstance are rolled back there. +func deleteVPCAttachment(ctx context.Context, k8s client.Client, vpc, vpcAttachment, namespace string) error { + attachment := &cloudv1alpha1.VPCAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: vpcAttachmentName(vpc, vpcAttachment), + Namespace: namespace, + }, + } + return client.IgnoreNotFound(k8s.Delete(ctx, attachment)) +} diff --git a/internal/cni/vpcattachment_test.go b/internal/cni/vpcattachment_test.go new file mode 100644 index 0000000..d92f7c3 --- /dev/null +++ b/internal/cni/vpcattachment_test.go @@ -0,0 +1,281 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cni + +import ( + "context" + "net" + "strings" + "testing" + + "github.com/containernetworking/cni/pkg/skel" + "sigs.k8s.io/controller-runtime/pkg/client" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" +) + +func TestVPCAttachmentName(t *testing.T) { + got := vpcAttachmentName(testVPC, testAttachment) + want := testVPC + "-" + testAttachment + if got != want { + t.Errorf("vpcAttachmentName(%q, %q) = %q, want %q", testVPC, testAttachment, got, want) + } +} + +func TestParsePodName(t *testing.T) { + tests := []struct { + name string + cniArgs string + expected string + }{ + {name: "empty string", cniArgs: "", expected: ""}, + {name: "name only", cniArgs: "K8S_POD_NAME=" + testPodName, expected: testPodName}, + { + name: "full multus args", + cniArgs: "K8S_POD_NAME=" + testPodName + ";K8S_POD_NAMESPACE=galactic-system;K8S_POD_INFRA_CONTAINER_ID=abc123", + expected: testPodName, + }, + {name: "name not present", cniArgs: "K8S_POD_NAMESPACE=" + testNamespace, expected: ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := parsePodName(tc.cniArgs) + if got != tc.expected { + t.Errorf("parsePodName(%q) = %q, want %q", tc.cniArgs, got, tc.expected) + } + }) + } +} + +func TestTruncateContainerID(t *testing.T) { + short := "abc123" + if got := truncateContainerID(short); got != short { + t.Errorf("truncateContainerID(%q) = %q, want unchanged", short, got) + } + + full := strings.Repeat("a", 64) // real container IDs are 64 hex chars + got := truncateContainerID(full) + if len(got) != containerIDStatusLen { + t.Errorf("truncateContainerID(64-char id) length = %d, want %d", len(got), containerIDStatusLen) + } + if got != full[:containerIDStatusLen] { + t.Errorf("truncateContainerID(%q) = %q, want prefix %q", full, got, full[:containerIDStatusLen]) + } +} + +func TestVPCAttachmentAddressesAndPodSubnet(t *testing.T) { + t.Run("nil ipamResult", func(t *testing.T) { + if addrs := vpcAttachmentAddresses(nil); addrs != nil { + t.Errorf("vpcAttachmentAddresses(nil) = %v, want nil", addrs) + } + if subnet := vpcAttachmentPodSubnet(nil); subnet != "" { + t.Errorf("vpcAttachmentPodSubnet(nil) = %q, want empty", subnet) + } + }) + + t.Run("IPv4 only", func(t *testing.T) { + res := &ipamResult{ipv4Address: net.ParseIP("10.128.0.5")} + addrs := vpcAttachmentAddresses(res) + if len(addrs) != 1 || addrs[0] != testIPv4Prefix { + t.Errorf("vpcAttachmentAddresses = %v, want [%q]", addrs, testIPv4Prefix) + } + if subnet := vpcAttachmentPodSubnet(res); subnet != "" { + t.Errorf("vpcAttachmentPodSubnet = %q, want empty (no IPv6 subnet)", subnet) + } + }) + + t.Run("dual stack", func(t *testing.T) { + ipv6Subnet := mustParseCIDR(t, "fd00:10:ff01::1234/96") + res := &ipamResult{ipv6Subnet: ipv6Subnet, ipv4Address: net.ParseIP("10.128.0.5")} + addrs := vpcAttachmentAddresses(res) + wantIPv6 := cloudv1alpha1.IPAddress(ipv6Subnet.String()) + if len(addrs) != 2 || addrs[0] != wantIPv6 || addrs[1] != testIPv4Prefix { + t.Errorf("vpcAttachmentAddresses = %v, want [%q, %q]", addrs, wantIPv6, testIPv4Prefix) + } + if subnet := vpcAttachmentPodSubnet(res); subnet != ipv6Subnet.String() { + t.Errorf("vpcAttachmentPodSubnet = %q, want %q", subnet, ipv6Subnet.String()) + } + }) +} + +// applyTestVPCAttachment is a thin wrapper around applyVPCAttachment fixing +// node/hostIf/guestIf to constant test values, so call sites in this file +// stay under the line-length limit. +func applyTestVPCAttachment( + k8s client.Client, args *skel.CmdArgs, conf *PluginConf, podNamespace string, + ipamRes *ipamResult, tracker *resourceTracker, +) error { + return applyVPCAttachment( + context.Background(), k8s, args, conf, podNamespace, "node-1", "hostIf", "guestIf", ipamRes, tracker, + ) +} + +func TestApplyVPCAttachment(t *testing.T) { + baseConf := func() *PluginConf { + return &PluginConf{ + VPC: testVPC, + VPCName: testVPCName, + VPCAttachment: testAttachment, + } + } + baseArgs := &skel.CmdArgs{ + ContainerID: strings.Repeat("a", 64), + IfName: testIfName, + Args: "K8S_POD_NAME=" + testPodName + ";K8S_POD_NAMESPACE=" + testNamespace, + } + dualStackIPAM := &ipamResult{ + ipv6Subnet: mustParseCIDR(t, "fd00:10:ff01::1234/96"), + ipv4Address: net.ParseIP("10.128.0.5"), + } + + t.Run("skipped when VPCName is empty", func(t *testing.T) { + k8s := fakeClient() + conf := baseConf() + conf.VPCName = "" + tracker := &resourceTracker{} + + if err := applyTestVPCAttachment(k8s, baseArgs, conf, testNamespace, dualStackIPAM, tracker); err != nil { + t.Fatalf("applyVPCAttachment() = %v, want nil (skip)", err) + } + if tracker.vpcAttachmentCreated { + t.Error("tracker.vpcAttachmentCreated = true, want false (skipped)") + } + var list cloudv1alpha1.VPCAttachmentList + if err := k8s.List(context.Background(), &list); err != nil { + t.Fatalf("list VPCAttachments: %v", err) + } + if len(list.Items) != 0 { + t.Errorf("VPCAttachments created = %d, want 0", len(list.Items)) + } + }) + + t.Run("skipped when no IPAM allocation", func(t *testing.T) { + k8s := fakeClient() + tracker := &resourceTracker{} + + if err := applyTestVPCAttachment(k8s, baseArgs, baseConf(), testNamespace, nil, tracker); err != nil { + t.Fatalf("applyVPCAttachment() = %v, want nil (skip)", err) + } + if tracker.vpcAttachmentCreated { + t.Error("tracker.vpcAttachmentCreated = true, want false (skipped)") + } + }) + + t.Run("errors when pod namespace is empty", func(t *testing.T) { + k8s := fakeClient() + tracker := &resourceTracker{} + + err := applyTestVPCAttachment(k8s, baseArgs, baseConf(), "", dualStackIPAM, tracker) + if err == nil { + t.Fatal("expected error for empty pod namespace, got nil") + } + }) + + t.Run("creates Spec and Status from IPAM result", func(t *testing.T) { + k8s := fakeClient() + conf := baseConf() + tracker := &resourceTracker{} + + err := applyTestVPCAttachment(k8s, baseArgs, conf, testNamespace, dualStackIPAM, tracker) + if err != nil { + t.Fatalf("applyVPCAttachment() = %v, want nil", err) + } + if !tracker.vpcAttachmentCreated { + t.Error("tracker.vpcAttachmentCreated = false, want true") + } + + var got cloudv1alpha1.VPCAttachment + name := vpcAttachmentName(conf.VPC, conf.VPCAttachment) + key := client.ObjectKey{Name: name, Namespace: testNamespace} + if err := k8s.Get(context.Background(), key, &got); err != nil { + t.Fatalf("get VPCAttachment: %v", err) + } + + if got.Spec.VPC.Name != conf.VPCName { + t.Errorf("Spec.VPC.Name = %q, want %q", got.Spec.VPC.Name, conf.VPCName) + } + if got.Spec.Interface.Name != testIfName { + t.Errorf("Spec.Interface.Name = %q, want %q", got.Spec.Interface.Name, testIfName) + } + if len(got.Spec.Interface.Addresses) != 2 { + t.Errorf("Spec.Interface.Addresses = %v, want 2 entries", got.Spec.Interface.Addresses) + } + + if got.Status.VPC != conf.VPC { + t.Errorf("Status.VPC = %q, want %q", got.Status.VPC, conf.VPC) + } + if got.Status.VPCAttachment != conf.VPCAttachment { + t.Errorf("Status.VPCAttachment = %q, want %q", got.Status.VPCAttachment, conf.VPCAttachment) + } + if got.Status.Node != "node-1" { + t.Errorf("Status.Node = %q, want %q", got.Status.Node, "node-1") + } + if len(got.Status.ContainerID) != containerIDStatusLen { + t.Errorf("Status.ContainerID length = %d, want %d", len(got.Status.ContainerID), containerIDStatusLen) + } + if got.Status.PodName != testPodName { + t.Errorf("Status.PodName = %q, want %q", got.Status.PodName, testPodName) + } + if got.Status.HostInterface != "hostIf" || got.Status.VRFInterface == "" || got.Status.GuestInterface != "guestIf" { + t.Errorf("Status interface names = %+v, unexpected", got.Status) + } + if got.Status.PodSubnet != dualStackIPAM.ipv6Subnet.String() { + t.Errorf("Status.PodSubnet = %q, want %q", got.Status.PodSubnet, dualStackIPAM.ipv6Subnet.String()) + } + }) + + t.Run("re-applying is idempotent (CreateOrUpdate)", func(t *testing.T) { + k8s := fakeClient() + conf := baseConf() + tracker := &resourceTracker{} + + for i := range 2 { + if err := applyTestVPCAttachment(k8s, baseArgs, conf, testNamespace, dualStackIPAM, tracker); err != nil { + t.Fatalf("applyVPCAttachment() attempt %d = %v, want nil", i, err) + } + } + var list cloudv1alpha1.VPCAttachmentList + if err := k8s.List(context.Background(), &list); err != nil { + t.Fatalf("list VPCAttachments: %v", err) + } + if len(list.Items) != 1 { + t.Errorf("VPCAttachments after 2 applies = %d, want 1", len(list.Items)) + } + }) +} + +func TestDeleteVPCAttachment(t *testing.T) { + t.Run("not found is not an error", func(t *testing.T) { + k8s := fakeClient() + if err := deleteVPCAttachment(context.Background(), k8s, testVPC, testAttachment, testNamespace); err != nil { + t.Fatalf("deleteVPCAttachment() = %v, want nil", err) + } + }) + + t.Run("deletes an existing VPCAttachment", func(t *testing.T) { + name := vpcAttachmentName(testVPC, testAttachment) + existing := &cloudv1alpha1.VPCAttachment{} + existing.SetName(name) + existing.SetNamespace(testNamespace) + existing.Spec = cloudv1alpha1.VPCAttachmentSpec{ + VPC: cloudv1alpha1.VPCRef{Name: testVPCName}, + Interface: cloudv1alpha1.VPCAttachmentInterface{ + Name: testIfName, + Addresses: []cloudv1alpha1.IPAddress{"10.0.0.1/32"}, + }, + } + k8s := fakeClient(existing) + + if err := deleteVPCAttachment(context.Background(), k8s, testVPC, testAttachment, testNamespace); err != nil { + t.Fatalf("deleteVPCAttachment() = %v, want nil", err) + } + + var got cloudv1alpha1.VPCAttachment + key := client.ObjectKey{Name: name, Namespace: testNamespace} + if err := k8s.Get(context.Background(), key, &got); err == nil { + t.Fatal("expected VPCAttachment to be deleted, but Get succeeded") + } + }) +} diff --git a/internal/config/webhook.go b/internal/config/webhook.go new file mode 100644 index 0000000..054e629 --- /dev/null +++ b/internal/config/webhook.go @@ -0,0 +1,130 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "errors" + + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +// --- Webhook defaults -------------------------------------------------- + +const ( + DefaultWebhookPort = 9443 // matches controller-runtime's webhook.Server default + DefaultWebhookMetricsPort = 8080 + DefaultWebhookHealthPort = 8081 + DefaultWebhookCertDir = "/etc/webhook/certs" // populated by the cert-manager-issued Secret volume mount + DefaultWebhookMTU = 1500 + DefaultWebhookInterfaceType = "veth" +) + +// --- Webhook environment variable keys ----------------------------------- + +const ( + EnvWebhookPort = "GALACTIC_WEBHOOK_PORT" + EnvWebhookMetricsPort = "GALACTIC_WEBHOOK_METRICS_PORT" + EnvWebhookHealthPort = "GALACTIC_WEBHOOK_HEALTH_PORT" + EnvWebhookCertDir = "GALACTIC_WEBHOOK_CERT_DIR" + EnvWebhookMTU = "GALACTIC_WEBHOOK_MTU" + EnvWebhookInterfaceType = "GALACTIC_WEBHOOK_INTERFACE_TYPE" +) + +// WebhookConfig resolves galactic-webhook configuration with three-tier +// precedence: CLI flag > env var > compiled-in default. Create once via +// NewWebhookConfig(), call BindFlags() to layer CLI flags, then read the +// exported fields. +type WebhookConfig struct { + v *viper.Viper + prefix string + + // Resolved fields. + Port int + MetricsPort int + HealthPort int + CertDir string + MTU int + InterfaceType string +} + +// NewWebhookConfig creates a webhook config resolver with the +// GALACTIC_WEBHOOK env prefix and AutomaticEnv enabled. +func NewWebhookConfig() *WebhookConfig { + v := viper.New() + v.SetEnvPrefix("GALACTIC_WEBHOOK") + v.AutomaticEnv() + + v.SetDefault("port", DefaultWebhookPort) + v.SetDefault("metrics_port", DefaultWebhookMetricsPort) + v.SetDefault("health_port", DefaultWebhookHealthPort) + v.SetDefault("cert_dir", DefaultWebhookCertDir) + v.SetDefault("mtu", DefaultWebhookMTU) + v.SetDefault("interface_type", DefaultWebhookInterfaceType) + + cfg := &WebhookConfig{ + v: v, + prefix: "GALACTIC_WEBHOOK", + } + cfg.readFields() + return cfg +} + +// BindFlags binds Cobra/pflag flags to the config resolver and re-reads the +// exported fields. +func (c *WebhookConfig) BindFlags(flags *pflag.FlagSet) { + bindings := []struct { + flag string + key string + }{ + {"port", "port"}, + {"metrics-port", "metrics_port"}, + {"health-port", "health_port"}, + {"cert-dir", "cert_dir"}, + {"mtu", "mtu"}, + {"interface-type", "interface_type"}, + } + for _, b := range bindings { + if flags.Changed(b.flag) { + c.v.Set(b.key, flags.Lookup(b.flag).Value.String()) + } else { + //nolint:errcheck // controlled keys, BindPFlag cannot fail here + c.v.BindPFlag(b.key, flags.Lookup(b.flag)) + } + } + c.readFields() +} + +// readFields populates the exported fields from the current Viper state. +func (c *WebhookConfig) readFields() { + c.Port = c.v.GetInt("port") + c.MetricsPort = c.v.GetInt("metrics_port") + c.HealthPort = c.v.GetInt("health_port") + c.CertDir = c.v.GetString("cert_dir") + c.MTU = c.v.GetInt("mtu") + c.InterfaceType = c.v.GetString("interface_type") +} + +// Validate checks that the required configuration fields are set and valid. +func (c *WebhookConfig) Validate() error { + if c.Port < 1 || c.Port > 65535 { + return errors.New("webhook port must be between 1 and 65535") + } + if c.MetricsPort < 1 || c.MetricsPort > 65535 { + return errors.New("metrics port must be between 1 and 65535") + } + if c.HealthPort < 1 || c.HealthPort > 65535 { + return errors.New("health port must be between 1 and 65535") + } + if c.CertDir == "" { + return errors.New("cert dir must not be empty") + } + switch c.InterfaceType { + case "veth", "tap": + default: + return errors.New("interface type must be \"veth\" or \"tap\"") + } + return nil +} diff --git a/internal/gc/gc.go b/internal/gc/gc.go index 415e99a..864f4eb 100644 --- a/internal/gc/gc.go +++ b/internal/gc/gc.go @@ -6,15 +6,22 @@ package gc import ( "context" + "encoding/json" "fmt" "log/slog" "regexp" "strings" "github.com/vishvananda/netlink" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" "go.datum.net/galactic/internal/plumbing/vrf" bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) @@ -24,13 +31,49 @@ import ( // This is the liveness signal GC uses — see cni.netnsAnnotationKey. const annotationNetNS = "galactic.datum.net/netns" -// OrphanedCRD represents a BGP CRD that appears to be orphaned because its -// associated container is no longer present on the node. +// OrphanedCRD.Kind values. +const ( + kindBGPAdvertisement = "BGPAdvertisement" + kindBGPVRFInstance = "BGPVRFInstance" + kindVPCAttachment = "VPCAttachment" + kindNetworkAttachmentDefinition = "NetworkAttachmentDefinition" +) + +// labelVPC marks a NetworkAttachmentDefinition as created by galactic-webhook +// for a specific VPC. Only NADs carrying this label are candidates for the +// reference-count orphan check below — NADs created any other way (manually, +// or by an external operator) are never touched by this GC pass. Must match +// the label galactic-webhook sets when it creates a NAD. +const labelVPC = "galactic.datumapis.com/vpc" + +// networksAnnotation is the Multus annotation a pod uses to reference +// NetworkAttachmentDefinitions by name (and, optionally, namespace). +const networksAnnotation = "k8s.v1.cni.cncf.io/networks" + +// nadGVK is the GroupVersionKind for NetworkAttachmentDefinition. Duplicated +// from internal/cni's nadGVK rather than imported — internal/gc and +// internal/cni are sibling packages with no existing dependency between them. +var nadGVK = schema.GroupVersionKind{ + Group: "k8s.cni.cncf.io", + Version: "v1", + Kind: kindNetworkAttachmentDefinition, +} + +// networkSelectionElement mirrors the fields of Multus's NetworkSelectionElement +// this package needs to read from a pod's networks annotation. Only Name and +// Namespace are needed to check whether a pod references a given NAD. +type networkSelectionElement struct { + Name string `json:"name"` + Namespace string `json:"namespace,omitempty"` +} + +// OrphanedCRD represents a CRD (or NAD) that appears to be orphaned because +// its associated container or pod is no longer present. type OrphanedCRD struct { Name string Namespace string - Kind string // "BGPAdvertisement" or "BGPVRFInstance" - ContainerID string // truncated container ID prefix from annotation + Kind string // kindBGPAdvertisement, kindBGPVRFInstance, kindVPCAttachment, or kindNetworkAttachmentDefinition + ContainerID string // truncated container ID prefix from annotation; empty for VPCAttachment/NAD } // CleanupResult tracks the outcome of a GC pass. @@ -135,7 +178,7 @@ func CollectOrphanedCRDs(ctx context.Context, k8s client.Client, namespace, node orphaned = append(orphaned, OrphanedCRD{ Name: adv.Name, Namespace: adv.Namespace, - Kind: "BGPAdvertisement", + Kind: kindBGPAdvertisement, ContainerID: anyContainerID, }) orphanedAdvNames[adv.Name] = struct{}{} @@ -149,10 +192,124 @@ func CollectOrphanedCRDs(ctx context.Context, k8s client.Client, namespace, node orphaned = append(orphaned, OrphanedCRD{ Name: name, Namespace: namespace, - Kind: "BGPVRFInstance", + Kind: kindBGPVRFInstance, + }) + } + + return orphaned, nil +} + +// podReferencesNAD reports whether any pod in namespace still lists nadName +// in its Multus k8s.v1.cni.cncf.io/networks annotation. Malformed annotation +// values are ignored (treated as not-referencing) rather than failing the +// whole scan — a single bad pod annotation should not block reclaiming an +// otherwise-orphaned NAD. +func podReferencesNAD(ctx context.Context, k8s client.Client, namespace, nadName string) (bool, error) { + var pods corev1.PodList + if err := k8s.List(ctx, &pods, client.InNamespace(namespace)); err != nil { + return false, fmt.Errorf("list pods in namespace %s: %w", namespace, err) + } + for _, pod := range pods.Items { + raw, ok := pod.Annotations[networksAnnotation] + if !ok || raw == "" { + continue + } + var elements []networkSelectionElement + if err := json.Unmarshal([]byte(raw), &elements); err != nil { + continue + } + for _, e := range elements { + if e.Name == nadName && (e.Namespace == "" || e.Namespace == namespace) { + return true, nil + } + } + } + return false, nil +} + +// CollectOrphanedNADs scans every NetworkAttachmentDefinition labeled with +// labelVPC (i.e. created by galactic-webhook, never a manually-created or +// externally-managed NAD) across all namespaces, and returns those no live +// pod in their own namespace still references via the Multus networks +// annotation. +// +// Unlike BGPAdvertisement/VPCAttachment orphan checks, this is not scoped to +// a single node: a NAD's liveness depends on whether any pod anywhere still +// points at it, not on which node ran CNI for it. Every node's GC pass runs +// this same check redundantly — acceptable (delete is idempotent, a +// not-found response from another node's earlier pass is not an error) given +// the alternative would be a separate cluster-scoped controller this plan +// does not otherwise need. +func CollectOrphanedNADs(ctx context.Context, k8s client.Client) ([]OrphanedCRD, error) { + nadList := &unstructured.UnstructuredList{} + nadList.SetGroupVersionKind(nadGVK) + if err := k8s.List(ctx, nadList, client.HasLabels{labelVPC}); err != nil { + return nil, fmt.Errorf("list NetworkAttachmentDefinitions: %w", err) + } + + var orphaned []OrphanedCRD + for _, nad := range nadList.Items { + referenced, err := podReferencesNAD(ctx, k8s, nad.GetNamespace(), nad.GetName()) + if err != nil { + slog.Error("GC: failed to check pod references for NAD", "err", err, + "name", nad.GetName(), "namespace", nad.GetNamespace()) + continue + } + if referenced { + continue + } + orphaned = append(orphaned, OrphanedCRD{ + Name: nad.GetName(), + Namespace: nad.GetNamespace(), + Kind: kindNetworkAttachmentDefinition, }) } + return orphaned, nil +} + +// CollectOrphanedVPCAttachments scans every VPCAttachment across all +// namespaces whose Status.Node matches nodeName (this node's own attachments +// only — the same per-node scoping philosophy routerNamesForNode already +// applies to BGP CRDs) and returns those whose Status.PodName no longer +// exists. +// +// VPCAttachments with an empty Status.PodName are skipped, not treated as +// orphaned: galactic-cni populates Status in the same call that creates +// Spec (see internal/cni's applyVPCAttachment), so an empty PodName here +// means either a stale read or a partially-failed create that +// resourceTracker's inline rollback already handles — not something this GC +// pass can safely judge with the same confidence as the pod-existence check +// below. +func CollectOrphanedVPCAttachments(ctx context.Context, k8s client.Client, nodeName string) ([]OrphanedCRD, error) { + var attachments cloudv1alpha1.VPCAttachmentList + if err := k8s.List(ctx, &attachments); err != nil { + return nil, fmt.Errorf("list VPCAttachments: %w", err) + } + var orphaned []OrphanedCRD + for _, a := range attachments.Items { + if a.Status.Node != nodeName { + continue + } + if a.Status.PodName == "" { + continue + } + var pod corev1.Pod + err := k8s.Get(ctx, types.NamespacedName{Name: a.Status.PodName, Namespace: a.Namespace}, &pod) + if err == nil { + continue // pod still exists — not orphaned + } + if !apierrors.IsNotFound(err) { + slog.Error("GC: failed to check pod existence for VPCAttachment", "err", err, + "name", a.Name, "namespace", a.Namespace, "podName", a.Status.PodName) + continue + } + orphaned = append(orphaned, OrphanedCRD{ + Name: a.Name, + Namespace: a.Namespace, + Kind: kindVPCAttachment, + }) + } return orphaned, nil } @@ -163,7 +320,7 @@ func RemoveOrphanedCRDs(ctx context.Context, k8s client.Client, orphans []Orphan for _, o := range orphans { switch o.Kind { - case "BGPAdvertisement": + case kindBGPAdvertisement: adv := &bgpv1alpha1.BGPAdvertisement{ ObjectMeta: metav1.ObjectMeta{ Name: o.Name, @@ -180,7 +337,7 @@ func RemoveOrphanedCRDs(ctx context.Context, k8s client.Client, orphans []Orphan "name", o.Name, "namespace", o.Namespace, "containerID", o.ContainerID) result.OrphanedCRDsRemoved++ - case "BGPVRFInstance": + case kindBGPVRFInstance: vrfInst := &bgpv1alpha1.BGPVRFInstance{ ObjectMeta: metav1.ObjectMeta{ Name: o.Name, @@ -196,6 +353,36 @@ func RemoveOrphanedCRDs(ctx context.Context, k8s client.Client, orphans []Orphan slog.Info("GC: removed orphaned BGPVRFInstance", "name", o.Name, "namespace", o.Namespace) result.OrphanedCRDsRemoved++ + + case kindVPCAttachment: + attachment := &cloudv1alpha1.VPCAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: o.Name, + Namespace: o.Namespace, + }, + } + if err := k8s.Delete(ctx, attachment); err != nil { + slog.Error("GC: failed to delete orphaned VPCAttachment", + "name", o.Name, "namespace", o.Namespace, "err", err) + result.Errors++ + continue + } + slog.Info("GC: removed orphaned VPCAttachment", "name", o.Name, "namespace", o.Namespace) + result.OrphanedCRDsRemoved++ + + case kindNetworkAttachmentDefinition: + nad := &unstructured.Unstructured{} + nad.SetGroupVersionKind(nadGVK) + nad.SetName(o.Name) + nad.SetNamespace(o.Namespace) + if err := k8s.Delete(ctx, nad); err != nil { + slog.Error("GC: failed to delete orphaned NetworkAttachmentDefinition", + "name", o.Name, "namespace", o.Namespace, "err", err) + result.Errors++ + continue + } + slog.Info("GC: removed orphaned NetworkAttachmentDefinition", "name", o.Name, "namespace", o.Namespace) + result.OrphanedCRDsRemoved++ } } @@ -314,7 +501,33 @@ func RunGC(ctx context.Context, k8s client.Client, namespace, nodeName string) C result.Errors += crResult.Errors } - // Phase 2: Remove orphaned VRF interfaces. + // Phase 2: Remove orphaned VPCAttachments (this node's own, per Status.Node) + // and NetworkAttachmentDefinitions (cluster-wide reference count) — see + // CollectOrphanedVPCAttachments/CollectOrphanedNADs for why these use + // different scoping than Phase 1's BGP CRDs. + vpcAttachmentOrphans, err := CollectOrphanedVPCAttachments(ctx, k8s, nodeName) + if err != nil { + slog.Error("GC: failed to collect orphaned VPCAttachments", "err", err) + result.Errors++ + } else if len(vpcAttachmentOrphans) > 0 { + slog.Info("GC: found orphaned VPCAttachments", "count", len(vpcAttachmentOrphans)) + vaResult := RemoveOrphanedCRDs(ctx, k8s, vpcAttachmentOrphans) + result.OrphanedCRDsRemoved += vaResult.OrphanedCRDsRemoved + result.Errors += vaResult.Errors + } + + nadOrphans, err := CollectOrphanedNADs(ctx, k8s) + if err != nil { + slog.Error("GC: failed to collect orphaned NetworkAttachmentDefinitions", "err", err) + result.Errors++ + } else if len(nadOrphans) > 0 { + slog.Info("GC: found orphaned NetworkAttachmentDefinitions", "count", len(nadOrphans)) + nadResult := RemoveOrphanedCRDs(ctx, k8s, nadOrphans) + result.OrphanedCRDsRemoved += nadResult.OrphanedCRDsRemoved + result.Errors += nadResult.Errors + } + + // Phase 3: Remove orphaned VRF interfaces. orphanedVRFs, err := CollectOrphanedVRFs(ctx, k8s, namespace, nodeName) if err != nil { slog.Error("GC: failed to collect orphaned VRFs", "err", err) diff --git a/internal/gc/orphans_test.go b/internal/gc/orphans_test.go new file mode 100644 index 0000000..e985673 --- /dev/null +++ b/internal/gc/orphans_test.go @@ -0,0 +1,228 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package gc + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" +) + +const ( + testNamespace = "default" + testVPCName = "my-vpc" + testNADName = "vpc-abc-def" + testAttachmentName = "abc-def" +) + +var gcTestScheme = func() *runtime.Scheme { + s := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(s)) + utilruntime.Must(cloudv1alpha1.AddToScheme(s)) + return s +}() + +func gcFakeClient(objs ...client.Object) client.Client { + return fake.NewClientBuilder().WithScheme(gcTestScheme).WithObjects(objs...).Build() +} + +func testNAD(name string, labels map[string]string) *unstructured.Unstructured { + nad := &unstructured.Unstructured{} + nad.SetGroupVersionKind(nadGVK) + nad.SetName(name) + nad.SetNamespace(testNamespace) + nad.SetLabels(labels) + return nad +} + +func podWithNetworks(name, networksJSON string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: testNamespace, + Annotations: map[string]string{ + networksAnnotation: networksJSON, + }, + }, + } +} + +func TestCollectOrphanedNADs(t *testing.T) { + t.Run("unlabeled NAD is never a candidate", func(t *testing.T) { + nad := testNAD("manual-nad", nil) + k8s := gcFakeClient(nad) + + orphans, err := CollectOrphanedNADs(context.Background(), k8s) + if err != nil { + t.Fatalf("CollectOrphanedNADs() = %v, want nil", err) + } + if len(orphans) != 0 { + t.Errorf("orphans = %v, want none (unlabeled NAD skipped)", orphans) + } + }) + + t.Run("labeled NAD referenced by a live pod is not orphaned", func(t *testing.T) { + nad := testNAD(testNADName, map[string]string{labelVPC: testVPCName}) + pod := podWithNetworks("my-pod", `[{"name":"vpc-abc-def","namespace":"default"}]`) + k8s := gcFakeClient(nad, pod) + + orphans, err := CollectOrphanedNADs(context.Background(), k8s) + if err != nil { + t.Fatalf("CollectOrphanedNADs() = %v, want nil", err) + } + if len(orphans) != 0 { + t.Errorf("orphans = %v, want none (referenced by live pod)", orphans) + } + }) + + t.Run("labeled NAD with no referencing pod is orphaned", func(t *testing.T) { + nad := testNAD(testNADName, map[string]string{labelVPC: testVPCName}) + unrelatedPod := podWithNetworks("other-pod", `[{"name":"some-other-nad"}]`) + k8s := gcFakeClient(nad, unrelatedPod) + + orphans, err := CollectOrphanedNADs(context.Background(), k8s) + if err != nil { + t.Fatalf("CollectOrphanedNADs() = %v, want nil", err) + } + if len(orphans) != 1 || orphans[0].Name != testNADName || orphans[0].Kind != kindNetworkAttachmentDefinition { + t.Errorf("orphans = %+v, want exactly one %s orphan named %s", orphans, kindNetworkAttachmentDefinition, testNADName) + } + }) + + t.Run("malformed networks annotation does not block reclaim", func(t *testing.T) { + nad := testNAD(testNADName, map[string]string{labelVPC: testVPCName}) + brokenPod := podWithNetworks("broken-pod", `not-json`) + k8s := gcFakeClient(nad, brokenPod) + + orphans, err := CollectOrphanedNADs(context.Background(), k8s) + if err != nil { + t.Fatalf("CollectOrphanedNADs() = %v, want nil", err) + } + if len(orphans) != 1 { + t.Errorf("orphans = %v, want one (malformed annotation ignored, not treated as a reference)", orphans) + } + }) +} + +func TestCollectOrphanedVPCAttachments(t *testing.T) { + const nodeName = "node-1" + + newAttachment := func(name, node, podName string) *cloudv1alpha1.VPCAttachment { + a := &cloudv1alpha1.VPCAttachment{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testNamespace}, + Spec: cloudv1alpha1.VPCAttachmentSpec{ + VPC: cloudv1alpha1.VPCRef{Name: testVPCName}, + Interface: cloudv1alpha1.VPCAttachmentInterface{ + Name: "eth0", + Addresses: []cloudv1alpha1.IPAddress{"10.0.0.1/32"}, + }, + }, + } + a.Status = cloudv1alpha1.VPCAttachmentStatus{Node: node, PodName: podName} + return a + } + + t.Run("different node is not ours to judge", func(t *testing.T) { + attachment := newAttachment(testAttachmentName, "other-node", "gone-pod") + k8s := gcFakeClient(attachment) + + orphans, err := CollectOrphanedVPCAttachments(context.Background(), k8s, nodeName) + if err != nil { + t.Fatalf("CollectOrphanedVPCAttachments() = %v, want nil", err) + } + if len(orphans) != 0 { + t.Errorf("orphans = %v, want none (belongs to another node)", orphans) + } + }) + + t.Run("empty PodName is skipped, not treated as orphaned", func(t *testing.T) { + attachment := newAttachment(testAttachmentName, nodeName, "") + k8s := gcFakeClient(attachment) + + orphans, err := CollectOrphanedVPCAttachments(context.Background(), k8s, nodeName) + if err != nil { + t.Fatalf("CollectOrphanedVPCAttachments() = %v, want nil", err) + } + if len(orphans) != 0 { + t.Errorf("orphans = %v, want none (empty PodName cannot be judged)", orphans) + } + }) + + t.Run("pod still exists is not orphaned", func(t *testing.T) { + attachment := newAttachment(testAttachmentName, nodeName, "my-pod") + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "my-pod", Namespace: testNamespace}} + k8s := gcFakeClient(attachment, pod) + + orphans, err := CollectOrphanedVPCAttachments(context.Background(), k8s, nodeName) + if err != nil { + t.Fatalf("CollectOrphanedVPCAttachments() = %v, want nil", err) + } + if len(orphans) != 0 { + t.Errorf("orphans = %v, want none (pod still exists)", orphans) + } + }) + + t.Run("pod gone is orphaned", func(t *testing.T) { + attachment := newAttachment(testAttachmentName, nodeName, "gone-pod") + k8s := gcFakeClient(attachment) + + orphans, err := CollectOrphanedVPCAttachments(context.Background(), k8s, nodeName) + if err != nil { + t.Fatalf("CollectOrphanedVPCAttachments() = %v, want nil", err) + } + if len(orphans) != 1 || orphans[0].Name != testAttachmentName || orphans[0].Kind != kindVPCAttachment { + t.Errorf("orphans = %+v, want exactly one %s orphan named %s", orphans, kindVPCAttachment, testAttachmentName) + } + }) +} + +func TestRemoveOrphanedCRDsVPCAttachmentAndNAD(t *testing.T) { + attachment := &cloudv1alpha1.VPCAttachment{ + ObjectMeta: metav1.ObjectMeta{Name: testAttachmentName, Namespace: testNamespace}, + Spec: cloudv1alpha1.VPCAttachmentSpec{ + VPC: cloudv1alpha1.VPCRef{Name: testVPCName}, + Interface: cloudv1alpha1.VPCAttachmentInterface{ + Name: "eth0", + Addresses: []cloudv1alpha1.IPAddress{"10.0.0.1/32"}, + }, + }, + } + nad := testNAD(testNADName, map[string]string{labelVPC: testVPCName}) + k8s := gcFakeClient(attachment, nad) + + orphans := []OrphanedCRD{ + {Name: testAttachmentName, Namespace: testNamespace, Kind: kindVPCAttachment}, + {Name: testNADName, Namespace: testNamespace, Kind: kindNetworkAttachmentDefinition}, + } + result := RemoveOrphanedCRDs(context.Background(), k8s, orphans) + if result.Errors != 0 { + t.Errorf("result.Errors = %d, want 0", result.Errors) + } + if result.OrphanedCRDsRemoved != 2 { + t.Errorf("result.OrphanedCRDsRemoved = %d, want 2", result.OrphanedCRDsRemoved) + } + + var gotAttachment cloudv1alpha1.VPCAttachment + attachmentKey := client.ObjectKey{Name: testAttachmentName, Namespace: testNamespace} + if err := k8s.Get(context.Background(), attachmentKey, &gotAttachment); err == nil { + t.Error("expected VPCAttachment to be deleted") + } + gotNAD := &unstructured.Unstructured{} + gotNAD.SetGroupVersionKind(nadGVK) + key := client.ObjectKey{Name: testNADName, Namespace: testNamespace} + if err := k8s.Get(context.Background(), key, gotNAD); err == nil { + t.Error("expected NAD to be deleted") + } +} diff --git a/internal/webhook/allocate.go b/internal/webhook/allocate.go new file mode 100644 index 0000000..38d7b68 --- /dev/null +++ b/internal/webhook/allocate.go @@ -0,0 +1,79 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package webhook + +import ( + "context" + "errors" + "fmt" + "strconv" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + "go.datum.net/galactic/internal/plumbing/intf" +) + +// ErrIDSpaceExhausted is returned by AllocateVPCAttachmentID when every +// 16-bit VPCAttachment ID (0-65535) for a VPC is already taken. +var ErrIDSpaceExhausted = errors.New("VPCAttachment ID space exhausted for this VPC") + +// AllocateVPCAttachmentID picks the lowest free 16-bit VPCAttachment ID for +// vpc, base62-encoded. It scans the NADs this webhook has already created +// for vpc (labelVPC + labelAttachmentID), not the VPCAttachment CRD: +// galactic-cni creates that object later, at ADD time, so it may not exist +// yet for a given ID — scanning it would risk double-allocating an ID whose +// NAD already exists but whose pod hasn't been scheduled/attached yet. The +// NAD is the object this webhook creates synchronously with the allocation +// decision, so it's the only object that can't lag behind it. +// +// This is scan-then-create, not a true compare-and-swap on the ID itself: +// callers are expected to retry (re-list, re-pick) on a NAD Create conflict, +// bounded — see the caller in pod_mutator.go. +func AllocateVPCAttachmentID( + ctx context.Context, k8s client.Client, vpc *cloudv1alpha1.VPC, namespace string, +) (string, error) { + nadList := &unstructured.UnstructuredList{} + nadList.SetGroupVersionKind(nadGVK) + err := k8s.List(ctx, nadList, client.InNamespace(namespace), client.MatchingLabels{labelVPC: vpc.Name}) + if err != nil { + return "", fmt.Errorf("list NADs for vpc %q: %w", vpc.Name, err) + } + + used := make(map[uint16]bool, len(nadList.Items)) + for _, nad := range nadList.Items { + hex, err := intf.Base62ToHex(nad.GetLabels()[labelAttachmentID]) + if err != nil { + continue // malformed/foreign label — ignore, don't block allocation + } + id, err := strconv.ParseUint(hex, 16, 16) + if err != nil { + continue + } + used[uint16(id)] = true + } + + id, ok := lowestFree(used) + if !ok { + return "", ErrIDSpaceExhausted + } + base62, err := intf.HexToBase62(fmt.Sprintf("%04x", id)) + if err != nil { + return "", fmt.Errorf("encode VPCAttachment id %d as base62: %w", id, err) + } + return base62, nil +} + +// lowestFree returns the smallest uint16 not present in used, and false if +// every value in [0, 65535] is taken. +func lowestFree(used map[uint16]bool) (uint16, bool) { + for id := range 65536 { + if !used[uint16(id)] { + return uint16(id), true + } + } + return 0, false +} diff --git a/internal/webhook/allocate_test.go b/internal/webhook/allocate_test.go new file mode 100644 index 0000000..9a258fe --- /dev/null +++ b/internal/webhook/allocate_test.go @@ -0,0 +1,125 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package webhook + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" +) + +func TestLowestFree(t *testing.T) { + tests := []struct { + name string + used map[uint16]bool + wantID uint16 + wantOK bool + }{ + {name: "empty", used: map[uint16]bool{}, wantID: 0, wantOK: true}, + {name: "0 taken", used: map[uint16]bool{0: true}, wantID: 1, wantOK: true}, + {name: "0 and 1 taken", used: map[uint16]bool{0: true, 1: true}, wantID: 2, wantOK: true}, + {name: "gap in the middle", used: map[uint16]bool{0: true, 1: true, 3: true}, wantID: 2, wantOK: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + id, ok := lowestFree(tc.used) + if ok != tc.wantOK || id != tc.wantID { + t.Errorf("lowestFree(%v) = (%d, %v), want (%d, %v)", tc.used, id, ok, tc.wantID, tc.wantOK) + } + }) + } + + t.Run("fully exhausted", func(t *testing.T) { + full := make(map[uint16]bool, 65536) + for i := range 65536 { + full[uint16(i)] = true + } + if _, ok := lowestFree(full); ok { + t.Error("lowestFree(full) ok = true, want false") + } + }) +} + +func webhookFakeClient(objs ...client.Object) client.Client { + return fake.NewClientBuilder().WithScheme(NewScheme()).WithObjects(objs...).Build() +} + +func nadWithLabels(name, vpcName, attachmentIDBase62 string) *unstructured.Unstructured { + nad := &unstructured.Unstructured{} + nad.SetGroupVersionKind(nadGVK) + nad.SetName(name) + nad.SetNamespace(testNamespace) + nad.SetLabels(map[string]string{ + labelVPC: vpcName, + labelAttachmentID: attachmentIDBase62, + }) + return nad +} + +func TestAllocateVPCAttachmentID(t *testing.T) { + vpc := &cloudv1alpha1.VPC{ObjectMeta: metav1.ObjectMeta{Name: testVPCName}} + + t.Run("no existing NADs returns the lowest ID", func(t *testing.T) { + k8s := webhookFakeClient() + id, err := AllocateVPCAttachmentID(context.Background(), k8s, vpc, testNamespace) + if err != nil { + t.Fatalf("AllocateVPCAttachmentID() = %v, want nil", err) + } + if id == "" { + t.Error("id is empty, want a base62 string") + } + }) + + t.Run("skips IDs already taken by NADs for this VPC", func(t *testing.T) { + firstID, err := AllocateVPCAttachmentID(context.Background(), webhookFakeClient(), vpc, testNamespace) + if err != nil { + t.Fatalf("allocate baseline id: %v", err) + } + nad := nadWithLabels(testVPCName+"-"+firstID, testVPCName, firstID) + k8s := webhookFakeClient(nad) + + secondID, err := AllocateVPCAttachmentID(context.Background(), k8s, vpc, testNamespace) + if err != nil { + t.Fatalf("AllocateVPCAttachmentID() = %v, want nil", err) + } + if secondID == firstID { + t.Errorf("second allocation = %q, want different from already-taken %q", secondID, firstID) + } + }) + + t.Run("NADs for a different VPC don't count against this one", func(t *testing.T) { + otherVPCNAD := nadWithLabels("other-vpc-0", "other-vpc", "0") + k8s := webhookFakeClient(otherVPCNAD) + + id, err := AllocateVPCAttachmentID(context.Background(), k8s, vpc, testNamespace) + if err != nil { + t.Fatalf("AllocateVPCAttachmentID() = %v, want nil", err) + } + // The lowest ID (base62 "0") must still be available for testVPCName + // even though "other-vpc" has already used it. + if id != "0" { + t.Errorf("id = %q, want %q (cross-VPC NAD must not count)", id, "0") + } + }) + + t.Run("malformed attachment-id label is ignored, not fatal", func(t *testing.T) { + nad := nadWithLabels(testVPCName+"-bad", testVPCName, "not-base62-!!!") + k8s := webhookFakeClient(nad) + + id, err := AllocateVPCAttachmentID(context.Background(), k8s, vpc, testNamespace) + if err != nil { + t.Fatalf("AllocateVPCAttachmentID() = %v, want nil", err) + } + if id == "" { + t.Error("id is empty, want a base62 string") + } + }) +} diff --git a/internal/webhook/nad.go b/internal/webhook/nad.go new file mode 100644 index 0000000..a5ff3a7 --- /dev/null +++ b/internal/webhook/nad.go @@ -0,0 +1,120 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package webhook + +import ( + "encoding/json" + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// cniVersion is the CNI spec version this webhook writes into every NAD's +// conflist. Matches the version internal/cni's config.go expects. +const cniVersion = "1.0.0" + +// pluginType is the CNI plugin type every NAD this webhook creates points at. +const pluginType = "galactic-cni" + +// interfaceTypeVeth is the default interface type baked into every NAD's +// conflist (see NADDefaults.InterfaceType) — matches internal/cni/cni.go's +// interfaceTypeVeth constant. Not imported directly: internal/webhook +// deliberately avoids importing internal/cni to keep netlink-heavy +// dependencies out of the webhook binary. +const interfaceTypeVeth = "veth" + +// termination mirrors internal/cni/types.go's Termination. Field tags MUST +// match exactly — this struct's JSON encoding is the wire contract between +// the NAD this webhook creates and the galactic-cni invocation Multus drives +// from it. +type termination struct { + Network string `json:"network"` + Via string `json:"via,omitempty"` +} + +// ipamConfig mirrors the subset of internal/cni/types.go's IPAM this webhook +// populates. See termination's doc comment on why the tags must match. +type ipamConfig struct { + Type string `json:"type"` + StaticIP string `json:"static_ip,omitempty"` +} + +// NADDefaults holds the galactic-owned constants baked into every NAD this +// webhook creates: MTU, interface type, IPAM policy, and terminations. +// VPCSpec has none of these fields by design — see this repo's design plan, +// "NAD content" (Option B: these come from galactic's own constants/config, +// not the VPC CR). A future revision could source per-VPC overrides from a +// ConfigMap; hardcoded defaults are sufficient for the first implementation. +type NADDefaults struct { + MTU int + InterfaceType string + IPAM *ipamConfig + Terminations []termination +} + +// DefaultNADDefaults returns the built-in NAD defaults used when the +// PodMutator isn't given an explicit NADDefaults. +func DefaultNADDefaults() NADDefaults { + return NADDefaults{ + MTU: 1500, + InterfaceType: interfaceTypeVeth, + } +} + +// cniConflist mirrors the subset of internal/cni/types.go's PluginConf this +// webhook populates. Field tags MUST match PluginConf exactly. +type cniConflist struct { + CNIVersion string `json:"cniVersion"` + Name string `json:"name"` + Type string `json:"type"` + VPC string `json:"vpc"` + VPCName string `json:"vpc_name,omitempty"` + VPCAttachment string `json:"vpcattachment"` + MTU int `json:"mtu,omitempty"` + InterfaceType string `json:"interface_type,omitempty"` + Terminations []termination `json:"terminations,omitempty"` + IPAM *ipamConfig `json:"ipam,omitempty"` +} + +// buildNAD constructs the NetworkAttachmentDefinition object this webhook +// creates for one pod's VPC attachment: name is the deterministic +// "-" (see AllocateVPCAttachmentID and +// pod_mutator.go), vpcName is the VPC CR's Kubernetes object name (needed by +// galactic-cni to build VPCAttachmentSpec.VPC.Name — see PluginConf.VPCName), +// and defaults supplies the galactic-owned conflist fields VPCSpec doesn't +// carry. +func buildNAD( + name, namespace, vpcBase62, vpcName, vpcAttachmentBase62 string, defaults NADDefaults, +) (*unstructured.Unstructured, error) { + conf := cniConflist{ + CNIVersion: cniVersion, + Name: name, + Type: pluginType, + VPC: vpcBase62, + VPCName: vpcName, + VPCAttachment: vpcAttachmentBase62, + MTU: defaults.MTU, + InterfaceType: defaults.InterfaceType, + Terminations: defaults.Terminations, + IPAM: defaults.IPAM, + } + configJSON, err := json.Marshal(conf) + if err != nil { + return nil, fmt.Errorf("marshal NAD conflist: %w", err) + } + + nad := &unstructured.Unstructured{} + nad.SetGroupVersionKind(nadGVK) + nad.SetName(name) + nad.SetNamespace(namespace) + nad.SetLabels(map[string]string{ + labelVPC: vpcName, + labelAttachmentID: vpcAttachmentBase62, + }) + if err := unstructured.SetNestedField(nad.Object, string(configJSON), "spec", "config"); err != nil { + return nil, fmt.Errorf("set NAD spec.config: %w", err) + } + return nad, nil +} diff --git a/internal/webhook/nad_test.go b/internal/webhook/nad_test.go new file mode 100644 index 0000000..aa64833 --- /dev/null +++ b/internal/webhook/nad_test.go @@ -0,0 +1,91 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package webhook + +import ( + "encoding/json" + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// testVPCName and testNamespace are shared across this package's test files. +const ( + testVPCName = "my-vpc" + testNamespace = "default" +) + +func TestBuildNAD(t *testing.T) { + defaults := NADDefaults{ + MTU: 1500, + InterfaceType: interfaceTypeVeth, + Terminations: []termination{{Network: "0.0.0.0/0", Via: "10.0.0.1"}}, + } + + nad, err := buildNAD("my-vpc-0", testNamespace, "vpcBase62", testVPCName, "0", defaults) + if err != nil { + t.Fatalf("buildNAD() = %v, want nil", err) + } + + if nad.GroupVersionKind() != nadGVK { + t.Errorf("GVK = %v, want %v", nad.GroupVersionKind(), nadGVK) + } + if nad.GetName() != "my-vpc-0" { + t.Errorf("name = %q, want %q", nad.GetName(), "my-vpc-0") + } + if nad.GetNamespace() != testNamespace { + t.Errorf("namespace = %q, want %q", nad.GetNamespace(), testNamespace) + } + if got := nad.GetLabels()[labelVPC]; got != testVPCName { + t.Errorf("label %s = %q, want %q", labelVPC, got, testVPCName) + } + if got := nad.GetLabels()[labelAttachmentID]; got != "0" { + t.Errorf("label %s = %q, want %q", labelAttachmentID, got, "0") + } + + configRaw, found, err := unstructured.NestedString(nad.Object, "spec", "config") + if err != nil || !found { + t.Fatalf("spec.config not found or errored: found=%v err=%v", found, err) + } + + var conf cniConflist + if err := json.Unmarshal([]byte(configRaw), &conf); err != nil { + t.Fatalf("unmarshal spec.config: %v", err) + } + if conf.Type != pluginType { + t.Errorf("conf.Type = %q, want %q", conf.Type, pluginType) + } + if conf.VPC != "vpcBase62" { + t.Errorf("conf.VPC = %q, want %q", conf.VPC, "vpcBase62") + } + if conf.VPCName != testVPCName { + t.Errorf("conf.VPCName = %q, want %q", conf.VPCName, testVPCName) + } + if conf.VPCAttachment != "0" { + t.Errorf("conf.VPCAttachment = %q, want %q", conf.VPCAttachment, "0") + } + if conf.MTU != 1500 { + t.Errorf("conf.MTU = %d, want 1500", conf.MTU) + } + if conf.InterfaceType != interfaceTypeVeth { + t.Errorf("conf.InterfaceType = %q, want %q", conf.InterfaceType, interfaceTypeVeth) + } + if len(conf.Terminations) != 1 || conf.Terminations[0].Network != "0.0.0.0/0" { + t.Errorf("conf.Terminations = %+v, want one entry for 0.0.0.0/0", conf.Terminations) + } + if conf.Name != "my-vpc-0" { + t.Errorf("conf.Name = %q, want %q (must match the NAD's own object name)", conf.Name, "my-vpc-0") + } +} + +func TestDefaultNADDefaults(t *testing.T) { + d := DefaultNADDefaults() + if d.MTU != 1500 { + t.Errorf("MTU = %d, want 1500", d.MTU) + } + if d.InterfaceType != interfaceTypeVeth { + t.Errorf("InterfaceType = %q, want %q", d.InterfaceType, interfaceTypeVeth) + } +} diff --git a/internal/webhook/pod_mutator.go b/internal/webhook/pod_mutator.go new file mode 100644 index 0000000..44c50b6 --- /dev/null +++ b/internal/webhook/pod_mutator.go @@ -0,0 +1,195 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package webhook + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" +) + +// maxAllocateAttempts bounds the allocate-then-create retry loop in +// createNAD: a NAD Create conflict means another concurrent admission +// request took the same ID between our List and our Create — see +// AllocateVPCAttachmentID's doc comment. +const maxAllocateAttempts = 5 + +// PodMutator implements admission.Handler for the VPC-attachment mutating +// webhook. On Pod CREATE, a pod carrying the annotationVPC annotation gets a +// VPCAttachment ID allocated and a NetworkAttachmentDefinition created, then +// gets patched to attach via that NAD — see package doc and this repo's +// design plan for the full rationale. +type PodMutator struct { + // Client is used for both reads (cache-backed) and writes (direct to + // the API server) — controller-runtime's client.Client already + // provides this split transparently. + Client client.Client + + // Decoder decodes the admitted Pod from the AdmissionRequest. + Decoder admission.Decoder + + // NADDefaults supplies the galactic-owned conflist fields (MTU, + // interface type, IPAM, terminations) VPCSpec doesn't carry. + NADDefaults NADDefaults +} + +var _ admission.Handler = &PodMutator{} + +// networkSelectionElement mirrors the fields of Multus's +// NetworkSelectionElement this package needs to write. internal/gc reads +// this same annotation independently with its own equivalent local type — +// see its doc comment for why the two aren't shared. +type networkSelectionElement struct { + Name string `json:"name"` + Namespace string `json:"namespace,omitempty"` +} + +// Handle implements admission.Handler. See this repo's design plan, +// "Handler logic," for the numbered steps this mirrors. +func (m *PodMutator) Handle(ctx context.Context, req admission.Request) admission.Response { + pod := &corev1.Pod{} + if err := m.Decoder.Decode(req, pod); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + // Step 2: reinvocation guard — a pod already carrying this annotation + // was already processed by an earlier invocation of this same webhook. + if _, done := pod.Annotations[annotationVPCAttachmentRef]; done { + return admission.Allowed("already processed (reinvocation guard)") + } + + // Step 3: silent no-op when the pod doesn't request a VPC. + vpcName := pod.Annotations[annotationVPC] + if vpcName == "" { + return admission.Allowed("no " + annotationVPC + " annotation") + } + + // Step 4: VPC attach is meaningless on host network. + if pod.Spec.HostNetwork { + return admission.Allowed("hostNetwork pod, VPC attach not applicable") + } + + // Step 5: the named VPC must exist. + var vpc cloudv1alpha1.VPC + if err := m.Client.Get(ctx, client.ObjectKey{Name: vpcName, Namespace: req.Namespace}, &vpc); err != nil { + if apierrors.IsNotFound(err) { + return admission.Denied(fmt.Sprintf("vpc %q not found in namespace %q: pod not admitted", vpcName, req.Namespace)) + } + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("get vpc %q: %w", vpcName, err)) + } + if vpc.Status.VPC == "" { + return admission.Denied(fmt.Sprintf( + "vpc %q has no assigned identifier yet (Status.VPC empty): pod not admitted", vpcName)) + } + + // Step 6: never create real objects on dry-run. + if req.DryRun != nil && *req.DryRun { + return admission.Allowed("dry-run, no changes made") + } + + // Steps 7-9: allocate a free VPCAttachment ID and create its NAD, + // bounded-retrying on an allocation collision. + nadName, err := m.createNAD(ctx, &vpc, req.Namespace) + if err != nil { + if errors.Is(err, ErrIDSpaceExhausted) { + return admission.Denied(fmt.Sprintf( + "vpc %q has no free VPCAttachment IDs (0-65535 exhausted): pod not admitted", vpcName)) + } + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("create NAD: %w", err)) + } + + // Step 10: patch the pod. + mutated := pod.DeepCopy() + if err := appendNetworksAnnotation(mutated, nadName, req.Namespace); err != nil { + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("append networks annotation: %w", err)) + } + if mutated.Annotations == nil { + mutated.Annotations = map[string]string{} + } + mutated.Annotations[annotationVPCAttachmentRef] = fmt.Sprintf("%s/%s", req.Namespace, nadName) + + marshaledPod, err := json.Marshal(mutated) + if err != nil { + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("marshal mutated pod: %w", err)) + } + + // Step 11. + return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) +} + +// createNAD allocates a VPCAttachment ID and creates the corresponding NAD, +// retrying (re-allocate, re-create) up to maxAllocateAttempts times if the +// Create collides with a concurrent admission request that took the same ID +// first (see AllocateVPCAttachmentID's doc comment). Returns the NAD's name. +func (m *PodMutator) createNAD(ctx context.Context, vpc *cloudv1alpha1.VPC, namespace string) (string, error) { + var lastErr error + for range maxAllocateAttempts { + id, err := AllocateVPCAttachmentID(ctx, m.Client, vpc, namespace) + if err != nil { + return "", err + } + + nadName := fmt.Sprintf("%s-%s", vpc.Status.VPC, id) + nad, err := buildNAD(nadName, namespace, vpc.Status.VPC, vpc.Name, id, m.NADDefaults) + if err != nil { + return "", err + } + + err = m.Client.Create(ctx, nad) + if err == nil { + return nadName, nil + } + if !apierrors.IsAlreadyExists(err) { + return "", err + } + // Someone else took this ID between our List and our Create — retry. + lastErr = err + } + return "", fmt.Errorf("allocate+create NAD for vpc %q: exhausted %d retries: %w", + vpc.Name, maxAllocateAttempts, lastErr) +} + +// appendNetworksAnnotation adds a NAD reference to the pod's Multus networks +// annotation, preserving whatever was already there (parse-merge, not +// string concat). The existing value may be a JSON array (the form this +// function itself produces) or Multus's comma-separated shorthand +// ("net1,net2", a legacy/manual form); either way the result is a JSON +// array, since that's the only form that can express namespace alongside name. +func appendNetworksAnnotation(pod *corev1.Pod, nadName, namespace string) error { + var elements []networkSelectionElement + if raw := pod.Annotations[networksAnnotation]; raw != "" { + if err := json.Unmarshal([]byte(raw), &elements); err != nil { + names := strings.Split(raw, ",") + elements = make([]networkSelectionElement, 0, len(names)) + for _, name := range names { + name = strings.TrimSpace(name) + if name != "" { + elements = append(elements, networkSelectionElement{Name: name}) + } + } + } + } + elements = append(elements, networkSelectionElement{Name: nadName, Namespace: namespace}) + + encoded, err := json.Marshal(elements) + if err != nil { + return fmt.Errorf("marshal networks annotation: %w", err) + } + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[networksAnnotation] = string(encoded) + return nil +} diff --git a/internal/webhook/pod_mutator_test.go b/internal/webhook/pod_mutator_test.go new file mode 100644 index 0000000..d06ec82 --- /dev/null +++ b/internal/webhook/pod_mutator_test.go @@ -0,0 +1,257 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package webhook + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + jsonpatch "github.com/evanphx/json-patch/v5" + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" +) + +func newPodMutator(objs ...client.Object) (*PodMutator, client.Client) { + k8s := webhookFakeClient(objs...) + return &PodMutator{ + Client: k8s, + Decoder: admission.NewDecoder(NewScheme()), + NADDefaults: DefaultNADDefaults(), + }, k8s +} + +// admissionRequest builds an admission.Request for pod in testNamespace. +func admissionRequest(t *testing.T, pod *corev1.Pod, dryRun bool) admission.Request { + t.Helper() + raw, err := json.Marshal(pod) + if err != nil { + t.Fatalf("marshal pod: %v", err) + } + return admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: testNamespace, + DryRun: &dryRun, + Object: runtime.RawExtension{Raw: raw}, + }, + } +} + +// testVPCObj builds a VPC named testVPCName in testNamespace with the given +// Status.VPC. +func testVPCObj(statusVPC string) *cloudv1alpha1.VPC { + return &cloudv1alpha1.VPC{ + ObjectMeta: metav1.ObjectMeta{Name: testVPCName, Namespace: testNamespace}, + Spec: cloudv1alpha1.VPCSpec{Networks: []cloudv1alpha1.Network{"10.0.0.0/8"}}, + Status: cloudv1alpha1.VPCStatus{VPC: statusVPC}, + } +} + +// applyPatches applies resp's JSON patch operations to the original raw pod +// JSON and unmarshals the result, so tests can assert on the actual patched +// annotations rather than parsing raw patch operations. +func applyPatches(t *testing.T, original []byte, resp admission.Response) *corev1.Pod { + t.Helper() + if len(resp.Patches) == 0 { + var pod corev1.Pod + if err := json.Unmarshal(original, &pod); err != nil { + t.Fatalf("unmarshal original pod: %v", err) + } + return &pod + } + patchJSON, err := json.Marshal(resp.Patches) + if err != nil { + t.Fatalf("marshal patches: %v", err) + } + patch, err := jsonpatch.DecodePatch(patchJSON) + if err != nil { + t.Fatalf("decode patch: %v", err) + } + patched, err := patch.Apply(original) + if err != nil { + t.Fatalf("apply patch: %v", err) + } + var pod corev1.Pod + if err := json.Unmarshal(patched, &pod); err != nil { + t.Fatalf("unmarshal patched pod: %v", err) + } + return &pod +} + +func TestPodMutatorHandle(t *testing.T) { + basePod := func() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + }, + } + } + + t.Run("no vpc annotation is a silent no-op", func(t *testing.T) { + m, _ := newPodMutator() + resp := m.Handle(context.Background(), admissionRequest(t, basePod(), false)) + if !resp.Allowed { + t.Fatalf("Allowed = false, want true: %+v", resp.Result) + } + if len(resp.Patches) != 0 { + t.Errorf("Patches = %v, want none", resp.Patches) + } + }) + + t.Run("reinvocation guard short-circuits", func(t *testing.T) { + pod := basePod() + pod.Annotations = map[string]string{annotationVPCAttachmentRef: testNamespace + "/my-vpc-0"} + m, _ := newPodMutator() + + resp := m.Handle(context.Background(), admissionRequest(t, pod, false)) + if !resp.Allowed { + t.Fatalf("Allowed = false, want true: %+v", resp.Result) + } + if len(resp.Patches) != 0 { + t.Errorf("Patches = %v, want none (already processed)", resp.Patches) + } + }) + + t.Run("hostNetwork pod is skipped", func(t *testing.T) { + pod := basePod() + pod.Annotations = map[string]string{annotationVPC: testVPCName} + pod.Spec.HostNetwork = true + m, _ := newPodMutator() + + resp := m.Handle(context.Background(), admissionRequest(t, pod, false)) + if !resp.Allowed { + t.Fatalf("Allowed = false, want true: %+v", resp.Result) + } + if len(resp.Patches) != 0 { + t.Errorf("Patches = %v, want none (hostNetwork)", resp.Patches) + } + }) + + t.Run("missing VPC is denied", func(t *testing.T) { + pod := basePod() + pod.Annotations = map[string]string{annotationVPC: "does-not-exist"} + m, _ := newPodMutator() + + resp := m.Handle(context.Background(), admissionRequest(t, pod, false)) + if resp.Allowed { + t.Fatal("Allowed = true, want false (VPC missing)") + } + if resp.Result.Code != http.StatusForbidden { + t.Errorf("Result.Code = %d, want %d", resp.Result.Code, http.StatusForbidden) + } + }) + + t.Run("VPC with no assigned identifier yet is denied", func(t *testing.T) { + pod := basePod() + pod.Annotations = map[string]string{annotationVPC: testVPCName} + vpc := testVPCObj("") // Status.VPC empty + m, _ := newPodMutator(vpc) + + resp := m.Handle(context.Background(), admissionRequest(t, pod, false)) + if resp.Allowed { + t.Fatal("Allowed = true, want false (no assigned identifier)") + } + }) + + t.Run("dry-run never creates real objects", func(t *testing.T) { + pod := basePod() + pod.Annotations = map[string]string{annotationVPC: testVPCName} + vpc := testVPCObj("vpcBase62") + m, k8s := newPodMutator(vpc) + + resp := m.Handle(context.Background(), admissionRequest(t, pod, true)) + if !resp.Allowed { + t.Fatalf("Allowed = false, want true: %+v", resp.Result) + } + if len(resp.Patches) != 0 { + t.Errorf("Patches = %v, want none on dry-run", resp.Patches) + } + var nadList unstructured.UnstructuredList + nadList.SetGroupVersionKind(nadGVK) + if err := k8s.List(context.Background(), &nadList); err != nil { + t.Fatalf("list NADs: %v", err) + } + if len(nadList.Items) != 0 { + t.Errorf("NADs created on dry-run = %d, want 0", len(nadList.Items)) + } + }) + + t.Run("happy path creates NAD and patches the pod", func(t *testing.T) { + pod := basePod() + pod.Annotations = map[string]string{annotationVPC: testVPCName} + vpc := testVPCObj("vpcBase62") + m, k8s := newPodMutator(vpc) + + req := admissionRequest(t, pod, false) + resp := m.Handle(context.Background(), req) + if !resp.Allowed { + t.Fatalf("Allowed = false, want true: %+v", resp.Result) + } + if len(resp.Patches) == 0 { + t.Fatal("Patches is empty, want a patch") + } + + var nadList unstructured.UnstructuredList + nadList.SetGroupVersionKind(nadGVK) + if err := k8s.List(context.Background(), &nadList); err != nil { + t.Fatalf("list NADs: %v", err) + } + if len(nadList.Items) != 1 { + t.Fatalf("NADs created = %d, want 1", len(nadList.Items)) + } + nadName := nadList.Items[0].GetName() + if nadName != "vpcBase62-0" { + t.Errorf("NAD name = %q, want %q", nadName, "vpcBase62-0") + } + + patched := applyPatches(t, req.Object.Raw, resp) + wantRef := testNamespace + "/" + nadName + if patched.Annotations[annotationVPCAttachmentRef] != wantRef { + t.Errorf("annotationVPCAttachmentRef = %q, want %q", + patched.Annotations[annotationVPCAttachmentRef], wantRef) + } + var elements []networkSelectionElement + if err := json.Unmarshal([]byte(patched.Annotations[networksAnnotation]), &elements); err != nil { + t.Fatalf("unmarshal networks annotation: %v", err) + } + if len(elements) != 1 || elements[0].Name != nadName { + t.Errorf("networks annotation = %+v, want one entry naming %q", elements, nadName) + } + }) + + t.Run("second pod against the same VPC gets a distinct ID", func(t *testing.T) { + vpc := testVPCObj("vpcBase62") + m, k8s := newPodMutator(vpc) + + for range 2 { + pod := basePod() + pod.Annotations = map[string]string{annotationVPC: testVPCName} + resp := m.Handle(context.Background(), admissionRequest(t, pod, false)) + if !resp.Allowed { + t.Fatalf("Allowed = false, want true: %+v", resp.Result) + } + } + + var nadList unstructured.UnstructuredList + nadList.SetGroupVersionKind(nadGVK) + if err := k8s.List(context.Background(), &nadList); err != nil { + t.Fatalf("list NADs: %v", err) + } + if len(nadList.Items) != 2 { + t.Fatalf("NADs created = %d, want 2 (distinct IDs)", len(nadList.Items)) + } + if nadList.Items[0].GetName() == nadList.Items[1].GetName() { + t.Error("both pods got the same NAD name, want distinct IDs") + } + }) +} diff --git a/internal/webhook/webhook.go b/internal/webhook/webhook.go new file mode 100644 index 0000000..a5152df --- /dev/null +++ b/internal/webhook/webhook.go @@ -0,0 +1,72 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package webhook implements galactic-webhook's mutating admission webhook: +// on Pod CREATE, a pod carrying the galactic.datumapis.com/vpc annotation +// gets a VPCAttachment ID allocated and a NetworkAttachmentDefinition +// created, then gets patched to attach via that NAD. See this repo's design +// plan (.local/plan-vpc-nad-webhook-plan.md) for the full rationale — +// notably why the webhook creates only the NAD and not the VPCAttachment CR +// itself (galactic-cni creates that, at ADD time, alongside +// BGPVRFInstance/BGPAdvertisement). +package webhook + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" +) + +const ( + // annotationVPC is the pod annotation requesting attachment to a VPC by + // name — the value is the VPC CR's Kubernetes object name. + annotationVPC = "galactic.datumapis.com/vpc" + + // annotationVPCAttachmentRef is the reinvocation guard: once set, this + // pod has already been processed by this webhook — see Handle's step 2. + // It is the only durable record of that fact available at admission + // time, since pod.Name may still be empty (generateName not yet + // resolved by the API server). + annotationVPCAttachmentRef = "galactic.datumapis.com/vpc-attachment-ref" + + // networksAnnotation is the Multus annotation used to attach a pod to + // one or more NetworkAttachmentDefinitions. + networksAnnotation = "k8s.v1.cni.cncf.io/networks" + + // labelVPC marks a NAD as belonging to a given VPC (value: the VPC CR's + // name) — both a query key for AllocateVPCAttachmentID's free-ID scan + // and, deliberately, the same annotation key used on the pod itself + // (annotationVPC): same meaning ("associated with this VPC") in both + // places, not a competing convention. + labelVPC = "galactic.datumapis.com/vpc" + + // labelAttachmentID records, on every NAD this webhook creates, the + // base62 VPCAttachment ID baked into that NAD's conflist. This is the + // allocator's only source of truth for which IDs are already taken — + // not VPCAttachmentStatus.VPCAttachment, which galactic-cni writes + // later and may not exist for a given ID yet. See AllocateVPCAttachmentID. + labelAttachmentID = "galactic.datumapis.com/vpcattachment-id" +) + +// nadGVK is the GroupVersionKind for NetworkAttachmentDefinition. +var nadGVK = schema.GroupVersionKind{ + Group: "k8s.cni.cncf.io", + Version: "v1", + Kind: "NetworkAttachmentDefinition", +} + +// NewScheme builds the runtime.Scheme galactic-webhook needs: core types +// (for decoding admitted Pods) and the cloud.datumapis.com VPC types (for +// looking up the VPC a pod requests attachment to). NAD is handled as +// unstructured.Unstructured and needs no scheme registration — the same +// pattern internal/cni's nad.go already uses. +func NewScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(cloudv1alpha1.AddToScheme(scheme)) + return scheme +} diff --git a/scripts/ci.sh b/scripts/ci.sh index b3b669f..a3df649 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -57,6 +57,30 @@ case "$COMMAND" in curl -sL "${NETWORK_CRD_URL}/${crd}" | kubectl apply -f - done + echo "--- Installing VPC CRDs (datum-cloud/cloud)" + # Same approach as the BGP CRD install above. galactic-cni creates + # VPCAttachment CRs itself (internal/cni/vpcattachment.go); this repo does + # not depend on a companion operator to install these CRDs, so the e2e + # cluster needs them installed directly. + CLOUD_SHA=$(awk '/go\.datum\.net\/cloud / {print $2}' go.mod | sed 's/.*-//') + CLOUD_CRD_URL="https://raw.githubusercontent.com/datum-cloud/cloud/${CLOUD_SHA}/config/crd" + for crd in \ + cloud.datumapis.com_vpcs.yaml \ + cloud.datumapis.com_vpcattachments.yaml; do + curl -sL "${CLOUD_CRD_URL}/${crd}" | kubectl apply -f - + done + + echo "--- Installing NetworkAttachmentDefinition CRD (Multus)" + # galactic-cni's annotateNAD (internal/cni/nad.go) patches a NAD once a + # pod namespace is known -- needed by TestCNIVPCAttachmentCreation, which + # (unlike the older tests) sets K8S_POD_NAMESPACE in CNI_ARGS. No Multus + # DaemonSet runs in this e2e cluster; only the CRD is needed for the CNI + # plugin's own patch call to resolve. Pinned to a specific commit, same + # reproducibility rationale as the BGP/VPC CRD installs above. + NAD_CRD_SHA="46fcc4112dcb9af0b277dc85fec8178acbddd5ab" + curl -sL "https://raw.githubusercontent.com/k8snetworkplumbingwg/network-attachment-definition-client/${NAD_CRD_SHA}/artifacts/networks-crd.yaml" \ + | kubectl apply -f - + echo "--- Applying galactic-system namespace and CNI RBAC" kubectl apply -k config/system kubectl apply -f config/cni/serviceaccount.yaml -f config/cni/rbac.yaml @@ -90,6 +114,23 @@ spec: safi: evpn EOF + echo "--- Creating VPC fixture for the VPCAttachment e2e test" + # galactic-cni's applyVPCAttachment (internal/cni/vpcattachment.go) never + # Gets/validates the VPC CR itself -- it only threads pluginConf.VPCName + # through as VPCAttachmentSpec.VPC.Name -- so this fixture exists purely + # to make the e2e cluster look like a real deployment, not because the + # CNI code path depends on it. + cat < /tmp/cni-vpcattach.json && "+ + "echo '"+script+"' > /tmp/run-cni-vpcattach.sh && "+ + "chmod +x /tmp/run-cni-vpcattach.sh", + ) + if err != nil { + t.Fatalf("write cni config and script: %v", err) + } + + if out, err := kubectl(t.Context(), "exec", podName, "-i", "--", "/tmp/run-cni-vpcattach.sh"); err != nil { + t.Logf("exec output: %s", out) + t.Fatalf("CNI ADD failed: %v", err) + } + + raw, err := kubectl(t.Context(), "get", "vpcattachments.cloud.datumapis.com", attachmentCR, "-o", "json") + if err != nil { + t.Fatalf("get VPCAttachment %q: %v\n%s", attachmentCR, err, raw) + } + + var attachment struct { + Spec struct { + VPC struct { + Name string `json:"name"` + } `json:"vpc"` + Interface struct { + Name string `json:"name"` + Addresses []string `json:"addresses"` + } `json:"interface"` + } `json:"spec"` + Status struct { + VPC string `json:"vpc"` + VPCAttachment string `json:"vpcAttachment"` + Node string `json:"node"` + ContainerID string `json:"containerID"` + PodName string `json:"podName"` + HostInterface string `json:"hostInterface"` + VRFInterface string `json:"vrfInterface"` + GuestInterface string `json:"guestInterface"` + PodSubnet string `json:"podSubnet"` + } `json:"status"` + } + if err := json.Unmarshal([]byte(raw), &attachment); err != nil { + t.Fatalf("unmarshal VPCAttachment: %v\n%s", err, raw) + } + + if attachment.Spec.VPC.Name != vpcName { + t.Errorf("Spec.VPC.Name = %q, want %q", attachment.Spec.VPC.Name, vpcName) + } + if len(attachment.Spec.Interface.Addresses) == 0 { + t.Error("Spec.Interface.Addresses is empty, want at least one allocated address") + } + if attachment.Status.VPC != vpc { + t.Errorf("Status.VPC = %q, want %q", attachment.Status.VPC, vpc) + } + if attachment.Status.VPCAttachment != vpcAttachment { + t.Errorf("Status.VPCAttachment = %q, want %q", attachment.Status.VPCAttachment, vpcAttachment) + } + if attachment.Status.Node != nodeName() { + t.Errorf("Status.Node = %q, want %q", attachment.Status.Node, nodeName()) + } + if attachment.Status.ContainerID != containerID { + t.Errorf("Status.ContainerID = %q, want %q", attachment.Status.ContainerID, containerID) + } + if attachment.Status.PodName != attachedPod { + t.Errorf("Status.PodName = %q, want %q", attachment.Status.PodName, attachedPod) + } + if attachment.Status.HostInterface == "" || attachment.Status.VRFInterface == "" { + t.Errorf("Status host/vrf interface names unexpectedly empty: %+v", attachment.Status) + } + if attachment.Status.PodSubnet == "" { + t.Error("Status.PodSubnet is empty, want an allocated subnet") + } +} + // nodeName returns the name of the node this pod runs on, or falls back to // "kind-worker" for single-node Kind clusters. func nodeName() string { @@ -307,6 +479,14 @@ func kubectl(ctx context.Context, args ...string) (string, error) { return strings.TrimSpace(string(out)), err } +// kubectlApply runs `kubectl apply -f -`, piping manifest in via stdin. +func kubectlApply(ctx context.Context, manifest string) (string, error) { + cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(manifest) + out, err := cmd.CombinedOutput() + return strings.TrimSpace(string(out)), err +} + // waitForPodPhase polls until the named pod reaches wantPhase or the timeout // expires. It returns an error describing the last observed phase on timeout. func waitForPodPhase(t *testing.T, name, wantPhase string) error { @@ -331,3 +511,20 @@ func deletePod(t *testing.T, name string) { t.Helper() kubectl(t.Context(), "delete", "pod", name, "--ignore-not-found", "--wait=false") //nolint:errcheck } + +// deleteVPCAttachment removes a VPCAttachment CR by name, ignoring +// not-found errors. Mirrors deletePod's cleanup pattern. +func deleteVPCAttachment(t *testing.T, name string) { + t.Helper() + //nolint:errcheck + kubectl(t.Context(), "delete", "vpcattachments.cloud.datumapis.com", name, "--ignore-not-found", "--wait=false") +} + +// deleteNAD removes a NetworkAttachmentDefinition by name, ignoring +// not-found errors. Mirrors deletePod's cleanup pattern. +func deleteNAD(t *testing.T, name string) { + t.Helper() + //nolint:errcheck + kubectl(t.Context(), "delete", "network-attachment-definitions.k8s.cni.cncf.io", name, + "--ignore-not-found", "--wait=false") +}