diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 7f082e8..52f7152 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -32,8 +32,21 @@ jobs: platforms: linux/amd64,linux/arm64 secrets: inherit + publish-fabric-router-image: + permissions: + id-token: write + contents: read + packages: write + attestations: write + uses: datum-cloud/actions/.github/workflows/publish-docker.yaml@v1.20.0 + with: + image-name: fabric-router + dockerfile-path: containers/fabric-router/Dockerfile + platforms: linux/amd64,linux/arm64 + secrets: inherit + publish-kustomize-bundles: - needs: [publish-galactic-cni-image, publish-galactic-router-image] + needs: [publish-galactic-cni-image, publish-galactic-router-image, publish-fabric-router-image] permissions: id-token: write contents: read @@ -45,4 +58,5 @@ jobs: images: | - { path: config/cni, name: ghcr.io/datum-cloud/galactic-cni, tag: "${{ needs.publish-galactic-cni-image.outputs.tag }}" } - { path: config/router/base, name: ghcr.io/datum-cloud/galactic-router, tag: "${{ needs.publish-galactic-router-image.outputs.tag }}" } + - { path: config/fabric, name: ghcr.io/datum-cloud/fabric-router, tag: "${{ needs.publish-fabric-router-image.outputs.tag }}" } secrets: inherit diff --git a/AGENTS.md b/AGENTS.md index e4ef492..c5b23b3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,6 +59,7 @@ Summary: - **`config/router/tenant-control/`** — the BGP route-reflector role (`galactic-router-control`, `GALACTIC_ROUTER_REFLECTOR=true`); opt-in only, requires nodes labeled `galactic.datumapis.com/node: control` (stays at zero replicas otherwise). `GALACTIC_ROUTER_BGP_LOCAL_ADDRESS` is auto-detected from the host's `lo` interface by default; see the comments in `daemonset-patch.yaml` for when to override it. - **`config/router/base/`** — the DaemonSet spec shared by both roles; not applied directly. - Apply the whole router component (both roles) with `kubectl apply -k config/router/`, or a single role with e.g. `kubectl apply -k config/router/tenant/`. +- **`config/fabric/`** — the FRR underlay eBGP DaemonSet (`fabric-router`; `galactic-router` needs a working underlay before it can start). Unlike `config/router/`, this is a single flat DaemonSet with no `tenant`/`tenant-control`-style role split — its affinity matches nodes labeled `galactic.datumapis.com/node` `In` `[edge, control]` directly, since (unlike galactic-router's route-reflector role) there's no env/config difference between running on a regular node vs. the control node. **Not** part of the root `config/kustomization.yaml` and not covered by `kubectl apply -k config/` — unlike every other component here, it has no generic default: the `frr.conf` it needs (interface addresses, remote-AS, etc.) is site-specific, so the deployer must hand-author a `fabric-config` ConfigMap (just `frr.conf` — `daemons`/`vtysh.conf` are baked into the `fabric-router` image) before applying `kubectl apply -k config/fabric/`. - **`deploy/containerlab/`** — ContainerLab topology (`gvpc.clab.yaml`) for three Kind clusters (dfw, iad, sjc) wired over an IPv6 SRv6 transit mesh. FRR runs as a hostNetwork DaemonSet on each worker for eBGP underlay; `galactic-router` (tenant role) handles EVPN path distribution over iBGP, and the iad route reflector builds on `config/router/tenant-control/`. See `deploy/containerlab/README.md` and `deploy/containerlab/Taskfile.yaml` for bring-up commands. ## New Developer Entry Points diff --git a/README.md b/README.md index 22edcb8..407620c 100644 --- a/README.md +++ b/README.md @@ -40,18 +40,27 @@ kubectl apply -k config/ Each component can also be applied on its own, e.g. `kubectl apply -k config/router` for just the router (both roles) or `kubectl apply -k config/router/tenant` for just the per-node role. +`config/fabric/` (the FRR underlay eBGP DaemonSet `galactic-router` depends on) is **not** included in `kubectl apply -k config/` and has its own prerequisite below — apply it separately once you've met that prerequisite: + +```bash +kubectl apply -k config/fabric/ +``` + #### Prerequisites -- **Container images.** `.github/workflows/publish.yaml` builds `ghcr.io/datum-cloud/galactic-cni` and `ghcr.io/datum-cloud/galactic-router` (from `containers/galactic-cni/Dockerfile` and `containers/galactic-router/Dockerfile` respectively) on every push and release — but it never publishes a `:latest` tag, only date-stamped tags per push/release (e.g. `v0.0.0-main-20260713-170924`) and, for tagged releases, semver tags. The `image:` references committed in `config/cni/daemonset.yaml` and `config/router/base/daemonset.yaml` say `:latest` only as a placeholder that CI substitutes with a real published tag when it builds the `ghcr.io/datum-cloud/galactic-kustomize` OCI Kustomize bundle — that substitution never happens in the git checkout itself. Applying `config/` directly from a clone will therefore fail to pull `:latest`. Before applying, resolve the current tag (check the [package pages](https://github.com/orgs/datum-cloud/packages?repo_name=galactic) or the latest successful run of `publish.yaml` on `main`) and pin it, e.g.: +- **Container images.** `.github/workflows/publish.yaml` builds `ghcr.io/datum-cloud/galactic-cni`, `ghcr.io/datum-cloud/galactic-router`, and `ghcr.io/datum-cloud/fabric-router` (from `containers/galactic-cni/Dockerfile`, `containers/galactic-router/Dockerfile`, and `containers/fabric-router/Dockerfile` respectively) on every push and release — but it never publishes a `:latest` tag, only date-stamped tags per push/release (e.g. `v0.0.0-main-20260713-170924`) and, for tagged releases, semver tags. The `image:` references committed in `config/cni/daemonset.yaml`, `config/router/base/daemonset.yaml`, and `config/fabric/daemonset.yaml` say `:latest` only as a placeholder that CI substitutes with a real published tag when it builds the `ghcr.io/datum-cloud/galactic-kustomize` OCI Kustomize bundle — that substitution never happens in the git checkout itself. Applying `config/` directly from a clone will therefore fail to pull `:latest`. Before applying, resolve the current tag (check the [package pages](https://github.com/orgs/datum-cloud/packages?repo_name=galactic) or the latest successful run of `publish.yaml` on `main`) and pin it, e.g.: ```bash cd config/cni && kustomize edit set image ghcr.io/datum-cloud/galactic-cni=ghcr.io/datum-cloud/galactic-cni: cd config/router/base && kustomize edit set image ghcr.io/datum-cloud/galactic-router=ghcr.io/datum-cloud/galactic-router: + cd config/fabric && kustomize edit set image ghcr.io/datum-cloud/fabric-router=ghcr.io/datum-cloud/fabric-router: ``` +- **`config/fabric/`: site-specific `frr.conf`.** Unlike every other component under `config/`, `config/fabric/daemonset.yaml` has no generic default config — the underlay eBGP session (interface addresses, remote-AS, etc.) is different for every site. Before applying `config/fabric/`, create a `fabric-config` ConfigMap in the `galactic-system` namespace with your site's `frr.conf` under that key (the other two files FRR needs, `daemons` and `vtysh.conf`, are already baked into the `fabric-router` image — see `containers/fabric-router/Dockerfile`). `deploy/containerlab/resources/fabric/{dfw,iad,sjc}/frr.conf` are worked examples from the lab, not something you can apply as-is. + - **Talos: gRPC health port.** `galactic-router` runs `hostNetwork: true` and defaults to gRPC health checks on port `5000`, which collides with Talos's built-in dashboard (`/sbin/dashboard` permanently binds `127.0.0.1:5000` on every Talos node). `config/router/base/daemonset.yaml` already ships with `GALACTIC_ROUTER_GRPC_HEALTH_PORT=5179` (and matching probe/containerPort) to avoid this; if you run `galactic-router` outside these manifests on Talos, set `GALACTIC_ROUTER_GRPC_HEALTH_PORT` to something other than `5000` yourself. -- **`galactic-router` tenant mode: BGP local address.** The node needs a global-unicast IPv6 address assigned to `lo` (typically by an underlay/fabric BGP daemon that starts before `galactic-router`), or you must set `GALACTIC_ROUTER_BGP_LOCAL_ADDRESS` explicitly — this is required even when `GALACTIC_ROUTER_BGP_LISTEN_PORT=-1` (no inbound listener), since `galactic-router` still needs a source address for outbound BGP connections. Without one of these, startup fails with `GALACTIC_ROUTER_BGP_LOCAL_ADDRESS not set and no address could be detected on lo: no global-unicast IPv6 address found on lo`. See [`docs/router/configuration.md`](./docs/router/configuration.md) for details. +- **`galactic-router` tenant mode: BGP local address.** The node needs a global-unicast IPv6 address assigned to `lo` (typically by `config/fabric/`'s underlay eBGP daemon, which must start and converge before `galactic-router`), or you must set `GALACTIC_ROUTER_BGP_LOCAL_ADDRESS` explicitly — this is required even when `GALACTIC_ROUTER_BGP_LISTEN_PORT=-1` (no inbound listener), since `galactic-router` still needs a source address for outbound BGP connections. Without one of these, startup fails with `GALACTIC_ROUTER_BGP_LOCAL_ADDRESS not set and no address could be detected on lo: no global-unicast IPv6 address found on lo`. See [`docs/router/configuration.md`](./docs/router/configuration.md) for details. See [`docs/router/configuration.md`](./docs/router/configuration.md) for the full `galactic-router` CLI flag / environment variable reference — note that env var names generally follow `GALACTIC_ROUTER_` but aren't always the naive uppercased guess (e.g. `--mode` is `GALACTIC_ROUTER_ROUTER_MODE`, not `GALACTIC_ROUTER_MODE`); the reference table has the exact name for every flag. diff --git a/deploy/containerlab/resources/control/fabric/iad/daemonset.yaml b/config/fabric/daemonset.yaml similarity index 52% rename from deploy/containerlab/resources/control/fabric/iad/daemonset.yaml rename to config/fabric/daemonset.yaml index e52ba65..35c6e84 100644 --- a/deploy/containerlab/resources/control/fabric/iad/daemonset.yaml +++ b/config/fabric/daemonset.yaml @@ -1,25 +1,33 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: iad-control-fabric + name: fabric-router namespace: galactic-system + labels: + app.kubernetes.io/name: fabric-router spec: selector: matchLabels: - app.kubernetes.io/name: iad-control-fabric + app.kubernetes.io/name: fabric-router template: metadata: labels: - app.kubernetes.io/name: iad-control-fabric + app.kubernetes.io/name: fabric-router spec: hostNetwork: true + # FRR here establishes the underlay eBGP session to the physical fabric + # and brings up the node's lo address, both of which galactic-router + # depends on before it can start — so this must tolerate NotReady the + # same way a CNI plugin does, or it never gets scheduled early enough. tolerations: - key: node.kubernetes.io/not-ready operator: Exists effect: NoSchedule - - key: galactic.datumapis.com/node - value: control - effect: NoSchedule + # Opt-in only, same as galactic-cni and galactic-router: runs on every + # node labeled either for regular tenant traffic or for the + # galactic-router route-reflector role, since both need underlay + # connectivity; new node types (GPU, monitoring, etc.) don't + # automatically get fabric. affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -30,14 +38,23 @@ spec: - key: galactic.datumapis.com/node operator: In values: + - edge - control initContainers: - name: frr-init - image: quay.io/frrouting/frr:10.2.0 + # ":latest" is a placeholder only: no such tag is ever pushed to + # GHCR. CI resolves and stamps a real version tag here when it + # publishes the ghcr.io/datum-cloud/galactic-kustomize OCI bundle + # (see .github/workflows/publish.yaml); applying this manifest + # directly from a git checkout will fail to pull the image unless + # you override the tag first — see the README's "Production + # Deployment" section. + image: ghcr.io/datum-cloud/fabric-router:latest command: - sh - -c - | + cp /etc/frr-defaults/* /etc/frr/ cp /tmp/frr-config/* /etc/frr/ install -d -o frr -g frr -m 775 /run/frr install -d -o frr -g frr -m 775 /var/log/frr @@ -57,7 +74,7 @@ spec: mountPath: /var/log/frr containers: - name: frr - image: quay.io/frrouting/frr:10.2.0 + image: ghcr.io/datum-cloud/fabric-router:latest securityContext: capabilities: add: @@ -72,9 +89,14 @@ spec: - name: frr-log mountPath: /var/log/frr volumes: + # fabric-config is NOT generated by this kustomization — the deployer + # must create it themselves with a site-specific frr.conf (interface + # addresses, remote-AS, etc. for the local underlay fabric). There is + # no generic default that peers correctly everywhere. See the + # README's "Production Deployment" section. - name: frr-config-source configMap: - name: iad-control-fabric-config + name: fabric-config - name: frr-etc emptyDir: {} - name: frr-run diff --git a/config/fabric/kustomization.yaml b/config/fabric/kustomization.yaml new file mode 100644 index 0000000..0987901 --- /dev/null +++ b/config/fabric/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - daemonset.yaml diff --git a/containers/fabric-router/Dockerfile b/containers/fabric-router/Dockerfile new file mode 100644 index 0000000..744351a --- /dev/null +++ b/containers/fabric-router/Dockerfile @@ -0,0 +1,21 @@ +# Fabric router image: the upstream FRR image used for the eBGP fabric/transit +# roles in deploy/containerlab, layered with common network debugging tools +# so issues can be triaged from inside the container without a separate +# toolbox image. +ARG FRR_VERSION=10.2.0 +FROM quay.io/frrouting/frr:${FRR_VERSION} + +RUN apk add --no-cache \ + tcpdump \ + mtr \ + busybox-extras \ + bind-tools \ + curl + +# Static FRR config common to every fabric/transit role. Copied to a +# non-standard path (rather than /etc/frr directly) because /etc/frr is an +# emptyDir in the fabric DaemonSets, populated at pod start by the frr-init +# container so frr.conf can still be swapped in per site; the daemonset's +# init command seeds it from here before layering the site-specific config +# in on top. +COPY containers/fabric-router/daemons containers/fabric-router/vtysh.conf /etc/frr-defaults/ diff --git a/deploy/containerlab/resources/fabric/base/daemons b/containers/fabric-router/daemons similarity index 100% rename from deploy/containerlab/resources/fabric/base/daemons rename to containers/fabric-router/daemons diff --git a/deploy/containerlab/resources/fabric/base/vtysh.conf b/containers/fabric-router/vtysh.conf similarity index 100% rename from deploy/containerlab/resources/fabric/base/vtysh.conf rename to containers/fabric-router/vtysh.conf diff --git a/deploy/containerlab/AGENTS.md b/deploy/containerlab/AGENTS.md index 13e77d0..6b14ec9 100644 --- a/deploy/containerlab/AGENTS.md +++ b/deploy/containerlab/AGENTS.md @@ -3,7 +3,7 @@ ## Important Conventions - **YAML extensions**: Always use `.yaml`, never `.yml`. -- **FRR image pinning**: Taskfile and DaemonSets pin FRR to `10.6.1`. Transit routers in `gvpc.clab.yaml` default to `frrouting/frr:latest` — this is a known mismatch (see review findings). +- **fabric-router image**: `containers/fabric-router/Dockerfile` layers `tcpdump`, `mtr-tiny`, `telnet`, `dnsutils`, and `curl` onto the upstream `quay.io/frrouting/frr` image (version pinned via the Taskfile's `FRR_VERSION`, currently `10.2.0`). Built and tagged `fabric-router:latest` by `task build:fabric-router`, then used by both fabric DaemonSets (`resources/fabric/base/` and `resources/control/fabric/iad/` both build on `config/fabric/` — see the "Shared manifests" bullet below — loaded into the Kind clusters via `deploy:images`) and the transit routers (`gvpc.clab.yaml`'s `kinds.linux.image`, picked up directly from the local Docker image cache). All reference sites set/require `imagePullPolicy: Never` where applicable — there's no registry to pull `fabric-router:latest` from. The image also bakes in the two files common to every fabric role, `containers/fabric-router/{daemons,vtysh.conf}`, to `/etc/frr-defaults/` (not `/etc/frr` directly, since that's an `emptyDir` the `frr-init` container populates at pod start so a per-site `frr.conf` can still be layered in from the `fabric-config` ConfigMap — see `config/fabric/daemonset.yaml`'s init command). Only `frr.conf` is still ConfigMap-sourced (`resources/fabric/{dfw,iad,sjc}/frr.conf`, `resources/control/fabric/iad/frr.conf`) — no raw manifests remain under `resources/fabric/` or `resources/control/fabric/`. - **Image loading**: Uses `ctr --namespace k8s.io images import` (not `kind load docker-image`) due to containerd v2 incompatibility. - **iad-worker-control**: Created as `iad-worker2` in the topology, renamed post-deploy via `deploy:rename-control`. The Kind config sets the hostname via `kubeadmConfigPatches`. Runs the galactic-router route reflector (RR) for the iad region. - **galactic-router image**: Uses `galactic-router:latest` with `imagePullPolicy: Never` — stale images persist across rebuilds. @@ -14,7 +14,7 @@ - **Node labels**: Workers use `galactic.datumapis.com/node: edge` (not `galactic.io/role: pop`). The control node uses `galactic.datumapis.com/node: control` with a matching `NoSchedule` taint. - **GC namespace**: The tenant DaemonSet sets `GALACTIC_ROUTER_GC_NAMESPACE=galactic-system` for namespace-scoped garbage collection. - **FRR config**: Transit router configs omit the `frr version` directive (managed by the FRR image, not the config). -- **Shared manifests**: `resources/cni/kustomization.yaml` builds on `config/cni/`, `resources/tenant/base/kustomization.yaml` builds on `config/router/tenant/`, and `resources/control/tenant/iad/kustomization.yaml` builds on `config/router/tenant-control/` (each of the router ones in turn pulls in `config/router/base/`) rather than forking them, patching in only what the lab needs to differ (image, env). Each role's node affinity and the base's blanket tolerations apply as-is — the lab doesn't need its own. `kubectl apply -k` refuses to load resource files from outside a kustomization's own root, so `deploy-cni.sh`/`deploy-tenant.sh` `docker cp` those directories into a local `base/` subdirectory on the node at deploy time (`resources/cni/base/`, `resources/tenant/base/{base,tenant}/`, `resources/control/tenant/iad/{base,tenant-control}/`) instead of referencing `config/` across that boundary. The `galactic-system` namespace and RBAC/ServiceAccount are applied straight from `config/system/namespace.yaml` and `config/*/{rbac,serviceaccount}.yaml` by `deploy-system.sh` via `lib.sh`'s `copy_config` (which copies all of `config/` to `/galactic/config/`) — not part of any kustomize build, so per-site `namePrefix` never touches the shared cluster-scoped RBAC. `config/system/namespace.yaml` is also the only place `galactic-system` gets created for production — there is no other bootstrap step in this repo, so if it's ever missing on a real cluster, deploying `config/router/` or `config/cni/` fails outright. +- **Shared manifests**: `resources/cni/kustomization.yaml` builds on `config/cni/`, `resources/tenant/base/kustomization.yaml` builds on `config/router/tenant/`, `resources/control/tenant/iad/kustomization.yaml` builds on `config/router/tenant-control/`, and both `resources/fabric/base/kustomization.yaml` and `resources/control/fabric/iad/kustomization.yaml` build on `config/fabric/` (each in turn pulls in its own `base/` where one exists: `config/router/base/`; `config/fabric/` has no role split, it's a single flat DaemonSet) rather than forking them, patching in only what the lab needs to differ (image, env — for both fabric overlays, just `image: fabric-router:latest` + `imagePullPolicy: Never` via each one's own `fabric-lab-patch.yaml`). `config/fabric/`'s own affinity already covers both node labels (`galactic.datumapis.com/node` `In` `[edge, control]`), but the *lab* still needs the router/control split back apart — iad's two nodes need different `frr.conf`, and a single shared ConfigMap can't serve both — so `resources/fabric/base/fabric-lab-patch.yaml` and `resources/control/fabric/iad/fabric-lab-patch.yaml` each narrow that affinity locally to just `edge` or just `control` respectively (on top of the image override), a lab-only concern with no production equivalent. Because both of those overlays land on the *same* iad cluster, they also can't share a DaemonSet name — but a strategic-merge `patches:` entry can't rename a resource (Kustomize keeps the `target`'s original identity regardless of what `metadata.name` the patch body says), so `resources/control/fabric/iad/kustomization.yaml` renames it to `fabric-control` via a second, JSON6902-style `patches:` entry (`op: replace, path: /metadata/name`) instead — `nameSuffix`/`namePrefix` transformers are the only other sanctioned way to rename, and neither fit here (nameSuffix would produce `fabric-router-control`, not `fabric-control`). `kubectl apply -k` refuses to load resource files from outside a kustomization's own root, so `deploy-cni.sh`/`deploy-tenant.sh`/`deploy-fabric.sh` `docker cp` those directories into a local `base/` subdirectory on the node at deploy time (`resources/cni/base/`, `resources/tenant/base/{base,tenant}/`, `resources/control/tenant/iad/{base,tenant-control}/`, `resources/fabric/base/fabric/`, `resources/control/fabric/iad/fabric/`) instead of referencing `config/` across that boundary. The `galactic-system` namespace and RBAC/ServiceAccount are applied straight from `config/system/namespace.yaml` and `config/*/{rbac,serviceaccount}.yaml` by `deploy-system.sh` via `lib.sh`'s `copy_config` (which copies all of `config/` to `/galactic/config/`) — not part of any kustomize build, so per-site `namePrefix` never touches the shared cluster-scoped RBAC. `config/system/namespace.yaml` is also the only place `galactic-system` gets created for production — there is no other bootstrap step in this repo, so if it's ever missing on a real cluster, deploying `config/router/` or `config/cni/` fails outright. ## Naming Layers diff --git a/deploy/containerlab/README.md b/deploy/containerlab/README.md index d4afca5..dcee139 100644 --- a/deploy/containerlab/README.md +++ b/deploy/containerlab/README.md @@ -36,10 +36,10 @@ topology nodes. Each cluster's `control-plane`/`worker` nodes above are its memb ### BGP design ``` -AS 65000 (dfw-fabric / FRR) ──eBGP── tr1 (AS 65100) -AS 65000 (iad-fabric / FRR) ──eBGP── tr3:eth5 (AS 65100) -AS 65000 (iad-control-fabric / FRR) ──eBGP── tr3:eth4 (AS 65100) -AS 65000 (sjc-fabric / FRR) ──eBGP── tr2 (AS 65100) +AS 65000 (dfw fabric-router / FRR) ──eBGP── tr1 (AS 65100) +AS 65000 (iad fabric-router / FRR) ──eBGP── tr3:eth5 (AS 65100) +AS 65000 (iad fabric-control / FRR) ──eBGP── tr3:eth4 (AS 65100) +AS 65000 (sjc fabric-router / FRR) ──eBGP── tr2 (AS 65100) AS 65000 (dfw-tenant / galactic-router) ──iBGP── iad-control-tenant (AS 65000 RR) AS 65000 (iad-tenant / galactic-router) ──iBGP── iad-control-tenant (AS 65000 RR) diff --git a/deploy/containerlab/Taskfile.yaml b/deploy/containerlab/Taskfile.yaml index fd3add7..8fc85ef 100644 --- a/deploy/containerlab/Taskfile.yaml +++ b/deploy/containerlab/Taskfile.yaml @@ -6,7 +6,7 @@ vars: TOPO: sh: echo *.clab.yaml FRR_VERSION: "10.2.0" - FRR_IMAGE: quay.io/frrouting/frr:{{.FRR_VERSION}} + FABRIC_ROUTER_IMAGE: fabric-router:latest tasks: default: @@ -16,23 +16,21 @@ tasks: build: desc: Build all container images - cmds: - - task: "build:node" - - task: "build:frr" - - task: "build:galactic-router" - - task: "build:galactic-cni" + deps: + - "build:node" + - "build:fabric-router" + - "build:galactic-router" + - "build:galactic-cni" "build:node": desc: Build the Kind node image cmds: - docker build --network=host -t kindest/node:galactic -f containers/kindest-node-galactic/Dockerfile ../.. - "build:frr": - desc: Pull the official FRR container from Quay.io (v{{.FRR_VERSION}}) - status: - - docker image inspect {{.FRR_IMAGE}} > /dev/null 2>&1 + "build:fabric-router": + desc: Build the fabric-router image (FRR v{{.FRR_VERSION}} + debug tooling) cmds: - - docker pull {{.FRR_IMAGE}} + - docker build --network=host --build-arg FRR_VERSION={{.FRR_VERSION}} -t {{.FABRIC_ROUTER_IMAGE}} -f ../../containers/fabric-router/Dockerfile ../.. "build:galactic-router": desc: Build the galactic-router container image @@ -90,13 +88,13 @@ tasks: desc: Load all container images into the Kind clusters cmds: - task: load-image - vars: {IMAGE: "{{.FRR_IMAGE}}", NODE: iad-worker} + vars: {IMAGE: "{{.FABRIC_ROUTER_IMAGE}}", NODE: iad-worker} - task: load-image - vars: {IMAGE: "{{.FRR_IMAGE}}", NODE: iad-worker-control} + vars: {IMAGE: "{{.FABRIC_ROUTER_IMAGE}}", NODE: iad-worker-control} - task: load-image - vars: {IMAGE: "{{.FRR_IMAGE}}", NODE: sjc-worker} + vars: {IMAGE: "{{.FABRIC_ROUTER_IMAGE}}", NODE: sjc-worker} - task: load-image - vars: {IMAGE: "{{.FRR_IMAGE}}", NODE: dfw-worker} + vars: {IMAGE: "{{.FABRIC_ROUTER_IMAGE}}", NODE: dfw-worker} - task: load-image vars: {IMAGE: galactic-router:latest, NODE: iad-worker} - task: load-image @@ -197,19 +195,19 @@ tasks: cmds: - | docker exec dfw-control-plane \ - kubectl exec -n galactic-system ds/dfw-fabric \ + kubectl exec -n galactic-system ds/fabric-router \ -- vtysh -c "show bgp ipv6 unicast summary" - | docker exec iad-control-plane \ - kubectl exec -n galactic-system ds/iad-fabric \ + kubectl exec -n galactic-system ds/fabric-router \ -- vtysh -c "show bgp ipv6 unicast summary" - | docker exec sjc-control-plane \ - kubectl exec -n galactic-system ds/sjc-fabric \ + kubectl exec -n galactic-system ds/fabric-router \ -- vtysh -c "show bgp ipv6 unicast summary" - | docker exec iad-control-plane \ - kubectl exec -n galactic-system ds/iad-control-fabric \ + kubectl exec -n galactic-system ds/fabric-control \ -- vtysh -c "show bgp ipv6 unicast summary" "test:bgp-peers": @@ -240,6 +238,6 @@ tasks: - docker rmi kindest/node:galactic || true - docker rmi galactic-router:latest || true - docker rmi galactic-cni:latest || true - - docker rmi {{.FRR_IMAGE}} || true + - docker rmi {{.FABRIC_ROUTER_IMAGE}} || true - rm -rf clab-{{.LAB}} build/ - rm -f dfw.kubeconfig sjc.kubeconfig iad.kubeconfig diff --git a/deploy/containerlab/docs/verification.md b/deploy/containerlab/docs/verification.md index 6f35731..571ab78 100644 --- a/deploy/containerlab/docs/verification.md +++ b/deploy/containerlab/docs/verification.md @@ -26,13 +26,13 @@ docker exec iad-control-plane kubectl get pods -n galactic-system docker exec sjc-control-plane kubectl get pods -n galactic-system # Run vtysh inside a pod -docker exec dfw-control-plane kubectl exec -n galactic-system ds/dfw-fabric \ +docker exec dfw-control-plane kubectl exec -n galactic-system ds/fabric-router \ -- vtysh -c "show bgp ipv6 unicast summary" -docker exec sjc-control-plane kubectl exec -n galactic-system ds/sjc-fabric \ +docker exec sjc-control-plane kubectl exec -n galactic-system ds/fabric-router \ -- vtysh -c "show bgp ipv6 unicast summary" -docker exec iad-control-plane kubectl exec -n galactic-system ds/iad-fabric \ +docker exec iad-control-plane kubectl exec -n galactic-system ds/fabric-router \ -- vtysh -c "show bgp ipv6 unicast summary" -docker exec iad-control-plane kubectl exec -n galactic-system ds/iad-control-fabric \ +docker exec iad-control-plane kubectl exec -n galactic-system ds/fabric-control \ -- vtysh -c "show bgp ipv6 unicast summary" ``` diff --git a/deploy/containerlab/gvpc.clab.yaml b/deploy/containerlab/gvpc.clab.yaml index cebdd3c..cb1988d 100644 --- a/deploy/containerlab/gvpc.clab.yaml +++ b/deploy/containerlab/gvpc.clab.yaml @@ -15,7 +15,7 @@ topology: kinds: linux: - image: frrouting/frr:latest + image: fabric-router:latest nodes: dfw-control-plane: diff --git a/deploy/containerlab/resources/control/fabric/iad/configmap.yaml b/deploy/containerlab/resources/control/fabric/iad/configmap.yaml deleted file mode 100644 index 2095dca..0000000 --- a/deploy/containerlab/resources/control/fabric/iad/configmap.yaml +++ /dev/null @@ -1,66 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: iad-control-fabric-config - namespace: galactic-system -data: - daemons: | - zebra=yes - bgpd=yes - ospfd=no - ospf6d=no - ripd=no - ripngd=no - isisd=no - pimd=no - ldpd=no - nhrpd=no - eigrpd=no - babeld=no - sharpd=no - pbrd=no - bfdd=no - fabricd=no - vrrpd=no - pathd=no - - vtysh_enable=yes - zebra_options=" -A 127.0.0.1 -s 90000000" - bgpd_options=" -A 127.0.0.1" - frr.conf: | - frr defaults traditional - hostname iad-control-fabric - log syslog informational - - interface lo - ! /128 provides a reachable address for galactic-router peers to connect to on port 1790. - ipv6 address fc00:0:8::1/128 - ! - interface eth1 - description tr3-facing - ipv6 address 2001:db8:1:31::2/64 - ! - - ipv6 route fc00:0:8::/48 Null0 - ! - route-map SET_SRC permit 10 - set src fc00:0:8::1 - ! - router bgp 65000 - bgp router-id 10.255.255.4 - no bgp default ipv4-unicast - no bgp ebgp-requires-policy - bgp log-neighbor-changes - neighbor 2001:db8:1:31::1 remote-as 65100 - ! - address-family ipv6 unicast - neighbor 2001:db8:1:31::1 activate - neighbor 2001:db8:1:31::1 allowas-in 1 - neighbor 2001:db8:1:31::1 route-map SET_SRC in - network fc00:0:8::/48 - exit-address-family - ! - - ipv6 forwarding - vtysh.conf: | - service integrated-vtysh-config diff --git a/deploy/containerlab/resources/control/fabric/iad/fabric-lab-patch.yaml b/deploy/containerlab/resources/control/fabric/iad/fabric-lab-patch.yaml new file mode 100644 index 0000000..a9f26fc --- /dev/null +++ b/deploy/containerlab/resources/control/fabric/iad/fabric-lab-patch.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fabric-router +spec: + template: + spec: + # Narrower than config/fabric/'s default (edge or control): this is + # the control-node-only fabric pod (iad-worker-control), which needs + # its own frr.conf distinct from iad's regular per-worker fabric + # (resources/fabric/iad/) — so this overlay sticks to control only, or + # the two would double-schedule onto the same node. + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist + - key: galactic.datumapis.com/node + operator: In + values: + - control + initContainers: + - name: frr-init + image: fabric-router:latest + imagePullPolicy: Never + containers: + - name: frr + image: fabric-router:latest + imagePullPolicy: Never diff --git a/deploy/containerlab/resources/control/fabric/iad/frr.conf b/deploy/containerlab/resources/control/fabric/iad/frr.conf new file mode 100644 index 0000000..a98ae00 --- /dev/null +++ b/deploy/containerlab/resources/control/fabric/iad/frr.conf @@ -0,0 +1,34 @@ +frr defaults traditional +hostname iad-fabric-control +log syslog informational + +interface lo + ! /128 provides a reachable address for galactic-router peers to connect to on port 1790. + ipv6 address fc00:0:8::1/128 +! +interface eth1 + description tr3-facing + ipv6 address 2001:db8:1:31::2/64 +! + +ipv6 route fc00:0:8::/48 Null0 +! +route-map SET_SRC permit 10 + set src fc00:0:8::1 +! +router bgp 65000 + bgp router-id 10.255.255.4 + no bgp default ipv4-unicast + no bgp ebgp-requires-policy + bgp log-neighbor-changes + neighbor 2001:db8:1:31::1 remote-as 65100 + ! + address-family ipv6 unicast + neighbor 2001:db8:1:31::1 activate + neighbor 2001:db8:1:31::1 allowas-in 1 + neighbor 2001:db8:1:31::1 route-map SET_SRC in + network fc00:0:8::/48 + exit-address-family +! + +ipv6 forwarding diff --git a/deploy/containerlab/resources/control/fabric/iad/kustomization.yaml b/deploy/containerlab/resources/control/fabric/iad/kustomization.yaml new file mode 100644 index 0000000..4721361 --- /dev/null +++ b/deploy/containerlab/resources/control/fabric/iad/kustomization.yaml @@ -0,0 +1,28 @@ +namespace: galactic-system +resources: + - fabric +patches: + - path: fabric-lab-patch.yaml + target: + kind: DaemonSet + name: fabric-router + # Strategic-merge patches can't rename a resource (Kustomize keeps the + # target's original identity regardless of what metadata.name the patch + # body says), so the rename to "fabric-control" needs a JSON6902 patch. + # iad is the one cluster where two fabric overlays land: this one (the + # control node) and resources/fabric/iad/ (the regular workers), each + # with its own frr.conf — without a distinct name both would produce a + # DaemonSet named "fabric-router" in the same namespace and silently + # clobber each other. + - target: + kind: DaemonSet + name: fabric-router + patch: |- + - op: replace + path: /metadata/name + value: fabric-control + +configMapGenerator: + - name: fabric-config + files: + - frr.conf diff --git a/deploy/containerlab/resources/fabric/base/daemonset.yaml b/deploy/containerlab/resources/fabric/base/daemonset.yaml deleted file mode 100644 index 7630e01..0000000 --- a/deploy/containerlab/resources/fabric/base/daemonset.yaml +++ /dev/null @@ -1,77 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: fabric - namespace: galactic-system -spec: - selector: - matchLabels: - app.kubernetes.io/name: fabric - template: - metadata: - labels: - app.kubernetes.io/name: fabric - spec: - hostNetwork: true - tolerations: - - key: node.kubernetes.io/not-ready - operator: Exists - effect: NoSchedule - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: DoesNotExist - initContainers: - - name: frr-init - image: quay.io/frrouting/frr:10.2.0 - command: - - sh - - -c - - | - cp /tmp/frr-config/* /etc/frr/ - install -d -o frr -g frr -m 775 /run/frr - install -d -o frr -g frr -m 775 /var/log/frr - securityContext: - capabilities: - add: - - NET_ADMIN - volumeMounts: - - name: frr-config-source - mountPath: /tmp/frr-config - readOnly: true - - name: frr-etc - mountPath: /etc/frr - - name: frr-run - mountPath: /run/frr - - name: frr-log - mountPath: /var/log/frr - containers: - - name: frr - image: quay.io/frrouting/frr:10.2.0 - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - - SYS_ADMIN - volumeMounts: - - name: frr-etc - mountPath: /etc/frr - - name: frr-run - mountPath: /run/frr - - name: frr-log - mountPath: /var/log/frr - volumes: - - name: frr-config-source - configMap: - name: fabric-config - - name: frr-etc - emptyDir: {} - - name: frr-run - emptyDir: - medium: Memory - - name: frr-log - emptyDir: {} diff --git a/deploy/containerlab/resources/fabric/base/fabric-lab-patch.yaml b/deploy/containerlab/resources/fabric/base/fabric-lab-patch.yaml new file mode 100644 index 0000000..cc04a2a --- /dev/null +++ b/deploy/containerlab/resources/fabric/base/fabric-lab-patch.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fabric-router +spec: + template: + spec: + # Narrower than config/fabric/'s default (edge or control): iad's + # control-node fabric pod is deployed separately by + # resources/control/fabric/iad/, with its own frr.conf for that node's + # distinct peering — so this per-site overlay sticks to edge only, or + # the control node would get two competing fabric pods. + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist + - key: galactic.datumapis.com/node + operator: In + values: + - edge + initContainers: + - name: frr-init + image: fabric-router:latest + imagePullPolicy: Never + containers: + - name: frr + image: fabric-router:latest + imagePullPolicy: Never diff --git a/deploy/containerlab/resources/fabric/base/kustomization.yaml b/deploy/containerlab/resources/fabric/base/kustomization.yaml index 66e77f8..110a203 100644 --- a/deploy/containerlab/resources/fabric/base/kustomization.yaml +++ b/deploy/containerlab/resources/fabric/base/kustomization.yaml @@ -1,9 +1,7 @@ resources: - - daemonset.yaml - -configMapGenerator: - - name: fabric-config - files: - - daemons - - vtysh.conf - + - fabric +patches: + - path: fabric-lab-patch.yaml + target: + kind: DaemonSet + name: fabric-router diff --git a/deploy/containerlab/resources/fabric/dfw/daemonset-patch.yaml b/deploy/containerlab/resources/fabric/dfw/daemonset-patch.yaml deleted file mode 100644 index c59e9f5..0000000 --- a/deploy/containerlab/resources/fabric/dfw/daemonset-patch.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: fabric -spec: - template: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: DoesNotExist - - key: galactic.datumapis.com/node - operator: In - values: - - edge diff --git a/deploy/containerlab/resources/fabric/dfw/kustomization.yaml b/deploy/containerlab/resources/fabric/dfw/kustomization.yaml index cabb8fd..73b4e24 100644 --- a/deploy/containerlab/resources/fabric/dfw/kustomization.yaml +++ b/deploy/containerlab/resources/fabric/dfw/kustomization.yaml @@ -1,16 +1,8 @@ -namePrefix: dfw- namespace: galactic-system resources: - ../base -patches: - - path: daemonset-patch.yaml - target: - kind: DaemonSet - name: fabric configMapGenerator: - name: fabric-config - behavior: merge files: - frr.conf - diff --git a/deploy/containerlab/resources/fabric/iad/daemonset-patch.yaml b/deploy/containerlab/resources/fabric/iad/daemonset-patch.yaml deleted file mode 100644 index c59e9f5..0000000 --- a/deploy/containerlab/resources/fabric/iad/daemonset-patch.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: fabric -spec: - template: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: DoesNotExist - - key: galactic.datumapis.com/node - operator: In - values: - - edge diff --git a/deploy/containerlab/resources/fabric/iad/kustomization.yaml b/deploy/containerlab/resources/fabric/iad/kustomization.yaml index 47373d3..73b4e24 100644 --- a/deploy/containerlab/resources/fabric/iad/kustomization.yaml +++ b/deploy/containerlab/resources/fabric/iad/kustomization.yaml @@ -1,16 +1,8 @@ -namePrefix: iad- namespace: galactic-system resources: - ../base -patches: - - path: daemonset-patch.yaml - target: - kind: DaemonSet - name: fabric configMapGenerator: - name: fabric-config - behavior: merge files: - frr.conf - diff --git a/deploy/containerlab/resources/fabric/sjc/daemonset-patch.yaml b/deploy/containerlab/resources/fabric/sjc/daemonset-patch.yaml deleted file mode 100644 index c59e9f5..0000000 --- a/deploy/containerlab/resources/fabric/sjc/daemonset-patch.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: fabric -spec: - template: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: DoesNotExist - - key: galactic.datumapis.com/node - operator: In - values: - - edge diff --git a/deploy/containerlab/resources/fabric/sjc/kustomization.yaml b/deploy/containerlab/resources/fabric/sjc/kustomization.yaml index 74f4743..73b4e24 100644 --- a/deploy/containerlab/resources/fabric/sjc/kustomization.yaml +++ b/deploy/containerlab/resources/fabric/sjc/kustomization.yaml @@ -1,16 +1,8 @@ -namePrefix: sjc- namespace: galactic-system resources: - ../base -patches: - - path: daemonset-patch.yaml - target: - kind: DaemonSet - name: fabric configMapGenerator: - name: fabric-config - behavior: merge files: - frr.conf - diff --git a/deploy/containerlab/scripts/deploy-fabric.sh b/deploy/containerlab/scripts/deploy-fabric.sh index 0df1c86..2c77938 100755 --- a/deploy/containerlab/scripts/deploy-fabric.sh +++ b/deploy/containerlab/scripts/deploy-fabric.sh @@ -6,10 +6,38 @@ set -euo pipefail SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) source "${SCRIPT_DIR}/lib.sh" +# config/fabric/ (shared with production) is a single DaemonSet whose +# affinity allows both the edge and control node labels. resources/fabric/ +# base/ and resources/control/fabric/iad/ each build on a copy of it and +# patch in the lab-only image/imagePullPolicy plus a narrower affinity +# (edge-only / control-only respectively) — iad needs the two split back +# apart because its two nodes need different frr.conf. Copied onto the node +# at deploy time nested under each consuming overlay's own root so its +# "fabric" resource reference resolves (kustomize requires resources in or +# below the overlay root). +FABRIC_DIR=$(cd "${SCRIPT_DIR}/../../../config/fabric" && pwd) + +# copy_fabric_config NODE copies config/fabric/ onto NODE, nested under +# resources/fabric/base/ so the base overlay's "fabric" resource reference +# resolves. +copy_fabric_config() { + local node="$1" + docker cp "${FABRIC_DIR}" "${node}:/galactic/resources/fabric/base/fabric" +} + +# copy_fabric_control_config NODE copies config/fabric/ onto NODE, nested +# under resources/control/fabric/iad/ so that overlay's "fabric" resource +# reference resolves. +copy_fabric_control_config() { + local node="$1" + docker cp "${FABRIC_DIR}" "${node}:/galactic/resources/control/fabric/iad/fabric" +} + # dfw and sjc only need the fabric overlay. for site in dfw sjc; do node=$(control_plane "${site}") copy_to "${node}" fabric + copy_fabric_config "${node}" apply_k "${node}" "/galactic/resources/fabric/${site}/" done @@ -17,13 +45,15 @@ done node=$(control_plane iad) echo "Copying resources to ${node}..." copy_to "${node}" fabric +copy_fabric_config "${node}" copy_to "${node}" control +copy_fabric_control_config "${node}" copy_to "${node}" tenant copy_to "${node}" bgp/tenant /galactic/resources/bgp-tenant/ copy_to "${node}" bgp/control /galactic/resources/bgp-control/ -# iad fabric is a kustomize overlay; control/fabric is raw manifests. +# Both fabric overlays (per-site and iad's control role) are kustomize now. apply_k "${node}" /galactic/resources/fabric/iad/ -apply_f "${node}" /galactic/resources/control/fabric/iad/ +apply_k "${node}" /galactic/resources/control/fabric/iad/ echo "Done."