diff --git a/Taskfile.yaml b/Taskfile.yaml index b1efca4..fad340b 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -27,6 +27,17 @@ vars: # In-cluster-rewritten kubeconfig for the control cluster, mounted into the # replicator (via a Secret) so it can reach control's API as its downstream. CONTROL_INCLUSTER_KUBECONFIG: 'dev/control.incluster.kubeconfig' + # Admin kubeconfig for the Milo core control plane (milo-apiserver), minted + # from the static test-admin-token. Used to install DNS CRDs and create the + # Org + projects on the core CP. Server points at a local port-forward. + MILO_ADMIN_KUBECONFIG: 'dev/milo.admin.kubeconfig' + # dns-operator image used by the replicator on dns-upstream. Loaded into the + # kind node; the overlays reference :latest with imagePullPolicy IfNotPresent. + DNSOP_IMG: 'ghcr.io/datum-cloud/dns-operator:latest' + # Cross-cluster kubeconfig the replicator uses to reach milo-apiserver on + # dns-control, via the envoy gateway NodePort on the control node's IP. + MILO_UPSTREAM_KUBECONFIG: 'dev/milo.upstream.kubeconfig' + MILO_GATEWAY_NODEPORT: '32648' # Dedicated kubeconfig for this environment (not the user's default # ~/.kube/config). go-task's `env:` blocks are implemented as process-wide # os.Setenv calls that leak between sibling task/cmd invocations (a known @@ -317,3 +328,182 @@ tasks: vars: CLUSTER_NAME: "{{.EDGE_CLUSTER_NAME}}" - rm -f {{.RUSTFS_NODEPORT_FILE}} {{.ENV_KUBECONFIG}} {{.CONTROL_INCLUSTER_KUBECONFIG}} + + # ---- Milo control-plane (drift-detection / Phase B) -------------------- + # Installs the Milo apiserver + controller-manager into the control cluster + # via Flux, so the replicator can run discovery.mode=milo and resource-metrics + # can collect project + core control planes. See config/dependencies/milo and + # docs/enhancements/controlplane-drift-detection.md. + env:milo-up: + desc: "Install Milo (apiserver + controller-manager) into the control cluster via Flux" + silent: true + cmds: + - echo "➡️ Installing Milo into kind-{{.CONTROL_CLUSTER_NAME}}..." + - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} apply -k config/dependencies/milo + - echo "⏳ waiting for Flux OCIRepository 'milo' to reconcile..." + - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n flux-system wait ocirepository/milo --for=condition=Ready --timeout=180s + - echo "⏳ waiting for Flux Kustomization 'milo-infra-crds'..." + - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n flux-system wait kustomization/milo-infra-crds --for=condition=Ready --timeout=300s + - echo "⏳ waiting for Flux Kustomization 'milo' (cold start applies CRDs + deploys apiserver, up to 10m)..." + - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n flux-system wait kustomization/milo --for=condition=Ready --timeout=600s + - echo "✅ Milo installed. milo-system objects:" + - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n milo-system get pods,svc + + env:derisk-control-milo: + desc: "De-risk: bring up ONLY the control cluster + Milo (no agent/rustfs) to inspect the live milo-apiserver" + cmds: + - task: env:test-infra-cluster-up + vars: + CLUSTER_NAME: "{{.CONTROL_CLUSTER_NAME}}" + - task: env:milo-up + + # internal: mint an admin kubeconfig for the Milo core CP via a transient + # port-forward, then run CMD (a kubectl invocation with KUBECONFIG=$mk). + # milo-apiserver's NodePort is not host-mapped, and on macOS the host cannot + # reach the kind docker-network IP, so a port-forward is the portable way to + # drive the core CP from the host (macOS dev + Linux CI). + env:with-milo-admin: + internal: true + silent: true + cmds: + - | + set -euo pipefail + export KUBECONFIG={{.ENV_KUBECONFIG}} + kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n milo-system port-forward svc/milo-apiserver 16443:6443 >/tmp/milo-pf.log 2>&1 & + pf=$!; trap 'kill $pf 2>/dev/null || true' EXIT + for i in $(seq 1 30); do + curl -sk -o /dev/null -m 2 https://127.0.0.1:16443/livez && break || sleep 1 + done + mk={{.MILO_ADMIN_KUBECONFIG}} + kubectl config --kubeconfig="$mk" set-cluster milo --server=https://127.0.0.1:16443 --insecure-skip-tls-verify=true >/dev/null + kubectl config --kubeconfig="$mk" set-credentials admin --token=test-admin-token >/dev/null + kubectl config --kubeconfig="$mk" set-context milo --cluster=milo --user=admin >/dev/null + kubectl config --kubeconfig="$mk" use-context milo >/dev/null + KUBECONFIG="$mk" {{.CMD}} + + env:observability-up: + desc: "Install the observability stack (Victoria Metrics + OTel + Prometheus CRDs) into the control cluster" + cmds: + - KUBECONFIG={{.ENV_KUBECONFIG}} TASK_X_REMOTE_TASKFILES=1 task --yes test-infra:install-observability + + env:metrics-up: + desc: "Deploy resource-metrics (mode:milo + collectRootControlPlane) + the dns-metrics policy + drift rules" + vars: + KCTL: KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} + cmds: + # All Flux objects live on the LOCAL (dns-control) cluster. The controller + # Kustomization deploys onto it; the CRD Kustomization targets the Milo + # core CP via kubeConfig. Both reference the same OCIRepository, and both + # come from resource-metrics' published bundle (no vendored manifests, no + # local image build — the pinned bundle ships a multi-arch image). + - "{{.KCTL}} apply -k config/dependencies/resource-metrics/controller" + - "{{.KCTL}} apply -k config/dependencies/resource-metrics/core-control-plane/crd" + - "{{.KCTL}} -n flux-system wait ocirepository/resource-metrics --for=condition=Ready --timeout=120s" + # Installs the ResourceMetricsPolicy CRD onto the Milo core CP from the bundle. + - "{{.KCTL}} -n flux-system wait kustomization/resource-metrics-crd --for=condition=Ready --timeout=180s" + # The dns-metrics policy is dns-operator-owned (not in the bundle), so it's + # applied directly to the core CP after its CRD is Established. + - task: env:with-milo-admin + vars: + CMD: kubectl apply -f config/observability/dns-metrics-policy.yaml + - "{{.KCTL}} -n flux-system wait kustomization/resource-metrics --for=condition=Ready --timeout=300s" + - "{{.KCTL}} -n resource-metrics-system rollout status deploy/resource-metrics-controller-manager --timeout=150s" + - "{{.KCTL}} -n telemetry-system apply -f config/observability/dns-drift-rules.yaml" + - echo "✅ resource-metrics + dns-metrics policy + drift rules deployed" + + env:milo-all-up: + desc: "Full clean bring-up: control+Milo, CRDs+projects, observability, resource-metrics, upstream replicator" + cmds: + - task: env:derisk-control-milo + - task: env:milo-bootstrap + - task: env:observability-up + - task: env:metrics-up + - task: env:upstream-milo-up + + env:chainsaw-milo: + desc: "Run the control-plane drift Chainsaw suite (happy-path/orphan/missing) against the running Milo env" + cmds: + - | + set -euo pipefail + root="$(pwd)" + cd test/e2e/controlplane-drift + # infra = dns-control kind (VM + OTel); replicator = dns-upstream kind. + # Both kind APIs are host-reachable. + kind get kubeconfig --name {{.CONTROL_CLUSTER_NAME}} > kubeconfig-infra + kind get kubeconfig --name {{.UPSTREAM_CLUSTER_NAME}} > kubeconfig-replicator + # alpha (project CP) + core (Milo core CP): reach milo-apiserver via a + # localhost port-forward — portable across macOS + Linux, since the + # gateway NodePort is not host-mapped and macOS can't reach the kind + # docker-network IP directly. + KUBECONFIG="$root/{{.ENV_KUBECONFIG}}" kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} \ + -n milo-system port-forward svc/milo-apiserver 16443:6443 >/tmp/milo-pf-chainsaw.log 2>&1 & + pf=$!; trap 'kill $pf 2>/dev/null || true' EXIT + for i in $(seq 1 30); do curl -sk -o /dev/null -m2 https://127.0.0.1:16443/livez && break || sleep 1; done + for name in core alpha; do + kubectl config --kubeconfig=kubeconfig-$name set-credentials a --token=test-admin-token >/dev/null + kubectl config --kubeconfig=kubeconfig-$name set-context $name --cluster=$name --user=a >/dev/null + kubectl config --kubeconfig=kubeconfig-$name use-context $name >/dev/null + done + kubectl config --kubeconfig=kubeconfig-core set-cluster core \ + --server=https://127.0.0.1:16443 --insecure-skip-tls-verify=true >/dev/null + kubectl config --kubeconfig=kubeconfig-alpha set-cluster alpha \ + --server=https://127.0.0.1:16443/apis/resourcemanager.miloapis.com/v1alpha1/projects/alpha/control-plane \ + --insecure-skip-tls-verify=true >/dev/null + KUBECONFIG=kubeconfig-infra chainsaw test --config .chainsaw.yaml {{.CHAINSAW_SUITE | default "."}} + + env:upstream-milo-up: + desc: "Create dns-upstream and deploy the replicator (discovery.mode=milo, 2 replicas) pointed at Milo on dns-control" + cmds: + - task: env:test-infra-cluster-up + vars: + CLUSTER_NAME: "{{.UPSTREAM_CLUSTER_NAME}}" + - kind load docker-image {{.DNSOP_IMG}} --name {{.UPSTREAM_CLUSTER_NAME}} + # Mint the cross-cluster milo kubeconfig (control node IP + gateway + # NodePort) and stash it as the milo-kubeconfig Secret the replicator-milo + # overlay mounts at /milo. NODE_IP via python to avoid go-task/docker + # template brace conflicts. + - | + set -euo pipefail + export KUBECONFIG={{.ENV_KUBECONFIG}} + NODE_IP=$(docker inspect {{.CONTROL_CLUSTER_NAME}}-control-plane | python3 -c "import json,sys; n=json.load(sys.stdin)[0]['NetworkSettings']['Networks']; print(next(iter(n.values()))['IPAddress'])") + # Derive the envoy gateway NodePort dynamically — test-infra's + # fix/remove-hardcoded-nodeports branch assigns it per-cluster, so it is + # NOT stable across rebuilds. Look up the nodePort mapped to gateway + # port 8443 (the HTTPS listener that fronts milo-apiserver). + NODEPORT=$(kubectl --context kind-{{.CONTROL_CLUSTER_NAME}} -n envoy-gateway-system get svc \ + -o jsonpath='{range .items[*]}{range .spec.ports[?(@.port==8443)]}{.nodePort}{end}{end}') + test -n "$NODEPORT" || { echo "could not resolve envoy gateway 8443 NodePort" >&2; exit 1; } + echo "➡️ milo gateway endpoint for replicator: https://${NODE_IP}:${NODEPORT}" + mk={{.MILO_UPSTREAM_KUBECONFIG}} + kubectl config --kubeconfig="$mk" set-cluster milo --server="https://${NODE_IP}:${NODEPORT}" --insecure-skip-tls-verify=true >/dev/null + kubectl config --kubeconfig="$mk" set-credentials admin --token=test-admin-token >/dev/null + kubectl config --kubeconfig="$mk" set-context milo --cluster=milo --user=admin >/dev/null + kubectl config --kubeconfig="$mk" use-context milo >/dev/null + kubectl --context kind-{{.UPSTREAM_CLUSTER_NAME}} create ns dns-replicator-system --dry-run=client -o yaml | kubectl --context kind-{{.UPSTREAM_CLUSTER_NAME}} apply -f - + kubectl --context kind-{{.UPSTREAM_CLUSTER_NAME}} -n dns-replicator-system create secret generic milo-kubeconfig --from-file=kubeconfig="$mk" --dry-run=client -o yaml | kubectl --context kind-{{.UPSTREAM_CLUSTER_NAME}} apply -f - + - KUBECONFIG={{.ENV_KUBECONFIG}} CONTEXT=kind-{{.UPSTREAM_CLUSTER_NAME}} KUSTOMIZE_DIR=config/overlays/replicator-milo make kustomize-apply + - KUBECONFIG={{.ENV_KUBECONFIG}} kubectl --context kind-{{.UPSTREAM_CLUSTER_NAME}} -n dns-replicator-system rollout status deploy/dns-operator-controller-manager --timeout=180s + - echo "✅ replicator up on kind-{{.UPSTREAM_CLUSTER_NAME}} (mode=milo, 2 replicas)" + + env:milo-bootstrap: + desc: "Install DNS + networking CRDs into the Milo core CP and create the drift Org + projects (alpha, beta)" + cmds: + - task: env:with-milo-admin + vars: + CMD: kubectl apply -k config/crd + # The replicator (dnszone/dnszonediscovery controllers) indexes Domain + # (networking.datumapis.com); without its CRD, project-CP engagement fails + # on cache-index setup. Generate the networking CRDs and install them into + # the core CP (shared to all project CPs). The make target also applies to + # the local kind API (harmless) and writes dev/crds/network-services. + - KUBECONFIG={{.ENV_KUBECONFIG}} make install-networking-crds CONTEXT=kind-{{.CONTROL_CLUSTER_NAME}} + - task: env:with-milo-admin + vars: + CMD: kubectl apply -f dev/crds/network-services + - task: env:with-milo-admin + vars: + CMD: kubectl apply -f test/e2e/controlplane-drift/fixtures/milo-projects.yaml + - task: env:with-milo-admin + vars: + CMD: kubectl wait --for=condition=Ready project/alpha project/beta --timeout=120s + - echo "✅ Milo core CP has DNS + networking CRDs; projects alpha, beta are Ready" diff --git a/config/dependencies/milo/flux-install-infra-crds.yaml b/config/dependencies/milo/flux-install-infra-crds.yaml new file mode 100644 index 0000000..d573c49 --- /dev/null +++ b/config/dependencies/milo/flux-install-infra-crds.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: milo-infra-crds + namespace: flux-system +# milo's overlays/test-infra does NOT install the ProjectControlPlane CRD, but +# milo-controller-manager runs with --control-plane-scope=core and watches +# ProjectControlPlane. Without this CRD, the project reconciler loops on +# "no matches for kind ProjectControlPlane", Projects never go Ready, and the +# drift chainsaw suite times out. Install the CRD separately from the same OCI +# bundle so it tracks the pinned milo tag. Remove once milo's test-infra +# overlay ships the infrastructure-group CRDs. +spec: + interval: 10m + retryInterval: 1m + timeout: 2m + prune: true + wait: true + sourceRef: + kind: OCIRepository + name: milo + # The published bundle flattens the config tree: milo's + # config/crd/bases/infrastructure is at crd/bases/infrastructure here. + path: crd/bases/infrastructure + dependsOn: [] diff --git a/config/dependencies/milo/flux-install.yaml b/config/dependencies/milo/flux-install.yaml new file mode 100644 index 0000000..19799c2 --- /dev/null +++ b/config/dependencies/milo/flux-install.yaml @@ -0,0 +1,54 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: milo + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 5m + prune: true + wait: true + targetNamespace: milo-system + # ProjectControlPlane CRD (from milo-infra-crds) must exist before the milo + # overlay starts, or milo-controller-manager crash-loops on the missing CRD. + dependsOn: + - name: milo-infra-crds + sourceRef: + kind: OCIRepository + name: milo + path: overlays/test-infra + # The upstream test-infra overlay assumes locally kind-loaded `dev` images; + # CI pulls from the registry, so override the tag to the OCI bundle pin. + # Keep in sync with ocirepository.yaml. + images: + - name: ghcr.io/datum-cloud/milo + newTag: v0.0.0-main + # Strip argo-system (argo-events HelmRelease + argo HelmRepository): the DNS + # drift e2e does not exercise event-driven flows, and argo-events pulls in a + # NATS JetStream dependency that bloats spin-up. Remove via $patch: delete. + patches: + - target: + group: helm.toolkit.fluxcd.io + version: v2 + kind: HelmRelease + name: argo-events + patch: | + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: argo-events + namespace: milo-system + $patch: delete + - target: + group: source.toolkit.fluxcd.io + version: v1 + kind: HelmRepository + name: argo + patch: | + apiVersion: source.toolkit.fluxcd.io/v1 + kind: HelmRepository + metadata: + name: argo + namespace: milo-system + $patch: delete diff --git a/config/dependencies/milo/kustomization.yaml b/config/dependencies/milo/kustomization.yaml new file mode 100644 index 0000000..c0f6171 --- /dev/null +++ b/config/dependencies/milo/kustomization.yaml @@ -0,0 +1,14 @@ +# Deploys the Milo apiserver + controller-manager into the control cluster via +# Flux, so the DNS drift e2e can run the replicator and resource-metrics in the +# production `discovery.mode: milo` topology (project control planes for the +# upstream side, the Milo core control plane as the downstream). Lifted from +# milo-os/resource-metrics config/dependencies/milo. +# +# See docs/enhancements/controlplane-drift-detection.md. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - ocirepository.yaml + - flux-install-infra-crds.yaml + - flux-install.yaml diff --git a/config/dependencies/milo/namespace.yaml b/config/dependencies/milo/namespace.yaml new file mode 100644 index 0000000..ac4a1e9 --- /dev/null +++ b/config/dependencies/milo/namespace.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: milo-system + labels: + app.kubernetes.io/name: milo + app.kubernetes.io/managed-by: kustomize +# milo's overlays/test-infra targets milo-system (via spec.namespace). Flux does +# not auto-create a Kustomization's target namespace, so pre-create it here +# alongside the OCIRepository + Flux Kustomizations. diff --git a/config/dependencies/milo/ocirepository.yaml b/config/dependencies/milo/ocirepository.yaml new file mode 100644 index 0000000..c452789 --- /dev/null +++ b/config/dependencies/milo/ocirepository.yaml @@ -0,0 +1,16 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: OCIRepository +metadata: + name: milo + namespace: flux-system +spec: + interval: 5m + url: oci://ghcr.io/datum-cloud/milo-kustomize + # milo publishes this bundle from milo/.github/workflows/build-apiserver.yaml. + # Tags: semver releases (v0.24.3) and per-merge v0.0.0- / + # v0.0.0-main-. Pin to a reproducible tag; keep in sync with the + # image newTag override in flux-install.yaml. Tracking the v0.0.0-main + # floating tag for now because the latest semver predates the multi-arch + # publish and ImagePullBackOffs on arm64 kind clusters. Bump deliberately. + ref: + tag: v0.0.0-main diff --git a/config/dependencies/resource-metrics/README.md b/config/dependencies/resource-metrics/README.md new file mode 100644 index 0000000..4d50e45 --- /dev/null +++ b/config/dependencies/resource-metrics/README.md @@ -0,0 +1,39 @@ +# resource-metrics (drift-detection dependency) + +Deploys the [resource-metrics](https://github.com/milo-os/resource-metrics) +controller that powers DNS control-plane drift detection: it emits a metric per +`DNSRecordSet`/`DNSZone` from every control plane so the alert rules can spot +records that have fallen out of sync between a customer's project and the +serving control plane. See `docs/enhancements/controlplane-drift-detection.md` +for the feature. + +You don't normally apply this by hand — `task env:metrics-up` does it while +bringing up the e2e. The rest of this file is for changing or debugging it. + +## What gets deployed, and where + +resource-metrics comes from its own published release (a Flux-managed kustomize +bundle), not copied into this repo, so it tracks upstream. It's split in two +because the pieces live on different clusters: + +- **`controller/`** — the controller, on the local kind cluster. This is the + upstream `overlays/test-infra` bundle with our image tag and OTel endpoint + patched in. +- **`core-control-plane/`** — its CRD and the `dns-metrics` policy, on the Milo + core control plane (where the controller reads them). The CRD comes from the + same bundle; the policy is ours and points back to + `config/observability/dns-metrics-policy.yaml`. + +## Changing it + +- **Image tag / OTel endpoint** — `controller/flux-install.yaml`. +- **Bundle version** — pinned in `controller/ocirepository.yaml` (and reused by + `core-control-plane/crd/`). + +> [!NOTE] +> The OTel endpoint is overridden with a full-ConfigMap patch because the pinned +> bundle hardcodes it. After resource-metrics +> [#14](https://github.com/milo-os/resource-metrics/pull/14) (configurable +> endpoint) and [#13](https://github.com/milo-os/resource-metrics/pull/13) +> (multi-arch image) merge, bump the pin to a `v0.0.0-main` tag and swap the +> patch for an `OTEL_EXPORTER_OTLP_ENDPOINT` env patch. diff --git a/config/dependencies/resource-metrics/controller/flux-install.yaml b/config/dependencies/resource-metrics/controller/flux-install.yaml new file mode 100644 index 0000000..1193e17 --- /dev/null +++ b/config/dependencies/resource-metrics/controller/flux-install.yaml @@ -0,0 +1,51 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: resource-metrics + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 5m + prune: true + wait: true + sourceRef: + kind: OCIRepository + name: resource-metrics + # resource-metrics' published overlays/test-infra already does what we need: + # discovery.mode=milo, collectRootControlPlane=true, the token-based + # milo-kubeconfig Secret, and the manager Deployment wiring. We only override + # the controller image (to the multi-arch tag) and the OTLP endpoint (our + # observability stack's collector lives in telemetry-system, not the overlay's + # default otel-collector-system). + path: overlays/test-infra + images: + - name: ghcr.io/milo-os/resource-metrics + newTag: v0.0.0-feat-multi-arch-arm64-image + patches: + # Point the operator at our OTel collector. Once milo-os/resource-metrics#14 + # lands, replace this whole ConfigMap patch with a Deployment env patch: + # env: [{name: OTEL_EXPORTER_OTLP_ENDPOINT, value: otel-collector-collector.telemetry-system:4317}] + - target: + kind: ConfigMap + name: resource-metrics-service-config + patch: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: resource-metrics-service-config + data: + server.yaml: | + apiVersion: apiserver.config.miloapis.com/v1alpha1 + kind: ResourceMetricsOperator + discovery: + mode: milo + internalServiceDiscovery: false + discoveryKubeconfigPath: /etc/milo/kubeconfig + projectKubeconfigPath: /etc/milo/kubeconfig + collectRootControlPlane: true + otel: + endpoint: otel-collector-collector.telemetry-system:4317 + insecure: true + collectionInterval: 5s + defaultMetricPrefix: "" diff --git a/config/dependencies/resource-metrics/controller/kustomization.yaml b/config/dependencies/resource-metrics/controller/kustomization.yaml new file mode 100644 index 0000000..1e66e3a --- /dev/null +++ b/config/dependencies/resource-metrics/controller/kustomization.yaml @@ -0,0 +1,9 @@ +# Deploys the resource-metrics controller by referencing its published kustomize +# bundle (overlays/test-infra) via Flux, instead of vendoring its manifests. +# Mirrors config/dependencies/milo. The controller runs on the control cluster +# and reads ResourceMetricsPolicy from the Milo core CP (see ../core-control-plane). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ocirepository.yaml + - flux-install.yaml diff --git a/config/dependencies/resource-metrics/controller/ocirepository.yaml b/config/dependencies/resource-metrics/controller/ocirepository.yaml new file mode 100644 index 0000000..6f1bc0e --- /dev/null +++ b/config/dependencies/resource-metrics/controller/ocirepository.yaml @@ -0,0 +1,14 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: OCIRepository +metadata: + name: resource-metrics + namespace: flux-system +spec: + interval: 5m + url: oci://ghcr.io/milo-os/resource-metrics-kustomize + # Pinned to a branch bundle that ships a multi-arch (amd64+arm64) controller + # image (milo-os/resource-metrics#13). Bump to a v0.0.0-main tag once #13 and + # #14 (configurable OTLP endpoint) merge — at which point the endpoint patch + # below collapses to an OTEL_EXPORTER_OTLP_ENDPOINT env patch. + ref: + tag: v0.0.0-feat-multi-arch-arm64-image diff --git a/config/dependencies/resource-metrics/core-control-plane/crd/flux-install.yaml b/config/dependencies/resource-metrics/core-control-plane/crd/flux-install.yaml new file mode 100644 index 0000000..95f30a8 --- /dev/null +++ b/config/dependencies/resource-metrics/core-control-plane/crd/flux-install.yaml @@ -0,0 +1,23 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: resource-metrics-crd + namespace: flux-system +# Installs the ResourceMetricsPolicy CRD onto the Milo CORE control plane +# straight from the operator's published bundle (path crd) — no vendored copy. +# Mirrors infra apps/resource-metrics-system/base/milo-control-plane.yaml. +# Targets the core CP via kubeConfig (a different API server than the local +# cluster this Kustomization object lives on). +spec: + interval: 10m + retryInterval: 1m + timeout: 2m + prune: true + wait: true + sourceRef: + kind: OCIRepository + name: resource-metrics # shared with controller/ocirepository.yaml + path: crd + kubeConfig: + secretRef: + name: resource-metrics-milo-kubeconfig diff --git a/config/dependencies/resource-metrics/core-control-plane/crd/kustomization.yaml b/config/dependencies/resource-metrics/core-control-plane/crd/kustomization.yaml new file mode 100644 index 0000000..47a7783 --- /dev/null +++ b/config/dependencies/resource-metrics/core-control-plane/crd/kustomization.yaml @@ -0,0 +1,8 @@ +# Installs the ResourceMetricsPolicy CRD onto the Milo core control plane from +# the published resource-metrics bundle via Flux (see flux-install.yaml). The +# milo-kubeconfig Secret it targets the core CP with is co-located here. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - milo-kubeconfig-secret.yaml + - flux-install.yaml diff --git a/config/dependencies/resource-metrics/core-control-plane/crd/milo-kubeconfig-secret.yaml b/config/dependencies/resource-metrics/core-control-plane/crd/milo-kubeconfig-secret.yaml new file mode 100644 index 0000000..50b284a --- /dev/null +++ b/config/dependencies/resource-metrics/core-control-plane/crd/milo-kubeconfig-secret.yaml @@ -0,0 +1,31 @@ +# Kubeconfig Flux uses to install resource-metrics CRDs onto the Milo core +# control plane (a different API server than the local kind cluster). Flux +# resolves kubeConfig.secretRef in the Kustomization's namespace (flux-system) +# and reads the kubeconfig from the `value` key. In-cluster milo-apiserver + +# the test-only static test-admin-token — mirrors infra's milo-configuration- +# kubeconfig, which uses a cert-based kubeconfig in production. +apiVersion: v1 +kind: Secret +metadata: + name: resource-metrics-milo-kubeconfig + namespace: flux-system +type: Opaque +stringData: + value: | + apiVersion: v1 + kind: Config + clusters: + - name: milo + cluster: + server: https://milo-apiserver.milo-system.svc.cluster.local:6443 + insecure-skip-tls-verify: true + users: + - name: milo-admin + user: + token: test-admin-token + contexts: + - name: milo + context: + cluster: milo + user: milo-admin + current-context: milo diff --git a/config/dependencies/resource-metrics/core-control-plane/policy/kustomization.yaml b/config/dependencies/resource-metrics/core-control-plane/policy/kustomization.yaml new file mode 100644 index 0000000..4fc681b --- /dev/null +++ b/config/dependencies/resource-metrics/core-control-plane/policy/kustomization.yaml @@ -0,0 +1,18 @@ +# Applies the dns-metrics ResourceMetricsPolicy onto the Milo core control +# plane. The policy itself is authored (and owned) elsewhere at +# config/observability/dns-metrics-policy.yaml — it is referenced here, not +# duplicated, so there is a single source of truth. +# +# TARGET: milo-apiserver (the core/downstream CP) — NOT the kind apiserver. +# Apply with a milo kubeconfig (see ../../README.md), AFTER the CRD in ../crd +# is Established. resource-metrics watches this policy on the core CP and, with +# discovery.collectRootControlPlane=true, emits the upstream/downstream series +# the drift recording rules diff. +# +# Referencing a path outside this kustomization root requires +# `--load-restrictor LoadRestrictionsNone` at build time. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../../observability/dns-metrics-policy.yaml diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 58967d9..e1290b2 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -62,7 +62,10 @@ spec: - /manager args: - --role=replicator - - --leader-elect + # Value comes from the LEADER_ELECT env var via Kubernetes' + # $(VAR) substitution, so overlays toggle leader election by patching + # that one env var instead of overriding this whole args list. + - --leader-elect=$(LEADER_ELECT) - --health-probe-bind-address=:8081 - --server-config=/config/server-config.yaml image: ghcr.io/datum-cloud/dns-operator:latest @@ -75,6 +78,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + # Substituted into --leader-elect=$(LEADER_ELECT) above. Overlays patch + # this env var (env lists merge by name) rather than restating args. + - name: LEADER_ELECT + value: "true" ports: [] securityContext: readOnlyRootFilesystem: true diff --git a/config/observability/dns-drift-rules.yaml b/config/observability/dns-drift-rules.yaml new file mode 100644 index 0000000..8e74721 --- /dev/null +++ b/config/observability/dns-drift-rules.yaml @@ -0,0 +1,103 @@ +# Recording + alerting rules that diff the dns-metrics series (dns-metrics-policy.yaml) +# to detect drift between upstream (project) and downstream (root) control planes. +# See docs/enhancements/controlplane-drift-detection.md. +# +# The policy runs on every control plane, so both generators emit everywhere. +# The rules partition on milo_control_plane_type ("root" vs "project") to keep +# each side's stray series out of the diff, and normalize both sides to a bare +# `proj` label to join on (proj, upstream_namespace, upstream_name). +# +# This is a standard PrometheusRule: victoria-metrics-operator converts it to a +# VMRule and preserves labels, so the resource-metrics-aggregator label below +# routes it to the vmalert that has the resource-metrics datasource. +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: dns-controlplane-drift + labels: + managed-by: flux + app.kubernetes.io/part-of: dns-operator + # Selector for the resource-metrics aggregator vmalert; presence is what + # matters, not the value. + telemetry.miloapis.com/resource-metrics-aggregator: "true" +spec: + groups: + - name: dns.drift.recording + interval: 30s + rules: + # Upstream project CPs, milo_project_name copied to the bare `proj` join + # key. Downstream's cluster label is normalized to match below: the + # annotation is "cluster-_" (the replicator maps the milo key + # "/alpha" to "_alpha"), verified live against milo_project_name="alpha". + - record: dns:recordset_upstream:normalized + expr: | + label_replace( + dns_recordset_upstream_info{milo_control_plane_type="project"}, + "proj", "$1", "milo_project_name", "(.*)" + ) + + # Downstream root CP records, cluster label stripped of "cluster-_" to + # recover the bare `proj`. + - record: dns:recordset_downstream:scoped + expr: | + label_replace( + dns_recordset_downstream_info{milo_control_plane_type="root"}, + "proj", "$1", "upstream_cluster", "cluster-_?(.*)" + ) + + # Orphan: a downstream record with no upstream owner — the engineering#346 + # leftover that keeps "reserving" a name after its owner is deleted. + - record: dns:recordset_downstream_orphan + expr: | + dns:recordset_downstream:scoped + unless on(proj, upstream_namespace, upstream_name) + dns:recordset_upstream:normalized + + # Missing: an upstream record with no downstream copy — replication stalled. + - record: dns:recordset_downstream_missing + expr: | + dns:recordset_upstream:normalized + unless on(proj, upstream_namespace, upstream_name) + dns:recordset_downstream:scoped + + - name: dns.drift.alerts + rules: + - alert: DNSDownstreamOrphanRecordSet + # for: rides out normal replication/staleness lag, so only persistent + # drift pages. + expr: dns:recordset_downstream_orphan > 0 + for: 10m + labels: + severity: warning + annotations: + summary: "Orphaned downstream DNSRecordSet (no upstream owner)" + description: >- + DNSRecordSet {{ $labels.upstream_name }} in namespace + {{ $labels.upstream_namespace }} (cluster {{ $labels.upstream_cluster }}) + exists on the downstream control plane with no matching upstream + owner. This is the failure mode from engineering#346 and can block + new records for the same name. Runbook: docs/enhancements/controlplane-drift-detection.md + + - alert: DNSDownstreamMissingRecordSet + expr: dns:recordset_downstream_missing > 0 + for: 10m + labels: + severity: warning + annotations: + summary: "Upstream DNSRecordSet not replicated downstream" + description: >- + DNSRecordSet {{ $labels.upstream_name }} in namespace + {{ $labels.upstream_namespace }} (cluster {{ $labels.upstream_cluster }}) + exists upstream but has no downstream copy — replication may be + stalled or a replica may not own this cluster. + + - alert: DNSRecordSetNotAccepted + expr: dns_recordset_upstream_info{accepted="false", milo_control_plane_type="project"} == 1 + for: 15m + labels: + severity: warning + annotations: + summary: "DNSRecordSet stuck not-Accepted for 15m" + description: >- + DNSRecordSet {{ $labels.upstream_name }} in namespace + {{ $labels.upstream_namespace }} has not reached Accepted=True. diff --git a/config/observability/dns-metrics-policy.yaml b/config/observability/dns-metrics-policy.yaml new file mode 100644 index 0000000..3c8cf00 --- /dev/null +++ b/config/observability/dns-metrics-policy.yaml @@ -0,0 +1,94 @@ +# ResourceMetricsPolicy for control-plane DNS drift detection. resource-metrics +# emits one gauge per DNSRecordSet/DNSZone from each control plane; recording +# rules diff the two sides. See docs/enhancements/controlplane-drift-detection.md. +# +# Upstream generators run on project control planes (desired state). Downstream +# generators run on the root control plane (replicated copies) and lift the +# replicator's meta.datumapis.com/upstream-* annotations onto labels, so both +# sides share the join key (upstream_cluster, upstream_namespace, upstream_name). +apiVersion: resourcemetrics.miloapis.com/v1alpha1 +kind: ResourceMetricsPolicy +metadata: + name: dns-metrics +spec: + generators: + # ---- Upstream (project control planes): desired state ----------------- + - name: dnsrecordset-upstream-info + resource: + group: dns.networking.miloapis.com + version: v1alpha1 + resource: dnsrecordsets + families: + - name: dns_recordset_upstream_info + help: "One series per DNSRecordSet on a source (upstream) control plane." + type: gauge + metrics: + - value: "1.0" + labels: + - name: upstream_namespace + value: "object.metadata.namespace" + - name: upstream_name + value: "object.metadata.name" + - name: accepted + # has() guards a record that has no status yet: reading a + # missing status.conditions is a CEL error, which drops the series. + value: "has(object.status) && has(object.status.conditions) && object.status.conditions.exists(c, c.type == 'Accepted' && c.status == 'True') ? 'true' : 'false'" + + - name: dnszone-upstream-info + resource: + group: dns.networking.miloapis.com + version: v1alpha1 + resource: dnszones + families: + - name: dns_zone_upstream_info + help: "One series per DNSZone on a source (upstream) control plane." + type: gauge + metrics: + - value: "1.0" + labels: + - name: upstream_namespace + value: "object.metadata.namespace" + - name: upstream_name + value: "object.metadata.name" + + # ---- Downstream (root control plane): replicated / actual state ------- + # Downstream namespaces are remapped, so the join labels come from the + # replicator's upstream-* annotations, not from metadata. The cluster-name + # annotation is prefixed "cluster-"; the recording rules normalize it. + - name: dnsrecordset-downstream-info + resource: + group: dns.networking.miloapis.com + version: v1alpha1 + resource: dnsrecordsets + families: + - name: dns_recordset_downstream_info + help: "One series per replicated DNSRecordSet on the downstream (root) control plane." + type: gauge + metrics: + - value: "1.0" + labels: + - name: upstream_cluster + value: "object.metadata.annotations['meta.datumapis.com/upstream-cluster-name']" + - name: upstream_namespace + value: "object.metadata.annotations['meta.datumapis.com/upstream-namespace']" + - name: upstream_name + value: "object.metadata.annotations['meta.datumapis.com/upstream-name']" + + - name: dnszone-downstream-info + resource: + group: dns.networking.miloapis.com + version: v1alpha1 + resource: dnszones + families: + - name: dns_zone_downstream_info + help: "One series per replicated DNSZone on the downstream (root) control plane." + type: gauge + metrics: + - value: "1.0" + labels: + - name: upstream_cluster + value: "object.metadata.annotations['meta.datumapis.com/upstream-cluster-name']" + - name: upstream_namespace + value: "object.metadata.annotations['meta.datumapis.com/upstream-namespace']" + - name: upstream_name + value: "object.metadata.annotations['meta.datumapis.com/upstream-name']" diff --git a/config/observability/kustomization.yaml b/config/observability/kustomization.yaml new file mode 100644 index 0000000..23eb333 --- /dev/null +++ b/config/observability/kustomization.yaml @@ -0,0 +1,21 @@ +# Observability configuration for detecting upstream/downstream control-plane +# drift in the DNS service. +# +# dns-metrics-policy.yaml ResourceMetricsPolicy (milo-os/resource-metrics) +# dns-drift-rules.yaml recording + alerting rules (PrometheusRule) +# +# These are platform/infra objects. Per datum-cloud/infra conventions: +# - The ResourceMetricsPolicy is applied to the Milo core control plane (where +# the resource-metrics CRDs are installed via milo-configuration-kubeconfig), +# NOT a plain cluster. resource-metrics must run with +# discovery.collectRootControlPlane: true (NOT yet enabled in infra) for the +# downstream/core-CP series this depends on. +# - The PrometheusRule (auto-converted to VMRule by victoria-metrics-operator) +# is picked up by the resource-metrics aggregator vmalert via its label. +# Kept in this repo co-located with the operator they observe; deployment wiring +# lives in the infra repo (or the e2e overlay). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - dns-metrics-policy.yaml + - dns-drift-rules.yaml diff --git a/config/overlays/agent-powerdns-milo/README.md b/config/overlays/agent-powerdns-milo/README.md new file mode 100644 index 0000000..f1309bd --- /dev/null +++ b/config/overlays/agent-powerdns-milo/README.md @@ -0,0 +1,39 @@ +# agent-powerdns-milo + +PowerDNS agent overlay that points the DNS agent at the **Milo core control +plane** (`milo-apiserver`, root scope) as its read and status-write source, for +the drift-detection e2e. Here the DNS CRDs and `DNSRecordSet`s live on the core +CP, not the local kind API, so the agent (`--role=downstream`) must read them +from there. + +## What changed vs. `agent-powerdns-federated` + +Bases on `../agent-powerdns-federated`, leaving the PowerDNS, Lightningstream, +and RustFS wiring intact. It only retargets the agent's API server: + +1. Replaces the `agent-server-config` ConfigMap with a milo-targeted `server-config.yaml`. +2. Mounts a `milo-kubeconfig` Secret at `/milo` on the `pdns-auth` StatefulSet's `manager` container and sets `KUBECONFIG=/milo/kubeconfig` (`deployment-patch.yaml`). +3. Ships that Secret (`milo-kubeconfig-secret.yaml`) — in-cluster `milo-apiserver` + the test-only `test-admin-token`. + +`disableNameSuffixHash: true` is kept so the ConfigMap replace keeps the +StatefulSet's existing volume reference valid. + +## How the retarget works + +> [!IMPORTANT] +> For `--role=downstream`, the read source is set by the `KUBECONFIG` env var, +> **not** a server-config field. + +The downstream branch in `cmd/main.go` builds its manager from +`ctrl.GetConfigOrDie()`; it never reads `discovery.*` or +`downstreamResourceManagement.kubeconfigPath` (only the replicator branch does). +`ctrl.GetConfig()` honors `KUBECONFIG` before the in-cluster config, and the +binary registers no `--kubeconfig` flag, so both the reads and the status writes +resolve to the mounted core-CP kubeconfig. The milo values in `server-config.yaml` +are inert under `--role=downstream` but stay correct if run as `--role=replicator`. + +## Validate + +```sh +kustomize build --load-restrictor LoadRestrictionsNone config/overlays/agent-powerdns-milo +``` diff --git a/config/overlays/agent-powerdns-milo/deployment-patch.yaml b/config/overlays/agent-powerdns-milo/deployment-patch.yaml new file mode 100644 index 0000000..db4e060 --- /dev/null +++ b/config/overlays/agent-powerdns-milo/deployment-patch.yaml @@ -0,0 +1,29 @@ +# Retarget the PowerDNS agent's read/write source at the Milo core control plane. +# +# For --role=downstream (cmd/main.go), the manager and all controllers are built +# from ctrl.GetConfigOrDie() + mgr.GetClient(); the server-config discovery/ +# downstream kubeconfig fields are NOT consulted in that branch. controller- +# runtime's ctrl.GetConfig() honors the KUBECONFIG env var before falling back to +# in-cluster config (no --kubeconfig flag is registered), so pointing KUBECONFIG +# at the mounted milo kubeconfig makes both the DNSRecordSet/DNSZone reads and the +# status writes resolve to the Milo core CP instead of the local kind API. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: pdns-auth +spec: + template: + spec: + containers: + - name: manager + env: + - name: KUBECONFIG + value: /milo/kubeconfig + volumeMounts: + - name: milo-kubeconfig + mountPath: /milo + readOnly: true + volumes: + - name: milo-kubeconfig + secret: + secretName: milo-kubeconfig diff --git a/config/overlays/agent-powerdns-milo/kustomization.yaml b/config/overlays/agent-powerdns-milo/kustomization.yaml new file mode 100644 index 0000000..4a30649 --- /dev/null +++ b/config/overlays/agent-powerdns-milo/kustomization.yaml @@ -0,0 +1,42 @@ +# PowerDNS agent overlay that points the agent at the Milo core control plane as +# its DNS read (and status-write) source, for the DNS drift-detection e2e. +# +# Bases on ../agent-powerdns-federated (PR #60: PowerDNS agent + Lightningstream +# + RustFS/S3 sync, deployed on the control cluster) and changes ONLY the agent's +# API target — the PowerDNS, Lightningstream, and s3-credentials wiring are left +# intact and inherited unchanged. +# +# What this overlay adds vs. agent-powerdns-federated: +# 1. Replaces the agent server-config (configMapGenerator behavior:replace) with +# a milo-targeted one. +# 2. Mounts a milo-kubeconfig Secret at /milo on the manager container and sets +# KUBECONFIG=/milo/kubeconfig (deployment-patch.yaml) — this is what actually +# retargets the downstream agent's read/write to the Milo core CP (see the +# note in server-config.yaml / deployment-patch.yaml). +# 3. Ships the milo-kubeconfig Secret (in-cluster milo-apiserver + test-admin-token). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: dns-agent-system + +resources: + - ../agent-powerdns-federated + - milo-kubeconfig-secret.yaml + +# Keep the deterministic name (base disables the name-suffix hash so the +# StatefulSet's `agent-server-config` volume ref stays valid). +generatorOptions: + disableNameSuffixHash: true + +# Replace the base agent server-config with the milo-targeted one. +configMapGenerator: + - name: agent-server-config + behavior: replace + files: + - server-config.yaml + +patches: + - path: deployment-patch.yaml + target: + kind: StatefulSet + name: pdns-auth diff --git a/config/overlays/agent-powerdns-milo/milo-kubeconfig-secret.yaml b/config/overlays/agent-powerdns-milo/milo-kubeconfig-secret.yaml new file mode 100644 index 0000000..404c023 --- /dev/null +++ b/config/overlays/agent-powerdns-milo/milo-kubeconfig-secret.yaml @@ -0,0 +1,42 @@ +# Kubeconfig the PowerDNS agent uses to talk to the Milo *core* control plane +# (milo-apiserver root scope), where the DNS CRDs and DNSRecordSets live in this +# drift-detection e2e. The base agent config defaults to the in-cluster +# ServiceAccount kubeconfig (the local kind apiserver), which does not host the +# DNSRecordSets the agent must program into PowerDNS — so without this Secret the +# agent reads an empty/wrong API. +# +# - server: in-cluster Service for milo-apiserver. The pod listens on port 6443 +# with a self-signed cert, so we skip TLS verification. This targets the +# milo-apiserver ROOT endpoint directly (no aggregation path), which serves +# dns.networking.miloapis.com/v1alpha1 DNSRecordSet/DNSZone/DNSZoneClass. +# - token: the static `test-admin-token` wired into milo's test-infra overlay +# (token-auth-file, secret milo-apiserver-auth-tokens key tokens.csv, ns +# milo-system). It grants system:masters in the test environment. +# +# Test-overlay-only construct; production deployments should use a least-privilege +# token (or mTLS) sourced from a sealed-secret / external secret store. +apiVersion: v1 +kind: Secret +metadata: + name: milo-kubeconfig + namespace: dns-agent-system +type: Opaque +stringData: + kubeconfig: | + apiVersion: v1 + kind: Config + clusters: + - name: milo + cluster: + server: https://milo-apiserver.milo-system.svc.cluster.local:6443 + insecure-skip-tls-verify: true + users: + - name: milo-admin + user: + token: test-admin-token + contexts: + - name: milo + context: + cluster: milo + user: milo-admin + current-context: milo diff --git a/config/overlays/agent-powerdns-milo/server-config.yaml b/config/overlays/agent-powerdns-milo/server-config.yaml new file mode 100644 index 0000000..1243ff9 --- /dev/null +++ b/config/overlays/agent-powerdns-milo/server-config.yaml @@ -0,0 +1,46 @@ +apiVersion: dns.networking.miloapis.com/v1alpha1 +kind: DNSOperator +# Milo-targeted PowerDNS agent config for the DNS drift-detection e2e. +# +# In this environment the DNS CRDs (DNSZone/DNSRecordSet/DNSZoneClass in +# dns.networking.miloapis.com/v1alpha1) live on the Milo *core* control plane +# (milo-apiserver root scope), NOT on the local kind API. The PowerDNS agent +# runs --role=downstream, so it must READ DNSRecordSets/DNSZones from the Milo +# core CP and program PowerDNS. +# +# IMPORTANT — how the read/write source is actually retargeted for role=downstream: +# cmd/main.go's `case "downstream":` branch builds its manager and *all* three +# controllers (DNSZone, DNSRecordSet, DNSRecordSetPowerDNS) from +# ctrl.GetConfigOrDie() and mgr.GetClient(). It does NOT consult +# discovery.* or downstreamResourceManagement.kubeconfigPath at all — those +# fields are only read by the `case "replicator":` branch. So for a downstream +# agent, mode:single + downstreamResourceManagement.kubeconfigPath CANNOT +# retarget the read source on their own. +# +# The actual retarget is done in kustomization/deployment-patch.yaml by setting +# the KUBECONFIG env var to /milo/kubeconfig on the manager container: +# ctrl.GetConfig() honors KUBECONFIG (no --kubeconfig flag is registered), so +# both the read (DNSRecordSet/DNSZone informers) and the write (status updates) +# resolve to the mounted Milo core-CP kubeconfig instead of the in-cluster kind +# API. +# +# The fields below are kept milo-targeted for consistency and to remain correct +# if this agent is ever run as --role=replicator; they are inert for +# --role=downstream. +discovery: + mode: single + internalServiceDiscovery: false + # Inert under --role=downstream (single mode uses ctrl.GetConfigOrDie()); set + # to the mounted milo kubeconfig so it is correct if run as replicator. + discoveryKubeconfigPath: "/milo/kubeconfig" + projectKubeconfigPath: "/milo/kubeconfig" +downstreamResourceManagement: + # Inert under --role=downstream (see note above); the KUBECONFIG env var in the + # deployment patch is what points the downstream client at the Milo core CP. + kubeconfigPath: "/milo/kubeconfig" + +controllers: + dnsRecordSetPowerDNS: + # maxConcurrentReconciles: 4 + # rateLimiterBaseDelay: 1s + # rateLimiterMaxDelay: 30s diff --git a/config/overlays/replicator-milo/kustomization.yaml b/config/overlays/replicator-milo/kustomization.yaml new file mode 100644 index 0000000..272bd5d --- /dev/null +++ b/config/overlays/replicator-milo/kustomization.yaml @@ -0,0 +1,48 @@ +# Replicator overlay for the production-accurate Milo drift-detection e2e. +# Mirrors config/overlays/replicator (namespace + webhook-cert patches) but runs +# discovery.mode=milo with 2 active replicas and a milo-apiserver kubeconfig +# instead of the single-mode downstream-kubeconfig Secret. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: dns-replicator-system + +resources: + - ../../default + +# Replace the base single-mode server-config with the milo-mode one. +configMapGenerator: + - name: server-config + behavior: replace + files: + - server-config.yaml + +patches: + - path: patch-milo.yaml + target: + kind: Deployment + name: controller-manager + - path: ../replicator/patch-namespace.yaml + target: + kind: ServiceAccount + name: controller-manager + - path: ../replicator/patch-namespace.yaml + target: + kind: Service + name: controller-manager-metrics-service + - path: ../replicator/patch-namespace.yaml + target: + kind: ClusterRoleBinding + name: manager-rolebinding + - path: ../replicator/patch-namespace.yaml + target: + kind: ClusterRoleBinding + name: dns-operator-metrics-auth-rolebinding + - path: ../replicator/patch-namespace.yaml + target: + kind: RoleBinding + name: leader-election-rolebinding + - path: ../replicator/patch-mwc-ca-injection.yaml + target: + kind: MutatingWebhookConfiguration + name: mutating-webhook-configuration diff --git a/config/overlays/replicator-milo/patch-milo.yaml b/config/overlays/replicator-milo/patch-milo.yaml new file mode 100644 index 0000000..1d30560 --- /dev/null +++ b/config/overlays/replicator-milo/patch-milo.yaml @@ -0,0 +1,28 @@ +# Milo-mode replicator: 2 replicas with leader election OFF so both replicas +# reconcile — this is what exercises the multicluster per-cluster ownership that +# regressed in engineering#346 — plus the milo-apiserver kubeconfig mount. +# +# Note we DON'T restate the container args here: env and volume/volumeMount +# lists merge by name under strategic merge, so we only patch what changes +# (LEADER_ELECT=false) and inherit the base args (--role, --server-config, …). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager +spec: + replicas: 2 + template: + spec: + containers: + - name: manager + env: + - name: LEADER_ELECT + value: "false" + volumeMounts: + - name: milo-kubeconfig + mountPath: /milo + readOnly: true + volumes: + - name: milo-kubeconfig + secret: + secretName: milo-kubeconfig diff --git a/config/overlays/replicator-milo/server-config.yaml b/config/overlays/replicator-milo/server-config.yaml new file mode 100644 index 0000000..80e6637 --- /dev/null +++ b/config/overlays/replicator-milo/server-config.yaml @@ -0,0 +1,26 @@ +apiVersion: dns.networking.miloapis.com/v1alpha1 +kind: DNSOperator +# Production-accurate Milo topology for the drift-detection e2e (Phase B). +# The replicator discovers project (upstream) control planes through +# milo-apiserver and writes replicated resources to the Milo core (downstream) +# control plane. All three paths target the same milo-apiserver kubeconfig: +# - discovery: lists ProjectControlPlane objects +# - project: template rest-config the provider rewrites to per-project +# aggregation paths +# - downstream: the core control plane where replicated DNSRecordSets land +# VERIFY against the live env that the core control plane is reachable via the +# same milo-apiserver endpoint (it is the root scope); if milo serves the core +# CP at a distinct path, split downstreamResourceManagement.kubeconfigPath out. +discovery: + mode: milo + internalServiceDiscovery: false + discoveryKubeconfigPath: "/milo/kubeconfig" + projectKubeconfigPath: "/milo/kubeconfig" +downstreamResourceManagement: + kubeconfigPath: "/milo/kubeconfig" + +controllers: + dnsRecordSetPowerDNS: + # maxConcurrentReconciles: 4 + # rateLimiterBaseDelay: 1s + # rateLimiterMaxDelay: 30s diff --git a/docs/enhancements/controlplane-drift-detection.md b/docs/enhancements/controlplane-drift-detection.md new file mode 100644 index 0000000..af67806 --- /dev/null +++ b/docs/enhancements/controlplane-drift-detection.md @@ -0,0 +1,59 @@ +# DNS Control-Plane Drift Detection + +Detects when a customer's DNS records fall out of sync between the control plane they manage (their project) and the control plane that actually serves DNS — and alerts on-call before it turns into a customer-visible outage. + +## Why this exists + +DNS records a customer creates in their project are replicated to a shared control plane that programs the live DNS servers. When that replication misbehaves, a record can be left behind on the serving side with nothing owning it anymore. A leftover like this keeps "reserving" a hostname, so the customer can no longer point that name anywhere — every new record is rejected as a duplicate. + +That is exactly what happened in [engineering#346](https://github.com/datum-cloud/engineering/issues/346): a customer's `www` record was unresolvable for ~24 hours, and it was only found by hand. There was no signal for the underlying condition. This feature makes that condition an alert. + +## What it detects + +Three alerts, each pointing at the specific record and customer project involved: + +| Alert | What it means | What to do | +|---|---|---| +| **DNSDownstreamOrphanRecordSet** | A record exists on the serving control plane with **no owner** in any customer project — a leftover that can block the customer from reusing that hostname (the #346 case). | Remove the orphaned record on the serving control plane. The alert labels name the project, namespace, and record. | +| **DNSDownstreamMissingRecordSet** | A record exists in a **customer project but was never replicated** to the serving side — the customer's change isn't taking effect. | Check replicator health for that project; the record isn't live until it replicates. | +| **DNSRecordSetNotAccepted** | A customer's record has been sitting **un-accepted** (e.g. a misconfigured zone). | Inspect the record's status conditions in the customer project. | + +Alerts only fire after the condition **persists** (`for: 10m`), so normal replication lag never pages anyone. + +## How it works + +``` +Customer projects ──replicate──► Serving control plane ──► Live DNS + (upstream) (downstream) + │ │ + └──────── resource-metrics ─────────┘ emits one metric per DNS record, per side + │ + ▼ + Victoria Metrics + alerts (compare the two sides; a mismatch is drift) +``` + +- [`milo-os/resource-metrics`](https://github.com/datum-cloud/resource-metrics) emits one metric per `DNSRecordSet`/`DNSZone` from every customer project and from the serving control plane — no new code in the DNS operator. +- Recording rules compare the two sides. A serving-side record with no matching project record is an **orphan**; a project record with no matching serving-side record is **missing**. +- Rules ship as a standard Prometheus `PrometheusRule` and evaluate in Victoria Metrics. + +## What's included + +- `config/observability/` — the `dns-metrics` metrics policy and the `dns-controlplane-drift` alert/recording rules. +- `config/dependencies/` and `config/overlays/` — the environment used to validate it end-to-end. +- `test/e2e/controlplane-drift/` — automated tests for all three cases (healthy, orphan, missing). + +## Try it + +Against a local Kubernetes (kind) setup: + +```sh +export TASK_X_REMOTE_TASKFILES=1 +task env:milo-all-up # bring up the full DNS platform + metrics + alerting +task env:chainsaw-milo # run the drift-detection tests (healthy / orphan / missing) +``` + +The orphan test reproduces #346 end-to-end: replicate a record, break replication, delete the customer's record, and watch the orphan alert fire on the leftover — then clear once it's removed. + +## Enabling in staging / production + +The metrics and rules are additive, but the serving-side metrics require one platform change: set `discovery.collectRootControlPlane: true` in the `resource-metrics` config (it collects only customer projects today). Validate the new series in staging for one metrics-retention window before wiring the alerts to paging. diff --git a/test/e2e/controlplane-drift/.chainsaw.yaml b/test/e2e/controlplane-drift/.chainsaw.yaml new file mode 100644 index 0000000..05691e6 --- /dev/null +++ b/test/e2e/controlplane-drift/.chainsaw.yaml @@ -0,0 +1,18 @@ +# Runner config for the control-plane drift suite. The scenarios share one +# replicator Deployment (scaled up/down) and one Victoria Metrics instance, so +# they MUST run sequentially. Delete/cleanup budgets are generous because +# deleting an upstream DNSRecordSet waits on the replicator removing its +# finalizer + cascading to the downstream copy. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Configuration +metadata: + name: controlplane-drift +spec: + parallel: 1 + timeouts: + apply: 60s + assert: 120s + error: 60s + delete: 150s + cleanup: 150s + exec: 600s diff --git a/test/e2e/controlplane-drift/.gitignore b/test/e2e/controlplane-drift/.gitignore new file mode 100644 index 0000000..5dafaf0 --- /dev/null +++ b/test/e2e/controlplane-drift/.gitignore @@ -0,0 +1,2 @@ +# Generated per-run by the Taskfile (env:chainsaw-milo) — never commit. +kubeconfig-* diff --git a/test/e2e/controlplane-drift/README.md b/test/e2e/controlplane-drift/README.md new file mode 100644 index 0000000..40f2cca --- /dev/null +++ b/test/e2e/controlplane-drift/README.md @@ -0,0 +1,57 @@ +# control-plane drift e2e + +Chainsaw scenarios that prove upstream/downstream control-plane DNS drift +detection end to end: resource-metrics emits per-object series from each control +plane, the `dns-controlplane-drift` rules diff them, and these tests assert the +results in Victoria Metrics. See +`docs/enhancements/controlplane-drift-detection.md`. + +Run with `task env:chainsaw-milo` against a running environment +(`task env:milo-all-up`). + +## Scenarios + +| Dir | Proves | +|---|---| +| `happy-path/` | A record on project CP `alpha` replicates to the core CP; both `*_upstream_info` and `*_downstream_info` land in VM and `dns:recordset_downstream_orphan == 0`. | +| `orphan/` | engineering#346: replicate a record, scale the replicator to 0, delete the upstream → the downstream copy is orphaned → `dns:recordset_downstream_orphan > 0` and `DNSDownstreamOrphanRecordSet` becomes active. Removing the leftover clears it. | +| `missing/` | Scale the replicator to 0, create an upstream record → never replicated → `dns:recordset_downstream_missing > 0`. | + +## Clusters + +Each `chainsaw-test.yaml` names four clusters, with kubeconfigs in this directory +(referenced as `../kubeconfig-*`). `env:chainsaw-milo` generates them; they are +gitignored. + +| Name | Kubeconfig | Cluster | +|---|---|---| +| `alpha` | `kubeconfig-alpha` | Project CP alpha (upstream), milo aggregation path `.../projects/alpha/control-plane`. | +| `core` | `kubeconfig-core` | Milo core CP (downstream); replicated copies land here. | +| `infra` | `kubeconfig-infra` | dns-control kind cluster — hosts VM + OTel; runs the in-cluster `curl` VM queries. | +| `replicator` | `kubeconfig-replicator` | dns-upstream kind cluster — runs the replicator; the scenarios scale `deployment/dns-operator-controller-manager` in `dns-replicator-system` here. | + +`alpha` and `core` both reach milo-apiserver on dns-control with the +`test-admin-token`, differing only in the server URL. + +## Victoria Metrics + +Scenarios query VM from a one-shot `curl` pod on the `infra` cluster. Default +endpoint (override with `VM_QUERY_URL`): + +``` +http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428/api/v1/query +``` + +VM must serve both the raw `dns_recordset_*` series and the vmalert +recording-rule / `ALERTS` series. + +## Timing + +- Recording rules (`dns:recordset_downstream_{orphan,missing}`, no `for:`, 30s + interval) are the primary gate — the tests assert those `> 0`. +- The alerts carry `for: 10m`, so `orphan/` accepts + `alertstate="pending"|"firing"` rather than waiting for `firing`. +- The orphan only materializes after the deleted upstream series ages out of + VM's ~5m staleness window, so `orphan/` polls for up to ~9m. +- The suite runs sequentially (`.chainsaw.yaml` `parallel: 1`): the scenarios + share clusters and scale the single replicator. diff --git a/test/e2e/controlplane-drift/fixtures/dnszone.yaml b/test/e2e/controlplane-drift/fixtures/dnszone.yaml new file mode 100644 index 0000000..ac0d2e3 --- /dev/null +++ b/test/e2e/controlplane-drift/fixtures/dnszone.yaml @@ -0,0 +1,13 @@ +# Shared DNSZone for the control-plane drift e2e. Namespaced; created on the +# upstream project CP (e.g. cluster: alpha) in the `default` namespace. The +# replicator copies it to the downstream (core) CP, remapping the namespace and +# stamping the meta.datumapis.com/upstream-* annotations that resource-metrics +# lifts onto the dns_zone_downstream_info series. +apiVersion: dns.networking.miloapis.com/v1alpha1 +kind: DNSZone +metadata: + name: drift-example + namespace: default +spec: + domainName: drift.example + dnsZoneClassName: drift-powerdns diff --git a/test/e2e/controlplane-drift/fixtures/dnszoneclass.yaml b/test/e2e/controlplane-drift/fixtures/dnszoneclass.yaml new file mode 100644 index 0000000..acac424 --- /dev/null +++ b/test/e2e/controlplane-drift/fixtures/dnszoneclass.yaml @@ -0,0 +1,19 @@ +# Shared DNSZoneClass for the control-plane drift e2e. DNSZoneClass is +# cluster-scoped, so it must be created on whichever control plane the +# DNSZone/DNSRecordSet objects live on (the project CPs / "upstream" side). +# Each scenario applies this to its upstream project CP (e.g. cluster: alpha) +# before creating a DNSZone that references it by name. +apiVersion: dns.networking.miloapis.com/v1alpha1 +kind: DNSZoneClass +metadata: + name: drift-powerdns +spec: + controllerName: powerdns + nameServerPolicy: + mode: Static + static: + servers: + - ns1.drift.example. + - ns2.drift.example. + defaults: + defaultTTL: 300 diff --git a/test/e2e/controlplane-drift/fixtures/milo-projects.yaml b/test/e2e/controlplane-drift/fixtures/milo-projects.yaml new file mode 100644 index 0000000..706af5d --- /dev/null +++ b/test/e2e/controlplane-drift/fixtures/milo-projects.yaml @@ -0,0 +1,33 @@ +# Org + two projects for the control-plane drift e2e. Applied to the Milo core +# control plane (milo-apiserver). Milo's project controller creates an isolated +# virtual control plane per Project, reachable at +# {milo}/apis/resourcemanager.miloapis.com/v1alpha1/projects//control-plane +# and shares the core-registered DNS CRDs. These two projects are the "upstream" +# control planes the replicator discovers in discovery.mode=milo; the core CP is +# the "downstream". See docs/enhancements/controlplane-drift-detection.md. +apiVersion: resourcemanager.miloapis.com/v1alpha1 +kind: Organization +metadata: + name: drift-org + annotations: + kubernetes.io/display-name: "DNS Drift E2E Org" +spec: + type: Standard +--- +apiVersion: resourcemanager.miloapis.com/v1alpha1 +kind: Project +metadata: + name: alpha +spec: + ownerRef: + kind: Organization + name: drift-org +--- +apiVersion: resourcemanager.miloapis.com/v1alpha1 +kind: Project +metadata: + name: beta +spec: + ownerRef: + kind: Organization + name: drift-org diff --git a/test/e2e/controlplane-drift/happy-path/chainsaw-test.yaml b/test/e2e/controlplane-drift/happy-path/chainsaw-test.yaml new file mode 100644 index 0000000..d0b15db --- /dev/null +++ b/test/e2e/controlplane-drift/happy-path/chainsaw-test.yaml @@ -0,0 +1,186 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# Happy path: create a DNSZone(class) + DNSRecordSet on project CP `alpha` +# (upstream), wait for the replicated copy to land on the core CP (downstream), +# then assert Victoria Metrics carries BOTH sides of the join and that the +# orphan recording rule is clean. +# +# Clusters (kubeconfigs provided by the Taskfile — see ../README.md): +# alpha -> project CP alpha (UPSTREAM) ../kubeconfig-alpha +# core -> Milo core control plane (DOWNSTREAM) ../kubeconfig-core +# infra -> kind cluster hosting Victoria Metrics + OTel ../kubeconfig-infra +# (used only to run in-cluster curl against VM) +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: controlplane-drift-happy-path +spec: + # The VM-poll scripts loop for several minutes to ride out the + # watch -> collect -> OTLP -> remote-write -> ingest pipeline latency. + timeouts: + exec: 600s + clusters: + alpha: + kubeconfig: ../kubeconfig-alpha + core: + kubeconfig: ../kubeconfig-core + infra: + kubeconfig: ../kubeconfig-infra + cluster: alpha + steps: + - name: Prereq - DNSZoneClass + DNSZone on upstream project CP alpha + try: + - apply: + cluster: alpha + file: ../fixtures/dnszoneclass.yaml + - apply: + cluster: alpha + file: ../fixtures/dnszone.yaml + - assert: + cluster: alpha + resource: + apiVersion: dns.networking.miloapis.com/v1alpha1 + kind: DNSZone + metadata: + name: drift-example + namespace: default + + - name: Create DNSRecordSet on upstream project CP alpha + try: + - create: + cluster: alpha + resource: + apiVersion: dns.networking.miloapis.com/v1alpha1 + kind: DNSRecordSet + metadata: + name: www-happy + namespace: default + spec: + dnsZoneRef: + name: drift-example + recordType: A + records: + - name: www + ttl: 60 + a: + content: 192.0.2.10 + - assert: + cluster: alpha + resource: + apiVersion: dns.networking.miloapis.com/v1alpha1 + kind: DNSRecordSet + metadata: + name: www-happy + namespace: default + + - name: Wait for the replicated copy to land on the downstream core CP + try: + - script: + cluster: core + timeout: 300s + content: | + set -eu + # The downstream namespace is remapped (ns-) and the real + # upstream identity lives only in annotations, so match on the + # upstream-name annotation across all namespaces rather than by + # name/namespace. + want="www-happy" + attempts=30 + i=0 + while [ "$i" -lt "$attempts" ]; do + i=$((i+1)) + got=$(kubectl get dnsrecordset -A \ + -o jsonpath='{range .items[*]}{.metadata.annotations.meta\.datumapis\.com/upstream-name}{"\n"}{end}' \ + 2>/dev/null || true) + echo "[attempt $i/$attempts] downstream upstream-name annotations: ${got}" + if printf '%s\n' "$got" | grep -qx "$want"; then + echo "replicated copy present on core CP" + exit 0 + fi + sleep 10 + done + echo "replicated DNSRecordSet ($want) never appeared on core CP" >&2 + exit 1 + + - name: Assert upstream + downstream series exist in Victoria Metrics + try: + - script: + cluster: infra + timeout: 300s + content: | + set -eu + vm_url="${VM_QUERY_URL:-http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428/api/v1/query}" + + # Poll VM for a PromQL query until it returns a non-empty result. + poll() { + query="$1"; label="$2" + attempt=0 + until [ "$attempt" -ge 24 ]; do + attempt=$((attempt+1)) + echo "[$label attempt $attempt] query: $query" + out=$(kubectl run e2e-vmq-happy-$label-$attempt \ + --rm -i --restart=Never --quiet \ + --image=curlimages/curl:8.10.1 \ + --timeout=20s \ + -- curl -fsS --max-time 5 \ + --data-urlencode "query=${query}" "${vm_url}" \ + || true) + json=$(printf '%s' "$out" | sed '/pod ".*" deleted/d') + echo "response: $json" + if printf '%s' "$json" | grep -Eq '"result":\s*\[\s*\{'; then + echo "[$label] series found" + return 0 + fi + sleep 10 + done + echo "[$label] series never appeared: $query" >&2 + return 1 + } + + poll 'dns_recordset_upstream_info{upstream_name="www-happy",milo_project_name="alpha"}' upstream + poll 'dns_recordset_downstream_info{upstream_name="www-happy",milo_control_plane_type="root"}' downstream + + - name: Assert the orphan recording rule is clean (== 0 / empty) + try: + - script: + cluster: infra + timeout: 120s + content: | + set -eu + vm_url="${VM_QUERY_URL:-http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428/api/v1/query}" + # The orphan recording rule only PRODUCES a series when an orphan + # exists (`unless` yields the leftover downstream vectors). A clean + # state is therefore an empty result. Scope to our record so an + # unrelated pre-existing orphan can't fail the happy path. + query='dns:recordset_downstream_orphan{upstream_name="www-happy"}' + out=$(kubectl run e2e-vmq-happy-orphan \ + --rm -i --restart=Never --quiet \ + --image=curlimages/curl:8.10.1 \ + --timeout=20s \ + -- curl -fsS --max-time 5 \ + --data-urlencode "query=${query}" "${vm_url}" \ + || true) + json=$(printf '%s' "$out" | sed '/pod ".*" deleted/d') + echo "response: $json" + if printf '%s' "$json" | grep -Eq '"result":\s*\[\s*\{'; then + echo "unexpected orphan series for www-happy — drift detected on happy path" >&2 + exit 1 + fi + echo "orphan recording rule clean (no series for www-happy)" + + - name: Teardown + try: + - delete: + cluster: alpha + ref: + apiVersion: dns.networking.miloapis.com/v1alpha1 + kind: DNSRecordSet + namespace: default + name: www-happy + - delete: + cluster: alpha + ref: + apiVersion: dns.networking.miloapis.com/v1alpha1 + kind: DNSZone + namespace: default + name: drift-example diff --git a/test/e2e/controlplane-drift/missing/chainsaw-test.yaml b/test/e2e/controlplane-drift/missing/chainsaw-test.yaml new file mode 100644 index 0000000..014e7be --- /dev/null +++ b/test/e2e/controlplane-drift/missing/chainsaw-test.yaml @@ -0,0 +1,173 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# Missing: an upstream DNSRecordSet that was never replicated downstream +# (stalled replication / shard-ownership gap). We reproduce it by scaling the +# replicator to 0 FIRST, then creating an upstream object — no downstream copy +# is ever made — and assert the missing recording rule fires. +# +# Clusters (kubeconfigs provided by the Taskfile — see ../README.md): +# alpha -> project CP alpha (UPSTREAM) ../kubeconfig-alpha +# core -> Milo core control plane (DOWNSTREAM) ../kubeconfig-core +# infra -> kind cluster hosting VM + OTel + the ../kubeconfig-infra +# replicator Deployment (ns dns-replicator-system) +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: controlplane-drift-missing +spec: + timeouts: + exec: 600s + clusters: + alpha: + kubeconfig: ../kubeconfig-alpha + core: + kubeconfig: ../kubeconfig-core + infra: + kubeconfig: ../kubeconfig-infra + replicator: + kubeconfig: ../kubeconfig-replicator + cluster: alpha + steps: + - name: Prereq - DNSZoneClass + DNSZone on upstream project CP alpha + try: + - apply: + cluster: alpha + file: ../fixtures/dnszoneclass.yaml + - apply: + cluster: alpha + file: ../fixtures/dnszone.yaml + - assert: + cluster: alpha + resource: + apiVersion: dns.networking.miloapis.com/v1alpha1 + kind: DNSZone + metadata: + name: drift-example + namespace: default + + - name: Scale the replicator to 0 so nothing gets replicated + try: + - script: + cluster: replicator + timeout: 120s + content: | + set -eu + kubectl -n dns-replicator-system scale deployment/dns-operator-controller-manager --replicas=0 + kubectl -n dns-replicator-system rollout status deployment/dns-operator-controller-manager --timeout=90s || true + kubectl -n dns-replicator-system get pods + + - name: Create an upstream DNSRecordSet that will never replicate + try: + - create: + cluster: alpha + resource: + apiVersion: dns.networking.miloapis.com/v1alpha1 + kind: DNSRecordSet + metadata: + name: www-missing + namespace: default + spec: + dnsZoneRef: + name: drift-example + recordType: A + records: + - name: www-missing + ttl: 60 + a: + content: 192.0.2.30 + - assert: + cluster: alpha + resource: + apiVersion: dns.networking.miloapis.com/v1alpha1 + kind: DNSRecordSet + metadata: + name: www-missing + namespace: default + + - name: Confirm no downstream copy exists on the core CP + try: + - script: + cluster: core + timeout: 60s + content: | + set -eu + got=$(kubectl get dnsrecordset -A \ + -o jsonpath='{range .items[*]}{.metadata.annotations.meta\.datumapis\.com/upstream-name}{"\n"}{end}' \ + 2>/dev/null || true) + echo "downstream upstream-name annotations: ${got}" + if printf '%s\n' "$got" | grep -qx "www-missing"; then + echo "unexpected: www-missing was replicated despite replicator scaled to 0" >&2 + exit 1 + fi + echo "confirmed: no downstream copy for www-missing" + + - name: Poll VM until the missing recording rule fires + try: + - script: + cluster: infra + timeout: 300s + content: | + set -eu + vm_url="${VM_QUERY_URL:-http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428/api/v1/query}" + query='dns:recordset_downstream_missing{upstream_name="www-missing"} > 0' + attempt=0 + until [ "$attempt" -ge 30 ]; do + attempt=$((attempt+1)) + echo "[missing attempt $attempt] query: $query" + out=$(kubectl run e2e-vmq-missing-$attempt \ + --rm -i --restart=Never --quiet \ + --image=curlimages/curl:8.10.1 \ + --timeout=20s \ + -- curl -fsS --max-time 5 \ + --data-urlencode "query=${query}" "${vm_url}" \ + || true) + json=$(printf '%s' "$out" | sed '/pod ".*" deleted/d') + echo "response: $json" + if printf '%s' "$json" | grep -Eq '"result":\s*\[\s*\{'; then + echo "missing detected by recording rule" + exit 0 + fi + sleep 10 + done + echo "dns:recordset_downstream_missing never went > 0 for www-missing" >&2 + exit 1 + + - name: Teardown - restore the replicator and clean up + try: + - script: + cluster: replicator + timeout: 180s + content: | + set -eu + # Restore the replicator BEFORE deleting the upstream object so the + # (now-replicated) downstream copy gets cascade-deleted normally and + # no orphan is left behind for the next scenario. + kubectl -n dns-replicator-system scale deployment/dns-operator-controller-manager --replicas=1 + kubectl -n dns-replicator-system rollout status deployment/dns-operator-controller-manager --timeout=120s + # rollout status only waits for pod readiness — wait for the + # multicluster provider to actually re-engage the project CPs before + # deleting upstream objects, otherwise the replicator can't release + # finalizers / cascade-delete the downstream copies and the deletes + # below time out. + for i in $(seq 1 20); do + if kubectl -n dns-replicator-system logs -l control-plane=controller-manager --since=90s 2>/dev/null \ + | grep -q "Successfully registered and engaged"; then + echo "replicator re-engaged project CPs"; break + fi + sleep 3 + done + sleep 10 + - delete: + cluster: alpha + ref: + apiVersion: dns.networking.miloapis.com/v1alpha1 + kind: DNSRecordSet + namespace: default + name: www-missing + - delete: + cluster: alpha + ref: + apiVersion: dns.networking.miloapis.com/v1alpha1 + kind: DNSZone + namespace: default + name: drift-example diff --git a/test/e2e/controlplane-drift/orphan/chainsaw-test.yaml b/test/e2e/controlplane-drift/orphan/chainsaw-test.yaml new file mode 100644 index 0000000..2e7019c --- /dev/null +++ b/test/e2e/controlplane-drift/orphan/chainsaw-test.yaml @@ -0,0 +1,280 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# Orphan / engineering#346 regression: a DNSRecordSet exists on the downstream +# (core) CP with no surviving upstream owner. We reproduce it by replicating a +# record, scaling the replicator to 0 (so it can't cascade-delete the downstream +# copy), then deleting the upstream object. The downstream copy is left behind => +# orphan. We assert the orphan recording rule fires and the alert becomes active, +# then restore the replicator and assert it clears. +# +# Clusters (kubeconfigs provided by the Taskfile — see ../README.md): +# alpha -> project CP alpha (UPSTREAM) ../kubeconfig-alpha +# core -> Milo core control plane (DOWNSTREAM) ../kubeconfig-core +# infra -> kind cluster hosting VM + OTel + the ../kubeconfig-infra +# replicator Deployment (ns dns-replicator-system) +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: controlplane-drift-orphan +spec: + timeouts: + exec: 600s + clusters: + alpha: + kubeconfig: ../kubeconfig-alpha + core: + kubeconfig: ../kubeconfig-core + infra: + kubeconfig: ../kubeconfig-infra + replicator: + kubeconfig: ../kubeconfig-replicator + cluster: alpha + steps: + - name: Prereq - DNSZoneClass + DNSZone on upstream project CP alpha + try: + - apply: + cluster: alpha + file: ../fixtures/dnszoneclass.yaml + - apply: + cluster: alpha + file: ../fixtures/dnszone.yaml + - assert: + cluster: alpha + resource: + apiVersion: dns.networking.miloapis.com/v1alpha1 + kind: DNSZone + metadata: + name: drift-example + namespace: default + + - name: Create DNSRecordSet on upstream and wait for downstream replica + try: + - create: + cluster: alpha + resource: + apiVersion: dns.networking.miloapis.com/v1alpha1 + kind: DNSRecordSet + metadata: + name: www-orphan + namespace: default + spec: + dnsZoneRef: + name: drift-example + recordType: A + records: + - name: www-orphan + ttl: 60 + a: + content: 192.0.2.20 + - script: + cluster: core + timeout: 300s + content: | + set -eu + want="www-orphan" + attempts=30 + i=0 + while [ "$i" -lt "$attempts" ]; do + i=$((i+1)) + got=$(kubectl get dnsrecordset -A \ + -o jsonpath='{range .items[*]}{.metadata.annotations.meta\.datumapis\.com/upstream-name}{"\n"}{end}' \ + 2>/dev/null || true) + echo "[attempt $i/$attempts] downstream upstream-name annotations: ${got}" + if printf '%s\n' "$got" | grep -qx "$want"; then + echo "replicated copy present on core CP" + exit 0 + fi + sleep 10 + done + echo "replicated DNSRecordSet ($want) never appeared on core CP" >&2 + exit 1 + + - name: Scale the replicator to 0 to prevent GC cascade + try: + - script: + cluster: replicator + timeout: 120s + content: | + set -eu + # Scaling every deployment in the replicator namespace avoids + # coupling the test to a specific deployment name. With no + # replicator running, deleting the upstream object cannot cascade + # to the downstream copy (they live on different control planes; + # only the replicator would remove it) — that leftover is the + # orphan. + kubectl -n dns-replicator-system scale deployment/dns-operator-controller-manager --replicas=0 + kubectl -n dns-replicator-system rollout status deployment/dns-operator-controller-manager --timeout=90s || true + kubectl -n dns-replicator-system get pods + + - name: Delete the upstream object, leaving the downstream copy orphaned + try: + - script: + cluster: alpha + timeout: 60s + content: | + set -eu + # The replicator is scaled to 0, so it will NOT remove its + # finalizer — a plain delete would hang. Strip the finalizer so the + # upstream fully deletes, leaving the downstream copy as a true + # orphan (the engineering#346 end state). + kubectl -n default patch dnsrecordset www-orphan \ + --type=merge -p '{"metadata":{"finalizers":[]}}' || true + kubectl -n default delete dnsrecordset www-orphan --wait=false || true + - script: + cluster: core + timeout: 60s + content: | + set -eu + # Confirm the downstream copy is still present (orphaned). + got=$(kubectl get dnsrecordset -A \ + -o jsonpath='{range .items[*]}{.metadata.annotations.meta\.datumapis\.com/upstream-name}{"\n"}{end}' \ + 2>/dev/null || true) + echo "downstream upstream-name annotations after upstream delete: ${got}" + printf '%s\n' "$got" | grep -qx "www-orphan" + + - name: Poll VM until the orphan recording rule fires + try: + - script: + cluster: infra + # Budget generously: the orphan only materializes once the deleted + # upstream series ages out of VM's staleness window (default ~5m) + # so `unless` no longer cancels the downstream vector. + timeout: 540s + content: | + set -eu + vm_url="${VM_QUERY_URL:-http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428/api/v1/query}" + query='dns:recordset_downstream_orphan{upstream_name="www-orphan"} > 0' + attempt=0 + until [ "$attempt" -ge 48 ]; do + attempt=$((attempt+1)) + echo "[orphan attempt $attempt] query: $query" + out=$(kubectl run e2e-vmq-orphan-$attempt \ + --rm -i --restart=Never --quiet \ + --image=curlimages/curl:8.10.1 \ + --timeout=20s \ + -- curl -fsS --max-time 5 \ + --data-urlencode "query=${query}" "${vm_url}" \ + || true) + json=$(printf '%s' "$out" | sed '/pod ".*" deleted/d') + echo "response: $json" + if printf '%s' "$json" | grep -Eq '"result":\s*\[\s*\{'; then + echo "orphan detected by recording rule" + exit 0 + fi + sleep 10 + done + echo "dns:recordset_downstream_orphan never went > 0 for www-orphan" >&2 + exit 1 + + - name: Assert the DNSDownstreamOrphanRecordSet alert is active + try: + - script: + cluster: infra + timeout: 180s + content: | + set -eu + vm_url="${VM_QUERY_URL:-http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428/api/v1/query}" + # The alert carries `for: 10m`, so it sits in `pending` for 10m + # before `firing`. Waiting the full period would blow the e2e + # budget, so we accept EITHER pending or firing here (the alert is + # active on the correct series). To assert `firing` specifically, + # deploy the rules with a shortened `for:` in a test overlay — see + # ../README.md. + query='ALERTS{alertname="DNSDownstreamOrphanRecordSet",upstream_name="www-orphan"}' + attempt=0 + until [ "$attempt" -ge 18 ]; do + attempt=$((attempt+1)) + echo "[alert attempt $attempt] query: $query" + out=$(kubectl run e2e-vmq-orphan-alert-$attempt \ + --rm -i --restart=Never --quiet \ + --image=curlimages/curl:8.10.1 \ + --timeout=20s \ + -- curl -fsS --max-time 5 \ + --data-urlencode "query=${query}" "${vm_url}" \ + || true) + json=$(printf '%s' "$out" | sed '/pod ".*" deleted/d') + echo "response: $json" + if printf '%s' "$json" | grep -Eq '"alertstate":"(pending|firing)"'; then + echo "DNSDownstreamOrphanRecordSet alert is active" + exit 0 + fi + sleep 10 + done + echo "DNSDownstreamOrphanRecordSet never became active for www-orphan" >&2 + exit 1 + + - name: Remediate (remove the leftover) and assert the orphan clears + try: + - script: + cluster: replicator + timeout: 180s + content: | + set -eu + # Restore the replicator for environment health. NOTE: it does NOT + # auto-GC the orphan — with the upstream owner gone there is no + # event to drive a reconcile of the leftover, so (exactly as in + # engineering#346) the downstream leftover PERSISTS until it is + # explicitly removed. That persistence is the whole reason this + # detection matters. + kubectl -n dns-replicator-system scale deployment/dns-operator-controller-manager --replicas=1 + kubectl -n dns-replicator-system rollout status deployment/dns-operator-controller-manager --timeout=120s + - script: + cluster: core + timeout: 120s + content: | + set -eu + # Remediation: remove the orphaned downstream leftover (the manual + # fix that resolved #346). Locate it by its upstream-name annotation + # and strip the finalizer so it fully deletes. + row=$(kubectl get dnsrecordset -A \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.metadata.annotations.meta\.datumapis\.com/upstream-name}{"\n"}{end}' \ + | awk '$3=="www-orphan"{print $1" "$2; exit}') + if [ -n "$row" ]; then + set -- $row + kubectl -n "$1" patch dnsrecordset "$2" --type=merge -p '{"metadata":{"finalizers":[]}}' || true + kubectl -n "$1" delete dnsrecordset "$2" --wait=false || true + echo "removed orphaned leftover $1/$2" + else + echo "no leftover found (already removed)" + fi + - script: + cluster: core + timeout: 90s + content: | + set -eu + # Assert remediation at the source of truth: the orphaned downstream + # object is gone from the core CP. We deliberately do NOT poll the + # VM recording rule here — its series only clears once the removed + # object's samples age out of VM's ~5m staleness window, which makes + # a rule-based clear assertion slow and flaky. Once the object is + # gone, the rule is guaranteed to clear after staleness. + for i in $(seq 1 12); do + got=$(kubectl get dnsrecordset -A \ + -o jsonpath='{range .items[*]}{.metadata.annotations.meta\.datumapis\.com/upstream-name}{"\n"}{end}' \ + 2>/dev/null || true) + if ! printf '%s\n' "$got" | grep -qx "www-orphan"; then + echo "orphaned leftover removed from core CP — drift resolved at source" + exit 0 + fi + echo "[clear attempt $i] leftover still present; waiting" + sleep 5 + done + echo "orphaned downstream leftover was not removed from the core CP" >&2 + exit 1 + + - name: Teardown + try: + # Ensure the replicator is back up even if an earlier step failed. + - script: + cluster: replicator + timeout: 120s + content: | + set -eu + kubectl -n dns-replicator-system scale deployment/dns-operator-controller-manager --replicas=1 || true + - delete: + cluster: alpha + ref: + apiVersion: dns.networking.miloapis.com/v1alpha1 + kind: DNSZone + namespace: default + name: drift-example