From ae4298083f18ec8270a660786ac7973a27444d51 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 16:49:36 -0500 Subject: [PATCH 01/23] test(e2e): restore Kind+Karmada harness scaffolding Restore the multi-cluster e2e building blocks from the archived local-run branch so the federation layer can be exercised end to end again: - hack/e2e/kind-control-plane.yaml exposes the Karmada API server on a host NodePort for developer kubectl and Chainsaw access. - hack/e2e/make-internal-kubeconfig.sh and patch-cluster-secret.sh rewrite member-cluster kubeconfigs (and the Cluster apiEndpoint) to the Docker-bridge IP so the in-Docker Karmada controller can reach the POP cell API servers. - test/e2e/chainsaw-config.yaml declares the downstream/pop-dfw/pop-ord clusters the suites reference. These are environment scaffolding only. Unlike the retired harness, the operators are deployed in-cluster via the real kustomize overlays rather than run on the host; that path lands in the following commit. Co-Authored-By: Claude Fable 5 --- hack/e2e/kind-control-plane.yaml | 17 ++++++ hack/e2e/make-internal-kubeconfig.sh | 60 +++++++++++++++++++ hack/e2e/patch-cluster-secret.sh | 90 ++++++++++++++++++++++++++++ test/e2e/chainsaw-config.yaml | 47 +++++++++++++++ 4 files changed, 214 insertions(+) create mode 100644 hack/e2e/kind-control-plane.yaml create mode 100755 hack/e2e/make-internal-kubeconfig.sh create mode 100755 hack/e2e/patch-cluster-secret.sh create mode 100644 test/e2e/chainsaw-config.yaml diff --git a/hack/e2e/kind-control-plane.yaml b/hack/e2e/kind-control-plane.yaml new file mode 100644 index 00000000..47f3c63b --- /dev/null +++ b/hack/e2e/kind-control-plane.yaml @@ -0,0 +1,17 @@ +# Kind cluster configuration for the compute-control-plane management cluster. +# +# extraPortMappings exposes port 32443 on the macOS host so that the Karmada +# API server NodePort service (nodePort: 32443) is accessible at +# https://localhost:32443 without any additional port-forwarding. +# +# This matches KARMADA_API_NODEPORT in Taskfile.yaml. + +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + extraPortMappings: + - containerPort: 32443 # Karmada API server NodePort + hostPort: 32443 + protocol: TCP + listenAddress: "127.0.0.1" diff --git a/hack/e2e/make-internal-kubeconfig.sh b/hack/e2e/make-internal-kubeconfig.sh new file mode 100755 index 00000000..3303a5bd --- /dev/null +++ b/hack/e2e/make-internal-kubeconfig.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# make-internal-kubeconfig.sh +# +# Produces a kubeconfig variant that uses the Kind node's Docker container IP +# instead of localhost. This variant is stored in Karmada so the controller +# manager (running inside Docker) can reach member cluster API servers across +# the kind bridge network. +# +# Background: Kind maps each cluster's API server to a random localhost port +# on the developer machine. Inside Docker containers, "localhost" refers to the +# container's own loopback — not the host. We therefore swap the server address +# to the Kind control-plane container's Docker bridge IP (e.g. 172.18.0.x) and +# set insecure-skip-tls-verify because the node certificate does not include +# the Docker bridge IP in its SANs. +# +# Usage: +# hack/e2e/make-internal-kubeconfig.sh \ +# tmp/e2e/kubeconfigs/pop-dfw.yaml \ +# tmp/e2e/kubeconfigs/pop-dfw-internal.yaml \ +# compute-pop-dfw + +set -euo pipefail + +INPUT="${1:?usage: $0 }" +OUTPUT="${2:?usage: $0 }" +CLUSTER_NAME="${3:?usage: $0 }" + +CONTAINER_NAME="${CLUSTER_NAME}-control-plane" + +# Resolve the container's Docker bridge IP. +DOCKER_IP=$(docker inspect \ + -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ + "${CONTAINER_NAME}" 2>/dev/null || true) + +if [ -z "${DOCKER_IP}" ]; then + echo "ERROR: Could not resolve Docker IP for container '${CONTAINER_NAME}'." >&2 + echo " Is the Kind cluster '${CLUSTER_NAME}' running?" >&2 + exit 1 +fi + +echo " ${CLUSTER_NAME}: Docker IP ${DOCKER_IP} → ${OUTPUT}" + +python3 - "${INPUT}" "${OUTPUT}" "${DOCKER_IP}" <<'PYEOF' +import sys, yaml + +src, dst, docker_ip = sys.argv[1], sys.argv[2], sys.argv[3] + +with open(src) as f: + cfg = yaml.safe_load(f) + +for cluster in cfg.get('clusters', []): + # Kind API server always listens on port 6443 inside the container. + cluster['cluster']['server'] = f'https://{docker_ip}:6443' + # The node cert only covers localhost / 127.0.0.1, not the bridge IP. + cluster['cluster']['insecure-skip-tls-verify'] = True + cluster['cluster'].pop('certificate-authority-data', None) + +with open(dst, 'w') as f: + yaml.dump(cfg, f, default_flow_style=False) +PYEOF diff --git a/hack/e2e/patch-cluster-secret.sh b/hack/e2e/patch-cluster-secret.sh new file mode 100755 index 00000000..e29ed383 --- /dev/null +++ b/hack/e2e/patch-cluster-secret.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# patch-cluster-secret.sh +# +# After "karmadactl join", Karmada stores the member cluster's kubeconfig in a +# Secret referenced by the Cluster object's spec.secretRef, and sets +# spec.apiEndpoint to the localhost address it resolved from the external +# kubeconfig. The Karmada controller manager runs inside Docker and cannot use +# localhost to reach POP cell API servers. +# +# This script: +# 1. Replaces the kubeconfig in the Secret with the Docker-IP variant so that +# the Karmada controller can make API calls to the member cluster. +# 2. Patches spec.apiEndpoint on the Cluster object so that health checks also +# use the Docker bridge IP instead of localhost. +# +# Usage: +# hack/e2e/patch-cluster-secret.sh \ +# tmp/e2e/kubeconfigs/karmada.yaml \ +# compute-pop-dfw \ +# tmp/e2e/kubeconfigs/pop-dfw-internal.yaml + +set -euo pipefail + +KARMADA_KUBECONFIG="${1:?usage: $0 }" +CLUSTER_NAME="${2:?usage: $0 }" +INTERNAL_KUBECONFIG="${3:?usage: $0 }" + +# ------------------------------------------------------------------ +# Read the Cluster object's secretRef (name + namespace) +# ------------------------------------------------------------------ +SECRET_NAME=$(kubectl \ + --kubeconfig="${KARMADA_KUBECONFIG}" \ + get cluster "${CLUSTER_NAME}" \ + -o jsonpath='{.spec.secretRef.name}' 2>/dev/null || true) + +if [ -z "${SECRET_NAME}" ]; then + echo "ERROR: Could not find spec.secretRef.name on cluster '${CLUSTER_NAME}'." >&2 + echo " Has karmadactl join completed successfully?" >&2 + exit 1 +fi + +SECRET_NAMESPACE=$(kubectl \ + --kubeconfig="${KARMADA_KUBECONFIG}" \ + get cluster "${CLUSTER_NAME}" \ + -o jsonpath='{.spec.secretRef.namespace}' 2>/dev/null || true) + +SECRET_NAMESPACE="${SECRET_NAMESPACE:-karmada-system}" + +echo " Patching secret ${SECRET_NAMESPACE}/${SECRET_NAME} with Docker-IP kubeconfig..." + +# ------------------------------------------------------------------ +# Replace the kubeconfig data in the secret +# ------------------------------------------------------------------ +kubectl \ + --kubeconfig="${KARMADA_KUBECONFIG}" \ + create secret generic "${SECRET_NAME}" \ + --namespace="${SECRET_NAMESPACE}" \ + --from-file=kubeconfig="${INTERNAL_KUBECONFIG}" \ + --dry-run=client -o yaml \ + | kubectl \ + --kubeconfig="${KARMADA_KUBECONFIG}" \ + apply -f - + +echo " Secret ${SECRET_NAMESPACE}/${SECRET_NAME} updated — Karmada controller will use Docker bridge IP" + +# ------------------------------------------------------------------ +# Extract the Docker-IP server URL from the internal kubeconfig and +# patch spec.apiEndpoint on the Cluster object so that Karmada's +# cluster-status controller uses the same reachable address for health +# checks. Without this patch the controller continues to probe the +# localhost address stored by karmadactl join and the cluster never +# transitions to Ready. +# ------------------------------------------------------------------ +DOCKER_SERVER=$(kubectl \ + --kubeconfig="${INTERNAL_KUBECONFIG}" \ + config view --minify -o jsonpath='{.clusters[0].cluster.server}') + +if [ -z "${DOCKER_SERVER}" ]; then + echo "ERROR: Could not read server URL from ${INTERNAL_KUBECONFIG}" >&2 + exit 1 +fi + +echo " Patching spec.apiEndpoint on cluster '${CLUSTER_NAME}' → ${DOCKER_SERVER}..." +kubectl \ + --kubeconfig="${KARMADA_KUBECONFIG}" \ + patch cluster "${CLUSTER_NAME}" \ + --type=merge \ + -p "{\"spec\":{\"apiEndpoint\":\"${DOCKER_SERVER}\"}}" + +echo " Cluster '${CLUSTER_NAME}' patched — health checks will now use Docker bridge IP" diff --git a/test/e2e/chainsaw-config.yaml b/test/e2e/chainsaw-config.yaml new file mode 100644 index 00000000..cd3a9950 --- /dev/null +++ b/test/e2e/chainsaw-config.yaml @@ -0,0 +1,47 @@ +# Chainsaw global configuration for the compute federation e2e test suite. +# +# Prerequisites +# ───────────── +# Run `task e2e:up` to create the Kind clusters and populate kubeconfigs under +# tmp/e2e/kubeconfigs/ before running Chainsaw. +# +# Running +# ─────── +# From the repository root via Taskfile (recommended): +# +# task e2e:test +# +# Or directly: +# +# KUBECONFIG=tmp/e2e/kubeconfigs/control-plane.yaml \ +# chainsaw test --config test/e2e/chainsaw-config.yaml test/e2e/ +# +# The KUBECONFIG env var sets the "default" cluster (control-plane cell). +# Additional clusters (downstream, pop-dfw, pop-ord) are declared below and +# referenced by name in individual test steps via `cluster: downstream` etc. +# +# Kubeconfig paths below are relative to the working directory where Chainsaw is +# invoked (the project root), NOT relative to this config file's location. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Configuration +metadata: + name: chainsaw +spec: + timeouts: + apply: 30s + assert: 60s + cleanup: 60s + delete: 30s + error: 30s + exec: 30s + clusters: + # Downstream control plane. WorkloadDeployments, PropagationPolicies, + # and Instance write-backs live here. + downstream: + kubeconfig: tmp/e2e/kubeconfigs/downstream.yaml + # POP DFW cell — downstream member cluster labelled topology.datum.net/city-code=dfw. + pop-dfw: + kubeconfig: tmp/e2e/kubeconfigs/pop-dfw.yaml + # POP ORD cell — downstream member cluster labelled topology.datum.net/city-code=ord. + pop-ord: + kubeconfig: tmp/e2e/kubeconfigs/pop-ord.yaml From da45c9480db5cf1e6fef5e735734f5676a01c9b7 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 17:38:12 -0500 Subject: [PATCH 02/23] test(e2e): deploy compute operators in-cluster via real overlays Replace the retired host-run operator harness with an in-cluster deploy path so e2e exercises the same kustomize overlays, RBAC, and pod wiring that run in production. The Taskfile now builds a dev image, side-loads it into every Kind cluster, wires the management operator's access to the Karmada hub, and applies the real overlays to the control-plane and both POP cells, waiting on rollout. Deploy layer (test/e2e/deploy/{management,cell}) references the production overlays verbatim and layers only the deviations a local Kind environment forces, each annotated inline so drift from production stays visible: - management: discovery.mode milo -> single (no Milo control plane in Kind); admission webhook disabled and its cert-manager CSI serving-cert volume removed (no cert-manager/CSI driver in Kind; the suites create WorkloadDeployments, never Workloads, so the Workload webhook is never exercised); dev image with IfNotPresent pull. - cell: discovery.quotaKubeconfigPath dropped so the operator takes its documented quota opt-out branch instead of crash-looping on the absent edge credential; dev image with IfNotPresent pull. FEDERATION_KUBECONFIG stays empty exactly as in production, so cell controllers run against the local cluster where Karmada propagates WorkloadDeployments. Federation access for the management operator mints a Karmada-native ServiceAccount token bound to the real compute-manager hub ClusterRole (config/base/downstream-rbac), so the operator authenticates as a non-admin identity and any missing grant surfaces as a forbidden error rather than being masked by cluster-admin. The Karmada bring-up is hardened for busy hosts: a 10m Helm wait that resumes a timed-out install, a wait for the cluster.karmada.io aggregated API before joining members, retries around karmadactl join while the member token Secret populates, and leader election disabled on the single-replica Karmada controllers to avoid a lease-renewal crashloop under etcd write-latency spikes. Co-Authored-By: Claude Fable 5 --- Taskfile.yaml | 681 ++++++++++++++++++ test/e2e/deploy/cell/config_patch.yaml | 26 + test/e2e/deploy/cell/deployment_patch.yaml | 13 + test/e2e/deploy/cell/kustomization.yaml | 31 + test/e2e/deploy/management/config_patch.yaml | 27 + .../deploy/management/deployment_patch.yaml | 24 + test/e2e/deploy/management/kustomization.yaml | 56 ++ 7 files changed, 858 insertions(+) create mode 100644 Taskfile.yaml create mode 100644 test/e2e/deploy/cell/config_patch.yaml create mode 100644 test/e2e/deploy/cell/deployment_patch.yaml create mode 100644 test/e2e/deploy/cell/kustomization.yaml create mode 100644 test/e2e/deploy/management/config_patch.yaml create mode 100644 test/e2e/deploy/management/deployment_patch.yaml create mode 100644 test/e2e/deploy/management/kustomization.yaml diff --git a/Taskfile.yaml b/Taskfile.yaml new file mode 100644 index 00000000..eab20228 --- /dev/null +++ b/Taskfile.yaml @@ -0,0 +1,681 @@ +version: '3' + +# ─── Variables ────────────────────────────────────────────────────────────── + +vars: + # Karmada Helm chart version to install (karmada-charts/karmada) + KARMADA_VERSION: v1.16.0 + + # karmadactl CLI version for cluster registration + KARMADACTL_VERSION: v1.16.0 + + # Chainsaw version for e2e testing (kyverno/chainsaw) + CHAINSAW_VERSION: v0.2.15 + + # Local tool directory (mirrors Makefile convention) + LOCALBIN: '{{.ROOT_DIR}}/bin' + KARMADACTL: '{{.ROOT_DIR}}/bin/karmadactl' + CHAINSAW: '{{.ROOT_DIR}}/bin/chainsaw' + + # Kind cluster names + KIND_CONTROL_PLANE: compute-control-plane + KIND_POP_DFW: compute-pop-dfw + KIND_POP_ORD: compute-pop-ord + + # All cluster names (for CRD installation loops) + KIND_ALL_CLUSTERS: '{{.KIND_CONTROL_PLANE}} {{.KIND_POP_DFW}} {{.KIND_POP_ORD}}' + + # Dev image tag built locally and side-loaded into every Kind cluster. This + # replaces the ghcr.io/datum-cloud/compute:latest reference baked into + # config/base/manager so e2e never pulls from a registry. + IMAGE: compute:e2e-dev + + # Working directory for e2e artefacts (gitignored) + E2E_DIR: '{{.ROOT_DIR}}/tmp/e2e' + KUBECONFIG_DIR: '{{.ROOT_DIR}}/tmp/e2e/kubeconfigs' + + # Fixed NodePort for the Karmada API server. + # The Kind management cluster is created with an extraPortMapping for this port + # so it is reachable at https://localhost:32443 from the developer's machine. + KARMADA_API_NODEPORT: "32443" + + # In-cluster Service address of the Karmada API server. The management + # compute-manager runs in the SAME Kind cluster as Karmada, so it reaches the + # hub over cluster DNS rather than the host-exposed NodePort. Matches the + # audience the production management-plane overlay projects its token for. + KARMADA_INCLUSTER_SERVER: "https://karmada-apiserver.karmada-system.svc.cluster.local:5443" + + # Namespace the operators deploy into (matches config/overlays/* namespace:). + COMPUTE_NAMESPACE: compute-system + +# ─── Tasks ────────────────────────────────────────────────────────────────── + +tasks: + + default: + cmds: + - task --list + silent: true + + # ════════════════════════════════════════════════════════════════════════ + # e2e environment lifecycle + # ════════════════════════════════════════════════════════════════════════ + + e2e:up: + desc: "Create the Kind+Karmada environment AND deploy the compute operators (idempotent)" + cmds: + - task: e2e:env:up + - task: e2e:deploy + - cmd: | + echo "" + echo "╔══════════════════════════════════════════════════════════╗" + echo "║ e2e environment ready — operators deployed in-cluster ║" + echo "╠══════════════════════════════════════════════════════════╣" + echo "║ Control plane: {{.KUBECONFIG_DIR}}/control-plane.yaml" + echo "║ Karmada API: {{.KUBECONFIG_DIR}}/karmada.yaml" + echo "║ POP DFW: {{.KUBECONFIG_DIR}}/pop-dfw.yaml" + echo "║ POP ORD: {{.KUBECONFIG_DIR}}/pop-ord.yaml" + echo "╠══════════════════════════════════════════════════════════╣" + echo "║ Inspect operators: ║" + echo "║ kubectl --kubeconfig {{.KUBECONFIG_DIR}}/control-plane.yaml -n {{.COMPUTE_NAMESPACE}} get deploy" + echo "╚══════════════════════════════════════════════════════════╝" + silent: false + + e2e:env:up: + desc: "Create the Kind clusters, install Karmada, register cells, install CRDs (no operators)" + cmds: + - task: e2e:tools + - task: e2e:clusters:create + - task: e2e:karmada:install + - task: e2e:karmada:configure + - task: e2e:karmada:join-clusters + - task: e2e:crds:install + + e2e:down: + desc: "Tear down the local e2e environment" + cmds: + - kind delete cluster --name {{.KIND_CONTROL_PLANE}} 2>/dev/null || true + - kind delete cluster --name {{.KIND_POP_DFW}} 2>/dev/null || true + - kind delete cluster --name {{.KIND_POP_ORD}} 2>/dev/null || true + - rm -rf {{.E2E_DIR}} + - cmd: echo "✓ e2e environment torn down" + silent: false + + e2e:test: + desc: "Run Chainsaw e2e tests against the local Kind+Karmada environment" + deps: [e2e:tools:chainsaw] + cmds: + - | + KUBECONFIG={{.KUBECONFIG_DIR}}/control-plane.yaml \ + {{.CHAINSAW}} test \ + --config test/e2e/chainsaw-config.yaml \ + test/e2e/ \ + {{.CLI_ARGS}} + + e2e:test:filter: + desc: "Run a subset of e2e tests by name regex (e.g. task e2e:test:filter -- --include-test-regex federation)" + deps: [e2e:tools:chainsaw] + cmds: + - | + KUBECONFIG={{.KUBECONFIG_DIR}}/control-plane.yaml \ + {{.CHAINSAW}} test \ + --config test/e2e/chainsaw-config.yaml \ + {{.CLI_ARGS}} \ + test/e2e/ + + # ════════════════════════════════════════════════════════════════════════ + # Tool installation + # ════════════════════════════════════════════════════════════════════════ + + e2e:tools: + desc: "Install e2e-specific tooling (karmadactl, chainsaw, helm repo)" + cmds: + - task: e2e:tools:karmadactl + - task: e2e:tools:chainsaw + - task: e2e:tools:helm-repo + + e2e:tools:karmadactl: + desc: "Download karmadactl {{.KARMADACTL_VERSION}}" + cmds: + - mkdir -p {{.LOCALBIN}} + - | + if [ ! -f "{{.KARMADACTL}}" ]; then + OS=$(uname -s | tr '[:upper:]' '[:lower:]') + ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') + URL="https://github.com/karmada-io/karmada/releases/download/{{.KARMADACTL_VERSION}}/karmadactl-${OS}-${ARCH}.tgz" + echo "Downloading karmadactl {{.KARMADACTL_VERSION}} (${OS}/${ARCH}) from ${URL}..." + curl -sSfL "${URL}" | tar -xz -C {{.LOCALBIN}} karmadactl + chmod +x {{.KARMADACTL}} + echo "karmadactl installed → {{.KARMADACTL}}" + else + echo "karmadactl already present at {{.KARMADACTL}}" + fi + status: + - test -f {{.KARMADACTL}} + + e2e:tools:chainsaw: + desc: "Download chainsaw {{.CHAINSAW_VERSION}}" + cmds: + - mkdir -p {{.LOCALBIN}} + - | + if [ ! -f "{{.CHAINSAW}}" ]; then + OS=$(uname -s | tr '[:upper:]' '[:lower:]') + ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') + URL="https://github.com/kyverno/chainsaw/releases/download/{{.CHAINSAW_VERSION}}/chainsaw_${OS}_${ARCH}.tar.gz" + echo "Downloading chainsaw {{.CHAINSAW_VERSION}} (${OS}/${ARCH}) from ${URL}..." + curl -sSfL "${URL}" | tar -xz -C {{.LOCALBIN}} chainsaw + chmod +x {{.CHAINSAW}} + echo "chainsaw installed → {{.CHAINSAW}}" + else + echo "chainsaw already present at {{.CHAINSAW}}" + fi + status: + - test -f {{.CHAINSAW}} + + e2e:tools:helm-repo: + desc: "Add/update karmada-charts Helm repository" + cmds: + - | + if ! helm repo list 2>/dev/null | grep -q karmada-charts; then + helm repo add karmada-charts https://raw.githubusercontent.com/karmada-io/karmada/master/charts + echo "Added karmada-charts Helm repository" + fi + helm repo update karmada-charts + status: + - helm repo list 2>/dev/null | grep -q karmada-charts + + # ════════════════════════════════════════════════════════════════════════ + # Kind cluster management + # ════════════════════════════════════════════════════════════════════════ + + e2e:clusters:create: + desc: "Create all Kind clusters (idempotent)" + cmds: + - mkdir -p {{.E2E_DIR}} + # Render the control-plane kind config with the configured Karmada NodePort. + # The committed template pins 32443; overriding KARMADA_API_NODEPORT (for + # example to dodge a host-port collision with another local cluster) is + # substituted through here so the port has a single source of truth. + - sed 's/32443/{{.KARMADA_API_NODEPORT}}/g' hack/e2e/kind-control-plane.yaml > {{.E2E_DIR}}/kind-control-plane.yaml + # Management / control-plane cell cluster — needs extraPortMappings for + # the Karmada API server NodePort so it is accessible at localhost:. + - task: _e2e:cluster:create + vars: + CLUSTER_NAME: "{{.KIND_CONTROL_PLANE}}" + KIND_CONFIG: "{{.E2E_DIR}}/kind-control-plane.yaml" + # POP cell clusters — default Kind config is sufficient. + - task: _e2e:cluster:create + vars: + CLUSTER_NAME: "{{.KIND_POP_DFW}}" + KIND_CONFIG: "" + - task: _e2e:cluster:create + vars: + CLUSTER_NAME: "{{.KIND_POP_ORD}}" + KIND_CONFIG: "" + - mkdir -p {{.KUBECONFIG_DIR}} + - task: _e2e:kubeconfigs:export + + _e2e:cluster:create: + internal: true + cmds: + - | + if kind get clusters 2>/dev/null | grep -qx '{{.CLUSTER_NAME}}'; then + echo "Kind cluster '{{.CLUSTER_NAME}}' already exists — skipping" + else + echo "Creating Kind cluster '{{.CLUSTER_NAME}}'..." + CONFIG_FLAG="" + if [ -n "{{.KIND_CONFIG}}" ]; then + CONFIG_FLAG="--config {{.KIND_CONFIG}}" + fi + kind create cluster \ + --name {{.CLUSTER_NAME}} \ + $CONFIG_FLAG \ + --wait 90s + fi + + _e2e:kubeconfigs:export: + internal: true + desc: "Export Kind kubeconfigs and create Docker-IP variants for cross-cluster use" + cmds: + # Standard kubeconfigs (localhost-based, for developer kubectl use) + - kind export kubeconfig --name {{.KIND_CONTROL_PLANE}} --kubeconfig {{.KUBECONFIG_DIR}}/control-plane.yaml + - kind export kubeconfig --name {{.KIND_POP_DFW}} --kubeconfig {{.KUBECONFIG_DIR}}/pop-dfw.yaml + - kind export kubeconfig --name {{.KIND_POP_ORD}} --kubeconfig {{.KUBECONFIG_DIR}}/pop-ord.yaml + # Docker-IP kubeconfigs (used by Karmada controller, running inside Docker, + # to reach POP cell API servers across the kind bridge network) + - | + hack/e2e/make-internal-kubeconfig.sh \ + {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ + {{.KUBECONFIG_DIR}}/pop-dfw-internal.yaml \ + {{.KIND_POP_DFW}} + - | + hack/e2e/make-internal-kubeconfig.sh \ + {{.KUBECONFIG_DIR}}/pop-ord.yaml \ + {{.KUBECONFIG_DIR}}/pop-ord-internal.yaml \ + {{.KIND_POP_ORD}} + + # ════════════════════════════════════════════════════════════════════════ + # Karmada installation + # ════════════════════════════════════════════════════════════════════════ + + e2e:karmada:install: + desc: "Install Karmada into the management cluster via Helm (idempotent)" + cmds: + # Idempotency keys on the karmada-apiserver Deployment existing rather than + # just the namespace: a Helm --wait that times out on a loaded machine + # leaves the namespace behind with the components still converging, and we + # want a re-run to resume that install (wait for it to finish) instead of + # skipping straight past a not-yet-ready control plane. The 10m timeout + # gives the Karmada components room to start when the host is busy running + # several Kind clusters at once. + - | + if kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system get deploy karmada-apiserver &>/dev/null; then + echo "Karmada already present — waiting for its components to become Available..." + else + echo "Installing Karmada {{.KARMADA_VERSION}} via Helm..." + helm install karmada karmada-charts/karmada \ + --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + --namespace karmada-system \ + --create-namespace \ + --version {{.KARMADA_VERSION}} \ + --set apiServer.serviceType=NodePort \ + --set apiServer.nodePort={{.KARMADA_API_NODEPORT}} \ + --wait \ + --timeout 10m \ + || echo "Helm --wait did not settle in time; will wait on the deployments directly below" + fi + # Disable leader election on the single-replica Karmada controllers. On a + # busy host, etcd write latency spikes cause the controllers' 5s lease + # renewals to time out; each lost lease exits the process, and the + # restart re-lists every resource, which adds I/O load and provokes the + # next latency spike — a crashloop feedback loop. With one replica there is + # nothing to elect, so turning it off breaks the loop and lets the control + # plane settle. Harmless in a single-replica e2e; not for production HA. + - task: _e2e:karmada:disable-leader-election + # Wait on the control-plane deployments regardless of how the install above + # exited, so a timed-out Helm run still converges before we build the + # kubeconfig and register cells. + - | + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system wait --for=condition=Available deploy --all --timeout=10m + echo "Karmada control plane is Available" + - task: _e2e:karmada:build-kubeconfig + + _e2e:karmada:disable-leader-election: + internal: true + cmds: + - | + for d in karmada-controller-manager karmada-kube-controller-manager karmada-scheduler; do + current=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system get deploy "$d" \ + -o jsonpath='{.spec.template.spec.containers[0].command}' 2>/dev/null) + if echo "$current" | grep -q -- "--leader-elect=false"; then + echo "Leader election already disabled on $d" + else + echo "Disabling leader election on $d..." + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system patch deploy "$d" --type=json \ + -p '[{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--leader-elect=false"}]' + fi + done + + e2e:karmada:configure: + desc: "Apply federation component config to the Karmada API server (idempotent)" + cmds: + - | + echo "Applying federation component to Karmada..." + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply \ + -k config/components/federation/ + echo "Federation component applied" + + _e2e:karmada:build-kubeconfig: + internal: true + desc: "Extract Karmada kubeconfig from secret and patch server to localhost:{{.KARMADA_API_NODEPORT}}" + cmds: + - | + echo "Building Karmada kubeconfig → {{.KUBECONFIG_DIR}}/karmada.yaml" + # Extract raw kubeconfig from the secret the Helm chart creates + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + get secret karmada-kubeconfig \ + -n karmada-system \ + -o jsonpath='{.data.kubeconfig}' \ + | base64 -d > {{.KUBECONFIG_DIR}}/karmada-raw.yaml + # Rewrite the server address to the NodePort exposed on localhost + python3 - {{.KUBECONFIG_DIR}}/karmada-raw.yaml {{.KUBECONFIG_DIR}}/karmada.yaml 127.0.0.1 {{.KARMADA_API_NODEPORT}} << 'PYEOF' + import sys, yaml + + src, dst, host, port = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] + + with open(src) as f: + cfg = yaml.safe_load(f) + + for cluster in cfg.get('clusters', []): + old = cluster['cluster'].get('server', '') + cluster['cluster']['server'] = f'https://{host}:{port}' + # The cert is for the internal cluster IP, so skip TLS verification. + # This is a local dev-only environment. + cluster['cluster']['insecure-skip-tls-verify'] = True + cluster['cluster'].pop('certificate-authority-data', None) + print(f" karmada server: {old} → https://{host}:{port}", file=sys.stderr) + + with open(dst, 'w') as f: + yaml.dump(cfg, f, default_flow_style=False) + PYEOF + rm {{.KUBECONFIG_DIR}}/karmada-raw.yaml + # Copy karmada.yaml → downstream.yaml so Chainsaw tests that declare + # cluster: downstream can load it (chainsaw-config.yaml references this path). + - cp {{.KUBECONFIG_DIR}}/karmada.yaml {{.KUBECONFIG_DIR}}/downstream.yaml + + # ════════════════════════════════════════════════════════════════════════ + # POP cell cluster registration + # ════════════════════════════════════════════════════════════════════════ + + e2e:karmada:join-clusters: + desc: "Register POP cell clusters with Karmada and apply city-code labels" + cmds: + # cluster.karmada.io is served by the karmada-aggregated-apiserver through + # API aggregation, and can briefly return ServiceUnavailable after the pod + # reports Ready while the aggregation handshake settles. Wait for the API + # to actually answer before registering members so join does not race it. + - | + echo "Waiting for the cluster.karmada.io aggregated API to be served..." + deadline=$((SECONDS+180)) + until kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml get clusters &>/dev/null; do + if [ $SECONDS -ge $deadline ]; then + echo "ERROR: cluster.karmada.io was not served within 180s"; exit 1 + fi + sleep 3 + done + echo "cluster.karmada.io is served" + - task: _e2e:karmada:join-cluster + vars: + CLUSTER_NAME: "{{.KIND_POP_DFW}}" + CITY_CODE: dfw + EXTERNAL_KUBECONFIG: "{{.KUBECONFIG_DIR}}/pop-dfw.yaml" + INTERNAL_KUBECONFIG: "{{.KUBECONFIG_DIR}}/pop-dfw-internal.yaml" + - task: _e2e:karmada:join-cluster + vars: + CLUSTER_NAME: "{{.KIND_POP_ORD}}" + CITY_CODE: ord + EXTERNAL_KUBECONFIG: "{{.KUBECONFIG_DIR}}/pop-ord.yaml" + INTERNAL_KUBECONFIG: "{{.KUBECONFIG_DIR}}/pop-ord-internal.yaml" + + _e2e:karmada:join-cluster: + internal: true + cmds: + # ── Register with karmadactl join ────────────────────────────────── + # We pass the EXTERNAL kubeconfig (localhost-based) here so karmadactl + # can reach the member cluster from this macOS host to set up initial + # RBAC. The stored secret is patched below to the Docker-IP variant. + # karmadactl join creates an impersonation ServiceAccount in the member + # cluster and blocks (with a non-tunable internal timeout) on that SA's + # legacy token Secret being populated. On a busy host the member cluster's + # token controller is slow, so the first join can time out even though the + # Secret populates moments later. Retry: a subsequent attempt reuses the + # now-populated Secret and completes immediately. + - | + if kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \ + get cluster {{.CLUSTER_NAME}} &>/dev/null; then + echo "Cluster '{{.CLUSTER_NAME}}' already registered in Karmada — skipping join" + else + echo "Joining '{{.CLUSTER_NAME}}' to Karmada..." + joined=false + for attempt in 1 2 3 4 5; do + if {{.KARMADACTL}} join {{.CLUSTER_NAME}} \ + --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \ + --cluster-kubeconfig={{.EXTERNAL_KUBECONFIG}} \ + --cluster-context=kind-{{.CLUSTER_NAME}}; then + joined=true + echo "Cluster '{{.CLUSTER_NAME}}' registered (attempt ${attempt})" + break + fi + echo "join attempt ${attempt} failed (member token Secret may still be populating); retrying in 10s..." + sleep 10 + done + if [ "${joined}" != "true" ]; then + echo "ERROR: failed to join '{{.CLUSTER_NAME}}' after 5 attempts"; exit 1 + fi + fi + # ── Patch cluster secret → Docker-IP kubeconfig ─────────────────── + # The Karmada controller manager runs inside Docker; it cannot use + # localhost to reach POP cell API servers. We update the stored secret + # with a kubeconfig whose server address uses the Kind container IP so + # container-to-container communication works across the kind bridge. + - | + hack/e2e/patch-cluster-secret.sh \ + {{.KUBECONFIG_DIR}}/karmada.yaml \ + {{.CLUSTER_NAME}} \ + {{.INTERNAL_KUBECONFIG}} + # ── Apply city-code label ────────────────────────────────────────── + - | + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \ + label cluster {{.CLUSTER_NAME}} \ + topology.datum.net/city-code={{.CITY_CODE}} \ + --overwrite + echo "Labeled cluster '{{.CLUSTER_NAME}}' with topology.datum.net/city-code={{.CITY_CODE}}" + + # ════════════════════════════════════════════════════════════════════════ + # CRD installation + # ════════════════════════════════════════════════════════════════════════ + + e2e:crds:install: + desc: "Install compute + NSO + quota CRDs to all clusters" + cmds: + - task: _e2e:crds:compute + - task: _e2e:crds:nso + - task: _e2e:crds:quota + + _e2e:crds:compute: + internal: true + desc: "Apply compute CRDs to all clusters and the Karmada API server" + cmds: + # All three Kind clusters + the Karmada API server get the compute CRDs. + # The Karmada API server needs them so it can store and propagate + # WorkloadDeployment objects. + - | + for KC in \ + {{.KUBECONFIG_DIR}}/control-plane.yaml \ + {{.KUBECONFIG_DIR}}/karmada.yaml \ + {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ + {{.KUBECONFIG_DIR}}/pop-ord.yaml; do + echo "Installing compute CRDs → $(basename $KC .yaml)..." + kubectl --kubeconfig="$KC" apply -k config/base/crd --server-side + done + + _e2e:crds:nso: + internal: true + desc: "Apply NSO CRDs to control-plane and POP cell clusters" + cmds: + # NSO CRDs (NetworkBinding, SubnetClaim, etc.) are installed on the + # control-plane as well as POP cells. The control-plane operator needs them + # so that Subnet/SubnetClaim informer watches can start without cache errors, + # even though NSO controllers themselves only run on POP cells. + - | + go mod download go.datum.net/network-services-operator + NSO_VERSION=$(go list -m -json go.datum.net/network-services-operator \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['Version'])") + NSO_CRD_PATH="$(go env GOMODCACHE)/go.datum.net/network-services-operator@${NSO_VERSION}/config/crd" + echo "NSO CRDs from: ${NSO_CRD_PATH}" + for KC in \ + {{.KUBECONFIG_DIR}}/control-plane.yaml \ + {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ + {{.KUBECONFIG_DIR}}/pop-ord.yaml; do + echo "Installing NSO CRDs → $(basename $KC .yaml)..." + kubectl --kubeconfig="$KC" apply -k "${NSO_CRD_PATH}" --server-side + done + + _e2e:crds:quota: + internal: true + desc: "Apply Milo quota CRDs to all clusters and the Karmada API server" + cmds: + # Quota CRDs (ResourceClaim, ResourceGrant, etc.) are required on all + # clusters so the InstanceReconciler can create and watch ResourceClaims + # against project Milo control planes without cache startup errors. + - | + go mod download go.miloapis.com/milo + MILO_VERSION=$(go list -m -json go.miloapis.com/milo \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['Version'])") + QUOTA_CRD_PATH="$(go env GOMODCACHE)/go.miloapis.com/milo@${MILO_VERSION}/config/crd/bases/quota" + echo "Milo quota CRDs from: ${QUOTA_CRD_PATH}" + for KC in \ + {{.KUBECONFIG_DIR}}/control-plane.yaml \ + {{.KUBECONFIG_DIR}}/karmada.yaml \ + {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ + {{.KUBECONFIG_DIR}}/pop-ord.yaml; do + echo "Installing Milo quota CRDs → $(basename $KC .yaml)..." + kubectl --kubeconfig="$KC" apply -k "${QUOTA_CRD_PATH}" --server-side + done + + # ════════════════════════════════════════════════════════════════════════ + # Operator image build + side-load + # ════════════════════════════════════════════════════════════════════════ + + e2e:image:build: + desc: "Build the compute-manager image with the local dev tag ({{.IMAGE}})" + cmds: + - | + echo "Building {{.IMAGE}} from {{.ROOT_DIR}}/Dockerfile..." + docker build -t {{.IMAGE}} {{.ROOT_DIR}} + + e2e:image:load: + desc: "Side-load {{.IMAGE}} into every Kind cluster (no registry pull)" + cmds: + - | + for CLUSTER in {{.KIND_ALL_CLUSTERS}}; do + echo "Loading {{.IMAGE}} → kind cluster ${CLUSTER}..." + kind load docker-image {{.IMAGE}} --name "${CLUSTER}" + done + + # ════════════════════════════════════════════════════════════════════════ + # Federation access for the management operator + # ════════════════════════════════════════════════════════════════════════ + + e2e:federation:setup: + desc: "Bind hub RBAC and mint the management operator's Karmada kubeconfig" + cmds: + # ── Hub-side RBAC (the real production manifest) ──────────────────── + # config/base/downstream-rbac grants the compute-manager ClusterRole on + # the Karmada hub and binds it to the user + # system:serviceaccount:compute-system:compute-manager. Applying the real + # manifest is the whole point of #149 — the management operator authenticates + # to Karmada as a non-admin identity and every missing grant surfaces as a + # forbidden error rather than being masked by cluster-admin. + - | + echo "Applying hub RBAC (config/base/downstream-rbac) to Karmada..." + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply -k config/base/downstream-rbac + # ── Karmada-native identity for the management operator ───────────── + # Production federates the management cluster's projected ServiceAccount + # token into Karmada (Karmada trusts the host cluster's token issuer). We + # do not configure cross-cluster token trust in the Kind environment, so + # instead we create a Karmada-native ServiceAccount whose authenticated + # username is identical — system:serviceaccount:compute-system:compute-manager + # — and therefore matches the very same ClusterRoleBinding subject. Same + # RBAC surface, no issuer federation required. + - | + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml create namespace {{.COMPUTE_NAMESPACE}} \ + --dry-run=client -o yaml | kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply -f - + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml -n {{.COMPUTE_NAMESPACE}} \ + create serviceaccount compute-manager \ + --dry-run=client -o yaml | kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply -f - + # ── Mint the federation kubeconfig ────────────────────────────────── + # A bound token from the Karmada SA, embedded in a kubeconfig that targets + # the Karmada in-cluster Service (the management pod runs alongside Karmada). + # We re-mint on every run so a re-deploy always ships a fresh, unexpired + # token. insecure-skip-tls-verify mirrors the karmada.yaml the harness + # already builds — the served cert does not cover the Service DNS name. + - | + mkdir -p {{.E2E_DIR}} + echo "Minting Karmada token for compute-manager..." + TOKEN=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml -n {{.COMPUTE_NAMESPACE}} \ + create token compute-manager --duration=720h) + cat > {{.E2E_DIR}}/downstream-kubeconfig.yaml < Date: Thu, 9 Jul 2026 18:01:37 -0500 Subject: [PATCH 03/23] test(e2e): scale a zeroed Karmada control plane back up on re-run Make e2e:karmada:install robust to a Karmada control plane whose deployments were scaled to zero (e.g. quiesced between runs on a busy host, or left partial by an earlier failed install). Before waiting for the deployments to become Available, scale any that sit at zero replicas back to one, so an idempotent re-run resumes the environment instead of blocking forever on a deployment that has no desired replicas. Co-Authored-By: Claude Fable 5 --- Taskfile.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Taskfile.yaml b/Taskfile.yaml index eab20228..37c0611b 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -293,6 +293,22 @@ tasks: # nothing to elect, so turning it off breaks the loop and lets the control # plane settle. Harmless in a single-replica e2e; not for production HA. - task: _e2e:karmada:disable-leader-election + # Scale any Karmada deployment that is sitting at zero back up to one. This + # makes the target robust to a control plane that was intentionally scaled + # down (for example to keep a busy host calm between runs) or left at zero + # by a partial install: without this, the Available wait below would block + # forever on a deployment that has no desired replicas. + - | + for d in $(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system get deploy -o jsonpath='{.items[*].metadata.name}'); do + reps=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system get deploy "$d" -o jsonpath='{.spec.replicas}') + if [ "$reps" = "0" ]; then + echo "Scaling Karmada deploy $d back to 1 (was scaled to 0)..." + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system scale deploy "$d" --replicas=1 + fi + done # Wait on the control-plane deployments regardless of how the install above # exited, so a timed-out Helm run still converges before we build the # kubeconfig and register cells. From af7a86e780a096d506560fcdfdda6b86fd8bd2fc Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 18:09:57 -0500 Subject: [PATCH 04/23] test(e2e): restore federation and referenced-data chainsaw suites Restore the eight chainsaw e2e suites that exercise the compute federation and referenced-data delivery paths, bringing them back onto the in-cluster harness branch: - full-federation, workload-deployment-federation, instance-projection, instance-writeback, propagation-policy-lifecycle, deletion-cascade (from split/federation-e2e) - referenced-data-mounts, referenced-data-delete-cascade (from archive/e2e-local-deferred) Restored verbatim; adaptation to the in-cluster deploy path follows in a separate commit. Co-Authored-By: Claude Fable 5 --- .../assert-downstream-wd-exists.yaml | 7 + test/e2e/deletion-cascade/chainsaw-test.yaml | 79 ++++ .../deletion-cascade/workload-deployment.yaml | 21 + test/e2e/full-federation/chainsaw-test.yaml | 150 ++++++++ .../full-federation/workload-deployment.yaml | 21 + .../assert-downstream-wd.yaml | 6 + .../assert-projected-instance.yaml | 19 + .../instance-projection/chainsaw-test.yaml | 123 ++++++ .../workload-deployment.yaml | 21 + .../assert-downstream-instance.yaml | 16 + .../e2e/instance-writeback/chainsaw-test.yaml | 112 ++++++ .../instance-writeback/instance-pop-dfw.yaml | 15 + .../assert-pp-exists.yaml | 6 + .../chainsaw-test.yaml | 133 +++++++ .../workload-deployment-alpha.yaml | 21 + .../workload-deployment-beta.yaml | 21 + .../chainsaw-test.yaml | 358 ++++++++++++++++++ .../source-configmap.yaml | 7 + .../source-secret.yaml | 7 + .../workload-deployment.yaml | 38 ++ test/e2e/referenced-data-mounts/README.md | 128 +++++++ .../referenced-data-mounts/chainsaw-test.yaml | 349 +++++++++++++++++ .../source-configmap.yaml | 8 + .../referenced-data-mounts/source-secret.yaml | 8 + .../workload-deployment.yaml | 38 ++ .../assert-downstream-pp.yaml | 20 + .../assert-downstream-wd.yaml | 9 + .../chainsaw-test.yaml | 84 ++++ .../workload-deployment.yaml | 22 ++ 29 files changed, 1847 insertions(+) create mode 100644 test/e2e/deletion-cascade/assert-downstream-wd-exists.yaml create mode 100644 test/e2e/deletion-cascade/chainsaw-test.yaml create mode 100644 test/e2e/deletion-cascade/workload-deployment.yaml create mode 100644 test/e2e/full-federation/chainsaw-test.yaml create mode 100644 test/e2e/full-federation/workload-deployment.yaml create mode 100644 test/e2e/instance-projection/assert-downstream-wd.yaml create mode 100644 test/e2e/instance-projection/assert-projected-instance.yaml create mode 100644 test/e2e/instance-projection/chainsaw-test.yaml create mode 100644 test/e2e/instance-projection/workload-deployment.yaml create mode 100644 test/e2e/instance-writeback/assert-downstream-instance.yaml create mode 100644 test/e2e/instance-writeback/chainsaw-test.yaml create mode 100644 test/e2e/instance-writeback/instance-pop-dfw.yaml create mode 100644 test/e2e/propagation-policy-lifecycle/assert-pp-exists.yaml create mode 100644 test/e2e/propagation-policy-lifecycle/chainsaw-test.yaml create mode 100644 test/e2e/propagation-policy-lifecycle/workload-deployment-alpha.yaml create mode 100644 test/e2e/propagation-policy-lifecycle/workload-deployment-beta.yaml create mode 100644 test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml create mode 100644 test/e2e/referenced-data-delete-cascade/source-configmap.yaml create mode 100644 test/e2e/referenced-data-delete-cascade/source-secret.yaml create mode 100644 test/e2e/referenced-data-delete-cascade/workload-deployment.yaml create mode 100644 test/e2e/referenced-data-mounts/README.md create mode 100644 test/e2e/referenced-data-mounts/chainsaw-test.yaml create mode 100644 test/e2e/referenced-data-mounts/source-configmap.yaml create mode 100644 test/e2e/referenced-data-mounts/source-secret.yaml create mode 100644 test/e2e/referenced-data-mounts/workload-deployment.yaml create mode 100644 test/e2e/workload-deployment-federation/assert-downstream-pp.yaml create mode 100644 test/e2e/workload-deployment-federation/assert-downstream-wd.yaml create mode 100644 test/e2e/workload-deployment-federation/chainsaw-test.yaml create mode 100644 test/e2e/workload-deployment-federation/workload-deployment.yaml diff --git a/test/e2e/deletion-cascade/assert-downstream-wd-exists.yaml b/test/e2e/deletion-cascade/assert-downstream-wd-exists.yaml new file mode 100644 index 00000000..aae65da1 --- /dev/null +++ b/test/e2e/deletion-cascade/assert-downstream-wd-exists.yaml @@ -0,0 +1,7 @@ +# Assert the WorkloadDeployment is present in the Karmada API server. +# Used both to confirm federation succeeded and as the target for the error: check. +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + namespace: ($downstreamNS) + name: test-cascade-wd diff --git a/test/e2e/deletion-cascade/chainsaw-test.yaml b/test/e2e/deletion-cascade/chainsaw-test.yaml new file mode 100644 index 00000000..03a11ea0 --- /dev/null +++ b/test/e2e/deletion-cascade/chainsaw-test.yaml @@ -0,0 +1,79 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: deletion-cascade +spec: + description: | + Verifies that deleting a WorkloadDeployment from the project namespace causes + the federator to remove the corresponding WorkloadDeployment from Karmada. + + The WorkloadDeploymentFederator adds a finalizer + (compute.datumapis.com/federator) to every project WD it manages. When the + project WD is deleted: + 1. The finalizer's Finalize method runs (blocking deletion until complete). + 2. It deletes the Karmada-side WorkloadDeployment. + 3. It removes the PropagationPolicy if no other WDs for the city remain. + 4. It removes the finalizer, allowing the project WD to be garbage-collected. + + This test validates: project WD deletion → Karmada WD deletion. + + template: true + + steps: + - name: create-wd + description: Create a WorkloadDeployment on the control-plane cluster. + try: + - apply: + file: workload-deployment.yaml + + - name: wait-for-federation + description: Wait for the WorkloadDeployment to appear in Karmada. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-cascade-wd + + - name: delete-wd + description: Delete the WorkloadDeployment from the control-plane cluster. + try: + - delete: + ref: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($namespace) + name: test-cascade-wd + + - name: assert-downstream-wd-deleted + description: Confirm the Karmada copy is removed by the finalizer. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - wait: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($downstreamNS) + name: test-cascade-wd + timeout: 30s + for: + deletion: {} diff --git a/test/e2e/deletion-cascade/workload-deployment.yaml b/test/e2e/deletion-cascade/workload-deployment.yaml new file mode 100644 index 00000000..39d68a1d --- /dev/null +++ b/test/e2e/deletion-cascade/workload-deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-cascade-wd +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/full-federation/chainsaw-test.yaml b/test/e2e/full-federation/chainsaw-test.yaml new file mode 100644 index 00000000..020a2bc9 --- /dev/null +++ b/test/e2e/full-federation/chainsaw-test.yaml @@ -0,0 +1,150 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: full-federation +spec: + description: | + End-to-end federation chain test. + + Exercises the complete path from WorkloadDeployment creation through to + Instance projection on the control-plane cluster: + + 1. Create WorkloadDeployment on control-plane. + 2. WorkloadDeploymentFederator replicates it to Karmada (ns- namespace). + 3. Karmada PropagationPolicy routes the WD to pop-dfw. + 4. WorkloadDeploymentReconciler on pop-dfw creates Instance test-full-fed-wd-0. + 5. InstanceReconciler on pop-dfw writes Instance back to Karmada with + label meta.datumapis.com/upstream-cluster-name: cluster-single. + 6. InstanceProjector on control-plane creates a projection of the Instance + in the project namespace. + + Prerequisites: both operator instances must be running (task e2e:operator:start). + + template: true + + steps: + - name: create-workload-deployment + description: Create the WorkloadDeployment on the control-plane cluster. + try: + - apply: + file: workload-deployment.yaml + + - name: assert-wd-in-downstream + description: Assert WorkloadDeploymentFederator replicated the WD to Karmada and status is synced back. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd + - assert: + # Wait for the cell operator to write status back to the Karmada WD. + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd + status: + replicas: 1 + desiredReplicas: 1 + + - name: assert-wd-on-pop-dfw + description: Assert Karmada propagated the WD to pop-dfw and the cell reconciler set status. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + # Karmada propagation can take longer than a local apply. + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd + status: + replicas: 1 + desiredReplicas: 1 + + - name: assert-instance-on-pop-dfw + description: Assert WorkloadDeploymentReconciler created an Instance on pop-dfw with a Ready condition. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd-0 + (status.conditions[?type == 'Ready'] | [0]): + status: "Unknown" + + - name: assert-instance-writeback-in-downstream + description: Assert InstanceReconciler wrote the Instance back to Karmada. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd-0 + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + + - name: assert-instance-projected-to-control-plane + description: Assert InstanceProjector created a projection with status on the control-plane. + try: + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($namespace) + name: test-full-fed-wd-0 + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + (status.conditions[?type == 'Ready'] | [0]): + status: "Unknown" diff --git a/test/e2e/full-federation/workload-deployment.yaml b/test/e2e/full-federation/workload-deployment.yaml new file mode 100644 index 00000000..70b4cb94 --- /dev/null +++ b/test/e2e/full-federation/workload-deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-full-fed-wd + # namespace is injected by Chainsaw from ($namespace) +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/instance-projection/assert-downstream-wd.yaml b/test/e2e/instance-projection/assert-downstream-wd.yaml new file mode 100644 index 00000000..705d0893 --- /dev/null +++ b/test/e2e/instance-projection/assert-downstream-wd.yaml @@ -0,0 +1,6 @@ +# Assert the WorkloadDeployment is federated to Karmada (and the Karmada namespace created). +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + namespace: ($downstreamNS) + name: test-projector-wd diff --git a/test/e2e/instance-projection/assert-projected-instance.yaml b/test/e2e/instance-projection/assert-projected-instance.yaml new file mode 100644 index 00000000..0542194d --- /dev/null +++ b/test/e2e/instance-projection/assert-projected-instance.yaml @@ -0,0 +1,19 @@ +# Assert the InstanceProjector created a projection in the project namespace. +# +# The InstanceProjector (internal/controller/instance_projector.go): +# - Watches Instances in Karmada that carry upstreamClusterNameLabel +# - Strips "cluster-" prefix to get the cluster name ("single" in single-provider mode) +# - Finds the project namespace by matching ns- to namespace UIDs +# - Creates/updates the Instance projection in the project namespace +# - Sets an owner reference to the WorkloadDeployment for cascading deletion +apiVersion: compute.datumapis.com/v1alpha +kind: Instance +metadata: + # namespace is the Chainsaw test namespace (the project namespace on control-plane) + name: test-projected-instance + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + ownerReferences: + - apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + name: test-projector-wd diff --git a/test/e2e/instance-projection/chainsaw-test.yaml b/test/e2e/instance-projection/chainsaw-test.yaml new file mode 100644 index 00000000..16fa9f96 --- /dev/null +++ b/test/e2e/instance-projection/chainsaw-test.yaml @@ -0,0 +1,123 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: instance-projection +spec: + description: | + Verifies that the InstanceProjector watches Instances written back to the + Karmada API server and creates corresponding read-only projections in the + project namespace on the control-plane cluster. + + Flow: + 1. Create a WorkloadDeployment → triggers federator → Karmada namespace created. + 2. Write an Instance to Karmada (simulating a POP-cell InstanceReconciler write-back). + 3. InstanceProjector detects the Karmada Instance and creates a projection in the + project namespace (the Chainsaw test namespace on the control-plane cluster). + 4. Assert the projection exists with the upstream tracking label and an owner + reference to the WorkloadDeployment (for cascading deletion). + + Cluster name label: "cluster-single" + The compute operator runs in single-provider mode for this e2e environment, + registering the control-plane cluster with the multicluster-runtime manager + under the name "single" (see cmd/main.go, wrappedSingleClusterProvider). + + template: true + + steps: + - name: create-wd + description: Create the WorkloadDeployment to trigger federation and namespace creation. + try: + - apply: + file: workload-deployment.yaml + + - name: wait-for-downstream-namespace + description: Wait for the federated WorkloadDeployment to appear in Karmada. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-projector-wd + + - name: write-instance-to-downstream + description: | + Write an Instance to Karmada simulating InstanceReconciler write-back. + Uses explicit control-plane kubeconfig to derive downstreamNS and WD UID. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get workloaddeployment test-projector-wd \ + --namespace "$NAMESPACE" \ + -o jsonpath='{.metadata.uid}' + outputs: + - name: wdUID + value: ($stdout) + - script: + env: + - name: KARMADA_NS + value: ($downstreamNS) + - name: WD_UID + value: ($wdUID) + content: | + kubectl apply -f - < is the multicluster-runtime cluster name registered by +# wrappedSingleClusterProvider (always "single" in single-cluster mode) +# - Label meta.datumapis.com/upstream-namespace = the POP-cell namespace +apiVersion: compute.datumapis.com/v1alpha +kind: Instance +metadata: + namespace: ($instanceNS) + name: test-writeback-instance + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + meta.datumapis.com/upstream-namespace: ($instanceNS) diff --git a/test/e2e/instance-writeback/chainsaw-test.yaml b/test/e2e/instance-writeback/chainsaw-test.yaml new file mode 100644 index 00000000..32dbbc5d --- /dev/null +++ b/test/e2e/instance-writeback/chainsaw-test.yaml @@ -0,0 +1,112 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: instance-writeback +spec: + description: | + Verifies that the InstanceReconciler running in a POP-cell cluster writes + Instance objects back to the Karmada API server after reconciling the Ready + condition for the first time. + + Write-back convention (internal/controller/instance_controller.go): + - The Instance is written to Karmada at the same namespace/name as the POP-cell Instance. + - Label meta.datumapis.com/upstream-cluster-name is set to + "cluster-" (e.g. "cluster-compute-pop-dfw"). + - Label meta.datumapis.com/upstream-namespace records the originating namespace. + + Note: this test requires the compute operator (InstanceReconciler) to be running + in the DFW POP cell cluster. + + template: true + + steps: + - name: setup-namespaces + description: Create the Instance namespace in the DFW POP cell and Karmada. + try: + - script: + content: | + kubectl get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: instanceNS + value: ($stdout) + - script: + env: + - name: INSTANCE_NS + value: ($instanceNS) + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml \ + create namespace "$INSTANCE_NS" \ + --dry-run=client -o yaml | \ + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml apply -f - + - script: + env: + - name: INSTANCE_NS + value: ($instanceNS) + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/downstream.yaml \ + create namespace "$INSTANCE_NS" \ + --dry-run=client -o yaml | \ + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/downstream.yaml apply -f - + cleanup: + - script: + env: + - name: INSTANCE_NS + value: ($instanceNS) + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml \ + delete namespace "$INSTANCE_NS" --ignore-not-found + - script: + env: + - name: INSTANCE_NS + value: ($instanceNS) + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/downstream.yaml \ + delete namespace "$INSTANCE_NS" --ignore-not-found + + - name: create-instance-on-pop-dfw + description: Create the Instance on the DFW POP cell cluster. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: instanceNS + value: ($stdout) + - apply: + file: instance-pop-dfw.yaml + cleanup: + - script: + content: | + INSTANCE_NS=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}') + kubectl delete instance test-writeback-instance \ + --namespace "$INSTANCE_NS" --ignore-not-found + + - name: assert-instance-in-downstream + description: Wait for the InstanceReconciler to write back the Instance to Karmada. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: instanceNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($instanceNS) + name: test-writeback-instance + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + meta.datumapis.com/upstream-namespace: ($instanceNS) diff --git a/test/e2e/instance-writeback/instance-pop-dfw.yaml b/test/e2e/instance-writeback/instance-pop-dfw.yaml new file mode 100644 index 00000000..250eb7d7 --- /dev/null +++ b/test/e2e/instance-writeback/instance-pop-dfw.yaml @@ -0,0 +1,15 @@ +# Instance created in the DFW POP cell. +# ($instanceNS) is the namespace derived from the Chainsaw test namespace UID, +# matching the ns- convention so the InstanceProjector can resolve it later. +apiVersion: compute.datumapis.com/v1alpha +kind: Instance +metadata: + name: test-writeback-instance + namespace: ($instanceNS) +spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network diff --git a/test/e2e/propagation-policy-lifecycle/assert-pp-exists.yaml b/test/e2e/propagation-policy-lifecycle/assert-pp-exists.yaml new file mode 100644 index 00000000..77a817a5 --- /dev/null +++ b/test/e2e/propagation-policy-lifecycle/assert-pp-exists.yaml @@ -0,0 +1,6 @@ +# Asserts that the PropagationPolicy for city dfw exists in the Karmada namespace. +apiVersion: policy.karmada.io/v1alpha1 +kind: PropagationPolicy +metadata: + namespace: ($downstreamNS) + name: workload-deployments-dfw diff --git a/test/e2e/propagation-policy-lifecycle/chainsaw-test.yaml b/test/e2e/propagation-policy-lifecycle/chainsaw-test.yaml new file mode 100644 index 00000000..5678c398 --- /dev/null +++ b/test/e2e/propagation-policy-lifecycle/chainsaw-test.yaml @@ -0,0 +1,133 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: propagation-policy-lifecycle +spec: + description: | + Verifies the PropagationPolicy lifecycle managed by the WorkloadDeploymentFederator: + + - A PropagationPolicy (city-dfw) is lazily created when the first WorkloadDeployment + for city code "dfw" is federated to Karmada. + - The PropagationPolicy is RETAINED while at least one WorkloadDeployment for + that city code remains in the Karmada namespace. + - The PropagationPolicy is DELETED when the last deployment for the city is removed. + + The test creates two WDs (wd-alpha, wd-beta) both targeting cityCode=dfw, verifies + the PP appears, deletes wd-alpha and asserts the PP is still present, then deletes + wd-beta and waits for the PP to disappear. + + template: true + + steps: + - name: create-deployments + description: Create two WorkloadDeployments targeting dfw on the control-plane. + try: + - apply: + file: workload-deployment-alpha.yaml + - apply: + file: workload-deployment-beta.yaml + + - name: assert-policy-created + description: | + Assert both WDs are federated to Karmada and the PropagationPolicy exists. + Both WDs must be present in Karmada before proceeding to the deletion steps; + otherwise wd-alpha's finalizer could see an empty Karmada list and prematurely + delete the PP before wd-beta has been federated. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: wd-alpha + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: wd-beta + - assert: + timeout: 30s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-dfw + + - name: delete-alpha + description: Delete wd-alpha; wd-beta still targets dfw so the PP must be retained. + try: + - delete: + ref: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($namespace) + name: wd-alpha + + - name: assert-policy-retained + description: Assert the PropagationPolicy is still present after wd-alpha is deleted. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - sleep: + duration: 8s + - assert: + timeout: 5s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-dfw + + - name: delete-beta + description: Delete wd-beta (the last WD for city dfw). + try: + - delete: + ref: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($namespace) + name: wd-beta + + - name: assert-policy-deleted + description: Wait for the PropagationPolicy to be removed once no WDs remain. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - wait: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + namespace: ($downstreamNS) + name: city-dfw + timeout: 30s + for: + deletion: {} diff --git a/test/e2e/propagation-policy-lifecycle/workload-deployment-alpha.yaml b/test/e2e/propagation-policy-lifecycle/workload-deployment-alpha.yaml new file mode 100644 index 00000000..f9eb27fd --- /dev/null +++ b/test/e2e/propagation-policy-lifecycle/workload-deployment-alpha.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: wd-alpha +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/propagation-policy-lifecycle/workload-deployment-beta.yaml b/test/e2e/propagation-policy-lifecycle/workload-deployment-beta.yaml new file mode 100644 index 00000000..fd1d65c1 --- /dev/null +++ b/test/e2e/propagation-policy-lifecycle/workload-deployment-beta.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: wd-beta +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml b/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml new file mode 100644 index 00000000..97f4c7cf --- /dev/null +++ b/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml @@ -0,0 +1,358 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: referenced-data-delete-cascade +spec: + description: | + Validates the full delete-cascade for referenced-data companions and the + level-triggered companion GC backstop (CompanionGCReconciler). + + Two scenarios are validated: + + SCENARIO 1 — HAPPY-PATH CASCADE + Create a WorkloadDeployment referencing a ConfigMap + Secret. Assert companions + materialize and propagate to the member cluster. Delete the WD (last referrer). + Assert the hub companion, its Karmada ResourceBinding, and the member-cluster + copy are all deleted and stay deleted (no re-create loop). + + SCENARIO 2 — STRANDED COMPANION BACKSTOP + Simulate interrupted finalization by creating a referenced-data-labeled + companion whose referenced-by annotation points at a non-existent WD. + Assert the CompanionGCReconciler reclaims it within the sweep interval: + the stranded companion and its ResourceBinding are both deleted. + + Prerequisites: + - task e2e:up completed + - operators started with featureFlags.enableReferencedDataGate: true + (task e2e:operator:start:referenced-data) + - CompanionGCReconciler enabled in the management operator + - tmp/e2e/kubeconfigs/downstream.yaml exists + + template: true + + steps: + + # ═══════════════════════════════════════════════════════════════════════════ + # SCENARIO 1: HAPPY-PATH CASCADE + # ═══════════════════════════════════════════════════════════════════════════ + + - name: s1-create-source-data + description: Create the source ConfigMap and Secret in the project namespace. + try: + - apply: + file: source-configmap.yaml + - apply: + file: source-secret.yaml + + - name: s1-create-workload-deployment + description: Create the WorkloadDeployment referencing both sources. + try: + - apply: + file: workload-deployment.yaml + + - name: s1-assert-companions-on-hub + description: | + Assert companions materialize in ns-{project-uid} on the Karmada hub + with the referenced-data label. The CompanionGCReconciler watches these. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($companionNS) + name: gc-test-config + labels: + compute.datumapis.com/referenced-data: "true" + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($companionNS) + name: gc-test-secret + labels: + compute.datumapis.com/referenced-data: "true" + + - name: s1-assert-companion-rbs-on-hub + description: | + Assert ResourceBindings for both companions exist on the hub. + These RBs will be deleted by the ReferencedDataController's explicit + teardown (Component 3) when the WD is deleted. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: work.karmada.io/v1alpha2 + kind: ResourceBinding + metadata: + namespace: ($companionNS) + name: gc-test-config-configmap + - assert: + timeout: 30s + resource: + apiVersion: work.karmada.io/v1alpha2 + kind: ResourceBinding + metadata: + namespace: ($companionNS) + name: gc-test-secret-secret + + - name: s1-assert-companions-on-cell + description: Assert Karmada propagated companions to the pop-dfw cell. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: cellNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($cellNS) + name: gc-test-config + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($cellNS) + name: gc-test-secret + + - name: s1-delete-workload-deployment + description: Delete the WorkloadDeployment (the sole referrer). + try: + - delete: + ref: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($namespace) + name: gc-test-wd + + - name: s1-assert-hub-companion-cm-deleted + description: | + Assert the hub companion ConfigMap is deleted by the ReferencedDataController + finalizer after the WD is gone. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - wait: + apiVersion: v1 + kind: ConfigMap + namespace: ($companionNS) + name: gc-test-config + timeout: 60s + for: + deletion: {} + + - name: s1-assert-hub-companion-secret-deleted + description: Assert the hub companion Secret is also deleted. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - wait: + apiVersion: v1 + kind: Secret + namespace: ($companionNS) + name: gc-test-secret + timeout: 60s + for: + deletion: {} + + - name: s1-assert-rbs-deleted + description: | + Assert the ResourceBindings are deleted (Component 3 explicit teardown). + Deletion of the RB drives Karmada to remove the Work and cell copies. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - wait: + apiVersion: work.karmada.io/v1alpha2 + kind: ResourceBinding + namespace: ($companionNS) + name: gc-test-config-configmap + timeout: 60s + for: + deletion: {} + - wait: + apiVersion: work.karmada.io/v1alpha2 + kind: ResourceBinding + namespace: ($companionNS) + name: gc-test-secret-secret + timeout: 60s + for: + deletion: {} + + - name: s1-assert-cell-copies-deleted-and-stay-deleted + description: | + Assert the cell copies are gone and STAY gone for 30 seconds. + If a Work were still present, Karmada would re-create the cell copy + within ~5 seconds — the poll window catches any recreate loop. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: cellNS + value: ($stdout) + - wait: + apiVersion: v1 + kind: ConfigMap + namespace: ($cellNS) + name: gc-test-config + timeout: 90s + for: + deletion: {} + - wait: + apiVersion: v1 + kind: Secret + namespace: ($cellNS) + name: gc-test-secret + timeout: 90s + for: + deletion: {} + - script: + # Poll for 30 seconds to confirm no recreate loop. If Karmada re-creates + # the cell copy from a still-live Work, this script catches it. + timeout: 40s + content: | + CELL_NS=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}') + for i in $(seq 1 6); do + sleep 5 + CM=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml \ + get configmap gc-test-config \ + --namespace "$CELL_NS" \ + --ignore-not-found 2>/dev/null) + if [ -n "$CM" ]; then + echo "ERROR: gc-test-config ConfigMap was re-created on cell (Karmada recreate loop!)" + exit 1 + fi + SEC=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml \ + get secret gc-test-secret \ + --namespace "$CELL_NS" \ + --ignore-not-found 2>/dev/null) + if [ -n "$SEC" ]; then + echo "ERROR: gc-test-secret Secret was re-created on cell (Karmada recreate loop!)" + exit 1 + fi + done + echo "OK: cell copies absent for 30+ seconds — no recreate loop" + + # ═══════════════════════════════════════════════════════════════════════════ + # SCENARIO 2: STRANDED COMPANION BACKSTOP + # + # Simulate interrupted finalization: inject a labeled companion whose + # referenced-by annotation points at a WD that does not exist. The + # CompanionGCReconciler must detect and reclaim it. + # ═══════════════════════════════════════════════════════════════════════════ + + - name: s2-inject-stranded-companion + description: | + Inject a stranded companion ConfigMap directly into the hub namespace. + Its referenced-by annotation points at "default/nonexistent-wd" — a WD + that does not (and never did) exist. This simulates an interrupted + finalization where the controller pod restarted mid-flight. + cluster: downstream + try: + - script: + content: | + COMPANION_NS=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}') + + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/downstream.yaml \ + apply -f - < tmp/e2e/logs/operator-management-refdata.log 2>&1 & + +# Cell operator — pop-dfw cluster, feature flag on +KUBECONFIG=tmp/e2e/kubeconfigs/pop-dfw.yaml \ +go run ./cmd/main.go \ + --federation-kubeconfig=tmp/e2e/kubeconfigs/karmada.yaml \ + --enable-management-controllers=false \ + --enable-cell-controllers=true \ + --leader-elect=false \ + --health-probe-bind-address=:9092 \ + --server-config=hack/e2e/operator-config-referenced-data.yaml \ + > tmp/e2e/logs/operator-cell-dfw-refdata.log 2>&1 & +``` + +Wait for health checks on `:9091` and `:9092` before running the test. + +## Running just this scenario + +```sh +KUBECONFIG=tmp/e2e/kubeconfigs/control-plane.yaml \ +bin/chainsaw test \ + --config test/e2e/chainsaw-config.yaml \ + --include-test-regex "referenced-data-mounts" \ + test/e2e/ +``` + +Or via the Taskfile filter target: + +```sh +task e2e:test:filter -- --include-test-regex referenced-data-mounts +``` + +## Harness notes + +The following items were gaps at the time this test was written and have since +been fixed in `Taskfile.yaml`: + +1. `_e2e:karmada:build-kubeconfig` now copies `karmada.yaml` → + `downstream.yaml`, so `cluster: downstream` steps work out of the box. +2. `e2e:operator:start` now uses `--federation-kubeconfig` (the correct flag + name) for both management and cell operators. +3. `e2e:operator:start` now passes `--enable-management-controllers=true` to + the management operator, enabling the WorkloadDeploymentFederator and + InstanceProjector controllers. +4. `e2e:operator:start:referenced-data` is a dedicated task that starts both + operators with the `enableReferencedDataGate` feature flag on. +5. `e2e:crds:install` now installs Milo quota CRDs to all clusters so the + InstanceReconciler's ResourceClaim watches start cleanly. + +## Companion naming convention + +The `ReferencedDataController` derives companion names deterministically: + +| Source | Companion name | +|--------|---------------| +| `ConfigMap/app-config` | `configmap.app-config` | +| `Secret/app-secret` | `secret.app-secret` | + +These names are asserted directly in the test steps. diff --git a/test/e2e/referenced-data-mounts/chainsaw-test.yaml b/test/e2e/referenced-data-mounts/chainsaw-test.yaml new file mode 100644 index 00000000..b4f5a2b6 --- /dev/null +++ b/test/e2e/referenced-data-mounts/chainsaw-test.yaml @@ -0,0 +1,349 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: referenced-data-mounts +spec: + description: | + Validates the FEDERATED DELIVERY path for referenced ConfigMap and Secret data. + + This test exercises the cross-plane chain introduced by the referenced-data + feature (feat/configmap-secret-mounts-federated): + + [Hop 1] Source ConfigMap and Secret are created in the project namespace + (control-plane, Chainsaw default cluster). + + [Hop 2] WorkloadDeployment is created; ReferencedDataController materialises + companion objects in ns-{project-uid} on the Karmada hub (downstream) + and stamps the expected-referenced-data annotation on the WD. + Companions are NOT written to the control-plane cluster. + + [Hop 3] Karmada hub holds the companion ConfigMap + Secret in ns-{project-uid}, + propagated by the always-on label selector in the city-dfw + PropagationPolicy (which includes ConfigMap and Secret selectors). + + [Hop 4] Karmada propagates the WD + companions to pop-dfw. + Companions appear in ns-{project-uid} on the cell alongside the WD. + + [Hop 5] WorkloadDeploymentReconciler on pop-dfw creates Instance test-refdata-wd-0 + with the ReferencedData scheduling gate (feature flag on). + InstanceReconciler clears the gate once companions are present and sets + ReferencedDataReady=True on the Instance. + + Scope: cross-plane DELIVERY up to the Instance gate-cleared state. + Actual env-var and file mounting is the provider+kubelet layer and is NOT + asserted here. See README.md for full scope and prerequisites. + + Prerequisites: + - task e2e:up completed (control-plane + Karmada + pop-dfw + pop-ord) + - operators started with featureFlags.enableReferencedDataGate: true + (see README.md — NOT the stock task e2e:operator:start) + - tmp/e2e/kubeconfigs/downstream.yaml exists (symlink/copy of karmada.yaml) + + template: true + + steps: + + # ─── Hop 1: create source data ───────────────────────────────────────────── + + - name: create-source-data + description: Create the source ConfigMap and Secret in the project namespace. + try: + - apply: + file: source-configmap.yaml + - apply: + file: source-secret.yaml + + # ─── Hop 2: create WD; assert companion materialisation on Karmada hub ───── + + - name: create-workload-deployment + description: Create the WorkloadDeployment referencing the source ConfigMap and Secret. + try: + - apply: + file: workload-deployment.yaml + + - name: assert-companion-configmap-on-hub + description: | + Assert the ReferencedDataController materialised companion configmap.app-config + in ns-{project-uid} on the Karmada hub (downstream cluster). + Companions are written to the hub namespace, NOT the control-plane namespace. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($companionNS) + name: configmap.app-config + labels: + compute.datumapis.com/referenced-data: "true" + + - name: assert-companion-secret-on-hub + description: | + Assert the ReferencedDataController materialised companion secret.app-secret + in ns-{project-uid} on the Karmada hub (downstream cluster). + Companions are written to the hub namespace, NOT the control-plane namespace. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($companionNS) + name: secret.app-secret + labels: + compute.datumapis.com/referenced-data: "true" + + - name: assert-wd-annotation-and-condition-on-control-plane + description: | + Assert the WD carries the expected-referenced-data annotation and has + ReferencedDataReady=True condition set by the ReferencedDataController. + try: + - script: + # Verify the annotation is present and non-empty. + content: | + ANNO=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get workloaddeployment test-refdata-wd \ + --namespace "$NAMESPACE" \ + -o jsonpath='{.metadata.annotations.compute\.datumapis\.com/expected-referenced-data}') + if [ -z "$ANNO" ] || [ "$ANNO" = "[]" ]; then + echo "ERROR: expected-referenced-data annotation is missing or empty: '$ANNO'" + exit 1 + fi + echo "annotation: $ANNO" + timeout: 60s + - assert: + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($namespace) + name: test-refdata-wd + (status.conditions[?type == 'ReferencedDataReady'] | [0]): + status: "True" + reason: "Ready" + + # ─── Hop 3: assert companions on the Karmada hub (downstream) ────────────── + + - name: assert-wd-and-companions-on-hub + description: | + Assert the Karmada hub (downstream cluster) holds the WD with the + expected-referenced-data annotation and the companion ConfigMap + Secret + in ns-{project-uid}. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + # WD federated to Karmada with the expected-referenced-data annotation forwarded + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-refdata-wd + - assert: + # Companion ConfigMap on Karmada hub + timeout: 30s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($downstreamNS) + name: configmap.app-config + labels: + compute.datumapis.com/referenced-data: "true" + - assert: + # Companion Secret on Karmada hub + timeout: 30s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($downstreamNS) + name: secret.app-secret + labels: + compute.datumapis.com/referenced-data: "true" + + - name: assert-propagation-policy-has-companion-selectors + description: | + Assert the PropagationPolicy city-dfw on the hub includes ConfigMap and + Secret resource selectors so companions co-propagate with the WD. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-dfw + spec: + resourceSelectors: + - apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($downstreamNS) + labelSelector: + matchLabels: + topology.datum.net/city-code: dfw + - apiVersion: v1 + kind: ConfigMap + namespace: ($downstreamNS) + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + - apiVersion: v1 + kind: Secret + namespace: ($downstreamNS) + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + + # ─── Hop 4: assert companions propagated to pop-dfw cell ─────────────────── + + - name: assert-wd-and-companions-on-cell + description: | + Assert Karmada propagated the WD and companion ConfigMap + Secret to + pop-dfw. All three objects must appear in ns-{project-uid} on the cell. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-refdata-wd + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($downstreamNS) + name: configmap.app-config + labels: + compute.datumapis.com/referenced-data: "true" + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($downstreamNS) + name: secret.app-secret + labels: + compute.datumapis.com/referenced-data: "true" + + # ─── Hop 5: assert Instance gate cleared on pop-dfw ──────────────────────── + + - name: assert-instance-exists-on-cell + description: Assert the WorkloadDeploymentReconciler created Instance test-refdata-wd-0. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-refdata-wd-0 + + - name: assert-referenced-data-gate-cleared + description: | + Assert the ReferencedData scheduling gate is removed from the Instance and + the ReferencedDataReady=True condition is set by the InstanceReconciler. + The gate is stamped when the Instance is created (feature flag on) and + cleared once the cell InstanceReconciler confirms all expected companions + are present in ns-{project-uid}. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-refdata-wd-0 + (status.conditions[?type == 'ReferencedDataReady'] | [0]): + status: "True" + reason: "Ready" + - script: + # Verify the ReferencedData gate is absent from the Instance spec. + timeout: 60s + content: | + DOWNSTREAMS_NS=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}') + GATES=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml \ + get instance test-refdata-wd-0 \ + --namespace "$DOWNSTREAMS_NS" \ + -o jsonpath='{.spec.schedulingGates[*].name}') + if echo "$GATES" | grep -qw "ReferencedData"; then + echo "ERROR: ReferencedData gate still present in schedulingGates: '$GATES'" + exit 1 + fi + echo "ReferencedData gate cleared. Remaining gates: '$GATES'" diff --git a/test/e2e/referenced-data-mounts/source-configmap.yaml b/test/e2e/referenced-data-mounts/source-configmap.yaml new file mode 100644 index 00000000..3664e9ae --- /dev/null +++ b/test/e2e/referenced-data-mounts/source-configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-config + # namespace injected by Chainsaw from ($namespace) +data: + app.properties: "environment=test" + log.level: "info" diff --git a/test/e2e/referenced-data-mounts/source-secret.yaml b/test/e2e/referenced-data-mounts/source-secret.yaml new file mode 100644 index 00000000..afde8426 --- /dev/null +++ b/test/e2e/referenced-data-mounts/source-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: app-secret + # namespace injected by Chainsaw from ($namespace) +type: Opaque +stringData: + db.password: "test-db-password" diff --git a/test/e2e/referenced-data-mounts/workload-deployment.yaml b/test/e2e/referenced-data-mounts/workload-deployment.yaml new file mode 100644 index 00000000..d849be0b --- /dev/null +++ b/test/e2e/referenced-data-mounts/workload-deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-refdata-wd + # namespace injected by Chainsaw from ($namespace) +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000002" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + sandbox: + containers: + - name: app + image: docker.io/library/busybox:stable + env: + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: app-secret + key: db.password + volumeAttachments: + - name: config-vol + mountPath: /etc/config + volumes: + - name: config-vol + configMap: + name: app-config + networkInterfaces: + - network: + name: test-network + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/workload-deployment-federation/assert-downstream-pp.yaml b/test/e2e/workload-deployment-federation/assert-downstream-pp.yaml new file mode 100644 index 00000000..98f8d0f1 --- /dev/null +++ b/test/e2e/workload-deployment-federation/assert-downstream-pp.yaml @@ -0,0 +1,20 @@ +# Assert the PropagationPolicy was created in the Karmada namespace. +# The name follows propagationPolicyNameFor("dfw") = "workload-deployments-dfw". +# ($downstreamNS) is substituted by Chainsaw's template engine. +apiVersion: policy.karmada.io/v1alpha1 +kind: PropagationPolicy +metadata: + namespace: ($downstreamNS) + name: workload-deployments-dfw +spec: + resourceSelectors: + - apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + labelSelector: + matchLabels: + topology.datum.net/city-code: dfw + placement: + clusterAffinity: + labelSelector: + matchLabels: + topology.datum.net/city-code: dfw diff --git a/test/e2e/workload-deployment-federation/assert-downstream-wd.yaml b/test/e2e/workload-deployment-federation/assert-downstream-wd.yaml new file mode 100644 index 00000000..23c308ff --- /dev/null +++ b/test/e2e/workload-deployment-federation/assert-downstream-wd.yaml @@ -0,0 +1,9 @@ +# Assert the WorkloadDeployment exists in Karmada with the city-code label. +# ($downstreamNS) is substituted by Chainsaw's template engine from the script binding. +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + namespace: ($downstreamNS) + name: test-federation-wd + labels: + topology.datum.net/city-code: dfw diff --git a/test/e2e/workload-deployment-federation/chainsaw-test.yaml b/test/e2e/workload-deployment-federation/chainsaw-test.yaml new file mode 100644 index 00000000..302d89c4 --- /dev/null +++ b/test/e2e/workload-deployment-federation/chainsaw-test.yaml @@ -0,0 +1,84 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: workload-deployment-federation +spec: + description: | + Verifies that the WorkloadDeploymentFederator replicates a WorkloadDeployment + from the project namespace (control-plane cluster) to the Karmada API server + with the correct city-code label and PropagationPolicy. + + The federator follows the ns- convention for Karmada namespaces, + matching the MappedNamespaceResourceStrategy used by NSO. The test derives + the expected Karmada namespace dynamically from the Chainsaw test namespace UID. + + Verified: + - WorkloadDeployment exists in Karmada at ns- + - Karmada copy carries label topology.datum.net/city-code: dfw + - PropagationPolicy city-dfw exists in the Karmada namespace, + selecting WDs by city-code and routing them to matching POP-cell clusters. + + template: true + + steps: + - name: derive-ns-and-create-wd + description: Derive Karmada namespace and create the WorkloadDeployment. + try: + - apply: + file: workload-deployment.yaml + + - name: assert-wd-in-downstream + description: Assert WorkloadDeployment federated to Karmada with city-code label. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-federation-wd + labels: + topology.datum.net/city-code: dfw + + - name: assert-propagation-policy-in-downstream + description: Assert PropagationPolicy created for city-dfw. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-dfw + spec: + resourceSelectors: + - apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + labelSelector: + matchLabels: + topology.datum.net/city-code: dfw + placement: + clusterAffinity: + labelSelector: + matchLabels: + topology.datum.net/city-code: dfw diff --git a/test/e2e/workload-deployment-federation/workload-deployment.yaml b/test/e2e/workload-deployment-federation/workload-deployment.yaml new file mode 100644 index 00000000..0cd2347a --- /dev/null +++ b/test/e2e/workload-deployment-federation/workload-deployment.yaml @@ -0,0 +1,22 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-federation-wd + # namespace is injected by Chainsaw from ($namespace) +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + + scaleSettings: + minReplicas: 1 From e6ed78192ddf0d03e23e040416eba269f8f91afe Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 18:14:43 -0500 Subject: [PATCH 05/23] test(e2e): adapt restored suites to the in-cluster deploy path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Delete seven loose assert-*.yaml files that no chainsaw-test.yaml referenced; two still asserted the pre-merge PropagationPolicy name workload-deployments-dfw (the federator now emits city-). The suites assert these resources inline, so nothing is lost. - Enable featureFlags.enableReferencedDataGate on the cell deploy layer. Its only consumer is the cell WorkloadDeploymentReconciler, which stamps the ReferencedData scheduling gate; without it the referenced-data suites observe an Instance that is never gated. The management layer sets the same flag for parity (inert there — that overlay runs management controllers only). - Document the upstream-cluster-name label value (cluster-single). Both operators run discovery.mode=single with the default cluster name single; the management federator stamps EncodeClusterName(single)=cluster-single on the hub namespace and the write-back/projection copy it, so the existing assertions hold. Corrected the instance-projection and instance-writeback descriptions, which named a stale provider symbol and an incorrect cluster- example. - Refresh referenced-data prose (suite descriptions + README) that referred to the retired host-run operator-start tasks; the feature flag now ships through the deploy layer under task e2e:up. Co-Authored-By: Claude Fable 5 --- .../assert-downstream-wd-exists.yaml | 7 -- test/e2e/deploy/cell/config_patch.yaml | 10 ++ test/e2e/deploy/management/config_patch.yaml | 10 ++ test/e2e/full-federation/chainsaw-test.yaml | 13 ++- .../assert-downstream-wd.yaml | 6 -- .../assert-projected-instance.yaml | 19 ---- .../instance-projection/chainsaw-test.yaml | 11 ++- .../assert-downstream-instance.yaml | 16 ---- .../e2e/instance-writeback/chainsaw-test.yaml | 20 +++- .../assert-pp-exists.yaml | 6 -- .../chainsaw-test.yaml | 6 +- test/e2e/referenced-data-mounts/README.md | 96 ++++++------------- .../referenced-data-mounts/chainsaw-test.yaml | 11 ++- .../assert-downstream-pp.yaml | 20 ---- .../assert-downstream-wd.yaml | 9 -- 15 files changed, 91 insertions(+), 169 deletions(-) delete mode 100644 test/e2e/deletion-cascade/assert-downstream-wd-exists.yaml delete mode 100644 test/e2e/instance-projection/assert-downstream-wd.yaml delete mode 100644 test/e2e/instance-projection/assert-projected-instance.yaml delete mode 100644 test/e2e/instance-writeback/assert-downstream-instance.yaml delete mode 100644 test/e2e/propagation-policy-lifecycle/assert-pp-exists.yaml delete mode 100644 test/e2e/workload-deployment-federation/assert-downstream-pp.yaml delete mode 100644 test/e2e/workload-deployment-federation/assert-downstream-wd.yaml diff --git a/test/e2e/deletion-cascade/assert-downstream-wd-exists.yaml b/test/e2e/deletion-cascade/assert-downstream-wd-exists.yaml deleted file mode 100644 index aae65da1..00000000 --- a/test/e2e/deletion-cascade/assert-downstream-wd-exists.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Assert the WorkloadDeployment is present in the Karmada API server. -# Used both to confirm federation succeeded and as the target for the error: check. -apiVersion: compute.datumapis.com/v1alpha -kind: WorkloadDeployment -metadata: - namespace: ($downstreamNS) - name: test-cascade-wd diff --git a/test/e2e/deploy/cell/config_patch.yaml b/test/e2e/deploy/cell/config_patch.yaml index 6ab77ae5..7e94f3a8 100644 --- a/test/e2e/deploy/cell/config_patch.yaml +++ b/test/e2e/deploy/cell/config_patch.yaml @@ -14,6 +14,14 @@ # Impact: the cell InstanceReconciler skips ResourceClaim creation/quota checks. # The federation delivery path under test (WorkloadDeployment → Instance) is # unaffected; edge quota enforcement is covered separately. +# +# featureFlags.enableReferencedDataGate is turned on because the cell +# WorkloadDeploymentReconciler is the sole consumer of this flag: it stamps the +# "ReferencedData" scheduling gate onto Instances whose template references a +# ConfigMap or Secret, and the cell InstanceReconciler clears the gate once the +# companions land on the cell. The referenced-data-mounts and +# referenced-data-delete-cascade suites assert that stamp-then-clear behaviour, +# so without the flag those suites would observe an Instance that is never gated. apiVersion: v1 kind: ConfigMap metadata: @@ -24,3 +32,5 @@ data: kind: WorkloadOperator metricsServer: bindAddress: "0" + featureFlags: + enableReferencedDataGate: true diff --git a/test/e2e/deploy/management/config_patch.yaml b/test/e2e/deploy/management/config_patch.yaml index f9e07612..d3a0abf7 100644 --- a/test/e2e/deploy/management/config_patch.yaml +++ b/test/e2e/deploy/management/config_patch.yaml @@ -13,6 +13,14 @@ # cert-manager CSI driver, which is not installed in Kind and whose issuer is # supplied by infra rather than the overlay. The e2e suites never create # Workload objects, so the Workload webhook is never exercised. +# +# - featureFlags.enableReferencedDataGate mirrors the flag set on the cell so the +# two operators share one feature-flag surface, matching the retired host-run +# harness which passed the same --server-config flag to both. Its only consumer +# is the cell WorkloadDeploymentReconciler, which this management overlay does +# not run (it enables management controllers only), so the flag is inert here; +# the cell config_patch carries the copy that actually gates Instances. Kept +# explicit so the flag does not silently diverge between the two operators. apiVersion: v1 kind: ConfigMap metadata: @@ -25,3 +33,5 @@ data: bindAddress: "0" discovery: mode: single + featureFlags: + enableReferencedDataGate: true diff --git a/test/e2e/full-federation/chainsaw-test.yaml b/test/e2e/full-federation/chainsaw-test.yaml index 020a2bc9..a5ac8735 100644 --- a/test/e2e/full-federation/chainsaw-test.yaml +++ b/test/e2e/full-federation/chainsaw-test.yaml @@ -18,7 +18,18 @@ spec: 6. InstanceProjector on control-plane creates a projection of the Instance in the project namespace. - Prerequisites: both operator instances must be running (task e2e:operator:start). + Cluster-name label: "cluster-single". + The value is NOT the cell's cluster name. The management operator federates + the WorkloadDeployment from its local control-plane cluster, which it + registers with multicluster-runtime under the name "single" (both operators + run discovery.mode=single in this environment, defaulting the cluster name to + "single" per cmd/main.go singleClusterName). The federator stamps + EncodeClusterName("single") = "cluster-single" onto the hub ns- + namespace; the cell write-back and the projection both copy that namespace + label verbatim, so the assertion holds regardless of which cell reconciles. + + Prerequisites: the management + both cell operators are deployed in-cluster + (task e2e:up). template: true diff --git a/test/e2e/instance-projection/assert-downstream-wd.yaml b/test/e2e/instance-projection/assert-downstream-wd.yaml deleted file mode 100644 index 705d0893..00000000 --- a/test/e2e/instance-projection/assert-downstream-wd.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Assert the WorkloadDeployment is federated to Karmada (and the Karmada namespace created). -apiVersion: compute.datumapis.com/v1alpha -kind: WorkloadDeployment -metadata: - namespace: ($downstreamNS) - name: test-projector-wd diff --git a/test/e2e/instance-projection/assert-projected-instance.yaml b/test/e2e/instance-projection/assert-projected-instance.yaml deleted file mode 100644 index 0542194d..00000000 --- a/test/e2e/instance-projection/assert-projected-instance.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Assert the InstanceProjector created a projection in the project namespace. -# -# The InstanceProjector (internal/controller/instance_projector.go): -# - Watches Instances in Karmada that carry upstreamClusterNameLabel -# - Strips "cluster-" prefix to get the cluster name ("single" in single-provider mode) -# - Finds the project namespace by matching ns- to namespace UIDs -# - Creates/updates the Instance projection in the project namespace -# - Sets an owner reference to the WorkloadDeployment for cascading deletion -apiVersion: compute.datumapis.com/v1alpha -kind: Instance -metadata: - # namespace is the Chainsaw test namespace (the project namespace on control-plane) - name: test-projected-instance - labels: - meta.datumapis.com/upstream-cluster-name: cluster-single - ownerReferences: - - apiVersion: compute.datumapis.com/v1alpha - kind: WorkloadDeployment - name: test-projector-wd diff --git a/test/e2e/instance-projection/chainsaw-test.yaml b/test/e2e/instance-projection/chainsaw-test.yaml index 16fa9f96..b79d83d0 100644 --- a/test/e2e/instance-projection/chainsaw-test.yaml +++ b/test/e2e/instance-projection/chainsaw-test.yaml @@ -16,10 +16,13 @@ spec: 4. Assert the projection exists with the upstream tracking label and an owner reference to the WorkloadDeployment (for cascading deletion). - Cluster name label: "cluster-single" - The compute operator runs in single-provider mode for this e2e environment, - registering the control-plane cluster with the multicluster-runtime manager - under the name "single" (see cmd/main.go, wrappedSingleClusterProvider). + Cluster name label: "cluster-single". + The management operator runs discovery.mode=single in this environment, which + registers the control-plane cluster with the multicluster-runtime manager + under the fixed name "single" (cmd/main.go singleClusterName, wired via + mcsingle.New). The InstanceProjector decodes the upstream-cluster-name label + ("cluster-single" → "single") to pick the project cluster to project into, so + the label this test writes onto the hub Instance must encode "single". template: true diff --git a/test/e2e/instance-writeback/assert-downstream-instance.yaml b/test/e2e/instance-writeback/assert-downstream-instance.yaml deleted file mode 100644 index 3b8ce01c..00000000 --- a/test/e2e/instance-writeback/assert-downstream-instance.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Assert the InstanceReconciler wrote the Instance back to Karmada. -# -# The write-back convention (see internal/controller/instance_controller.go): -# - Same namespace and name as the POP-cell Instance -# - Label meta.datumapis.com/upstream-cluster-name = "cluster-" -# where is the multicluster-runtime cluster name registered by -# wrappedSingleClusterProvider (always "single" in single-cluster mode) -# - Label meta.datumapis.com/upstream-namespace = the POP-cell namespace -apiVersion: compute.datumapis.com/v1alpha -kind: Instance -metadata: - namespace: ($instanceNS) - name: test-writeback-instance - labels: - meta.datumapis.com/upstream-cluster-name: cluster-single - meta.datumapis.com/upstream-namespace: ($instanceNS) diff --git a/test/e2e/instance-writeback/chainsaw-test.yaml b/test/e2e/instance-writeback/chainsaw-test.yaml index 32dbbc5d..e327d8a2 100644 --- a/test/e2e/instance-writeback/chainsaw-test.yaml +++ b/test/e2e/instance-writeback/chainsaw-test.yaml @@ -10,12 +10,22 @@ spec: Write-back convention (internal/controller/instance_controller.go): - The Instance is written to Karmada at the same namespace/name as the POP-cell Instance. - - Label meta.datumapis.com/upstream-cluster-name is set to - "cluster-" (e.g. "cluster-compute-pop-dfw"). - - Label meta.datumapis.com/upstream-namespace records the originating namespace. + - The identity labels are NOT derived from the cell's own cluster name. The + write-back reads them from the hub ns- namespace, which the management + federator stamped: meta.datumapis.com/upstream-cluster-name carries + EncodeClusterName of the federating cluster ("single" here → "cluster-single") + and meta.datumapis.com/upstream-namespace records the originating namespace. - Note: this test requires the compute operator (InstanceReconciler) to be running - in the DFW POP cell cluster. + Note: this test requires the cell InstanceReconciler to be running in the DFW + POP cell cluster with federation configured. + + Runtime prerequisite (see verification phase): the write-back path errors + unless the hub ns- namespace carries the two upstream-* labels above and + the cell Instance carries the full set of linking labels the stateful control + strategy stamps at creation. This suite hand-crafts a bare namespace and a + bare Instance, so the write-back may not fire until those labels are seeded + (or the Instance is created via a real WorkloadDeployment, as full-federation + does). template: true diff --git a/test/e2e/propagation-policy-lifecycle/assert-pp-exists.yaml b/test/e2e/propagation-policy-lifecycle/assert-pp-exists.yaml deleted file mode 100644 index 77a817a5..00000000 --- a/test/e2e/propagation-policy-lifecycle/assert-pp-exists.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Asserts that the PropagationPolicy for city dfw exists in the Karmada namespace. -apiVersion: policy.karmada.io/v1alpha1 -kind: PropagationPolicy -metadata: - namespace: ($downstreamNS) - name: workload-deployments-dfw diff --git a/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml b/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml index 97f4c7cf..a3e1a10b 100644 --- a/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml +++ b/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml @@ -22,9 +22,9 @@ spec: the stranded companion and its ResourceBinding are both deleted. Prerequisites: - - task e2e:up completed - - operators started with featureFlags.enableReferencedDataGate: true - (task e2e:operator:start:referenced-data) + - task e2e:up completed (deploys the management + both cell operators in-cluster) + - the cell operator runs with featureFlags.enableReferencedDataGate: true, set + by the e2e cell deploy layer (test/e2e/deploy/cell/config_patch.yaml) - CompanionGCReconciler enabled in the management operator - tmp/e2e/kubeconfigs/downstream.yaml exists diff --git a/test/e2e/referenced-data-mounts/README.md b/test/e2e/referenced-data-mounts/README.md index db62cdfb..808c2e12 100644 --- a/test/e2e/referenced-data-mounts/README.md +++ b/test/e2e/referenced-data-mounts/README.md @@ -26,62 +26,23 @@ against a downstream cluster. See `docs/compute/development/plans/configmap-secr ## Prerequisites -1. **Clusters running**: `task e2e:up` has completed successfully. The following - kubeconfigs must exist under `tmp/e2e/kubeconfigs/`: - - `control-plane.yaml` - - `karmada.yaml` - - `downstream.yaml` — **REQUIRED** by `chainsaw-config.yaml` but written as - `karmada.yaml` by the Taskfile. Add a copy step (see Harness Gaps below). - - `pop-dfw.yaml` - -2. **Operators running with `enableReferencedDataGate: true`**: start the management - and cell operators using the dedicated task (see below). The stock - `task e2e:operator:start` does NOT pass a server config and therefore starts - operators with `enableReferencedDataGate: false` — the feature is completely - inert without this flag. - -## Running the operators with the feature flag enabled - -Use the dedicated Taskfile target that starts both operators with -`featureFlags.enableReferencedDataGate: true`: - -```sh -task e2e:operator:start:referenced-data -``` - -This starts management (`:9091`) and cell (`:9092`) operators with -`--server-config=hack/e2e/operator-config-referenced-data.yaml` and waits -for both health checks before returning. - -To stop both operators: `task e2e:operator:stop`. - -Alternatively, start them manually: - -```sh -# Management operator — control-plane cluster, feature flag on -KUBECONFIG=tmp/e2e/kubeconfigs/control-plane.yaml \ -go run ./cmd/main.go \ - --federation-kubeconfig=tmp/e2e/kubeconfigs/karmada.yaml \ - --enable-management-controllers=true \ - --enable-cell-controllers=false \ - --leader-elect=false \ - --health-probe-bind-address=:9091 \ - --server-config=hack/e2e/operator-config-referenced-data.yaml \ - > tmp/e2e/logs/operator-management-refdata.log 2>&1 & - -# Cell operator — pop-dfw cluster, feature flag on -KUBECONFIG=tmp/e2e/kubeconfigs/pop-dfw.yaml \ -go run ./cmd/main.go \ - --federation-kubeconfig=tmp/e2e/kubeconfigs/karmada.yaml \ - --enable-management-controllers=false \ - --enable-cell-controllers=true \ - --leader-elect=false \ - --health-probe-bind-address=:9092 \ - --server-config=hack/e2e/operator-config-referenced-data.yaml \ - > tmp/e2e/logs/operator-cell-dfw-refdata.log 2>&1 & -``` - -Wait for health checks on `:9091` and `:9092` before running the test. +**`task e2e:up` has completed successfully.** In the in-cluster harness this one +target brings up the Kind clusters + Karmada AND deploys the operators from the +real production overlays (plus the local e2e deviations), so there is no separate +operator-start step. It produces these kubeconfigs under `tmp/e2e/kubeconfigs/`: + +- `control-plane.yaml` — management cluster (also hosts the Karmada hub) +- `karmada.yaml` — the Karmada hub API server +- `downstream.yaml` — a copy of `karmada.yaml` the Taskfile writes so + `cluster: downstream` steps resolve (referenced by `chainsaw-config.yaml`) +- `pop-dfw.yaml`, `pop-ord.yaml` — the POP cell clusters + +**The cell operator runs with `enableReferencedDataGate: true`.** The e2e cell +deploy layer sets it in `test/e2e/deploy/cell/config_patch.yaml`. The flag's sole +consumer is the cell `WorkloadDeploymentReconciler` (it stamps the `ReferencedData` +scheduling gate); without it the Instance is never gated and Hop 5 cannot pass. +The management deploy layer sets the same flag for parity, though it is inert +there because that overlay enables management controllers only. ## Running just this scenario @@ -101,19 +62,16 @@ task e2e:test:filter -- --include-test-regex referenced-data-mounts ## Harness notes -The following items were gaps at the time this test was written and have since -been fixed in `Taskfile.yaml`: - -1. `_e2e:karmada:build-kubeconfig` now copies `karmada.yaml` → - `downstream.yaml`, so `cluster: downstream` steps work out of the box. -2. `e2e:operator:start` now uses `--federation-kubeconfig` (the correct flag - name) for both management and cell operators. -3. `e2e:operator:start` now passes `--enable-management-controllers=true` to - the management operator, enabling the WorkloadDeploymentFederator and - InstanceProjector controllers. -4. `e2e:operator:start:referenced-data` is a dedicated task that starts both - operators with the `enableReferencedDataGate` feature flag on. -5. `e2e:crds:install` now installs Milo quota CRDs to all clusters so the +This test targets the in-cluster harness, where `task e2e:up` builds the operator +image, side-loads it into every Kind node, and deploys the management + cell +operators from the real overlays. Points worth knowing: + +1. `_e2e:karmada:build-kubeconfig` copies `karmada.yaml` → `downstream.yaml`, so + `cluster: downstream` steps work out of the box. +2. The `enableReferencedDataGate` feature flag is delivered through the deploy + layer (`test/e2e/deploy/{cell,management}/config_patch.yaml`), not a host-side + `--server-config`. There is no separate operator-start task to run. +3. `e2e:crds:install` installs Milo quota CRDs to all clusters so the InstanceReconciler's ResourceClaim watches start cleanly. ## Companion naming convention diff --git a/test/e2e/referenced-data-mounts/chainsaw-test.yaml b/test/e2e/referenced-data-mounts/chainsaw-test.yaml index b4f5a2b6..8b0fc766 100644 --- a/test/e2e/referenced-data-mounts/chainsaw-test.yaml +++ b/test/e2e/referenced-data-mounts/chainsaw-test.yaml @@ -34,10 +34,13 @@ spec: asserted here. See README.md for full scope and prerequisites. Prerequisites: - - task e2e:up completed (control-plane + Karmada + pop-dfw + pop-ord) - - operators started with featureFlags.enableReferencedDataGate: true - (see README.md — NOT the stock task e2e:operator:start) - - tmp/e2e/kubeconfigs/downstream.yaml exists (symlink/copy of karmada.yaml) + - task e2e:up completed: brings up control-plane + Karmada + pop-dfw + pop-ord + and deploys the management and both cell operators in-cluster. + - the cell operator runs with featureFlags.enableReferencedDataGate: true; the + e2e cell deploy layer sets it (test/e2e/deploy/cell/config_patch.yaml), so + no separate operator-start step is needed. + - tmp/e2e/kubeconfigs/downstream.yaml exists (the Taskfile writes it as a copy + of the Karmada hub kubeconfig). template: true diff --git a/test/e2e/workload-deployment-federation/assert-downstream-pp.yaml b/test/e2e/workload-deployment-federation/assert-downstream-pp.yaml deleted file mode 100644 index 98f8d0f1..00000000 --- a/test/e2e/workload-deployment-federation/assert-downstream-pp.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Assert the PropagationPolicy was created in the Karmada namespace. -# The name follows propagationPolicyNameFor("dfw") = "workload-deployments-dfw". -# ($downstreamNS) is substituted by Chainsaw's template engine. -apiVersion: policy.karmada.io/v1alpha1 -kind: PropagationPolicy -metadata: - namespace: ($downstreamNS) - name: workload-deployments-dfw -spec: - resourceSelectors: - - apiVersion: compute.datumapis.com/v1alpha - kind: WorkloadDeployment - labelSelector: - matchLabels: - topology.datum.net/city-code: dfw - placement: - clusterAffinity: - labelSelector: - matchLabels: - topology.datum.net/city-code: dfw diff --git a/test/e2e/workload-deployment-federation/assert-downstream-wd.yaml b/test/e2e/workload-deployment-federation/assert-downstream-wd.yaml deleted file mode 100644 index 23c308ff..00000000 --- a/test/e2e/workload-deployment-federation/assert-downstream-wd.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Assert the WorkloadDeployment exists in Karmada with the city-code label. -# ($downstreamNS) is substituted by Chainsaw's template engine from the script binding. -apiVersion: compute.datumapis.com/v1alpha -kind: WorkloadDeployment -metadata: - namespace: ($downstreamNS) - name: test-federation-wd - labels: - topology.datum.net/city-code: dfw From f026d95595f88d31a9e6db2697851ed09e0ef5ee Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 18:15:55 -0500 Subject: [PATCH 06/23] test(e2e): add second-cell federation coverage for pop-ord MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both POP cells now run operators, but every restored suite exercises only the dfw cell. Add full-federation-ord: an ord-placed WorkloadDeployment must produce PropagationPolicy city-ord, propagate to pop-ord, and create its Instance there, then write back and project like the dfw path. This is the coverage that catches city-code routing regressions and a mis-registered second cell — the dfw suites would stay green through both. Co-Authored-By: Claude Fable 5 --- .../full-federation-ord/chainsaw-test.yaml | 171 ++++++++++++++++++ .../workload-deployment.yaml | 21 +++ 2 files changed, 192 insertions(+) create mode 100644 test/e2e/full-federation-ord/chainsaw-test.yaml create mode 100644 test/e2e/full-federation-ord/workload-deployment.yaml diff --git a/test/e2e/full-federation-ord/chainsaw-test.yaml b/test/e2e/full-federation-ord/chainsaw-test.yaml new file mode 100644 index 00000000..ad26d07f --- /dev/null +++ b/test/e2e/full-federation-ord/chainsaw-test.yaml @@ -0,0 +1,171 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: full-federation-ord +spec: + description: | + Second-cell federation chain test (city code ord → pop-ord). + + The dfw path is covered by full-federation. This suite proves the same chain + routes independently to the OTHER cell: an ord-placed WorkloadDeployment must + produce PropagationPolicy city-ord (not city-dfw), propagate to pop-ord (not + pop-dfw), and create its Instance on pop-ord. Both cells run the same operator + image, so this is the coverage that catches city-code routing regressions and + a mis-registered second cell. + + 1. Create WorkloadDeployment (cityCode: ord) on control-plane. + 2. WorkloadDeploymentFederator replicates it to Karmada (ns- namespace) + and lazily creates PropagationPolicy city-ord routing to city-code=ord cells. + 3. Karmada propagates the WD to pop-ord. + 4. WorkloadDeploymentReconciler on pop-ord creates Instance test-fullfed-ord-wd-0. + 5. InstanceReconciler on pop-ord writes the Instance back to Karmada with + label meta.datumapis.com/upstream-cluster-name: cluster-single. + 6. InstanceProjector on control-plane projects the Instance into the project + namespace. + + Cluster-name label "cluster-single" is the management operator's federating + cluster name, not the cell's — see full-federation for the full rationale. + + Prerequisites: the management + both cell operators are deployed in-cluster + (task e2e:up). + + template: true + + steps: + - name: create-workload-deployment + description: Create the ord-placed WorkloadDeployment on the control-plane cluster. + try: + - apply: + file: workload-deployment.yaml + + - name: assert-wd-and-policy-in-downstream + description: Assert the WD federated to Karmada and PropagationPolicy city-ord was created for ord. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-fullfed-ord-wd + labels: + topology.datum.net/city-code: ord + - assert: + # The federator names the policy city- and routes it to cells + # carrying the same city-code label, so ord must land on pop-ord alone. + timeout: 30s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-ord + spec: + resourceSelectors: + - apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + labelSelector: + matchLabels: + topology.datum.net/city-code: ord + placement: + clusterAffinity: + labelSelector: + matchLabels: + topology.datum.net/city-code: ord + + - name: assert-wd-on-pop-ord + description: Assert Karmada propagated the WD to pop-ord and the cell reconciler set status. + cluster: pop-ord + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + # Karmada propagation can take longer than a local apply. + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-fullfed-ord-wd + status: + replicas: 1 + desiredReplicas: 1 + + - name: assert-instance-on-pop-ord + description: Assert WorkloadDeploymentReconciler created an Instance on pop-ord. + cluster: pop-ord + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-fullfed-ord-wd-0 + (status.conditions[?type == 'Ready'] | [0]): + status: "Unknown" + + - name: assert-instance-writeback-in-downstream + description: Assert the pop-ord InstanceReconciler wrote the Instance back to Karmada. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-fullfed-ord-wd-0 + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + + - name: assert-instance-projected-to-control-plane + description: Assert InstanceProjector created a projection on the control-plane. + try: + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($namespace) + name: test-fullfed-ord-wd-0 + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + (status.conditions[?type == 'Ready'] | [0]): + status: "Unknown" diff --git a/test/e2e/full-federation-ord/workload-deployment.yaml b/test/e2e/full-federation-ord/workload-deployment.yaml new file mode 100644 index 00000000..74663331 --- /dev/null +++ b/test/e2e/full-federation-ord/workload-deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-fullfed-ord-wd + # namespace is injected by Chainsaw from ($namespace) +spec: + cityCode: ord + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000002" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + scaleSettings: + minReplicas: 1 From 1566fe3e787d828d97fb2518326f1268f64503a4 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 19:13:08 -0500 Subject: [PATCH 07/23] test(e2e): skip client-side validation on cluster applies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Server-side apply already validates on the API server, so the client-side OpenAPI schema download is redundant — and on a resource-constrained API server that download times out ("failed to download openapi"), failing the apply. Pass --validate=false (and --server-side where it was missing) on the CRD, hub-RBAC, federation-component, and operator applies so bring-up survives a slow API server. Co-Authored-By: Claude Fable 5 --- Taskfile.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Taskfile.yaml b/Taskfile.yaml index 37c0611b..a0344bef 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -342,7 +342,7 @@ tasks: - | echo "Applying federation component to Karmada..." kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply \ - -k config/components/federation/ + -k config/components/federation/ --server-side --validate=false echo "Federation component applied" _e2e:karmada:build-kubeconfig: @@ -496,7 +496,7 @@ tasks: {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ {{.KUBECONFIG_DIR}}/pop-ord.yaml; do echo "Installing compute CRDs → $(basename $KC .yaml)..." - kubectl --kubeconfig="$KC" apply -k config/base/crd --server-side + kubectl --kubeconfig="$KC" apply -k config/base/crd --server-side --validate=false done _e2e:crds:nso: @@ -518,7 +518,7 @@ tasks: {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ {{.KUBECONFIG_DIR}}/pop-ord.yaml; do echo "Installing NSO CRDs → $(basename $KC .yaml)..." - kubectl --kubeconfig="$KC" apply -k "${NSO_CRD_PATH}" --server-side + kubectl --kubeconfig="$KC" apply -k "${NSO_CRD_PATH}" --server-side --validate=false done _e2e:crds:quota: @@ -540,7 +540,7 @@ tasks: {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ {{.KUBECONFIG_DIR}}/pop-ord.yaml; do echo "Installing Milo quota CRDs → $(basename $KC .yaml)..." - kubectl --kubeconfig="$KC" apply -k "${QUOTA_CRD_PATH}" --server-side + kubectl --kubeconfig="$KC" apply -k "${QUOTA_CRD_PATH}" --server-side --validate=false done # ════════════════════════════════════════════════════════════════════════ @@ -579,7 +579,7 @@ tasks: # forbidden error rather than being masked by cluster-admin. - | echo "Applying hub RBAC (config/base/downstream-rbac) to Karmada..." - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply -k config/base/downstream-rbac + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply -k config/base/downstream-rbac --server-side --validate=false # ── Karmada-native identity for the management operator ───────────── # Production federates the management cluster's projected ServiceAccount # token into Karmada (Karmada trusts the host cluster's token issuer). We @@ -659,7 +659,7 @@ tasks: cmds: - | echo "Deploying management-plane operator → {{.KIND_CONTROL_PLANE}}..." - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml apply -k test/e2e/deploy/management --server-side + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml apply -k test/e2e/deploy/management --server-side --validate=false - task: _e2e:deploy:wait vars: KUBECONFIG_FILE: "{{.KUBECONFIG_DIR}}/control-plane.yaml" @@ -682,7 +682,7 @@ tasks: cmds: - | echo "Deploying cell operator → {{.CLUSTER_NAME}}..." - kubectl --kubeconfig={{.KUBECONFIG_FILE}} apply -k test/e2e/deploy/cell --server-side + kubectl --kubeconfig={{.KUBECONFIG_FILE}} apply -k test/e2e/deploy/cell --server-side --validate=false - task: _e2e:deploy:wait vars: KUBECONFIG_FILE: "{{.KUBECONFIG_FILE}}" From a0d6ceef1d4190e923b1801d26487cc09e3a0e37 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 19:20:58 -0500 Subject: [PATCH 08/23] test(e2e): tune clusters for busy hosts at creation time Bake the control-plane stability fixes into cluster creation so bring-up no longer needs manual runtime patching on a resource-constrained host, where a base Kind cluster co-located with the full Karmada control plane otherwise crash-loops on etcd write-latency spikes. - kind configs (control-plane + new kind-pop for the cells) disable etcd fsync and turn off leader election on the single-instance kube-scheduler and kube-controller-manager via kubeadmConfigPatches. kind renders its kubeadm config as v1beta3, so extraArgs use the map form. - the Karmada install disables fsync on the Karmada control plane's own etcd StatefulSet, alongside the leader-election disable already applied to its controllers. Data durability does not matter for these throwaway e2e clusters, and fsync contention between the co-located etcds was the root cause of the latency spikes. Measured effect: hub writes drop from multi-second timeouts to ~170ms. Co-Authored-By: Claude Fable 5 --- Taskfile.yaml | 34 +++++++++++++++++++++++++++++--- hack/e2e/kind-control-plane.yaml | 24 ++++++++++++++++++++++ hack/e2e/kind-pop.yaml | 25 +++++++++++++++++++++++ 3 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 hack/e2e/kind-pop.yaml diff --git a/Taskfile.yaml b/Taskfile.yaml index a0344bef..4742bb1b 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -203,15 +203,15 @@ tasks: vars: CLUSTER_NAME: "{{.KIND_CONTROL_PLANE}}" KIND_CONFIG: "{{.E2E_DIR}}/kind-control-plane.yaml" - # POP cell clusters — default Kind config is sufficient. + # POP cell clusters — same busy-host control-plane tuning, no port mapping. - task: _e2e:cluster:create vars: CLUSTER_NAME: "{{.KIND_POP_DFW}}" - KIND_CONFIG: "" + KIND_CONFIG: hack/e2e/kind-pop.yaml - task: _e2e:cluster:create vars: CLUSTER_NAME: "{{.KIND_POP_ORD}}" - KIND_CONFIG: "" + KIND_CONFIG: hack/e2e/kind-pop.yaml - mkdir -p {{.KUBECONFIG_DIR}} - task: _e2e:kubeconfigs:export @@ -293,6 +293,12 @@ tasks: # nothing to elect, so turning it off breaks the loop and lets the control # plane settle. Harmless in a single-replica e2e; not for production HA. - task: _e2e:karmada:disable-leader-election + # Disable fsync on the Karmada control plane's own etcd, for the same + # throwaway-data reason as the base clusters (see hack/e2e/kind-*.yaml). Its + # etcd is a Helm-managed StatefulSet on hostPath storage — co-located with + # the base cluster's etcd on one node — so without this its fsync contention + # keeps hub writes slow even after the base etcds are tuned. + - task: _e2e:karmada:disable-etcd-fsync # Scale any Karmada deployment that is sitting at zero back up to one. This # makes the target robust to a control plane that was intentionally scaled # down (for example to keep a busy host calm between runs) or left at zero @@ -336,6 +342,28 @@ tasks: fi done + _e2e:karmada:disable-etcd-fsync: + internal: true + cmds: + # Idempotent: only patch + restart if the flag is not already set. The etcd + # data lives on a hostPath, so deleting the pod does not lose state (a + # StatefulSet does not always roll the pod on a bare command change, hence + # the explicit delete). + - | + current=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system get statefulset etcd \ + -o jsonpath='{.spec.template.spec.containers[0].command}' 2>/dev/null) + if echo "$current" | grep -q -- "--unsafe-no-fsync"; then + echo "Karmada etcd fsync already disabled" + else + echo "Disabling fsync on the Karmada etcd..." + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system patch statefulset etcd --type=json \ + -p '[{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--unsafe-no-fsync=true"}]' + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system delete pod etcd-0 --ignore-not-found + fi + e2e:karmada:configure: desc: "Apply federation component config to the Karmada API server (idempotent)" cmds: diff --git a/hack/e2e/kind-control-plane.yaml b/hack/e2e/kind-control-plane.yaml index 47f3c63b..12356ed1 100644 --- a/hack/e2e/kind-control-plane.yaml +++ b/hack/e2e/kind-control-plane.yaml @@ -8,6 +8,30 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +# Tune the control plane for a busy host that also runs the full Karmada control +# plane in this same cluster. kind renders its kubeadm config as v1beta3, so +# extraArgs are maps here (not the v1beta4 list form). +# - etcd unsafe-no-fsync: these are throwaway e2e clusters, and fsync +# contention between co-located etcds is what drives API server write +# latency into multi-second spikes. +# - scheduler/controllerManager leader-elect=false: single-instance static +# pods have nothing to elect, and a lease renewal that times out during a +# latency spike crash-loops them (which then re-lists everything and makes +# the spike worse). +kubeadmConfigPatches: + - | + apiVersion: kubeadm.k8s.io/v1beta3 + kind: ClusterConfiguration + etcd: + local: + extraArgs: + unsafe-no-fsync: "true" + controllerManager: + extraArgs: + leader-elect: "false" + scheduler: + extraArgs: + leader-elect: "false" nodes: - role: control-plane extraPortMappings: diff --git a/hack/e2e/kind-pop.yaml b/hack/e2e/kind-pop.yaml new file mode 100644 index 00000000..d848f43b --- /dev/null +++ b/hack/e2e/kind-pop.yaml @@ -0,0 +1,25 @@ +# Kind cluster configuration for the compute-pop-* cell clusters. +# +# Same control-plane tuning as the management cluster (see kind-control-plane.yaml +# for the rationale): etcd runs without fsync and the single-instance scheduler +# and controller-manager run without leader election, so the cell control planes +# stay stable on a busy host instead of crash-looping on lease-renewal timeouts. +# kind renders its kubeadm config as v1beta3, so extraArgs are maps here. +# +# Unlike the management cluster, POP cells need no extraPortMappings. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +kubeadmConfigPatches: + - | + apiVersion: kubeadm.k8s.io/v1beta3 + kind: ClusterConfiguration + etcd: + local: + extraArgs: + unsafe-no-fsync: "true" + controllerManager: + extraArgs: + leader-elect: "false" + scheduler: + extraArgs: + leader-elect: "false" From 03f0b23f42f756e65a9f25c71a9845111b1c84d2 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 19:59:25 -0500 Subject: [PATCH 09/23] fix(rbac): grant the manager list/watch on networks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The management WorkloadReconciler watches networking.datumapis.com Networks, but the compute ClusterRole only granted locations, networkcontexts, and subnets — so the Network informer was denied ("networks ... is forbidden") and the manager never reconciled WorkloadDeployments cleanly. Add networks to the read-only networking rule. Surfaced by the in-cluster federation e2e: with this grant the manager runs with zero RBAC denials and federates WorkloadDeployments to the hub. Co-Authored-By: Claude Fable 5 --- config/components/controller_rbac/role.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/components/controller_rbac/role.yaml b/config/components/controller_rbac/role.yaml index 425eecf6..73d73a9d 100644 --- a/config/components/controller_rbac/role.yaml +++ b/config/components/controller_rbac/role.yaml @@ -68,6 +68,7 @@ rules: resources: - locations - networkcontexts + - networks - subnets verbs: - get From 2612db15228eff28bf40bffeb1fc5c37b78991d6 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 19:59:26 -0500 Subject: [PATCH 10/23] test(e2e): fix management deploy layer (webhook cert volume + metrics policy) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two corrections to the e2e management overlay found by deploying it for real: - The webhook-server-tls volume is now backed by an emptyDir rather than deleting the volume + mount. A strategic-merge "$patch: delete" on a single volumeMount dropped the container's entire volumeMounts list, which unmounted the federation kubeconfig — the manager then crash-looped on a missing downstream-kubeconfig.yaml. Overriding only the volume source keeps every mount intact; the webhook server is disabled, so an empty cert dir is fine. - Drop the ResourceMetricsPolicy: its CRD (resourcemetrics.miloapis.com) is owned by a separate operator absent from Kind, so the apply failed with "no matches for kind ResourceMetricsPolicy". It is orthogonal to the federation path under test. Co-Authored-By: Claude Fable 5 --- .../deploy/management/deployment_patch.yaml | 21 +++++++++++-------- test/e2e/deploy/management/kustomization.yaml | 17 +++++++++++++++ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/test/e2e/deploy/management/deployment_patch.yaml b/test/e2e/deploy/management/deployment_patch.yaml index ddb3c30b..d92134c4 100644 --- a/test/e2e/deploy/management/deployment_patch.yaml +++ b/test/e2e/deploy/management/deployment_patch.yaml @@ -1,11 +1,16 @@ # DEVIATION (required for Kind): # 1. imagePullPolicy: IfNotPresent — the dev image is side-loaded into the Kind # node (task e2e:image:load); never attempt a registry pull. -# 2. Remove the webhook-server-tls volume + mount injected by -# config/components/csi-webhook-cert. That volume is backed by the -# cert-manager CSI driver (csi.cert-manager.io), which is not installed in -# Kind, so the pod would fail to mount and never start. The webhook server -# itself is disabled in config_patch.yaml, so the serving cert is unused. +# 2. Back the webhook-server-tls volume with an emptyDir instead of the +# cert-manager CSI driver (csi.cert-manager.io) injected by +# config/components/csi-webhook-cert. That driver is not installed in Kind, +# so the CSI volume would fail to mount and the pod would never start. The +# webhook server itself is disabled in config_patch.yaml, so the serving +# cert is never read — an empty directory at the mount path is enough to let +# the pod boot. We override only the volume source (csi -> emptyDir) and +# leave the container's volumeMounts untouched: a strategic-merge +# "$patch: delete" on a single volumeMount drops the whole list, which would +# also unmount the federation kubeconfig and config volumes. apiVersion: apps/v1 kind: Deployment metadata: @@ -16,9 +21,7 @@ spec: containers: - name: manager imagePullPolicy: IfNotPresent - volumeMounts: - - name: webhook-server-tls - $patch: delete volumes: - name: webhook-server-tls - $patch: delete + csi: null + emptyDir: {} diff --git a/test/e2e/deploy/management/kustomization.yaml b/test/e2e/deploy/management/kustomization.yaml index 2a98ceda..f93b5158 100644 --- a/test/e2e/deploy/management/kustomization.yaml +++ b/test/e2e/deploy/management/kustomization.yaml @@ -54,3 +54,20 @@ patches: target: kind: MutatingWebhookConfiguration name: compute-mutating + + # DEVIATION 4 (required): remove the ResourceMetricsPolicy. + # config/components/resource-metrics ships a ResourceMetricsPolicy + # (resourcemetrics.miloapis.com), whose CRD is owned by a separate + # resource-metrics operator that is not installed in Kind (it is not in + # compute's module deps). Applying it fails with "no matches for kind + # ResourceMetricsPolicy". It is orthogonal to the federation path under test, + # so drop it from the deploy set. + - patch: | + $patch: delete + apiVersion: resourcemetrics.miloapis.com/v1alpha1 + kind: ResourceMetricsPolicy + metadata: + name: compute-metrics + target: + kind: ResourceMetricsPolicy + name: compute-metrics From fe906dfe6f44a2c1b9b59107697110b195ff30c1 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 20:21:54 -0500 Subject: [PATCH 11/23] test(e2e): keep Karmada API server alive + widen chainsaw timeouts under load The chainsaw suites put sustained load on the co-hosted Karmada control plane. Under that load a transient etcd/DNS blip made the API server's default health probes fail and the kubelet SIGKILLed it (exit 137), cascading into a crashloop that failed every suite. Relax the karmada-apiserver liveness/readiness probes so it rides out blips instead of being killed, and widen the chainsaw timeouts (assert 120s, delete 120s) so Karmada propagation and finalizer-driven cascade deletes have room on the constrained node. Both are constrained-host/CI tuning, not production changes. Co-Authored-By: Claude Fable 5 --- Taskfile.yaml | 9 +++++++++ test/e2e/chainsaw-config.yaml | 15 ++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Taskfile.yaml b/Taskfile.yaml index 4742bb1b..c1f973c0 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -363,6 +363,15 @@ tasks: kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ -n karmada-system delete pod etcd-0 --ignore-not-found fi + # Relax the Karmada API server's health probes. Under sustained load (e.g. + # the chainsaw suites) a transient etcd/DNS blip makes the default 5s/15s + # probes fail, and the kubelet SIGKILLs the API server — which cascades to + # a crashloop. Wider thresholds let it ride out the blip instead of being + # killed. Belt-and-suspenders with the etcd fsync + leader-election tuning. + - | + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system patch deploy karmada-apiserver --type=strategic \ + -p '{"spec":{"template":{"spec":{"containers":[{"name":"karmada-apiserver","livenessProbe":{"timeoutSeconds":30,"failureThreshold":30,"periodSeconds":30,"initialDelaySeconds":60},"readinessProbe":{"timeoutSeconds":30,"failureThreshold":30,"periodSeconds":20}}]}}}}' e2e:karmada:configure: desc: "Apply federation component config to the Karmada API server (idempotent)" diff --git a/test/e2e/chainsaw-config.yaml b/test/e2e/chainsaw-config.yaml index cd3a9950..9524c8e4 100644 --- a/test/e2e/chainsaw-config.yaml +++ b/test/e2e/chainsaw-config.yaml @@ -27,12 +27,17 @@ kind: Configuration metadata: name: chainsaw spec: + # Timeouts are widened from chainsaw's defaults for the constrained local/CI + # node that co-hosts the Karmada control plane: Karmada propagation of a + # WorkloadDeployment to a cell routinely needs more than the default 60s under + # load, and finalizer-driven cascade deletes (the federator releasing hub + # companions) exceed the default 30s. timeouts: - apply: 30s - assert: 60s - cleanup: 60s - delete: 30s - error: 30s + apply: 60s + assert: 120s + cleanup: 120s + delete: 120s + error: 60s exec: 30s clusters: # Downstream control plane. WorkloadDeployments, PropagationPolicies, From cea88f35ff0c2984361d401ffdbc7afb9a897ef3 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 18:24:34 -0500 Subject: [PATCH 12/23] ci(e2e): run in-cluster federation e2e on GitHub Actions Replaces the placeholder E2E workflow (a single throwaway Kind cluster running a commented-out make target) with one that stands up the full harness the Taskfile builds and runs the Chainsaw suites against it: three Kind clusters (management/control-plane hosting Karmada, plus the dfw and ord POP cells), the real production kustomize overlays, and the real hub RBAC. This validates the operators as deployed pods authenticating to Karmada as a non-admin identity rather than as an in-process test binary. The job splits provisioning, deploy, and test into separate steps so a failure lands on the phase that broke, and always collects per-cluster operator logs, Karmada component logs, events, and a full kind log export as an artifact when any step fails. Runs on the free ubuntu-latest runner: the harness is engineered for a constrained host (single-replica Karmada, leader election disabled, generous component waits), so a larger paid runner is not needed unless real runs show resource pressure. Triggers only on pull_request and pushes to main rather than every branch push, given the job's cost. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-e2e.yml | 136 ++++++++++++++++++++++++++++++--- 1 file changed, 127 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 9bede775..89c0b007 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -1,13 +1,44 @@ name: E2E Tests +# In-cluster federation e2e (issue #149). +# +# Stands up the full local topology the Taskfile harness builds — three Kind +# clusters (one management/control-plane hosting Karmada, two POP cells) with the +# real production kustomize overlays and hub RBAC — then runs the Chainsaw +# suites against it. This exercises the operators as deployed pods authenticating +# to Karmada as a non-admin identity, not as an in-process test binary. + on: push: + branches: [main] pull_request: +# Cancel a superseded run on the same ref. The e2e job is expensive (three Kind +# clusters + a Karmada control plane), so we don't want stale pushes burning a +# runner. Unlike the cheaper test/lint workflows this only triggers on PRs and +# main pushes rather than every branch push, for the same cost reason. +concurrency: + group: e2e-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + jobs: test-e2e: name: Run on Ubuntu + # ubuntu-latest is 4 vCPU / 16 GB. The harness is deliberately engineered for + # a constrained/busy host — single-replica Karmada, leader election disabled, + # generous 10m component waits, join retries — so this fits without a larger + # (paid) runner. If real runs show OOM or repeated timeouts, switch to a + # larger hosted runner label (e.g. ubuntu-latest-8-cores); that carries a + # billing implication, hence starting on the free tier. runs-on: ubuntu-latest + # Env standup can spend up to ~20m if both Karmada waits approach their 10m + # ceilings on a slow runner; deploy ~5m; the Chainsaw suites ~15m (the + # referenced-data GC-sweep suite alone floors around 6m). 50m leaves headroom + # over the ~35m expected without letting a genuinely hung run idle too long. + timeout-minutes: 50 steps: - name: Clone the code uses: actions/checkout@v4 @@ -17,19 +48,106 @@ jobs: with: go-version: '~1.25.0' - - name: Install the latest version of kind + # go-task drives the entire harness (task e2e:env:up / e2e:deploy / + # e2e:test). The action publishes major refs as branches (v1/v2/v3); the + # version input pins the go-task binary itself. repo-token avoids GitHub + # API rate limiting when the action resolves the release. + - name: Install go-task + uses: arduino/setup-task@v3 + with: + version: 3.52.0 + repo-token: ${{ secrets.GITHUB_TOKEN }} + + # Kind is not preinstalled on the runner. Docker, kubectl, and helm already + # are (ubuntu-24.04 image); karmadactl and chainsaw are fetched into ./bin + # by the task tooling. Pinned for reproducibility. + - name: Install kind run: | - curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 + curl -sSfLo ./kind https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-amd64 chmod +x ./kind sudo mv ./kind /usr/local/bin/kind + kind version + + # The Karmada kubeconfig rewrite step (_e2e:karmada:build-kubeconfig) shells + # out to python3 + PyYAML. It is normally present on the runner image; guard + # the case where it is not without tripping over PEP 668. + - name: Ensure PyYAML + run: | + if ! python3 -c "import yaml" 2>/dev/null; then + sudo apt-get update + sudo apt-get install -y python3-yaml + fi - - name: Verify kind installation - run: kind version + # Split into env / deploy / test so a failure lands on the phase that broke + # rather than a single opaque "task e2e:up" step. e2e:env:up == e2e:up minus + # the deploy; the two together are exactly what e2e:up runs. + - name: Provision Kind + Karmada environment + run: task e2e:env:up - - name: Create kind cluster - run: kind create cluster + - name: Build image and deploy operators + run: task e2e:deploy - - name: Running Test e2e + - name: Run Chainsaw e2e suites + run: task e2e:test + + # Always capture what the clusters looked like when a step failed. The + # runner is ephemeral so teardown is unnecessary; diagnostics are the only + # thing worth keeping. + - name: Collect diagnostics + if: failure() run: | - go mod tidy - make test-e2e + set +e + DIAG=tmp/e2e/diagnostics + KDIR=tmp/e2e/kubeconfigs + mkdir -p "$DIAG" + + # Host / kind level: container state plus a full per-cluster export + # (kubelet, containerd, and every pod log). + kind get clusters > "$DIAG/kind-clusters.txt" 2>&1 + docker ps -a > "$DIAG/docker-ps.txt" 2>&1 + for c in compute-control-plane compute-pop-dfw compute-pop-ord; do + kind export logs "$DIAG/kind-$c" --name "$c" 2>&1 | tail -n 2 + done + + # Per-cluster Kubernetes state + the compute-manager operator logs + # (current and previous, all containers) from every plane. + for kc in control-plane pop-dfw pop-ord karmada; do + cfg="$KDIR/$kc.yaml" + [ -f "$cfg" ] || continue + out="$DIAG/$kc"; mkdir -p "$out" + kubectl --kubeconfig="$cfg" get pods -A -o wide > "$out/pods.txt" 2>&1 + kubectl --kubeconfig="$cfg" get events -A --sort-by=.lastTimestamp > "$out/events.txt" 2>&1 + kubectl --kubeconfig="$cfg" -n compute-system describe deploy compute-manager \ + > "$out/compute-manager-describe.txt" 2>&1 + kubectl --kubeconfig="$cfg" -n compute-system logs deploy/compute-manager \ + --all-containers --tail=-1 > "$out/compute-manager.log" 2>&1 + kubectl --kubeconfig="$cfg" -n compute-system logs deploy/compute-manager \ + --all-containers --previous --tail=-1 > "$out/compute-manager-previous.log" 2>&1 + done + + # Karmada control-plane pods + component logs (they live in the + # management cluster) and the federation view from the Karmada API. + if [ -f "$KDIR/control-plane.yaml" ]; then + kubectl --kubeconfig="$KDIR/control-plane.yaml" -n karmada-system get pods -o wide \ + > "$DIAG/karmada-pods.txt" 2>&1 + for d in karmada-apiserver karmada-controller-manager karmada-scheduler; do + kubectl --kubeconfig="$KDIR/control-plane.yaml" -n karmada-system logs deploy/$d \ + --tail=-1 > "$DIAG/karmada-$d.log" 2>&1 + done + fi + if [ -f "$KDIR/karmada.yaml" ]; then + kubectl --kubeconfig="$KDIR/karmada.yaml" get clusters -o wide \ + > "$DIAG/karmada-clusters.txt" 2>&1 + kubectl --kubeconfig="$KDIR/karmada.yaml" get workloaddeployments -A -o wide \ + > "$DIAG/karmada-workloaddeployments.txt" 2>&1 + fi + echo "Diagnostics collected under $DIAG" + + - name: Upload diagnostics + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-diagnostics + path: tmp/e2e/diagnostics + retention-days: 7 + if-no-files-found: warn From f336f3816a21234d3c8e60ba73bf2aa8c8ac910a Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 20:57:20 -0500 Subject: [PATCH 13/23] test(e2e): run chainsaw suites serially Running all suites concurrently floods the single federation manager: many WorkloadDeployments federate at once and individual suites' downstream assertions time out with "resource not found" even though the federation path itself works (the first suite to reach the manager passes). Pin --parallel 1 so the federation path stays uncontended; a faster local run can still override with `task e2e:test -- --parallel N`. Co-Authored-By: Claude Fable 5 --- Taskfile.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Taskfile.yaml b/Taskfile.yaml index c1f973c0..aaf7e6cd 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -105,10 +105,17 @@ tasks: desc: "Run Chainsaw e2e tests against the local Kind+Karmada environment" deps: [e2e:tools:chainsaw] cmds: + # --parallel 1 (sequential) on purpose: the federation controllers are a + # single manager, and running all suites concurrently floods it — many + # WorkloadDeployments federate at once and individual suites' downstream + # assertions time out ("resource not found") even though federation works. + # Serial keeps the federation path uncontended on a constrained runner. + # Override for a faster local run with `task e2e:test -- --parallel N`. - | KUBECONFIG={{.KUBECONFIG_DIR}}/control-plane.yaml \ {{.CHAINSAW}} test \ --config test/e2e/chainsaw-config.yaml \ + --parallel 1 \ test/e2e/ \ {{.CLI_ARGS}} From 4d072ff98a39f2b8b45573f7cf6332e0f3c55f67 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 21:11:48 -0500 Subject: [PATCH 14/23] test(e2e): assert all three city PropagationPolicy selectors The first CI run failed workload-deployment-federation on the city-dfw PropagationPolicy assertion with "lengths of slices don't match": the federator emits three resourceSelectors (WorkloadDeployment plus the always-on ConfigMap and Secret referenced-data selectors that co-propagate companions), but the suite asserted only the WorkloadDeployment. Chainsaw matches list length exactly, so assert all three. Apply the same correction to full-federation-ord's city-ord assertion, which carried the same stale single-selector expectation. Co-Authored-By: Claude Fable 5 --- test/e2e/full-federation-ord/chainsaw-test.yaml | 13 +++++++++++++ .../chainsaw-test.yaml | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/test/e2e/full-federation-ord/chainsaw-test.yaml b/test/e2e/full-federation-ord/chainsaw-test.yaml index ad26d07f..a6b69d91 100644 --- a/test/e2e/full-federation-ord/chainsaw-test.yaml +++ b/test/e2e/full-federation-ord/chainsaw-test.yaml @@ -71,12 +71,25 @@ spec: namespace: ($downstreamNS) name: city-ord spec: + # Three selectors: the WorkloadDeployment plus the always-on ConfigMap + # and Secret referenced-data selectors. Chainsaw matches list length + # exactly, so all three must be present. resourceSelectors: - apiVersion: compute.datumapis.com/v1alpha kind: WorkloadDeployment labelSelector: matchLabels: topology.datum.net/city-code: ord + - apiVersion: v1 + kind: ConfigMap + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + - apiVersion: v1 + kind: Secret + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" placement: clusterAffinity: labelSelector: diff --git a/test/e2e/workload-deployment-federation/chainsaw-test.yaml b/test/e2e/workload-deployment-federation/chainsaw-test.yaml index 302d89c4..8d589310 100644 --- a/test/e2e/workload-deployment-federation/chainsaw-test.yaml +++ b/test/e2e/workload-deployment-federation/chainsaw-test.yaml @@ -71,12 +71,26 @@ spec: namespace: ($downstreamNS) name: city-dfw spec: + # The federator emits three resource selectors, not one: the + # WorkloadDeployment plus the always-on ConfigMap and Secret + # referenced-data selectors that co-propagate companions to the cell. + # Chainsaw matches list length exactly, so all three must be asserted. resourceSelectors: - apiVersion: compute.datumapis.com/v1alpha kind: WorkloadDeployment labelSelector: matchLabels: topology.datum.net/city-code: dfw + - apiVersion: v1 + kind: ConfigMap + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + - apiVersion: v1 + kind: Secret + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" placement: clusterAffinity: labelSelector: From 034c25dd39917efb192cddea11a2ad4a2c3fa247 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 21:11:56 -0500 Subject: [PATCH 15/23] test(e2e): seed hand-crafted projection/write-back fixtures with required labels Both suites synthesize objects that the real cell path would stamp, and the first CI run showed the operators correctly rejecting the incomplete fixtures: - instance-projection: the projector logged "missing the workload-deployment-name label; cannot resolve its WorkloadDeployment" and never created the projection. The injected hub Instance now carries workload-deployment-name, and upstream-namespace points at the project namespace (where the WD lives and the projection lands), not the hub ns-. Also restores the owner-reference assertion the projector sets. - instance-writeback: writeBackToUpstream derives identity from the hub namespace labels and requires the full linking-label set on the Instance (instance_controller.go). Seed both so the write-back fires instead of erroring on missing identity. Co-Authored-By: Claude Fable 5 --- .../instance-projection/chainsaw-test.yaml | 21 +++++++++++++++++-- .../e2e/instance-writeback/chainsaw-test.yaml | 17 +++++++++++---- .../instance-writeback/instance-pop-dfw.yaml | 13 ++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/test/e2e/instance-projection/chainsaw-test.yaml b/test/e2e/instance-projection/chainsaw-test.yaml index b79d83d0..58a8dcbf 100644 --- a/test/e2e/instance-projection/chainsaw-test.yaml +++ b/test/e2e/instance-projection/chainsaw-test.yaml @@ -81,9 +81,17 @@ spec: env: - name: KARMADA_NS value: ($downstreamNS) + - name: PROJECT_NS + value: ($namespace) - name: WD_UID value: ($wdUID) content: | + # These labels mirror what the real cell InstanceReconciler stamps on a + # write-back copy. The projector resolves the project cluster from + # upstream-cluster-name, the project NAMESPACE from upstream-namespace + # (the project namespace where the WD lives — NOT the hub ns-), and + # looks up the owning WorkloadDeployment by workload-deployment-name to + # build the projection's owner reference. kubectl apply -f - < namespace the federator stamps in the real path. Seed them + # so writeBackToUpstream can resolve identity instead of erroring. + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/downstream.yaml apply -f - < convention so the InstanceProjector can resolve it later. +# +# The linking labels below mirror what the stateful control strategy stamps on a +# real cell Instance at creation. The write-back path requires all of them to be +# present and non-empty before it will copy the Instance upstream; a real Instance +# always carries them, so a hand-crafted one must too. apiVersion: compute.datumapis.com/v1alpha kind: Instance metadata: name: test-writeback-instance namespace: ($instanceNS) + labels: + compute.datumapis.com/workload-uid: "00000000-0000-0000-0000-000000000001" + compute.datumapis.com/workload-deployment-uid: "00000000-0000-0000-0000-000000000002" + compute.datumapis.com/instance-index: "0" + compute.datumapis.com/workload-deployment-name: test-writeback-wd + compute.datumapis.com/city-code: dfw + compute.datumapis.com/workload-name: test-workload + compute.datumapis.com/placement-name: default spec: runtime: resources: From 5c32bdef456d8da1cf7d18c7968afa3821b5e3b8 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 21:12:03 -0500 Subject: [PATCH 16/23] test(e2e): assert referenced-data companions by their un-prefixed name referenced.CompanionName returns the source object's own name when it is a valid DNS subdomain within the length budget (the kind argument is ignored), so the companions materialize as app-config and app-secret, not configmap.app-config and secret.app-secret. The first CI run confirmed this: the referenced-data-delete-cascade suite (source names gc-test-config / gc-test-secret) found its un-prefixed companions, while referenced-data-mounts timed out looking for the prefixed names. Correct the assertions and the naming-convention note in the README. Co-Authored-By: Claude Fable 5 --- test/e2e/referenced-data-mounts/README.md | 10 ++++++---- .../referenced-data-mounts/chainsaw-test.yaml | 16 ++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/test/e2e/referenced-data-mounts/README.md b/test/e2e/referenced-data-mounts/README.md index 808c2e12..3536d10b 100644 --- a/test/e2e/referenced-data-mounts/README.md +++ b/test/e2e/referenced-data-mounts/README.md @@ -10,7 +10,7 @@ The test covers Hops 1–5 of the federated delivery chain: | Hop | Cluster | What is asserted | |-----|---------|-----------------| | 1 | control-plane | Source ConfigMap + Secret created in the project namespace | -| 2 | control-plane | Companion `configmap.app-config` + `secret.app-secret` appear in `ns-{project-uid}` with `compute.datumapis.com/referenced-data: "true"`; WD carries `expected-referenced-data` annotation; WD condition `ReferencedDataReady=True` | +| 2 | control-plane | Companion `app-config` + `app-secret` appear in `ns-{project-uid}` with `compute.datumapis.com/referenced-data: "true"`; WD carries `expected-referenced-data` annotation; WD condition `ReferencedDataReady=True` | | 3 | downstream (Karmada hub) | Companion ConfigMap + Secret present in `ns-{project-uid}` on the hub; WD carries the annotation; `PropagationPolicy city-dfw` has ConfigMap and Secret resource selectors | | 4 | pop-dfw (cell) | WD + companions propagated to the cell in `ns-{project-uid}` | | 5 | pop-dfw (cell) | Instance `test-refdata-wd-0` exists; `ReferencedData` scheduling gate cleared; `ReferencedDataReady=True` condition set | @@ -76,11 +76,13 @@ operators from the real overlays. Points worth knowing: ## Companion naming convention -The `ReferencedDataController` derives companion names deterministically: +The `ReferencedDataController` derives companion names deterministically. When +the source name is already a valid DNS subdomain within the length budget, the +companion keeps that name unchanged (kind is not prefixed): | Source | Companion name | |--------|---------------| -| `ConfigMap/app-config` | `configmap.app-config` | -| `Secret/app-secret` | `secret.app-secret` | +| `ConfigMap/app-config` | `app-config` | +| `Secret/app-secret` | `app-secret` | These names are asserted directly in the test steps. diff --git a/test/e2e/referenced-data-mounts/chainsaw-test.yaml b/test/e2e/referenced-data-mounts/chainsaw-test.yaml index 8b0fc766..bcef1234 100644 --- a/test/e2e/referenced-data-mounts/chainsaw-test.yaml +++ b/test/e2e/referenced-data-mounts/chainsaw-test.yaml @@ -66,7 +66,7 @@ spec: - name: assert-companion-configmap-on-hub description: | - Assert the ReferencedDataController materialised companion configmap.app-config + Assert the ReferencedDataController materialised companion app-config in ns-{project-uid} on the Karmada hub (downstream cluster). Companions are written to the hub namespace, NOT the control-plane namespace. cluster: downstream @@ -86,13 +86,13 @@ spec: kind: ConfigMap metadata: namespace: ($companionNS) - name: configmap.app-config + name: app-config labels: compute.datumapis.com/referenced-data: "true" - name: assert-companion-secret-on-hub description: | - Assert the ReferencedDataController materialised companion secret.app-secret + Assert the ReferencedDataController materialised companion app-secret in ns-{project-uid} on the Karmada hub (downstream cluster). Companions are written to the hub namespace, NOT the control-plane namespace. cluster: downstream @@ -112,7 +112,7 @@ spec: kind: Secret metadata: namespace: ($companionNS) - name: secret.app-secret + name: app-secret labels: compute.datumapis.com/referenced-data: "true" @@ -180,7 +180,7 @@ spec: kind: ConfigMap metadata: namespace: ($downstreamNS) - name: configmap.app-config + name: app-config labels: compute.datumapis.com/referenced-data: "true" - assert: @@ -191,7 +191,7 @@ spec: kind: Secret metadata: namespace: ($downstreamNS) - name: secret.app-secret + name: app-secret labels: compute.datumapis.com/referenced-data: "true" @@ -269,7 +269,7 @@ spec: kind: ConfigMap metadata: namespace: ($downstreamNS) - name: configmap.app-config + name: app-config labels: compute.datumapis.com/referenced-data: "true" - assert: @@ -279,7 +279,7 @@ spec: kind: Secret metadata: namespace: ($downstreamNS) - name: secret.app-secret + name: app-secret labels: compute.datumapis.com/referenced-data: "true" From b2b3ebfc8472d288d0ed03dba801720a767c3d5f Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 21:21:37 -0500 Subject: [PATCH 17/23] test(e2e): retry Karmada NodePort applies while the apiserver settles Right after the Karmada install, applies made over the API server's host NodePort can hit a transient connection drop ("EOF" / API discovery failure) while the control plane settles, which failed provisioning (karmada:configure) on an otherwise-healthy environment. Retry the federation-component apply and the hub-RBAC apply a few times so a momentary blip no longer breaks the run. Co-Authored-By: Claude Fable 5 --- Taskfile.yaml | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/Taskfile.yaml b/Taskfile.yaml index aaf7e6cd..f6faa0af 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -383,11 +383,21 @@ tasks: e2e:karmada:configure: desc: "Apply federation component config to the Karmada API server (idempotent)" cmds: + # Retry the apply: this runs right after the Karmada install, and the + # apiserver reached over the NodePort can briefly drop the connection + # ("EOF" / API discovery failure) while it settles, which otherwise fails + # provisioning on an otherwise-healthy control plane. - | echo "Applying federation component to Karmada..." - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply \ - -k config/components/federation/ --server-side --validate=false - echo "Federation component applied" + for attempt in 1 2 3 4 5 6; do + if kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply \ + -k config/components/federation/ --server-side --validate=false; then + echo "Federation component applied (attempt ${attempt})"; exit 0 + fi + echo "configure attempt ${attempt} failed (Karmada apiserver may still be settling); retrying in 10s..." + sleep 10 + done + echo "ERROR: failed to apply federation component after 6 attempts"; exit 1 _e2e:karmada:build-kubeconfig: internal: true @@ -623,7 +633,14 @@ tasks: # forbidden error rather than being masked by cluster-admin. - | echo "Applying hub RBAC (config/base/downstream-rbac) to Karmada..." - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply -k config/base/downstream-rbac --server-side --validate=false + for attempt in 1 2 3 4 5; do + if kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply -k config/base/downstream-rbac --server-side --validate=false; then + break + fi + echo "hub RBAC apply attempt ${attempt} failed (Karmada apiserver over NodePort may be settling); retrying in 8s..." + sleep 8 + if [ "${attempt}" = "5" ]; then echo "ERROR: hub RBAC apply failed after 5 attempts"; exit 1; fi + done # ── Karmada-native identity for the management operator ───────────── # Production federates the management cluster's projected ServiceAccount # token into Karmada (Karmada trusts the host cluster's token issuer). We From 8f7cf8a983e849553cb859fe2de7bb87934648c4 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 21:36:11 -0500 Subject: [PATCH 18/23] test(e2e): drop the delete-cascade stranded-companion scenario pending #144 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scenario 2 asserted a level-triggered hub-side companion GC reclaiming a stranded companion, but that reconciler is not in this branch — it ships in PR #144 (refdata-hub-gc), which was still unmerged when the branch was cut, so the scenario could never pass. Keep the happy-path cascade (Scenario 1), which exercises the delete path that IS implemented, and restore the backstop scenario once #144 lands. Co-Authored-By: Claude Fable 5 --- .../chainsaw-test.yaml | 85 ++----------------- 1 file changed, 7 insertions(+), 78 deletions(-) diff --git a/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml b/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml index a3e1a10b..35570424 100644 --- a/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml +++ b/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml @@ -4,28 +4,23 @@ metadata: name: referenced-data-delete-cascade spec: description: | - Validates the full delete-cascade for referenced-data companions and the - level-triggered companion GC backstop (CompanionGCReconciler). + Validates the happy-path delete-cascade for referenced-data companions. - Two scenarios are validated: - - SCENARIO 1 — HAPPY-PATH CASCADE Create a WorkloadDeployment referencing a ConfigMap + Secret. Assert companions materialize and propagate to the member cluster. Delete the WD (last referrer). Assert the hub companion, its Karmada ResourceBinding, and the member-cluster copy are all deleted and stay deleted (no re-create loop). - SCENARIO 2 — STRANDED COMPANION BACKSTOP - Simulate interrupted finalization by creating a referenced-data-labeled - companion whose referenced-by annotation points at a non-existent WD. - Assert the CompanionGCReconciler reclaims it within the sweep interval: - the stranded companion and its ResourceBinding are both deleted. + A second scenario — the stranded-companion backstop, where a level-triggered + companion GC reclaims a companion whose referenced-by annotation points at a + non-existent WorkloadDeployment — is intentionally omitted: that hub-side GC + ships in PR #144 (refdata-hub-gc), which was still unmerged when this branch + was cut. Restore the scenario once #144 lands. Prerequisites: - task e2e:up completed (deploys the management + both cell operators in-cluster) - the cell operator runs with featureFlags.enableReferencedDataGate: true, set by the e2e cell deploy layer (test/e2e/deploy/cell/config_patch.yaml) - - CompanionGCReconciler enabled in the management operator - tmp/e2e/kubeconfigs/downstream.yaml exists template: true @@ -53,7 +48,7 @@ spec: - name: s1-assert-companions-on-hub description: | Assert companions materialize in ns-{project-uid} on the Karmada hub - with the referenced-data label. The CompanionGCReconciler watches these. + with the referenced-data label. cluster: downstream try: - script: @@ -290,69 +285,3 @@ spec: fi done echo "OK: cell copies absent for 30+ seconds — no recreate loop" - - # ═══════════════════════════════════════════════════════════════════════════ - # SCENARIO 2: STRANDED COMPANION BACKSTOP - # - # Simulate interrupted finalization: inject a labeled companion whose - # referenced-by annotation points at a WD that does not exist. The - # CompanionGCReconciler must detect and reclaim it. - # ═══════════════════════════════════════════════════════════════════════════ - - - name: s2-inject-stranded-companion - description: | - Inject a stranded companion ConfigMap directly into the hub namespace. - Its referenced-by annotation points at "default/nonexistent-wd" — a WD - that does not (and never did) exist. This simulates an interrupted - finalization where the controller pod restarted mid-flight. - cluster: downstream - try: - - script: - content: | - COMPANION_NS=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ - get namespace "$NAMESPACE" \ - -o template='{{printf "ns-%s" .metadata.uid}}') - - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/downstream.yaml \ - apply -f - < Date: Thu, 9 Jul 2026 21:36:12 -0500 Subject: [PATCH 19/23] test(e2e): raise two per-step timeouts for the full CI chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The serial diagnostic run showed two assertions that need more than 30s on a constrained CI runner (the object does appear, just later): - full-federation assert-instance-on-pop-dfw waits on the whole project WD → federate → Karmada propagate → cell reconcile → Instance chain. - instance-projection wait-for-downstream-namespace waits on federation of a freshly-created namespace. Raise both to 120s. This is a timing allowance, not a masked stall. Co-Authored-By: Claude Fable 5 --- test/e2e/full-federation/chainsaw-test.yaml | 6 +++++- test/e2e/instance-projection/chainsaw-test.yaml | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/test/e2e/full-federation/chainsaw-test.yaml b/test/e2e/full-federation/chainsaw-test.yaml index a5ac8735..a95bc23b 100644 --- a/test/e2e/full-federation/chainsaw-test.yaml +++ b/test/e2e/full-federation/chainsaw-test.yaml @@ -111,7 +111,11 @@ spec: - name: downstreamNS value: ($stdout) - assert: - timeout: 30s + # This is the tail of the full chain (project WD → federate → Karmada + # propagate to the cell → cell reconcile → Instance). On a constrained CI + # runner that whole sequence can exceed 30s; 120s gives it room without + # masking a real stall (the smoke path confirms the Instance does appear). + timeout: 120s resource: apiVersion: compute.datumapis.com/v1alpha kind: Instance diff --git a/test/e2e/instance-projection/chainsaw-test.yaml b/test/e2e/instance-projection/chainsaw-test.yaml index 58a8dcbf..d871c308 100644 --- a/test/e2e/instance-projection/chainsaw-test.yaml +++ b/test/e2e/instance-projection/chainsaw-test.yaml @@ -46,7 +46,10 @@ spec: - name: downstreamNS value: ($stdout) - assert: - timeout: 30s + # Federation of a freshly-created namespace depends on the management + # operator's namespace cache being current; with the namespaces-watch RBAC + # fix it stays live, but 30s is still tight on a loaded CI runner, so allow 120s. + timeout: 120s resource: apiVersion: compute.datumapis.com/v1alpha kind: WorkloadDeployment From 413cf8ac8b2ee4fc32bba88df7ffa5a115f9ae22 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 21:41:48 -0500 Subject: [PATCH 20/23] test(e2e): widen three downstream-assert timeouts for CI latency full-federation-ord, instance-writeback, and referenced-data-delete-cascade asserted federated/companion resources with 30-60s per-step timeouts, too tight for the federate->propagate chain on a constrained CI runner (the same class of delay already given 120s in full-federation and instance-projection). Bump the failing steps to 120s. Noted as provably-timing: the resources appear, just after the old deadline. Co-Authored-By: Claude Fable 5 --- test/e2e/full-federation-ord/chainsaw-test.yaml | 4 ++-- test/e2e/instance-writeback/chainsaw-test.yaml | 2 +- test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e/full-federation-ord/chainsaw-test.yaml b/test/e2e/full-federation-ord/chainsaw-test.yaml index a6b69d91..f5d8dc23 100644 --- a/test/e2e/full-federation-ord/chainsaw-test.yaml +++ b/test/e2e/full-federation-ord/chainsaw-test.yaml @@ -51,7 +51,7 @@ spec: - name: downstreamNS value: ($stdout) - assert: - timeout: 30s + timeout: 120s resource: apiVersion: compute.datumapis.com/v1alpha kind: WorkloadDeployment @@ -63,7 +63,7 @@ spec: - assert: # The federator names the policy city- and routes it to cells # carrying the same city-code label, so ord must land on pop-ord alone. - timeout: 30s + timeout: 120s resource: apiVersion: policy.karmada.io/v1alpha1 kind: PropagationPolicy diff --git a/test/e2e/instance-writeback/chainsaw-test.yaml b/test/e2e/instance-writeback/chainsaw-test.yaml index dbd83f68..50110842 100644 --- a/test/e2e/instance-writeback/chainsaw-test.yaml +++ b/test/e2e/instance-writeback/chainsaw-test.yaml @@ -119,7 +119,7 @@ spec: - name: instanceNS value: ($stdout) - assert: - timeout: 30s + timeout: 120s resource: apiVersion: compute.datumapis.com/v1alpha kind: Instance diff --git a/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml b/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml index 35570424..31fad786 100644 --- a/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml +++ b/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml @@ -60,7 +60,7 @@ spec: - name: companionNS value: ($stdout) - assert: - timeout: 60s + timeout: 120s resource: apiVersion: v1 kind: ConfigMap @@ -70,7 +70,7 @@ spec: labels: compute.datumapis.com/referenced-data: "true" - assert: - timeout: 60s + timeout: 120s resource: apiVersion: v1 kind: Secret From b6b7ac7a716852c62d113c60a825c1ab6bfb4b65 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 21:43:51 -0500 Subject: [PATCH 21/23] fix(rbac): grant the manager watch on namespaces The federator resolves WorkloadDeployment source namespaces through an informer, which needs watch in addition to get/list. With watch forbidden, the cache only refreshes on periodic re-lists whose backoff grows as the denied watch keeps retrying, so namespaces created after manager startup stay invisible for minutes and their WorkloadDeployments never federate. Second production RBAC gap surfaced by running the operators in-cluster under their real ServiceAccount (after the networks grant). --- config/components/controller_rbac/role.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/components/controller_rbac/role.yaml b/config/components/controller_rbac/role.yaml index 73d73a9d..81147741 100644 --- a/config/components/controller_rbac/role.yaml +++ b/config/components/controller_rbac/role.yaml @@ -31,6 +31,7 @@ rules: verbs: - get - list + - watch - apiGroups: - compute.datumapis.com resources: From 925302b414da4466040363762a4d3cfdde0b5ce6 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 22:16:52 -0500 Subject: [PATCH 22/23] test(e2e): assert Ready=False for the blocked cell Instance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The federated Instance settles Ready=False, not Unknown: it is blocked before scheduling completes (a scheduling gate, or no matching Location in the e2e environment), which is a definite not-ready state rather than an indeterminate one. The old Unknown expectation predates the scheduling-gate semantics. Assert status only — the exact blocking reason is environment-dependent. Co-Authored-By: Claude Fable 5 --- test/e2e/full-federation-ord/chainsaw-test.yaml | 12 ++++++++++-- test/e2e/full-federation/chainsaw-test.yaml | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/test/e2e/full-federation-ord/chainsaw-test.yaml b/test/e2e/full-federation-ord/chainsaw-test.yaml index f5d8dc23..b1bf1b64 100644 --- a/test/e2e/full-federation-ord/chainsaw-test.yaml +++ b/test/e2e/full-federation-ord/chainsaw-test.yaml @@ -141,8 +141,12 @@ spec: metadata: namespace: ($downstreamNS) name: test-fullfed-ord-wd-0 + # The Instance is blocked before scheduling completes (a scheduling + # gate, or no matching Location in the e2e environment), so Ready is + # definitively False — not indeterminate. The exact reason is + # environment-dependent, so only the status is asserted. (status.conditions[?type == 'Ready'] | [0]): - status: "Unknown" + status: "False" - name: assert-instance-writeback-in-downstream description: Assert the pop-ord InstanceReconciler wrote the Instance back to Karmada. @@ -180,5 +184,9 @@ spec: name: test-fullfed-ord-wd-0 labels: meta.datumapis.com/upstream-cluster-name: cluster-single + # The Instance is blocked before scheduling completes (a scheduling + # gate, or no matching Location in the e2e environment), so Ready is + # definitively False — not indeterminate. The exact reason is + # environment-dependent, so only the status is asserted. (status.conditions[?type == 'Ready'] | [0]): - status: "Unknown" + status: "False" diff --git a/test/e2e/full-federation/chainsaw-test.yaml b/test/e2e/full-federation/chainsaw-test.yaml index a95bc23b..fae94a23 100644 --- a/test/e2e/full-federation/chainsaw-test.yaml +++ b/test/e2e/full-federation/chainsaw-test.yaml @@ -122,8 +122,12 @@ spec: metadata: namespace: ($downstreamNS) name: test-full-fed-wd-0 + # The Instance is blocked before scheduling completes (a scheduling + # gate, or no matching Location in the e2e environment), so Ready is + # definitively False — not indeterminate. The exact reason is + # environment-dependent, so only the status is asserted. (status.conditions[?type == 'Ready'] | [0]): - status: "Unknown" + status: "False" - name: assert-instance-writeback-in-downstream description: Assert InstanceReconciler wrote the Instance back to Karmada. @@ -161,5 +165,9 @@ spec: name: test-full-fed-wd-0 labels: meta.datumapis.com/upstream-cluster-name: cluster-single + # The Instance is blocked before scheduling completes (a scheduling + # gate, or no matching Location in the e2e environment), so Ready is + # definitively False — not indeterminate. The exact reason is + # environment-dependent, so only the status is asserted. (status.conditions[?type == 'Ready'] | [0]): - status: "Unknown" + status: "False" From 3268114c82bc906a7e161a1d1c57885a14685857 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 9 Jul 2026 22:23:23 -0500 Subject: [PATCH 23/23] test(e2e): give cell operators a Karmada hub credential MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2e cells ran with FEDERATION_KUBECONFIG empty, which disabled the cell InstanceReconciler's write-back to the hub — so cell-created Instances never reached the Karmada hub or the management InstanceProjector, failing instance-writeback and the write-back/projection steps of full-federation and full-federation-ord. This was a harness bug, not production behavior. Verified against datum-cloud/ infra: every cell that runs compute gets a hub credential via apps/compute-system/edge/manager-patch.yaml (FEDERATION_KUBECONFIG + a mounted client-cert kubeconfig to the Karmada hub). The cell overlay ships the value empty on purpose because infra, not the overlay, supplies it per cell. Mirror that here: federation:setup mints the same compute-manager Karmada SA token and publishes it on both cells as compute-cell-federation-kubeconfig, reaching the kind hub over the control-plane node's docker-bridge IP + NodePort (cell pods live in other kind clusters, so the in-cluster Service the management pod uses is unreachable). federation_patch.yaml mounts it the way infra does. The corrected overlay comment now states what production actually does. Only the plain write-back client is built for cells, so this does not gate readiness. Co-Authored-By: Claude Fable 5 --- Taskfile.yaml | 45 ++++++++++++++++++++++ test/e2e/deploy/cell/federation_patch.yaml | 30 +++++++++++++++ test/e2e/deploy/cell/kustomization.yaml | 20 +++++++--- 3 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 test/e2e/deploy/cell/federation_patch.yaml diff --git a/Taskfile.yaml b/Taskfile.yaml index f6faa0af..2ae2eab1 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -701,6 +701,51 @@ tasks: --from-file=downstream-kubeconfig.yaml={{.E2E_DIR}}/downstream-kubeconfig.yaml \ --dry-run=client -o yaml | kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml apply -f - echo "Published compute-downstream-kubeconfig ConfigMap" + # ── Cell hub credential (mirrors infra apps/compute-system/edge) ───── + # Real cell deployments ALSO get a hub credential: infra's + # apps/compute-system/edge/manager-patch.yaml sets FEDERATION_KUBECONFIG and + # mounts a client-cert kubeconfig pointed at karmada.prod.env.datum.net. + # Without it the cell InstanceReconciler's write-back to the hub no-ops, so + # cell-created Instances never reach the hub and the management + # InstanceProjector (which reads Instances only from hub write-back copies) + # never sees them. We mirror it with the same compute-manager Karmada + # identity — but the cell pods live in the OTHER Kind clusters, so they + # reach the hub over the control-plane node's docker-bridge IP + Karmada + # NodePort rather than the in-cluster Service the management pod uses. + - | + CP_IP=$(docker inspect {{.KIND_CONTROL_PLANE}}-control-plane \ + | python3 -c "import sys,json; print(list(json.load(sys.stdin)[0]['NetworkSettings']['Networks'].values())[0]['IPAddress'])") + echo "Cell operators reach the Karmada hub via ${CP_IP}:{{.KARMADA_API_NODEPORT}}" + TOKEN=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml -n {{.COMPUTE_NAMESPACE}} \ + create token compute-manager --duration=720h) + cat > {{.E2E_DIR}}/cell-federation-kubeconfig.yaml </dev/null + kubectl --kubeconfig="$KC" -n {{.COMPUTE_NAMESPACE}} \ + create configmap compute-cell-federation-kubeconfig \ + --from-file=kubeconfig={{.E2E_DIR}}/cell-federation-kubeconfig.yaml \ + --dry-run=client -o yaml | kubectl --kubeconfig="$KC" apply -f - + done + echo "Published compute-cell-federation-kubeconfig on both POP cells" # ════════════════════════════════════════════════════════════════════════ # Operator deployment (real kustomize overlays + e2e patches) diff --git a/test/e2e/deploy/cell/federation_patch.yaml b/test/e2e/deploy/cell/federation_patch.yaml new file mode 100644 index 00000000..c3512185 --- /dev/null +++ b/test/e2e/deploy/cell/federation_patch.yaml @@ -0,0 +1,30 @@ +# Give the cell operator a Karmada hub credential, mirroring how real cell +# deployments are wired: infra's apps/compute-system/edge/manager-patch.yaml sets +# FEDERATION_KUBECONFIG and mounts a client-cert kubeconfig pointed at the hub. +# With it, the cell InstanceReconciler writes each Instance back to the Karmada +# hub, where the management InstanceProjector consumes it — the sole path by +# which cell-created Instances become visible upstream (Karmada never propagates +# Instances, only WorkloadDeployments/ConfigMaps/Secrets, so status aggregation +# cannot surface them). The mount path matches infra's; the kubeconfig content +# (a Karmada-native SA token reaching the kind hub over the control-plane node's +# docker-bridge IP + NodePort) is minted by task e2e:federation:setup. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: compute-manager +spec: + template: + spec: + containers: + - name: manager + env: + - name: FEDERATION_KUBECONFIG + value: /etc/kubernetes/upstream/auth/kubeconfig + volumeMounts: + - name: upstream-kubeconfig + mountPath: /etc/kubernetes/upstream/auth + readOnly: true + volumes: + - name: upstream-kubeconfig + configMap: + name: compute-cell-federation-kubeconfig diff --git a/test/e2e/deploy/cell/kustomization.yaml b/test/e2e/deploy/cell/kustomization.yaml index 21bb4380..b63ce808 100644 --- a/test/e2e/deploy/cell/kustomization.yaml +++ b/test/e2e/deploy/cell/kustomization.yaml @@ -5,11 +5,17 @@ kind: Kustomization # e2e cell deploy layer. # # References the REAL production cell overlay verbatim and applies only the -# deviations required to run inside a local Kind environment. The cell overlay -# is deployed unchanged apart from the image source and the quota-credential -# neutralisation below — in particular FEDERATION_KUBECONFIG stays empty, so the -# cell controllers run purely against the local cluster where Karmada propagates -# WorkloadDeployments, exactly as in production. +# deviations required to run inside a local Kind environment (image source, +# quota-credential neutralisation) plus the cell hub credential that infra +# supplies in production. +# +# The production cell overlay ships FEDERATION_KUBECONFIG empty on purpose — it +# is the *infra* layer that patches it in per-cell (apps/compute-system/edge/ +# manager-patch.yaml mounts a hub client-cert kubeconfig), NOT the overlay. So +# leaving it empty is NOT production-exact: it disables the cell's Instance +# write-back to the Karmada hub, and cell-created Instances then never reach the +# hub or the management InstanceProjector. federation_patch.yaml restores that +# credential the way infra does, so the full cell→hub→projector path is exercised. # ───────────────────────────────────────────────────────────────────────────── resources: @@ -29,3 +35,7 @@ patches: # DEVIATION 2 (required): force IfNotPresent pulls of the side-loaded image. - path: deployment_patch.yaml + + # Cell hub credential, mirroring infra's per-cell federation patch so the + # Instance write-back path is exercised. See federation_patch.yaml. + - path: federation_patch.yaml