diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 9bede775..89c0b007 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -1,13 +1,44 @@ name: E2E Tests +# In-cluster federation e2e (issue #149). +# +# Stands up the full local topology the Taskfile harness builds — three Kind +# clusters (one management/control-plane hosting Karmada, two POP cells) with the +# real production kustomize overlays and hub RBAC — then runs the Chainsaw +# suites against it. This exercises the operators as deployed pods authenticating +# to Karmada as a non-admin identity, not as an in-process test binary. + on: push: + branches: [main] pull_request: +# Cancel a superseded run on the same ref. The e2e job is expensive (three Kind +# clusters + a Karmada control plane), so we don't want stale pushes burning a +# runner. Unlike the cheaper test/lint workflows this only triggers on PRs and +# main pushes rather than every branch push, for the same cost reason. +concurrency: + group: e2e-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + jobs: test-e2e: name: Run on Ubuntu + # ubuntu-latest is 4 vCPU / 16 GB. The harness is deliberately engineered for + # a constrained/busy host — single-replica Karmada, leader election disabled, + # generous 10m component waits, join retries — so this fits without a larger + # (paid) runner. If real runs show OOM or repeated timeouts, switch to a + # larger hosted runner label (e.g. ubuntu-latest-8-cores); that carries a + # billing implication, hence starting on the free tier. runs-on: ubuntu-latest + # Env standup can spend up to ~20m if both Karmada waits approach their 10m + # ceilings on a slow runner; deploy ~5m; the Chainsaw suites ~15m (the + # referenced-data GC-sweep suite alone floors around 6m). 50m leaves headroom + # over the ~35m expected without letting a genuinely hung run idle too long. + timeout-minutes: 50 steps: - name: Clone the code uses: actions/checkout@v4 @@ -17,19 +48,106 @@ jobs: with: go-version: '~1.25.0' - - name: Install the latest version of kind + # go-task drives the entire harness (task e2e:env:up / e2e:deploy / + # e2e:test). The action publishes major refs as branches (v1/v2/v3); the + # version input pins the go-task binary itself. repo-token avoids GitHub + # API rate limiting when the action resolves the release. + - name: Install go-task + uses: arduino/setup-task@v3 + with: + version: 3.52.0 + repo-token: ${{ secrets.GITHUB_TOKEN }} + + # Kind is not preinstalled on the runner. Docker, kubectl, and helm already + # are (ubuntu-24.04 image); karmadactl and chainsaw are fetched into ./bin + # by the task tooling. Pinned for reproducibility. + - name: Install kind run: | - curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 + curl -sSfLo ./kind https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-amd64 chmod +x ./kind sudo mv ./kind /usr/local/bin/kind + kind version + + # The Karmada kubeconfig rewrite step (_e2e:karmada:build-kubeconfig) shells + # out to python3 + PyYAML. It is normally present on the runner image; guard + # the case where it is not without tripping over PEP 668. + - name: Ensure PyYAML + run: | + if ! python3 -c "import yaml" 2>/dev/null; then + sudo apt-get update + sudo apt-get install -y python3-yaml + fi - - name: Verify kind installation - run: kind version + # Split into env / deploy / test so a failure lands on the phase that broke + # rather than a single opaque "task e2e:up" step. e2e:env:up == e2e:up minus + # the deploy; the two together are exactly what e2e:up runs. + - name: Provision Kind + Karmada environment + run: task e2e:env:up - - name: Create kind cluster - run: kind create cluster + - name: Build image and deploy operators + run: task e2e:deploy - - name: Running Test e2e + - name: Run Chainsaw e2e suites + run: task e2e:test + + # Always capture what the clusters looked like when a step failed. The + # runner is ephemeral so teardown is unnecessary; diagnostics are the only + # thing worth keeping. + - name: Collect diagnostics + if: failure() run: | - go mod tidy - make test-e2e + set +e + DIAG=tmp/e2e/diagnostics + KDIR=tmp/e2e/kubeconfigs + mkdir -p "$DIAG" + + # Host / kind level: container state plus a full per-cluster export + # (kubelet, containerd, and every pod log). + kind get clusters > "$DIAG/kind-clusters.txt" 2>&1 + docker ps -a > "$DIAG/docker-ps.txt" 2>&1 + for c in compute-control-plane compute-pop-dfw compute-pop-ord; do + kind export logs "$DIAG/kind-$c" --name "$c" 2>&1 | tail -n 2 + done + + # Per-cluster Kubernetes state + the compute-manager operator logs + # (current and previous, all containers) from every plane. + for kc in control-plane pop-dfw pop-ord karmada; do + cfg="$KDIR/$kc.yaml" + [ -f "$cfg" ] || continue + out="$DIAG/$kc"; mkdir -p "$out" + kubectl --kubeconfig="$cfg" get pods -A -o wide > "$out/pods.txt" 2>&1 + kubectl --kubeconfig="$cfg" get events -A --sort-by=.lastTimestamp > "$out/events.txt" 2>&1 + kubectl --kubeconfig="$cfg" -n compute-system describe deploy compute-manager \ + > "$out/compute-manager-describe.txt" 2>&1 + kubectl --kubeconfig="$cfg" -n compute-system logs deploy/compute-manager \ + --all-containers --tail=-1 > "$out/compute-manager.log" 2>&1 + kubectl --kubeconfig="$cfg" -n compute-system logs deploy/compute-manager \ + --all-containers --previous --tail=-1 > "$out/compute-manager-previous.log" 2>&1 + done + + # Karmada control-plane pods + component logs (they live in the + # management cluster) and the federation view from the Karmada API. + if [ -f "$KDIR/control-plane.yaml" ]; then + kubectl --kubeconfig="$KDIR/control-plane.yaml" -n karmada-system get pods -o wide \ + > "$DIAG/karmada-pods.txt" 2>&1 + for d in karmada-apiserver karmada-controller-manager karmada-scheduler; do + kubectl --kubeconfig="$KDIR/control-plane.yaml" -n karmada-system logs deploy/$d \ + --tail=-1 > "$DIAG/karmada-$d.log" 2>&1 + done + fi + if [ -f "$KDIR/karmada.yaml" ]; then + kubectl --kubeconfig="$KDIR/karmada.yaml" get clusters -o wide \ + > "$DIAG/karmada-clusters.txt" 2>&1 + kubectl --kubeconfig="$KDIR/karmada.yaml" get workloaddeployments -A -o wide \ + > "$DIAG/karmada-workloaddeployments.txt" 2>&1 + fi + echo "Diagnostics collected under $DIAG" + + - name: Upload diagnostics + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-diagnostics + path: tmp/e2e/diagnostics + retention-days: 7 + if-no-files-found: warn diff --git a/Taskfile.yaml b/Taskfile.yaml new file mode 100644 index 00000000..2ae2eab1 --- /dev/null +++ b/Taskfile.yaml @@ -0,0 +1,803 @@ +version: '3' + +# ─── Variables ────────────────────────────────────────────────────────────── + +vars: + # Karmada Helm chart version to install (karmada-charts/karmada) + KARMADA_VERSION: v1.16.0 + + # karmadactl CLI version for cluster registration + KARMADACTL_VERSION: v1.16.0 + + # Chainsaw version for e2e testing (kyverno/chainsaw) + CHAINSAW_VERSION: v0.2.15 + + # Local tool directory (mirrors Makefile convention) + LOCALBIN: '{{.ROOT_DIR}}/bin' + KARMADACTL: '{{.ROOT_DIR}}/bin/karmadactl' + CHAINSAW: '{{.ROOT_DIR}}/bin/chainsaw' + + # Kind cluster names + KIND_CONTROL_PLANE: compute-control-plane + KIND_POP_DFW: compute-pop-dfw + KIND_POP_ORD: compute-pop-ord + + # All cluster names (for CRD installation loops) + KIND_ALL_CLUSTERS: '{{.KIND_CONTROL_PLANE}} {{.KIND_POP_DFW}} {{.KIND_POP_ORD}}' + + # Dev image tag built locally and side-loaded into every Kind cluster. This + # replaces the ghcr.io/datum-cloud/compute:latest reference baked into + # config/base/manager so e2e never pulls from a registry. + IMAGE: compute:e2e-dev + + # Working directory for e2e artefacts (gitignored) + E2E_DIR: '{{.ROOT_DIR}}/tmp/e2e' + KUBECONFIG_DIR: '{{.ROOT_DIR}}/tmp/e2e/kubeconfigs' + + # Fixed NodePort for the Karmada API server. + # The Kind management cluster is created with an extraPortMapping for this port + # so it is reachable at https://localhost:32443 from the developer's machine. + KARMADA_API_NODEPORT: "32443" + + # In-cluster Service address of the Karmada API server. The management + # compute-manager runs in the SAME Kind cluster as Karmada, so it reaches the + # hub over cluster DNS rather than the host-exposed NodePort. Matches the + # audience the production management-plane overlay projects its token for. + KARMADA_INCLUSTER_SERVER: "https://karmada-apiserver.karmada-system.svc.cluster.local:5443" + + # Namespace the operators deploy into (matches config/overlays/* namespace:). + COMPUTE_NAMESPACE: compute-system + +# ─── Tasks ────────────────────────────────────────────────────────────────── + +tasks: + + default: + cmds: + - task --list + silent: true + + # ════════════════════════════════════════════════════════════════════════ + # e2e environment lifecycle + # ════════════════════════════════════════════════════════════════════════ + + e2e:up: + desc: "Create the Kind+Karmada environment AND deploy the compute operators (idempotent)" + cmds: + - task: e2e:env:up + - task: e2e:deploy + - cmd: | + echo "" + echo "╔══════════════════════════════════════════════════════════╗" + echo "║ e2e environment ready — operators deployed in-cluster ║" + echo "╠══════════════════════════════════════════════════════════╣" + echo "║ Control plane: {{.KUBECONFIG_DIR}}/control-plane.yaml" + echo "║ Karmada API: {{.KUBECONFIG_DIR}}/karmada.yaml" + echo "║ POP DFW: {{.KUBECONFIG_DIR}}/pop-dfw.yaml" + echo "║ POP ORD: {{.KUBECONFIG_DIR}}/pop-ord.yaml" + echo "╠══════════════════════════════════════════════════════════╣" + echo "║ Inspect operators: ║" + echo "║ kubectl --kubeconfig {{.KUBECONFIG_DIR}}/control-plane.yaml -n {{.COMPUTE_NAMESPACE}} get deploy" + echo "╚══════════════════════════════════════════════════════════╝" + silent: false + + e2e:env:up: + desc: "Create the Kind clusters, install Karmada, register cells, install CRDs (no operators)" + cmds: + - task: e2e:tools + - task: e2e:clusters:create + - task: e2e:karmada:install + - task: e2e:karmada:configure + - task: e2e:karmada:join-clusters + - task: e2e:crds:install + + e2e:down: + desc: "Tear down the local e2e environment" + cmds: + - kind delete cluster --name {{.KIND_CONTROL_PLANE}} 2>/dev/null || true + - kind delete cluster --name {{.KIND_POP_DFW}} 2>/dev/null || true + - kind delete cluster --name {{.KIND_POP_ORD}} 2>/dev/null || true + - rm -rf {{.E2E_DIR}} + - cmd: echo "✓ e2e environment torn down" + silent: false + + e2e:test: + desc: "Run Chainsaw e2e tests against the local Kind+Karmada environment" + deps: [e2e:tools:chainsaw] + cmds: + # --parallel 1 (sequential) on purpose: the federation controllers are a + # single manager, and running all suites concurrently floods it — many + # WorkloadDeployments federate at once and individual suites' downstream + # assertions time out ("resource not found") even though federation works. + # Serial keeps the federation path uncontended on a constrained runner. + # Override for a faster local run with `task e2e:test -- --parallel N`. + - | + KUBECONFIG={{.KUBECONFIG_DIR}}/control-plane.yaml \ + {{.CHAINSAW}} test \ + --config test/e2e/chainsaw-config.yaml \ + --parallel 1 \ + test/e2e/ \ + {{.CLI_ARGS}} + + e2e:test:filter: + desc: "Run a subset of e2e tests by name regex (e.g. task e2e:test:filter -- --include-test-regex federation)" + deps: [e2e:tools:chainsaw] + cmds: + - | + KUBECONFIG={{.KUBECONFIG_DIR}}/control-plane.yaml \ + {{.CHAINSAW}} test \ + --config test/e2e/chainsaw-config.yaml \ + {{.CLI_ARGS}} \ + test/e2e/ + + # ════════════════════════════════════════════════════════════════════════ + # Tool installation + # ════════════════════════════════════════════════════════════════════════ + + e2e:tools: + desc: "Install e2e-specific tooling (karmadactl, chainsaw, helm repo)" + cmds: + - task: e2e:tools:karmadactl + - task: e2e:tools:chainsaw + - task: e2e:tools:helm-repo + + e2e:tools:karmadactl: + desc: "Download karmadactl {{.KARMADACTL_VERSION}}" + cmds: + - mkdir -p {{.LOCALBIN}} + - | + if [ ! -f "{{.KARMADACTL}}" ]; then + OS=$(uname -s | tr '[:upper:]' '[:lower:]') + ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') + URL="https://github.com/karmada-io/karmada/releases/download/{{.KARMADACTL_VERSION}}/karmadactl-${OS}-${ARCH}.tgz" + echo "Downloading karmadactl {{.KARMADACTL_VERSION}} (${OS}/${ARCH}) from ${URL}..." + curl -sSfL "${URL}" | tar -xz -C {{.LOCALBIN}} karmadactl + chmod +x {{.KARMADACTL}} + echo "karmadactl installed → {{.KARMADACTL}}" + else + echo "karmadactl already present at {{.KARMADACTL}}" + fi + status: + - test -f {{.KARMADACTL}} + + e2e:tools:chainsaw: + desc: "Download chainsaw {{.CHAINSAW_VERSION}}" + cmds: + - mkdir -p {{.LOCALBIN}} + - | + if [ ! -f "{{.CHAINSAW}}" ]; then + OS=$(uname -s | tr '[:upper:]' '[:lower:]') + ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') + URL="https://github.com/kyverno/chainsaw/releases/download/{{.CHAINSAW_VERSION}}/chainsaw_${OS}_${ARCH}.tar.gz" + echo "Downloading chainsaw {{.CHAINSAW_VERSION}} (${OS}/${ARCH}) from ${URL}..." + curl -sSfL "${URL}" | tar -xz -C {{.LOCALBIN}} chainsaw + chmod +x {{.CHAINSAW}} + echo "chainsaw installed → {{.CHAINSAW}}" + else + echo "chainsaw already present at {{.CHAINSAW}}" + fi + status: + - test -f {{.CHAINSAW}} + + e2e:tools:helm-repo: + desc: "Add/update karmada-charts Helm repository" + cmds: + - | + if ! helm repo list 2>/dev/null | grep -q karmada-charts; then + helm repo add karmada-charts https://raw.githubusercontent.com/karmada-io/karmada/master/charts + echo "Added karmada-charts Helm repository" + fi + helm repo update karmada-charts + status: + - helm repo list 2>/dev/null | grep -q karmada-charts + + # ════════════════════════════════════════════════════════════════════════ + # Kind cluster management + # ════════════════════════════════════════════════════════════════════════ + + e2e:clusters:create: + desc: "Create all Kind clusters (idempotent)" + cmds: + - mkdir -p {{.E2E_DIR}} + # Render the control-plane kind config with the configured Karmada NodePort. + # The committed template pins 32443; overriding KARMADA_API_NODEPORT (for + # example to dodge a host-port collision with another local cluster) is + # substituted through here so the port has a single source of truth. + - sed 's/32443/{{.KARMADA_API_NODEPORT}}/g' hack/e2e/kind-control-plane.yaml > {{.E2E_DIR}}/kind-control-plane.yaml + # Management / control-plane cell cluster — needs extraPortMappings for + # the Karmada API server NodePort so it is accessible at localhost:. + - task: _e2e:cluster:create + vars: + CLUSTER_NAME: "{{.KIND_CONTROL_PLANE}}" + KIND_CONFIG: "{{.E2E_DIR}}/kind-control-plane.yaml" + # POP cell clusters — same busy-host control-plane tuning, no port mapping. + - task: _e2e:cluster:create + vars: + CLUSTER_NAME: "{{.KIND_POP_DFW}}" + KIND_CONFIG: hack/e2e/kind-pop.yaml + - task: _e2e:cluster:create + vars: + CLUSTER_NAME: "{{.KIND_POP_ORD}}" + KIND_CONFIG: hack/e2e/kind-pop.yaml + - mkdir -p {{.KUBECONFIG_DIR}} + - task: _e2e:kubeconfigs:export + + _e2e:cluster:create: + internal: true + cmds: + - | + if kind get clusters 2>/dev/null | grep -qx '{{.CLUSTER_NAME}}'; then + echo "Kind cluster '{{.CLUSTER_NAME}}' already exists — skipping" + else + echo "Creating Kind cluster '{{.CLUSTER_NAME}}'..." + CONFIG_FLAG="" + if [ -n "{{.KIND_CONFIG}}" ]; then + CONFIG_FLAG="--config {{.KIND_CONFIG}}" + fi + kind create cluster \ + --name {{.CLUSTER_NAME}} \ + $CONFIG_FLAG \ + --wait 90s + fi + + _e2e:kubeconfigs:export: + internal: true + desc: "Export Kind kubeconfigs and create Docker-IP variants for cross-cluster use" + cmds: + # Standard kubeconfigs (localhost-based, for developer kubectl use) + - kind export kubeconfig --name {{.KIND_CONTROL_PLANE}} --kubeconfig {{.KUBECONFIG_DIR}}/control-plane.yaml + - kind export kubeconfig --name {{.KIND_POP_DFW}} --kubeconfig {{.KUBECONFIG_DIR}}/pop-dfw.yaml + - kind export kubeconfig --name {{.KIND_POP_ORD}} --kubeconfig {{.KUBECONFIG_DIR}}/pop-ord.yaml + # Docker-IP kubeconfigs (used by Karmada controller, running inside Docker, + # to reach POP cell API servers across the kind bridge network) + - | + hack/e2e/make-internal-kubeconfig.sh \ + {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ + {{.KUBECONFIG_DIR}}/pop-dfw-internal.yaml \ + {{.KIND_POP_DFW}} + - | + hack/e2e/make-internal-kubeconfig.sh \ + {{.KUBECONFIG_DIR}}/pop-ord.yaml \ + {{.KUBECONFIG_DIR}}/pop-ord-internal.yaml \ + {{.KIND_POP_ORD}} + + # ════════════════════════════════════════════════════════════════════════ + # Karmada installation + # ════════════════════════════════════════════════════════════════════════ + + e2e:karmada:install: + desc: "Install Karmada into the management cluster via Helm (idempotent)" + cmds: + # Idempotency keys on the karmada-apiserver Deployment existing rather than + # just the namespace: a Helm --wait that times out on a loaded machine + # leaves the namespace behind with the components still converging, and we + # want a re-run to resume that install (wait for it to finish) instead of + # skipping straight past a not-yet-ready control plane. The 10m timeout + # gives the Karmada components room to start when the host is busy running + # several Kind clusters at once. + - | + if kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system get deploy karmada-apiserver &>/dev/null; then + echo "Karmada already present — waiting for its components to become Available..." + else + echo "Installing Karmada {{.KARMADA_VERSION}} via Helm..." + helm install karmada karmada-charts/karmada \ + --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + --namespace karmada-system \ + --create-namespace \ + --version {{.KARMADA_VERSION}} \ + --set apiServer.serviceType=NodePort \ + --set apiServer.nodePort={{.KARMADA_API_NODEPORT}} \ + --wait \ + --timeout 10m \ + || echo "Helm --wait did not settle in time; will wait on the deployments directly below" + fi + # Disable leader election on the single-replica Karmada controllers. On a + # busy host, etcd write latency spikes cause the controllers' 5s lease + # renewals to time out; each lost lease exits the process, and the + # restart re-lists every resource, which adds I/O load and provokes the + # next latency spike — a crashloop feedback loop. With one replica there is + # nothing to elect, so turning it off breaks the loop and lets the control + # plane settle. Harmless in a single-replica e2e; not for production HA. + - task: _e2e:karmada:disable-leader-election + # Disable fsync on the Karmada control plane's own etcd, for the same + # throwaway-data reason as the base clusters (see hack/e2e/kind-*.yaml). Its + # etcd is a Helm-managed StatefulSet on hostPath storage — co-located with + # the base cluster's etcd on one node — so without this its fsync contention + # keeps hub writes slow even after the base etcds are tuned. + - task: _e2e:karmada:disable-etcd-fsync + # Scale any Karmada deployment that is sitting at zero back up to one. This + # makes the target robust to a control plane that was intentionally scaled + # down (for example to keep a busy host calm between runs) or left at zero + # by a partial install: without this, the Available wait below would block + # forever on a deployment that has no desired replicas. + - | + for d in $(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system get deploy -o jsonpath='{.items[*].metadata.name}'); do + reps=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system get deploy "$d" -o jsonpath='{.spec.replicas}') + if [ "$reps" = "0" ]; then + echo "Scaling Karmada deploy $d back to 1 (was scaled to 0)..." + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system scale deploy "$d" --replicas=1 + fi + done + # Wait on the control-plane deployments regardless of how the install above + # exited, so a timed-out Helm run still converges before we build the + # kubeconfig and register cells. + - | + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system wait --for=condition=Available deploy --all --timeout=10m + echo "Karmada control plane is Available" + - task: _e2e:karmada:build-kubeconfig + + _e2e:karmada:disable-leader-election: + internal: true + cmds: + - | + for d in karmada-controller-manager karmada-kube-controller-manager karmada-scheduler; do + current=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system get deploy "$d" \ + -o jsonpath='{.spec.template.spec.containers[0].command}' 2>/dev/null) + if echo "$current" | grep -q -- "--leader-elect=false"; then + echo "Leader election already disabled on $d" + else + echo "Disabling leader election on $d..." + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system patch deploy "$d" --type=json \ + -p '[{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--leader-elect=false"}]' + fi + done + + _e2e:karmada:disable-etcd-fsync: + internal: true + cmds: + # Idempotent: only patch + restart if the flag is not already set. The etcd + # data lives on a hostPath, so deleting the pod does not lose state (a + # StatefulSet does not always roll the pod on a bare command change, hence + # the explicit delete). + - | + current=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system get statefulset etcd \ + -o jsonpath='{.spec.template.spec.containers[0].command}' 2>/dev/null) + if echo "$current" | grep -q -- "--unsafe-no-fsync"; then + echo "Karmada etcd fsync already disabled" + else + echo "Disabling fsync on the Karmada etcd..." + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system patch statefulset etcd --type=json \ + -p '[{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--unsafe-no-fsync=true"}]' + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system delete pod etcd-0 --ignore-not-found + fi + # Relax the Karmada API server's health probes. Under sustained load (e.g. + # the chainsaw suites) a transient etcd/DNS blip makes the default 5s/15s + # probes fail, and the kubelet SIGKILLs the API server — which cascades to + # a crashloop. Wider thresholds let it ride out the blip instead of being + # killed. Belt-and-suspenders with the etcd fsync + leader-election tuning. + - | + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + -n karmada-system patch deploy karmada-apiserver --type=strategic \ + -p '{"spec":{"template":{"spec":{"containers":[{"name":"karmada-apiserver","livenessProbe":{"timeoutSeconds":30,"failureThreshold":30,"periodSeconds":30,"initialDelaySeconds":60},"readinessProbe":{"timeoutSeconds":30,"failureThreshold":30,"periodSeconds":20}}]}}}}' + + e2e:karmada:configure: + desc: "Apply federation component config to the Karmada API server (idempotent)" + cmds: + # Retry the apply: this runs right after the Karmada install, and the + # apiserver reached over the NodePort can briefly drop the connection + # ("EOF" / API discovery failure) while it settles, which otherwise fails + # provisioning on an otherwise-healthy control plane. + - | + echo "Applying federation component to Karmada..." + for attempt in 1 2 3 4 5 6; do + if kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply \ + -k config/components/federation/ --server-side --validate=false; then + echo "Federation component applied (attempt ${attempt})"; exit 0 + fi + echo "configure attempt ${attempt} failed (Karmada apiserver may still be settling); retrying in 10s..." + sleep 10 + done + echo "ERROR: failed to apply federation component after 6 attempts"; exit 1 + + _e2e:karmada:build-kubeconfig: + internal: true + desc: "Extract Karmada kubeconfig from secret and patch server to localhost:{{.KARMADA_API_NODEPORT}}" + cmds: + - | + echo "Building Karmada kubeconfig → {{.KUBECONFIG_DIR}}/karmada.yaml" + # Extract raw kubeconfig from the secret the Helm chart creates + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ + get secret karmada-kubeconfig \ + -n karmada-system \ + -o jsonpath='{.data.kubeconfig}' \ + | base64 -d > {{.KUBECONFIG_DIR}}/karmada-raw.yaml + # Rewrite the server address to the NodePort exposed on localhost + python3 - {{.KUBECONFIG_DIR}}/karmada-raw.yaml {{.KUBECONFIG_DIR}}/karmada.yaml 127.0.0.1 {{.KARMADA_API_NODEPORT}} << 'PYEOF' + import sys, yaml + + src, dst, host, port = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] + + with open(src) as f: + cfg = yaml.safe_load(f) + + for cluster in cfg.get('clusters', []): + old = cluster['cluster'].get('server', '') + cluster['cluster']['server'] = f'https://{host}:{port}' + # The cert is for the internal cluster IP, so skip TLS verification. + # This is a local dev-only environment. + cluster['cluster']['insecure-skip-tls-verify'] = True + cluster['cluster'].pop('certificate-authority-data', None) + print(f" karmada server: {old} → https://{host}:{port}", file=sys.stderr) + + with open(dst, 'w') as f: + yaml.dump(cfg, f, default_flow_style=False) + PYEOF + rm {{.KUBECONFIG_DIR}}/karmada-raw.yaml + # Copy karmada.yaml → downstream.yaml so Chainsaw tests that declare + # cluster: downstream can load it (chainsaw-config.yaml references this path). + - cp {{.KUBECONFIG_DIR}}/karmada.yaml {{.KUBECONFIG_DIR}}/downstream.yaml + + # ════════════════════════════════════════════════════════════════════════ + # POP cell cluster registration + # ════════════════════════════════════════════════════════════════════════ + + e2e:karmada:join-clusters: + desc: "Register POP cell clusters with Karmada and apply city-code labels" + cmds: + # cluster.karmada.io is served by the karmada-aggregated-apiserver through + # API aggregation, and can briefly return ServiceUnavailable after the pod + # reports Ready while the aggregation handshake settles. Wait for the API + # to actually answer before registering members so join does not race it. + - | + echo "Waiting for the cluster.karmada.io aggregated API to be served..." + deadline=$((SECONDS+180)) + until kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml get clusters &>/dev/null; do + if [ $SECONDS -ge $deadline ]; then + echo "ERROR: cluster.karmada.io was not served within 180s"; exit 1 + fi + sleep 3 + done + echo "cluster.karmada.io is served" + - task: _e2e:karmada:join-cluster + vars: + CLUSTER_NAME: "{{.KIND_POP_DFW}}" + CITY_CODE: dfw + EXTERNAL_KUBECONFIG: "{{.KUBECONFIG_DIR}}/pop-dfw.yaml" + INTERNAL_KUBECONFIG: "{{.KUBECONFIG_DIR}}/pop-dfw-internal.yaml" + - task: _e2e:karmada:join-cluster + vars: + CLUSTER_NAME: "{{.KIND_POP_ORD}}" + CITY_CODE: ord + EXTERNAL_KUBECONFIG: "{{.KUBECONFIG_DIR}}/pop-ord.yaml" + INTERNAL_KUBECONFIG: "{{.KUBECONFIG_DIR}}/pop-ord-internal.yaml" + + _e2e:karmada:join-cluster: + internal: true + cmds: + # ── Register with karmadactl join ────────────────────────────────── + # We pass the EXTERNAL kubeconfig (localhost-based) here so karmadactl + # can reach the member cluster from this macOS host to set up initial + # RBAC. The stored secret is patched below to the Docker-IP variant. + # karmadactl join creates an impersonation ServiceAccount in the member + # cluster and blocks (with a non-tunable internal timeout) on that SA's + # legacy token Secret being populated. On a busy host the member cluster's + # token controller is slow, so the first join can time out even though the + # Secret populates moments later. Retry: a subsequent attempt reuses the + # now-populated Secret and completes immediately. + - | + if kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \ + get cluster {{.CLUSTER_NAME}} &>/dev/null; then + echo "Cluster '{{.CLUSTER_NAME}}' already registered in Karmada — skipping join" + else + echo "Joining '{{.CLUSTER_NAME}}' to Karmada..." + joined=false + for attempt in 1 2 3 4 5; do + if {{.KARMADACTL}} join {{.CLUSTER_NAME}} \ + --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \ + --cluster-kubeconfig={{.EXTERNAL_KUBECONFIG}} \ + --cluster-context=kind-{{.CLUSTER_NAME}}; then + joined=true + echo "Cluster '{{.CLUSTER_NAME}}' registered (attempt ${attempt})" + break + fi + echo "join attempt ${attempt} failed (member token Secret may still be populating); retrying in 10s..." + sleep 10 + done + if [ "${joined}" != "true" ]; then + echo "ERROR: failed to join '{{.CLUSTER_NAME}}' after 5 attempts"; exit 1 + fi + fi + # ── Patch cluster secret → Docker-IP kubeconfig ─────────────────── + # The Karmada controller manager runs inside Docker; it cannot use + # localhost to reach POP cell API servers. We update the stored secret + # with a kubeconfig whose server address uses the Kind container IP so + # container-to-container communication works across the kind bridge. + - | + hack/e2e/patch-cluster-secret.sh \ + {{.KUBECONFIG_DIR}}/karmada.yaml \ + {{.CLUSTER_NAME}} \ + {{.INTERNAL_KUBECONFIG}} + # ── Apply city-code label ────────────────────────────────────────── + - | + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \ + label cluster {{.CLUSTER_NAME}} \ + topology.datum.net/city-code={{.CITY_CODE}} \ + --overwrite + echo "Labeled cluster '{{.CLUSTER_NAME}}' with topology.datum.net/city-code={{.CITY_CODE}}" + + # ════════════════════════════════════════════════════════════════════════ + # CRD installation + # ════════════════════════════════════════════════════════════════════════ + + e2e:crds:install: + desc: "Install compute + NSO + quota CRDs to all clusters" + cmds: + - task: _e2e:crds:compute + - task: _e2e:crds:nso + - task: _e2e:crds:quota + + _e2e:crds:compute: + internal: true + desc: "Apply compute CRDs to all clusters and the Karmada API server" + cmds: + # All three Kind clusters + the Karmada API server get the compute CRDs. + # The Karmada API server needs them so it can store and propagate + # WorkloadDeployment objects. + - | + for KC in \ + {{.KUBECONFIG_DIR}}/control-plane.yaml \ + {{.KUBECONFIG_DIR}}/karmada.yaml \ + {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ + {{.KUBECONFIG_DIR}}/pop-ord.yaml; do + echo "Installing compute CRDs → $(basename $KC .yaml)..." + kubectl --kubeconfig="$KC" apply -k config/base/crd --server-side --validate=false + done + + _e2e:crds:nso: + internal: true + desc: "Apply NSO CRDs to control-plane and POP cell clusters" + cmds: + # NSO CRDs (NetworkBinding, SubnetClaim, etc.) are installed on the + # control-plane as well as POP cells. The control-plane operator needs them + # so that Subnet/SubnetClaim informer watches can start without cache errors, + # even though NSO controllers themselves only run on POP cells. + - | + go mod download go.datum.net/network-services-operator + NSO_VERSION=$(go list -m -json go.datum.net/network-services-operator \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['Version'])") + NSO_CRD_PATH="$(go env GOMODCACHE)/go.datum.net/network-services-operator@${NSO_VERSION}/config/crd" + echo "NSO CRDs from: ${NSO_CRD_PATH}" + for KC in \ + {{.KUBECONFIG_DIR}}/control-plane.yaml \ + {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ + {{.KUBECONFIG_DIR}}/pop-ord.yaml; do + echo "Installing NSO CRDs → $(basename $KC .yaml)..." + kubectl --kubeconfig="$KC" apply -k "${NSO_CRD_PATH}" --server-side --validate=false + done + + _e2e:crds:quota: + internal: true + desc: "Apply Milo quota CRDs to all clusters and the Karmada API server" + cmds: + # Quota CRDs (ResourceClaim, ResourceGrant, etc.) are required on all + # clusters so the InstanceReconciler can create and watch ResourceClaims + # against project Milo control planes without cache startup errors. + - | + go mod download go.miloapis.com/milo + MILO_VERSION=$(go list -m -json go.miloapis.com/milo \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['Version'])") + QUOTA_CRD_PATH="$(go env GOMODCACHE)/go.miloapis.com/milo@${MILO_VERSION}/config/crd/bases/quota" + echo "Milo quota CRDs from: ${QUOTA_CRD_PATH}" + for KC in \ + {{.KUBECONFIG_DIR}}/control-plane.yaml \ + {{.KUBECONFIG_DIR}}/karmada.yaml \ + {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ + {{.KUBECONFIG_DIR}}/pop-ord.yaml; do + echo "Installing Milo quota CRDs → $(basename $KC .yaml)..." + kubectl --kubeconfig="$KC" apply -k "${QUOTA_CRD_PATH}" --server-side --validate=false + done + + # ════════════════════════════════════════════════════════════════════════ + # Operator image build + side-load + # ════════════════════════════════════════════════════════════════════════ + + e2e:image:build: + desc: "Build the compute-manager image with the local dev tag ({{.IMAGE}})" + cmds: + - | + echo "Building {{.IMAGE}} from {{.ROOT_DIR}}/Dockerfile..." + docker build -t {{.IMAGE}} {{.ROOT_DIR}} + + e2e:image:load: + desc: "Side-load {{.IMAGE}} into every Kind cluster (no registry pull)" + cmds: + - | + for CLUSTER in {{.KIND_ALL_CLUSTERS}}; do + echo "Loading {{.IMAGE}} → kind cluster ${CLUSTER}..." + kind load docker-image {{.IMAGE}} --name "${CLUSTER}" + done + + # ════════════════════════════════════════════════════════════════════════ + # Federation access for the management operator + # ════════════════════════════════════════════════════════════════════════ + + e2e:federation:setup: + desc: "Bind hub RBAC and mint the management operator's Karmada kubeconfig" + cmds: + # ── Hub-side RBAC (the real production manifest) ──────────────────── + # config/base/downstream-rbac grants the compute-manager ClusterRole on + # the Karmada hub and binds it to the user + # system:serviceaccount:compute-system:compute-manager. Applying the real + # manifest is the whole point of #149 — the management operator authenticates + # to Karmada as a non-admin identity and every missing grant surfaces as a + # forbidden error rather than being masked by cluster-admin. + - | + echo "Applying hub RBAC (config/base/downstream-rbac) to Karmada..." + for attempt in 1 2 3 4 5; do + if kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply -k config/base/downstream-rbac --server-side --validate=false; then + break + fi + echo "hub RBAC apply attempt ${attempt} failed (Karmada apiserver over NodePort may be settling); retrying in 8s..." + sleep 8 + if [ "${attempt}" = "5" ]; then echo "ERROR: hub RBAC apply failed after 5 attempts"; exit 1; fi + done + # ── Karmada-native identity for the management operator ───────────── + # Production federates the management cluster's projected ServiceAccount + # token into Karmada (Karmada trusts the host cluster's token issuer). We + # do not configure cross-cluster token trust in the Kind environment, so + # instead we create a Karmada-native ServiceAccount whose authenticated + # username is identical — system:serviceaccount:compute-system:compute-manager + # — and therefore matches the very same ClusterRoleBinding subject. Same + # RBAC surface, no issuer federation required. + - | + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml create namespace {{.COMPUTE_NAMESPACE}} \ + --dry-run=client -o yaml | kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply -f - + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml -n {{.COMPUTE_NAMESPACE}} \ + create serviceaccount compute-manager \ + --dry-run=client -o yaml | kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply -f - + # ── Mint the federation kubeconfig ────────────────────────────────── + # A bound token from the Karmada SA, embedded in a kubeconfig that targets + # the Karmada in-cluster Service (the management pod runs alongside Karmada). + # We re-mint on every run so a re-deploy always ships a fresh, unexpired + # token. insecure-skip-tls-verify mirrors the karmada.yaml the harness + # already builds — the served cert does not cover the Service DNS name. + - | + mkdir -p {{.E2E_DIR}} + echo "Minting Karmada token for compute-manager..." + TOKEN=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml -n {{.COMPUTE_NAMESPACE}} \ + create token compute-manager --duration=720h) + cat > {{.E2E_DIR}}/downstream-kubeconfig.yaml < {{.E2E_DIR}}/cell-federation-kubeconfig.yaml </dev/null + kubectl --kubeconfig="$KC" -n {{.COMPUTE_NAMESPACE}} \ + create configmap compute-cell-federation-kubeconfig \ + --from-file=kubeconfig={{.E2E_DIR}}/cell-federation-kubeconfig.yaml \ + --dry-run=client -o yaml | kubectl --kubeconfig="$KC" apply -f - + done + echo "Published compute-cell-federation-kubeconfig on both POP cells" + + # ════════════════════════════════════════════════════════════════════════ + # Operator deployment (real kustomize overlays + e2e patches) + # ════════════════════════════════════════════════════════════════════════ + + e2e:deploy: + desc: "Build+load the image, wire federation, and deploy the operators to all clusters" + cmds: + - task: e2e:image:build + - task: e2e:image:load + - task: e2e:federation:setup + - task: e2e:deploy:management + - task: e2e:deploy:cells + + e2e:deploy:management: + desc: "Deploy the management-plane overlay to the control-plane cluster" + cmds: + - | + echo "Deploying management-plane operator → {{.KIND_CONTROL_PLANE}}..." + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml apply -k test/e2e/deploy/management --server-side --validate=false + - task: _e2e:deploy:wait + vars: + KUBECONFIG_FILE: "{{.KUBECONFIG_DIR}}/control-plane.yaml" + LABEL: control-plane + + e2e:deploy:cells: + desc: "Deploy the cell overlay to both POP cell clusters" + cmds: + - task: _e2e:deploy:cell + vars: + KUBECONFIG_FILE: "{{.KUBECONFIG_DIR}}/pop-dfw.yaml" + CLUSTER_NAME: "{{.KIND_POP_DFW}}" + - task: _e2e:deploy:cell + vars: + KUBECONFIG_FILE: "{{.KUBECONFIG_DIR}}/pop-ord.yaml" + CLUSTER_NAME: "{{.KIND_POP_ORD}}" + + _e2e:deploy:cell: + internal: true + cmds: + - | + echo "Deploying cell operator → {{.CLUSTER_NAME}}..." + kubectl --kubeconfig={{.KUBECONFIG_FILE}} apply -k test/e2e/deploy/cell --server-side --validate=false + - task: _e2e:deploy:wait + vars: + KUBECONFIG_FILE: "{{.KUBECONFIG_FILE}}" + LABEL: "{{.CLUSTER_NAME}}" + + _e2e:deploy:wait: + internal: true + cmds: + - | + echo "Waiting for compute-manager rollout on {{.LABEL}}..." + kubectl --kubeconfig={{.KUBECONFIG_FILE}} -n {{.COMPUTE_NAMESPACE}} \ + rollout status deployment/compute-manager --timeout=180s diff --git a/config/components/controller_rbac/role.yaml b/config/components/controller_rbac/role.yaml index 425eecf6..81147741 100644 --- a/config/components/controller_rbac/role.yaml +++ b/config/components/controller_rbac/role.yaml @@ -31,6 +31,7 @@ rules: verbs: - get - list + - watch - apiGroups: - compute.datumapis.com resources: @@ -68,6 +69,7 @@ rules: resources: - locations - networkcontexts + - networks - subnets verbs: - get diff --git a/hack/e2e/kind-control-plane.yaml b/hack/e2e/kind-control-plane.yaml new file mode 100644 index 00000000..12356ed1 --- /dev/null +++ b/hack/e2e/kind-control-plane.yaml @@ -0,0 +1,41 @@ +# Kind cluster configuration for the compute-control-plane management cluster. +# +# extraPortMappings exposes port 32443 on the macOS host so that the Karmada +# API server NodePort service (nodePort: 32443) is accessible at +# https://localhost:32443 without any additional port-forwarding. +# +# This matches KARMADA_API_NODEPORT in Taskfile.yaml. + +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +# Tune the control plane for a busy host that also runs the full Karmada control +# plane in this same cluster. kind renders its kubeadm config as v1beta3, so +# extraArgs are maps here (not the v1beta4 list form). +# - etcd unsafe-no-fsync: these are throwaway e2e clusters, and fsync +# contention between co-located etcds is what drives API server write +# latency into multi-second spikes. +# - scheduler/controllerManager leader-elect=false: single-instance static +# pods have nothing to elect, and a lease renewal that times out during a +# latency spike crash-loops them (which then re-lists everything and makes +# the spike worse). +kubeadmConfigPatches: + - | + apiVersion: kubeadm.k8s.io/v1beta3 + kind: ClusterConfiguration + etcd: + local: + extraArgs: + unsafe-no-fsync: "true" + controllerManager: + extraArgs: + leader-elect: "false" + scheduler: + extraArgs: + leader-elect: "false" +nodes: + - role: control-plane + extraPortMappings: + - containerPort: 32443 # Karmada API server NodePort + hostPort: 32443 + protocol: TCP + listenAddress: "127.0.0.1" diff --git a/hack/e2e/kind-pop.yaml b/hack/e2e/kind-pop.yaml new file mode 100644 index 00000000..d848f43b --- /dev/null +++ b/hack/e2e/kind-pop.yaml @@ -0,0 +1,25 @@ +# Kind cluster configuration for the compute-pop-* cell clusters. +# +# Same control-plane tuning as the management cluster (see kind-control-plane.yaml +# for the rationale): etcd runs without fsync and the single-instance scheduler +# and controller-manager run without leader election, so the cell control planes +# stay stable on a busy host instead of crash-looping on lease-renewal timeouts. +# kind renders its kubeadm config as v1beta3, so extraArgs are maps here. +# +# Unlike the management cluster, POP cells need no extraPortMappings. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +kubeadmConfigPatches: + - | + apiVersion: kubeadm.k8s.io/v1beta3 + kind: ClusterConfiguration + etcd: + local: + extraArgs: + unsafe-no-fsync: "true" + controllerManager: + extraArgs: + leader-elect: "false" + scheduler: + extraArgs: + leader-elect: "false" diff --git a/hack/e2e/make-internal-kubeconfig.sh b/hack/e2e/make-internal-kubeconfig.sh new file mode 100755 index 00000000..3303a5bd --- /dev/null +++ b/hack/e2e/make-internal-kubeconfig.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# make-internal-kubeconfig.sh +# +# Produces a kubeconfig variant that uses the Kind node's Docker container IP +# instead of localhost. This variant is stored in Karmada so the controller +# manager (running inside Docker) can reach member cluster API servers across +# the kind bridge network. +# +# Background: Kind maps each cluster's API server to a random localhost port +# on the developer machine. Inside Docker containers, "localhost" refers to the +# container's own loopback — not the host. We therefore swap the server address +# to the Kind control-plane container's Docker bridge IP (e.g. 172.18.0.x) and +# set insecure-skip-tls-verify because the node certificate does not include +# the Docker bridge IP in its SANs. +# +# Usage: +# hack/e2e/make-internal-kubeconfig.sh \ +# tmp/e2e/kubeconfigs/pop-dfw.yaml \ +# tmp/e2e/kubeconfigs/pop-dfw-internal.yaml \ +# compute-pop-dfw + +set -euo pipefail + +INPUT="${1:?usage: $0 }" +OUTPUT="${2:?usage: $0 }" +CLUSTER_NAME="${3:?usage: $0 }" + +CONTAINER_NAME="${CLUSTER_NAME}-control-plane" + +# Resolve the container's Docker bridge IP. +DOCKER_IP=$(docker inspect \ + -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ + "${CONTAINER_NAME}" 2>/dev/null || true) + +if [ -z "${DOCKER_IP}" ]; then + echo "ERROR: Could not resolve Docker IP for container '${CONTAINER_NAME}'." >&2 + echo " Is the Kind cluster '${CLUSTER_NAME}' running?" >&2 + exit 1 +fi + +echo " ${CLUSTER_NAME}: Docker IP ${DOCKER_IP} → ${OUTPUT}" + +python3 - "${INPUT}" "${OUTPUT}" "${DOCKER_IP}" <<'PYEOF' +import sys, yaml + +src, dst, docker_ip = sys.argv[1], sys.argv[2], sys.argv[3] + +with open(src) as f: + cfg = yaml.safe_load(f) + +for cluster in cfg.get('clusters', []): + # Kind API server always listens on port 6443 inside the container. + cluster['cluster']['server'] = f'https://{docker_ip}:6443' + # The node cert only covers localhost / 127.0.0.1, not the bridge IP. + cluster['cluster']['insecure-skip-tls-verify'] = True + cluster['cluster'].pop('certificate-authority-data', None) + +with open(dst, 'w') as f: + yaml.dump(cfg, f, default_flow_style=False) +PYEOF diff --git a/hack/e2e/patch-cluster-secret.sh b/hack/e2e/patch-cluster-secret.sh new file mode 100755 index 00000000..e29ed383 --- /dev/null +++ b/hack/e2e/patch-cluster-secret.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# patch-cluster-secret.sh +# +# After "karmadactl join", Karmada stores the member cluster's kubeconfig in a +# Secret referenced by the Cluster object's spec.secretRef, and sets +# spec.apiEndpoint to the localhost address it resolved from the external +# kubeconfig. The Karmada controller manager runs inside Docker and cannot use +# localhost to reach POP cell API servers. +# +# This script: +# 1. Replaces the kubeconfig in the Secret with the Docker-IP variant so that +# the Karmada controller can make API calls to the member cluster. +# 2. Patches spec.apiEndpoint on the Cluster object so that health checks also +# use the Docker bridge IP instead of localhost. +# +# Usage: +# hack/e2e/patch-cluster-secret.sh \ +# tmp/e2e/kubeconfigs/karmada.yaml \ +# compute-pop-dfw \ +# tmp/e2e/kubeconfigs/pop-dfw-internal.yaml + +set -euo pipefail + +KARMADA_KUBECONFIG="${1:?usage: $0 }" +CLUSTER_NAME="${2:?usage: $0 }" +INTERNAL_KUBECONFIG="${3:?usage: $0 }" + +# ------------------------------------------------------------------ +# Read the Cluster object's secretRef (name + namespace) +# ------------------------------------------------------------------ +SECRET_NAME=$(kubectl \ + --kubeconfig="${KARMADA_KUBECONFIG}" \ + get cluster "${CLUSTER_NAME}" \ + -o jsonpath='{.spec.secretRef.name}' 2>/dev/null || true) + +if [ -z "${SECRET_NAME}" ]; then + echo "ERROR: Could not find spec.secretRef.name on cluster '${CLUSTER_NAME}'." >&2 + echo " Has karmadactl join completed successfully?" >&2 + exit 1 +fi + +SECRET_NAMESPACE=$(kubectl \ + --kubeconfig="${KARMADA_KUBECONFIG}" \ + get cluster "${CLUSTER_NAME}" \ + -o jsonpath='{.spec.secretRef.namespace}' 2>/dev/null || true) + +SECRET_NAMESPACE="${SECRET_NAMESPACE:-karmada-system}" + +echo " Patching secret ${SECRET_NAMESPACE}/${SECRET_NAME} with Docker-IP kubeconfig..." + +# ------------------------------------------------------------------ +# Replace the kubeconfig data in the secret +# ------------------------------------------------------------------ +kubectl \ + --kubeconfig="${KARMADA_KUBECONFIG}" \ + create secret generic "${SECRET_NAME}" \ + --namespace="${SECRET_NAMESPACE}" \ + --from-file=kubeconfig="${INTERNAL_KUBECONFIG}" \ + --dry-run=client -o yaml \ + | kubectl \ + --kubeconfig="${KARMADA_KUBECONFIG}" \ + apply -f - + +echo " Secret ${SECRET_NAMESPACE}/${SECRET_NAME} updated — Karmada controller will use Docker bridge IP" + +# ------------------------------------------------------------------ +# Extract the Docker-IP server URL from the internal kubeconfig and +# patch spec.apiEndpoint on the Cluster object so that Karmada's +# cluster-status controller uses the same reachable address for health +# checks. Without this patch the controller continues to probe the +# localhost address stored by karmadactl join and the cluster never +# transitions to Ready. +# ------------------------------------------------------------------ +DOCKER_SERVER=$(kubectl \ + --kubeconfig="${INTERNAL_KUBECONFIG}" \ + config view --minify -o jsonpath='{.clusters[0].cluster.server}') + +if [ -z "${DOCKER_SERVER}" ]; then + echo "ERROR: Could not read server URL from ${INTERNAL_KUBECONFIG}" >&2 + exit 1 +fi + +echo " Patching spec.apiEndpoint on cluster '${CLUSTER_NAME}' → ${DOCKER_SERVER}..." +kubectl \ + --kubeconfig="${KARMADA_KUBECONFIG}" \ + patch cluster "${CLUSTER_NAME}" \ + --type=merge \ + -p "{\"spec\":{\"apiEndpoint\":\"${DOCKER_SERVER}\"}}" + +echo " Cluster '${CLUSTER_NAME}' patched — health checks will now use Docker bridge IP" diff --git a/test/e2e/chainsaw-config.yaml b/test/e2e/chainsaw-config.yaml new file mode 100644 index 00000000..9524c8e4 --- /dev/null +++ b/test/e2e/chainsaw-config.yaml @@ -0,0 +1,52 @@ +# Chainsaw global configuration for the compute federation e2e test suite. +# +# Prerequisites +# ───────────── +# Run `task e2e:up` to create the Kind clusters and populate kubeconfigs under +# tmp/e2e/kubeconfigs/ before running Chainsaw. +# +# Running +# ─────── +# From the repository root via Taskfile (recommended): +# +# task e2e:test +# +# Or directly: +# +# KUBECONFIG=tmp/e2e/kubeconfigs/control-plane.yaml \ +# chainsaw test --config test/e2e/chainsaw-config.yaml test/e2e/ +# +# The KUBECONFIG env var sets the "default" cluster (control-plane cell). +# Additional clusters (downstream, pop-dfw, pop-ord) are declared below and +# referenced by name in individual test steps via `cluster: downstream` etc. +# +# Kubeconfig paths below are relative to the working directory where Chainsaw is +# invoked (the project root), NOT relative to this config file's location. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Configuration +metadata: + name: chainsaw +spec: + # Timeouts are widened from chainsaw's defaults for the constrained local/CI + # node that co-hosts the Karmada control plane: Karmada propagation of a + # WorkloadDeployment to a cell routinely needs more than the default 60s under + # load, and finalizer-driven cascade deletes (the federator releasing hub + # companions) exceed the default 30s. + timeouts: + apply: 60s + assert: 120s + cleanup: 120s + delete: 120s + error: 60s + exec: 30s + clusters: + # Downstream control plane. WorkloadDeployments, PropagationPolicies, + # and Instance write-backs live here. + downstream: + kubeconfig: tmp/e2e/kubeconfigs/downstream.yaml + # POP DFW cell — downstream member cluster labelled topology.datum.net/city-code=dfw. + pop-dfw: + kubeconfig: tmp/e2e/kubeconfigs/pop-dfw.yaml + # POP ORD cell — downstream member cluster labelled topology.datum.net/city-code=ord. + pop-ord: + kubeconfig: tmp/e2e/kubeconfigs/pop-ord.yaml diff --git a/test/e2e/deletion-cascade/chainsaw-test.yaml b/test/e2e/deletion-cascade/chainsaw-test.yaml new file mode 100644 index 00000000..03a11ea0 --- /dev/null +++ b/test/e2e/deletion-cascade/chainsaw-test.yaml @@ -0,0 +1,79 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: deletion-cascade +spec: + description: | + Verifies that deleting a WorkloadDeployment from the project namespace causes + the federator to remove the corresponding WorkloadDeployment from Karmada. + + The WorkloadDeploymentFederator adds a finalizer + (compute.datumapis.com/federator) to every project WD it manages. When the + project WD is deleted: + 1. The finalizer's Finalize method runs (blocking deletion until complete). + 2. It deletes the Karmada-side WorkloadDeployment. + 3. It removes the PropagationPolicy if no other WDs for the city remain. + 4. It removes the finalizer, allowing the project WD to be garbage-collected. + + This test validates: project WD deletion → Karmada WD deletion. + + template: true + + steps: + - name: create-wd + description: Create a WorkloadDeployment on the control-plane cluster. + try: + - apply: + file: workload-deployment.yaml + + - name: wait-for-federation + description: Wait for the WorkloadDeployment to appear in Karmada. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-cascade-wd + + - name: delete-wd + description: Delete the WorkloadDeployment from the control-plane cluster. + try: + - delete: + ref: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($namespace) + name: test-cascade-wd + + - name: assert-downstream-wd-deleted + description: Confirm the Karmada copy is removed by the finalizer. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - wait: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($downstreamNS) + name: test-cascade-wd + timeout: 30s + for: + deletion: {} diff --git a/test/e2e/deletion-cascade/workload-deployment.yaml b/test/e2e/deletion-cascade/workload-deployment.yaml new file mode 100644 index 00000000..39d68a1d --- /dev/null +++ b/test/e2e/deletion-cascade/workload-deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-cascade-wd +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/deploy/cell/config_patch.yaml b/test/e2e/deploy/cell/config_patch.yaml new file mode 100644 index 00000000..7e94f3a8 --- /dev/null +++ b/test/e2e/deploy/cell/config_patch.yaml @@ -0,0 +1,36 @@ +# DEVIATION (required for Kind): overrides the operator config the production +# cell overlay ships (config/overlays/cell/disable_webhook_patch.yaml) to drop +# discovery.quotaKubeconfigPath. +# +# The production cell config points quotaKubeconfigPath at +# /etc/quota-credentials/kubeconfig, delivered by config/components/quota-credentials +# from the compute-edge-milo-client-cert Secret + compute-quota-kubeconfig +# ConfigMap. Those are cluster-environment secrets that do not exist in Kind, and +# the projected volume sources are optional — so the file is simply absent. The +# operator treats a configured-but-missing quota kubeconfig as fatal (os.Exit), +# which would crash-loop the cell pod. Dropping the path takes the documented +# opt-out branch instead: quota enforcement is disabled and the operator boots. +# +# Impact: the cell InstanceReconciler skips ResourceClaim creation/quota checks. +# The federation delivery path under test (WorkloadDeployment → Instance) is +# unaffected; edge quota enforcement is covered separately. +# +# featureFlags.enableReferencedDataGate is turned on because the cell +# WorkloadDeploymentReconciler is the sole consumer of this flag: it stamps the +# "ReferencedData" scheduling gate onto Instances whose template references a +# ConfigMap or Secret, and the cell InstanceReconciler clears the gate once the +# companions land on the cell. The referenced-data-mounts and +# referenced-data-delete-cascade suites assert that stamp-then-clear behaviour, +# so without the flag those suites would observe an Instance that is never gated. +apiVersion: v1 +kind: ConfigMap +metadata: + name: compute-config +data: + config.yaml: | + apiVersion: apiserver.config.datumapis.com/v1alpha1 + kind: WorkloadOperator + metricsServer: + bindAddress: "0" + featureFlags: + enableReferencedDataGate: true diff --git a/test/e2e/deploy/cell/deployment_patch.yaml b/test/e2e/deploy/cell/deployment_patch.yaml new file mode 100644 index 00000000..9fd08dd5 --- /dev/null +++ b/test/e2e/deploy/cell/deployment_patch.yaml @@ -0,0 +1,13 @@ +# DEVIATION (required for Kind): imagePullPolicy: IfNotPresent — the dev image is +# side-loaded into the Kind node (task e2e:image:load); never attempt a registry +# pull. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: compute-manager +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: IfNotPresent diff --git a/test/e2e/deploy/cell/federation_patch.yaml b/test/e2e/deploy/cell/federation_patch.yaml new file mode 100644 index 00000000..c3512185 --- /dev/null +++ b/test/e2e/deploy/cell/federation_patch.yaml @@ -0,0 +1,30 @@ +# Give the cell operator a Karmada hub credential, mirroring how real cell +# deployments are wired: infra's apps/compute-system/edge/manager-patch.yaml sets +# FEDERATION_KUBECONFIG and mounts a client-cert kubeconfig pointed at the hub. +# With it, the cell InstanceReconciler writes each Instance back to the Karmada +# hub, where the management InstanceProjector consumes it — the sole path by +# which cell-created Instances become visible upstream (Karmada never propagates +# Instances, only WorkloadDeployments/ConfigMaps/Secrets, so status aggregation +# cannot surface them). The mount path matches infra's; the kubeconfig content +# (a Karmada-native SA token reaching the kind hub over the control-plane node's +# docker-bridge IP + NodePort) is minted by task e2e:federation:setup. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: compute-manager +spec: + template: + spec: + containers: + - name: manager + env: + - name: FEDERATION_KUBECONFIG + value: /etc/kubernetes/upstream/auth/kubeconfig + volumeMounts: + - name: upstream-kubeconfig + mountPath: /etc/kubernetes/upstream/auth + readOnly: true + volumes: + - name: upstream-kubeconfig + configMap: + name: compute-cell-federation-kubeconfig diff --git a/test/e2e/deploy/cell/kustomization.yaml b/test/e2e/deploy/cell/kustomization.yaml new file mode 100644 index 00000000..b63ce808 --- /dev/null +++ b/test/e2e/deploy/cell/kustomization.yaml @@ -0,0 +1,41 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# ───────────────────────────────────────────────────────────────────────────── +# e2e cell deploy layer. +# +# References the REAL production cell overlay verbatim and applies only the +# deviations required to run inside a local Kind environment (image source, +# quota-credential neutralisation) plus the cell hub credential that infra +# supplies in production. +# +# The production cell overlay ships FEDERATION_KUBECONFIG empty on purpose — it +# is the *infra* layer that patches it in per-cell (apps/compute-system/edge/ +# manager-patch.yaml mounts a hub client-cert kubeconfig), NOT the overlay. So +# leaving it empty is NOT production-exact: it disables the cell's Instance +# write-back to the Karmada hub, and cell-created Instances then never reach the +# hub or the management InstanceProjector. federation_patch.yaml restores that +# credential the way infra does, so the full cell→hub→projector path is exercised. +# ───────────────────────────────────────────────────────────────────────────── + +resources: + - ../../../../config/overlays/cell + +# Use the locally built image side-loaded into the Kind node (task e2e:image:load) +# instead of pulling ghcr.io/datum-cloud/compute:latest from a registry. +images: + - name: ghcr.io/datum-cloud/compute + newName: compute + newTag: e2e-dev + +patches: + # DEVIATION 1 (required): drop discovery.quotaKubeconfigPath. + # See config_patch.yaml for the full rationale. + - path: config_patch.yaml + + # DEVIATION 2 (required): force IfNotPresent pulls of the side-loaded image. + - path: deployment_patch.yaml + + # Cell hub credential, mirroring infra's per-cell federation patch so the + # Instance write-back path is exercised. See federation_patch.yaml. + - path: federation_patch.yaml diff --git a/test/e2e/deploy/management/config_patch.yaml b/test/e2e/deploy/management/config_patch.yaml new file mode 100644 index 00000000..d3a0abf7 --- /dev/null +++ b/test/e2e/deploy/management/config_patch.yaml @@ -0,0 +1,37 @@ +# DEVIATION (required for Kind): overrides the operator config the production +# management-plane overlay ships (config/overlays/management-plane/discovery_mode_patch.yaml). +# +# - discovery.mode: milo → single. Milo discovery enumerates Projects from a Milo +# control plane to build per-project clients; the Kind environment has no Milo. +# Single-cluster discovery runs the management controllers against the local +# control-plane cluster while still federating WorkloadDeployments to Karmada +# through FEDERATION_KUBECONFIG (unchanged from the overlay). This matches how +# the retired host-run harness drove the management operator. +# +# - webhookServer is intentionally omitted, which disables the admission webhook +# server. The production overlay serves it with a cert delivered by the +# cert-manager CSI driver, which is not installed in Kind and whose issuer is +# supplied by infra rather than the overlay. The e2e suites never create +# Workload objects, so the Workload webhook is never exercised. +# +# - featureFlags.enableReferencedDataGate mirrors the flag set on the cell so the +# two operators share one feature-flag surface, matching the retired host-run +# harness which passed the same --server-config flag to both. Its only consumer +# is the cell WorkloadDeploymentReconciler, which this management overlay does +# not run (it enables management controllers only), so the flag is inert here; +# the cell config_patch carries the copy that actually gates Instances. Kept +# explicit so the flag does not silently diverge between the two operators. +apiVersion: v1 +kind: ConfigMap +metadata: + name: compute-config +data: + config.yaml: | + apiVersion: apiserver.config.datumapis.com/v1alpha1 + kind: WorkloadOperator + metricsServer: + bindAddress: "0" + discovery: + mode: single + featureFlags: + enableReferencedDataGate: true diff --git a/test/e2e/deploy/management/deployment_patch.yaml b/test/e2e/deploy/management/deployment_patch.yaml new file mode 100644 index 00000000..d92134c4 --- /dev/null +++ b/test/e2e/deploy/management/deployment_patch.yaml @@ -0,0 +1,27 @@ +# DEVIATION (required for Kind): +# 1. imagePullPolicy: IfNotPresent — the dev image is side-loaded into the Kind +# node (task e2e:image:load); never attempt a registry pull. +# 2. Back the webhook-server-tls volume with an emptyDir instead of the +# cert-manager CSI driver (csi.cert-manager.io) injected by +# config/components/csi-webhook-cert. That driver is not installed in Kind, +# so the CSI volume would fail to mount and the pod would never start. The +# webhook server itself is disabled in config_patch.yaml, so the serving +# cert is never read — an empty directory at the mount path is enough to let +# the pod boot. We override only the volume source (csi -> emptyDir) and +# leave the container's volumeMounts untouched: a strategic-merge +# "$patch: delete" on a single volumeMount drops the whole list, which would +# also unmount the federation kubeconfig and config volumes. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: compute-manager +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: IfNotPresent + volumes: + - name: webhook-server-tls + csi: null + emptyDir: {} diff --git a/test/e2e/deploy/management/kustomization.yaml b/test/e2e/deploy/management/kustomization.yaml new file mode 100644 index 00000000..f93b5158 --- /dev/null +++ b/test/e2e/deploy/management/kustomization.yaml @@ -0,0 +1,73 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# ───────────────────────────────────────────────────────────────────────────── +# e2e management-plane deploy layer. +# +# References the REAL production management-plane overlay verbatim and applies +# only the deviations required to run inside a local Kind environment (no Milo +# control plane, no cert-manager, no image registry). Every deviation is +# annotated below with the reason it exists so drift from production stays +# visible. Exercising the real overlay + real hub RBAC is the point of #149. +# ───────────────────────────────────────────────────────────────────────────── + +resources: + - ../../../../config/overlays/management-plane + +# Use the locally built image side-loaded into the Kind node (task e2e:image:load) +# instead of pulling ghcr.io/datum-cloud/compute:latest from a registry. +images: + - name: ghcr.io/datum-cloud/compute + newName: compute + newTag: e2e-dev + +patches: + # DEVIATION 1 (required): discovery.mode milo → single, webhook server off. + # See config_patch.yaml for the full rationale. + - path: config_patch.yaml + + # DEVIATION 2 (required): drop the cert-manager CSI serving-cert volume and + # force IfNotPresent pulls. See deployment_patch.yaml. + - path: deployment_patch.yaml + + # DEVIATION 3 (required): remove the admission webhook configurations. + # The webhook server is disabled (DEVIATION 1) because no serving cert is + # available in Kind, so these failurePolicy=Fail configurations would reject + # every Workload write. The e2e suites create WorkloadDeployments, never + # Workloads, so nothing here is exercised; removing the configs keeps the + # cluster clean rather than leaving them dangling against a dead endpoint. + - patch: | + $patch: delete + apiVersion: admissionregistration.k8s.io/v1 + kind: ValidatingWebhookConfiguration + metadata: + name: compute-validating + target: + kind: ValidatingWebhookConfiguration + name: compute-validating + - patch: | + $patch: delete + apiVersion: admissionregistration.k8s.io/v1 + kind: MutatingWebhookConfiguration + metadata: + name: compute-mutating + target: + kind: MutatingWebhookConfiguration + name: compute-mutating + + # DEVIATION 4 (required): remove the ResourceMetricsPolicy. + # config/components/resource-metrics ships a ResourceMetricsPolicy + # (resourcemetrics.miloapis.com), whose CRD is owned by a separate + # resource-metrics operator that is not installed in Kind (it is not in + # compute's module deps). Applying it fails with "no matches for kind + # ResourceMetricsPolicy". It is orthogonal to the federation path under test, + # so drop it from the deploy set. + - patch: | + $patch: delete + apiVersion: resourcemetrics.miloapis.com/v1alpha1 + kind: ResourceMetricsPolicy + metadata: + name: compute-metrics + target: + kind: ResourceMetricsPolicy + name: compute-metrics diff --git a/test/e2e/full-federation-ord/chainsaw-test.yaml b/test/e2e/full-federation-ord/chainsaw-test.yaml new file mode 100644 index 00000000..b1bf1b64 --- /dev/null +++ b/test/e2e/full-federation-ord/chainsaw-test.yaml @@ -0,0 +1,192 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: full-federation-ord +spec: + description: | + Second-cell federation chain test (city code ord → pop-ord). + + The dfw path is covered by full-federation. This suite proves the same chain + routes independently to the OTHER cell: an ord-placed WorkloadDeployment must + produce PropagationPolicy city-ord (not city-dfw), propagate to pop-ord (not + pop-dfw), and create its Instance on pop-ord. Both cells run the same operator + image, so this is the coverage that catches city-code routing regressions and + a mis-registered second cell. + + 1. Create WorkloadDeployment (cityCode: ord) on control-plane. + 2. WorkloadDeploymentFederator replicates it to Karmada (ns- namespace) + and lazily creates PropagationPolicy city-ord routing to city-code=ord cells. + 3. Karmada propagates the WD to pop-ord. + 4. WorkloadDeploymentReconciler on pop-ord creates Instance test-fullfed-ord-wd-0. + 5. InstanceReconciler on pop-ord writes the Instance back to Karmada with + label meta.datumapis.com/upstream-cluster-name: cluster-single. + 6. InstanceProjector on control-plane projects the Instance into the project + namespace. + + Cluster-name label "cluster-single" is the management operator's federating + cluster name, not the cell's — see full-federation for the full rationale. + + Prerequisites: the management + both cell operators are deployed in-cluster + (task e2e:up). + + template: true + + steps: + - name: create-workload-deployment + description: Create the ord-placed WorkloadDeployment on the control-plane cluster. + try: + - apply: + file: workload-deployment.yaml + + - name: assert-wd-and-policy-in-downstream + description: Assert the WD federated to Karmada and PropagationPolicy city-ord was created for ord. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 120s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-fullfed-ord-wd + labels: + topology.datum.net/city-code: ord + - assert: + # The federator names the policy city- and routes it to cells + # carrying the same city-code label, so ord must land on pop-ord alone. + timeout: 120s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-ord + spec: + # Three selectors: the WorkloadDeployment plus the always-on ConfigMap + # and Secret referenced-data selectors. Chainsaw matches list length + # exactly, so all three must be present. + resourceSelectors: + - apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + labelSelector: + matchLabels: + topology.datum.net/city-code: ord + - apiVersion: v1 + kind: ConfigMap + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + - apiVersion: v1 + kind: Secret + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + placement: + clusterAffinity: + labelSelector: + matchLabels: + topology.datum.net/city-code: ord + + - name: assert-wd-on-pop-ord + description: Assert Karmada propagated the WD to pop-ord and the cell reconciler set status. + cluster: pop-ord + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + # Karmada propagation can take longer than a local apply. + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-fullfed-ord-wd + status: + replicas: 1 + desiredReplicas: 1 + + - name: assert-instance-on-pop-ord + description: Assert WorkloadDeploymentReconciler created an Instance on pop-ord. + cluster: pop-ord + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-fullfed-ord-wd-0 + # The Instance is blocked before scheduling completes (a scheduling + # gate, or no matching Location in the e2e environment), so Ready is + # definitively False — not indeterminate. The exact reason is + # environment-dependent, so only the status is asserted. + (status.conditions[?type == 'Ready'] | [0]): + status: "False" + + - name: assert-instance-writeback-in-downstream + description: Assert the pop-ord InstanceReconciler wrote the Instance back to Karmada. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-fullfed-ord-wd-0 + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + + - name: assert-instance-projected-to-control-plane + description: Assert InstanceProjector created a projection on the control-plane. + try: + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($namespace) + name: test-fullfed-ord-wd-0 + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + # The Instance is blocked before scheduling completes (a scheduling + # gate, or no matching Location in the e2e environment), so Ready is + # definitively False — not indeterminate. The exact reason is + # environment-dependent, so only the status is asserted. + (status.conditions[?type == 'Ready'] | [0]): + status: "False" diff --git a/test/e2e/full-federation-ord/workload-deployment.yaml b/test/e2e/full-federation-ord/workload-deployment.yaml new file mode 100644 index 00000000..74663331 --- /dev/null +++ b/test/e2e/full-federation-ord/workload-deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-fullfed-ord-wd + # namespace is injected by Chainsaw from ($namespace) +spec: + cityCode: ord + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000002" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/full-federation/chainsaw-test.yaml b/test/e2e/full-federation/chainsaw-test.yaml new file mode 100644 index 00000000..fae94a23 --- /dev/null +++ b/test/e2e/full-federation/chainsaw-test.yaml @@ -0,0 +1,173 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: full-federation +spec: + description: | + End-to-end federation chain test. + + Exercises the complete path from WorkloadDeployment creation through to + Instance projection on the control-plane cluster: + + 1. Create WorkloadDeployment on control-plane. + 2. WorkloadDeploymentFederator replicates it to Karmada (ns- namespace). + 3. Karmada PropagationPolicy routes the WD to pop-dfw. + 4. WorkloadDeploymentReconciler on pop-dfw creates Instance test-full-fed-wd-0. + 5. InstanceReconciler on pop-dfw writes Instance back to Karmada with + label meta.datumapis.com/upstream-cluster-name: cluster-single. + 6. InstanceProjector on control-plane creates a projection of the Instance + in the project namespace. + + Cluster-name label: "cluster-single". + The value is NOT the cell's cluster name. The management operator federates + the WorkloadDeployment from its local control-plane cluster, which it + registers with multicluster-runtime under the name "single" (both operators + run discovery.mode=single in this environment, defaulting the cluster name to + "single" per cmd/main.go singleClusterName). The federator stamps + EncodeClusterName("single") = "cluster-single" onto the hub ns- + namespace; the cell write-back and the projection both copy that namespace + label verbatim, so the assertion holds regardless of which cell reconciles. + + Prerequisites: the management + both cell operators are deployed in-cluster + (task e2e:up). + + template: true + + steps: + - name: create-workload-deployment + description: Create the WorkloadDeployment on the control-plane cluster. + try: + - apply: + file: workload-deployment.yaml + + - name: assert-wd-in-downstream + description: Assert WorkloadDeploymentFederator replicated the WD to Karmada and status is synced back. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd + - assert: + # Wait for the cell operator to write status back to the Karmada WD. + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd + status: + replicas: 1 + desiredReplicas: 1 + + - name: assert-wd-on-pop-dfw + description: Assert Karmada propagated the WD to pop-dfw and the cell reconciler set status. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + # Karmada propagation can take longer than a local apply. + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd + status: + replicas: 1 + desiredReplicas: 1 + + - name: assert-instance-on-pop-dfw + description: Assert WorkloadDeploymentReconciler created an Instance on pop-dfw with a Ready condition. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + # This is the tail of the full chain (project WD → federate → Karmada + # propagate to the cell → cell reconcile → Instance). On a constrained CI + # runner that whole sequence can exceed 30s; 120s gives it room without + # masking a real stall (the smoke path confirms the Instance does appear). + timeout: 120s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd-0 + # The Instance is blocked before scheduling completes (a scheduling + # gate, or no matching Location in the e2e environment), so Ready is + # definitively False — not indeterminate. The exact reason is + # environment-dependent, so only the status is asserted. + (status.conditions[?type == 'Ready'] | [0]): + status: "False" + + - name: assert-instance-writeback-in-downstream + description: Assert InstanceReconciler wrote the Instance back to Karmada. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd-0 + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + + - name: assert-instance-projected-to-control-plane + description: Assert InstanceProjector created a projection with status on the control-plane. + try: + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($namespace) + name: test-full-fed-wd-0 + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + # The Instance is blocked before scheduling completes (a scheduling + # gate, or no matching Location in the e2e environment), so Ready is + # definitively False — not indeterminate. The exact reason is + # environment-dependent, so only the status is asserted. + (status.conditions[?type == 'Ready'] | [0]): + status: "False" diff --git a/test/e2e/full-federation/workload-deployment.yaml b/test/e2e/full-federation/workload-deployment.yaml new file mode 100644 index 00000000..70b4cb94 --- /dev/null +++ b/test/e2e/full-federation/workload-deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-full-fed-wd + # namespace is injected by Chainsaw from ($namespace) +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/instance-projection/chainsaw-test.yaml b/test/e2e/instance-projection/chainsaw-test.yaml new file mode 100644 index 00000000..d871c308 --- /dev/null +++ b/test/e2e/instance-projection/chainsaw-test.yaml @@ -0,0 +1,146 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: instance-projection +spec: + description: | + Verifies that the InstanceProjector watches Instances written back to the + Karmada API server and creates corresponding read-only projections in the + project namespace on the control-plane cluster. + + Flow: + 1. Create a WorkloadDeployment → triggers federator → Karmada namespace created. + 2. Write an Instance to Karmada (simulating a POP-cell InstanceReconciler write-back). + 3. InstanceProjector detects the Karmada Instance and creates a projection in the + project namespace (the Chainsaw test namespace on the control-plane cluster). + 4. Assert the projection exists with the upstream tracking label and an owner + reference to the WorkloadDeployment (for cascading deletion). + + Cluster name label: "cluster-single". + The management operator runs discovery.mode=single in this environment, which + registers the control-plane cluster with the multicluster-runtime manager + under the fixed name "single" (cmd/main.go singleClusterName, wired via + mcsingle.New). The InstanceProjector decodes the upstream-cluster-name label + ("cluster-single" → "single") to pick the project cluster to project into, so + the label this test writes onto the hub Instance must encode "single". + + template: true + + steps: + - name: create-wd + description: Create the WorkloadDeployment to trigger federation and namespace creation. + try: + - apply: + file: workload-deployment.yaml + + - name: wait-for-downstream-namespace + description: Wait for the federated WorkloadDeployment to appear in Karmada. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + # Federation of a freshly-created namespace depends on the management + # operator's namespace cache being current; with the namespaces-watch RBAC + # fix it stays live, but 30s is still tight on a loaded CI runner, so allow 120s. + timeout: 120s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-projector-wd + + - name: write-instance-to-downstream + description: | + Write an Instance to Karmada simulating InstanceReconciler write-back. + Uses explicit control-plane kubeconfig to derive downstreamNS and WD UID. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get workloaddeployment test-projector-wd \ + --namespace "$NAMESPACE" \ + -o jsonpath='{.metadata.uid}' + outputs: + - name: wdUID + value: ($stdout) + - script: + env: + - name: KARMADA_NS + value: ($downstreamNS) + - name: PROJECT_NS + value: ($namespace) + - name: WD_UID + value: ($wdUID) + content: | + # These labels mirror what the real cell InstanceReconciler stamps on a + # write-back copy. The projector resolves the project cluster from + # upstream-cluster-name, the project NAMESPACE from upstream-namespace + # (the project namespace where the WD lives — NOT the hub ns-), and + # looks up the owning WorkloadDeployment by workload-deployment-name to + # build the projection's owner reference. + kubectl apply -f - < namespace, which the management + federator stamped: meta.datumapis.com/upstream-cluster-name carries + EncodeClusterName of the federating cluster ("single" here → "cluster-single") + and meta.datumapis.com/upstream-namespace records the originating namespace. + + Note: this test requires the cell InstanceReconciler to be running in the DFW + POP cell cluster with federation configured. + + Runtime prerequisite (see verification phase): the write-back path errors + unless the hub ns- namespace carries the two upstream-* labels above and + the cell Instance carries the full set of linking labels the stateful control + strategy stamps at creation. This suite hand-crafts a bare namespace and a + bare Instance, so the write-back may not fire until those labels are seeded + (or the Instance is created via a real WorkloadDeployment, as full-federation + does). + + template: true + + steps: + - name: setup-namespaces + description: Create the Instance namespace in the DFW POP cell and Karmada. + try: + - script: + content: | + kubectl get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: instanceNS + value: ($stdout) + - script: + env: + - name: INSTANCE_NS + value: ($instanceNS) + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml \ + create namespace "$INSTANCE_NS" \ + --dry-run=client -o yaml | \ + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml apply -f - + - script: + env: + - name: INSTANCE_NS + value: ($instanceNS) + content: | + # The cell write-back reads its identity (upstream cluster + project + # namespace) from THIS hub namespace's labels, exactly as it would from + # the ns- namespace the federator stamps in the real path. Seed them + # so writeBackToUpstream can resolve identity instead of erroring. + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/downstream.yaml apply -f - < convention so the InstanceProjector can resolve it later. +# +# The linking labels below mirror what the stateful control strategy stamps on a +# real cell Instance at creation. The write-back path requires all of them to be +# present and non-empty before it will copy the Instance upstream; a real Instance +# always carries them, so a hand-crafted one must too. +apiVersion: compute.datumapis.com/v1alpha +kind: Instance +metadata: + name: test-writeback-instance + namespace: ($instanceNS) + labels: + compute.datumapis.com/workload-uid: "00000000-0000-0000-0000-000000000001" + compute.datumapis.com/workload-deployment-uid: "00000000-0000-0000-0000-000000000002" + compute.datumapis.com/instance-index: "0" + compute.datumapis.com/workload-deployment-name: test-writeback-wd + compute.datumapis.com/city-code: dfw + compute.datumapis.com/workload-name: test-workload + compute.datumapis.com/placement-name: default +spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network diff --git a/test/e2e/propagation-policy-lifecycle/chainsaw-test.yaml b/test/e2e/propagation-policy-lifecycle/chainsaw-test.yaml new file mode 100644 index 00000000..5678c398 --- /dev/null +++ b/test/e2e/propagation-policy-lifecycle/chainsaw-test.yaml @@ -0,0 +1,133 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: propagation-policy-lifecycle +spec: + description: | + Verifies the PropagationPolicy lifecycle managed by the WorkloadDeploymentFederator: + + - A PropagationPolicy (city-dfw) is lazily created when the first WorkloadDeployment + for city code "dfw" is federated to Karmada. + - The PropagationPolicy is RETAINED while at least one WorkloadDeployment for + that city code remains in the Karmada namespace. + - The PropagationPolicy is DELETED when the last deployment for the city is removed. + + The test creates two WDs (wd-alpha, wd-beta) both targeting cityCode=dfw, verifies + the PP appears, deletes wd-alpha and asserts the PP is still present, then deletes + wd-beta and waits for the PP to disappear. + + template: true + + steps: + - name: create-deployments + description: Create two WorkloadDeployments targeting dfw on the control-plane. + try: + - apply: + file: workload-deployment-alpha.yaml + - apply: + file: workload-deployment-beta.yaml + + - name: assert-policy-created + description: | + Assert both WDs are federated to Karmada and the PropagationPolicy exists. + Both WDs must be present in Karmada before proceeding to the deletion steps; + otherwise wd-alpha's finalizer could see an empty Karmada list and prematurely + delete the PP before wd-beta has been federated. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: wd-alpha + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: wd-beta + - assert: + timeout: 30s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-dfw + + - name: delete-alpha + description: Delete wd-alpha; wd-beta still targets dfw so the PP must be retained. + try: + - delete: + ref: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($namespace) + name: wd-alpha + + - name: assert-policy-retained + description: Assert the PropagationPolicy is still present after wd-alpha is deleted. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - sleep: + duration: 8s + - assert: + timeout: 5s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-dfw + + - name: delete-beta + description: Delete wd-beta (the last WD for city dfw). + try: + - delete: + ref: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($namespace) + name: wd-beta + + - name: assert-policy-deleted + description: Wait for the PropagationPolicy to be removed once no WDs remain. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - wait: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + namespace: ($downstreamNS) + name: city-dfw + timeout: 30s + for: + deletion: {} diff --git a/test/e2e/propagation-policy-lifecycle/workload-deployment-alpha.yaml b/test/e2e/propagation-policy-lifecycle/workload-deployment-alpha.yaml new file mode 100644 index 00000000..f9eb27fd --- /dev/null +++ b/test/e2e/propagation-policy-lifecycle/workload-deployment-alpha.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: wd-alpha +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/propagation-policy-lifecycle/workload-deployment-beta.yaml b/test/e2e/propagation-policy-lifecycle/workload-deployment-beta.yaml new file mode 100644 index 00000000..fd1d65c1 --- /dev/null +++ b/test/e2e/propagation-policy-lifecycle/workload-deployment-beta.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: wd-beta +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml b/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml new file mode 100644 index 00000000..31fad786 --- /dev/null +++ b/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml @@ -0,0 +1,287 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: referenced-data-delete-cascade +spec: + description: | + Validates the happy-path delete-cascade for referenced-data companions. + + Create a WorkloadDeployment referencing a ConfigMap + Secret. Assert companions + materialize and propagate to the member cluster. Delete the WD (last referrer). + Assert the hub companion, its Karmada ResourceBinding, and the member-cluster + copy are all deleted and stay deleted (no re-create loop). + + A second scenario — the stranded-companion backstop, where a level-triggered + companion GC reclaims a companion whose referenced-by annotation points at a + non-existent WorkloadDeployment — is intentionally omitted: that hub-side GC + ships in PR #144 (refdata-hub-gc), which was still unmerged when this branch + was cut. Restore the scenario once #144 lands. + + Prerequisites: + - task e2e:up completed (deploys the management + both cell operators in-cluster) + - the cell operator runs with featureFlags.enableReferencedDataGate: true, set + by the e2e cell deploy layer (test/e2e/deploy/cell/config_patch.yaml) + - tmp/e2e/kubeconfigs/downstream.yaml exists + + template: true + + steps: + + # ═══════════════════════════════════════════════════════════════════════════ + # SCENARIO 1: HAPPY-PATH CASCADE + # ═══════════════════════════════════════════════════════════════════════════ + + - name: s1-create-source-data + description: Create the source ConfigMap and Secret in the project namespace. + try: + - apply: + file: source-configmap.yaml + - apply: + file: source-secret.yaml + + - name: s1-create-workload-deployment + description: Create the WorkloadDeployment referencing both sources. + try: + - apply: + file: workload-deployment.yaml + + - name: s1-assert-companions-on-hub + description: | + Assert companions materialize in ns-{project-uid} on the Karmada hub + with the referenced-data label. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - assert: + timeout: 120s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($companionNS) + name: gc-test-config + labels: + compute.datumapis.com/referenced-data: "true" + - assert: + timeout: 120s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($companionNS) + name: gc-test-secret + labels: + compute.datumapis.com/referenced-data: "true" + + - name: s1-assert-companion-rbs-on-hub + description: | + Assert ResourceBindings for both companions exist on the hub. + These RBs will be deleted by the ReferencedDataController's explicit + teardown (Component 3) when the WD is deleted. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: work.karmada.io/v1alpha2 + kind: ResourceBinding + metadata: + namespace: ($companionNS) + name: gc-test-config-configmap + - assert: + timeout: 30s + resource: + apiVersion: work.karmada.io/v1alpha2 + kind: ResourceBinding + metadata: + namespace: ($companionNS) + name: gc-test-secret-secret + + - name: s1-assert-companions-on-cell + description: Assert Karmada propagated companions to the pop-dfw cell. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: cellNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($cellNS) + name: gc-test-config + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($cellNS) + name: gc-test-secret + + - name: s1-delete-workload-deployment + description: Delete the WorkloadDeployment (the sole referrer). + try: + - delete: + ref: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($namespace) + name: gc-test-wd + + - name: s1-assert-hub-companion-cm-deleted + description: | + Assert the hub companion ConfigMap is deleted by the ReferencedDataController + finalizer after the WD is gone. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - wait: + apiVersion: v1 + kind: ConfigMap + namespace: ($companionNS) + name: gc-test-config + timeout: 60s + for: + deletion: {} + + - name: s1-assert-hub-companion-secret-deleted + description: Assert the hub companion Secret is also deleted. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - wait: + apiVersion: v1 + kind: Secret + namespace: ($companionNS) + name: gc-test-secret + timeout: 60s + for: + deletion: {} + + - name: s1-assert-rbs-deleted + description: | + Assert the ResourceBindings are deleted (Component 3 explicit teardown). + Deletion of the RB drives Karmada to remove the Work and cell copies. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - wait: + apiVersion: work.karmada.io/v1alpha2 + kind: ResourceBinding + namespace: ($companionNS) + name: gc-test-config-configmap + timeout: 60s + for: + deletion: {} + - wait: + apiVersion: work.karmada.io/v1alpha2 + kind: ResourceBinding + namespace: ($companionNS) + name: gc-test-secret-secret + timeout: 60s + for: + deletion: {} + + - name: s1-assert-cell-copies-deleted-and-stay-deleted + description: | + Assert the cell copies are gone and STAY gone for 30 seconds. + If a Work were still present, Karmada would re-create the cell copy + within ~5 seconds — the poll window catches any recreate loop. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: cellNS + value: ($stdout) + - wait: + apiVersion: v1 + kind: ConfigMap + namespace: ($cellNS) + name: gc-test-config + timeout: 90s + for: + deletion: {} + - wait: + apiVersion: v1 + kind: Secret + namespace: ($cellNS) + name: gc-test-secret + timeout: 90s + for: + deletion: {} + - script: + # Poll for 30 seconds to confirm no recreate loop. If Karmada re-creates + # the cell copy from a still-live Work, this script catches it. + timeout: 40s + content: | + CELL_NS=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}') + for i in $(seq 1 6); do + sleep 5 + CM=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml \ + get configmap gc-test-config \ + --namespace "$CELL_NS" \ + --ignore-not-found 2>/dev/null) + if [ -n "$CM" ]; then + echo "ERROR: gc-test-config ConfigMap was re-created on cell (Karmada recreate loop!)" + exit 1 + fi + SEC=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml \ + get secret gc-test-secret \ + --namespace "$CELL_NS" \ + --ignore-not-found 2>/dev/null) + if [ -n "$SEC" ]; then + echo "ERROR: gc-test-secret Secret was re-created on cell (Karmada recreate loop!)" + exit 1 + fi + done + echo "OK: cell copies absent for 30+ seconds — no recreate loop" diff --git a/test/e2e/referenced-data-delete-cascade/source-configmap.yaml b/test/e2e/referenced-data-delete-cascade/source-configmap.yaml new file mode 100644 index 00000000..e9da4de8 --- /dev/null +++ b/test/e2e/referenced-data-delete-cascade/source-configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: gc-test-config + # namespace injected by Chainsaw from ($namespace) +data: + config.yaml: "env=test" diff --git a/test/e2e/referenced-data-delete-cascade/source-secret.yaml b/test/e2e/referenced-data-delete-cascade/source-secret.yaml new file mode 100644 index 00000000..e5eb7500 --- /dev/null +++ b/test/e2e/referenced-data-delete-cascade/source-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: gc-test-secret + # namespace injected by Chainsaw from ($namespace) +stringData: + token: "test-token-value" diff --git a/test/e2e/referenced-data-delete-cascade/workload-deployment.yaml b/test/e2e/referenced-data-delete-cascade/workload-deployment.yaml new file mode 100644 index 00000000..99534bae --- /dev/null +++ b/test/e2e/referenced-data-delete-cascade/workload-deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: gc-test-wd + # namespace injected by Chainsaw from ($namespace) +spec: + cityCode: dfw + placementName: default + workloadRef: + name: gc-test-workload + uid: "00000000-0000-0000-0000-000000000099" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + sandbox: + containers: + - name: app + image: docker.io/library/busybox:stable + env: + - name: TOKEN + valueFrom: + secretKeyRef: + name: gc-test-secret + key: token + volumeAttachments: + - name: cfg-vol + mountPath: /etc/config + volumes: + - name: cfg-vol + configMap: + name: gc-test-config + networkInterfaces: + - network: + name: test-network + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/referenced-data-mounts/README.md b/test/e2e/referenced-data-mounts/README.md new file mode 100644 index 00000000..3536d10b --- /dev/null +++ b/test/e2e/referenced-data-mounts/README.md @@ -0,0 +1,88 @@ +# referenced-data-mounts — Federated Delivery E2E Test + +This Chainsaw scenario validates the **cross-plane delivery path** for referenced +ConfigMap and Secret data, exercised end-to-end across the Kind+Karmada topology. + +## What this test validates + +The test covers Hops 1–5 of the federated delivery chain: + +| Hop | Cluster | What is asserted | +|-----|---------|-----------------| +| 1 | control-plane | Source ConfigMap + Secret created in the project namespace | +| 2 | control-plane | Companion `app-config` + `app-secret` appear in `ns-{project-uid}` with `compute.datumapis.com/referenced-data: "true"`; WD carries `expected-referenced-data` annotation; WD condition `ReferencedDataReady=True` | +| 3 | downstream (Karmada hub) | Companion ConfigMap + Secret present in `ns-{project-uid}` on the hub; WD carries the annotation; `PropagationPolicy city-dfw` has ConfigMap and Secret resource selectors | +| 4 | pop-dfw (cell) | WD + companions propagated to the cell in `ns-{project-uid}` | +| 5 | pop-dfw (cell) | Instance `test-refdata-wd-0` exists; `ReferencedData` scheduling gate cleared; `ReferencedDataReady=True` condition set | + +## What this test does NOT validate + +Actual env-var injection and file mounting inside a running Instance is the +**provider + kubelet layer**, not the delivery layer. That path requires the +unikraft-provider to be running with `SameCluster=true` or `SameCluster=false` +against a downstream cluster. See `docs/compute/development/plans/configmap-secret-mounts-e2e.md` +(same-cluster provider path) and `configmap-secret-mounts-e2e-multicluster.md` +(cross-cluster provider path, 4-cluster topology) for the full mount-validation scope. + +## Prerequisites + +**`task e2e:up` has completed successfully.** In the in-cluster harness this one +target brings up the Kind clusters + Karmada AND deploys the operators from the +real production overlays (plus the local e2e deviations), so there is no separate +operator-start step. It produces these kubeconfigs under `tmp/e2e/kubeconfigs/`: + +- `control-plane.yaml` — management cluster (also hosts the Karmada hub) +- `karmada.yaml` — the Karmada hub API server +- `downstream.yaml` — a copy of `karmada.yaml` the Taskfile writes so + `cluster: downstream` steps resolve (referenced by `chainsaw-config.yaml`) +- `pop-dfw.yaml`, `pop-ord.yaml` — the POP cell clusters + +**The cell operator runs with `enableReferencedDataGate: true`.** The e2e cell +deploy layer sets it in `test/e2e/deploy/cell/config_patch.yaml`. The flag's sole +consumer is the cell `WorkloadDeploymentReconciler` (it stamps the `ReferencedData` +scheduling gate); without it the Instance is never gated and Hop 5 cannot pass. +The management deploy layer sets the same flag for parity, though it is inert +there because that overlay enables management controllers only. + +## Running just this scenario + +```sh +KUBECONFIG=tmp/e2e/kubeconfigs/control-plane.yaml \ +bin/chainsaw test \ + --config test/e2e/chainsaw-config.yaml \ + --include-test-regex "referenced-data-mounts" \ + test/e2e/ +``` + +Or via the Taskfile filter target: + +```sh +task e2e:test:filter -- --include-test-regex referenced-data-mounts +``` + +## Harness notes + +This test targets the in-cluster harness, where `task e2e:up` builds the operator +image, side-loads it into every Kind node, and deploys the management + cell +operators from the real overlays. Points worth knowing: + +1. `_e2e:karmada:build-kubeconfig` copies `karmada.yaml` → `downstream.yaml`, so + `cluster: downstream` steps work out of the box. +2. The `enableReferencedDataGate` feature flag is delivered through the deploy + layer (`test/e2e/deploy/{cell,management}/config_patch.yaml`), not a host-side + `--server-config`. There is no separate operator-start task to run. +3. `e2e:crds:install` installs Milo quota CRDs to all clusters so the + InstanceReconciler's ResourceClaim watches start cleanly. + +## Companion naming convention + +The `ReferencedDataController` derives companion names deterministically. When +the source name is already a valid DNS subdomain within the length budget, the +companion keeps that name unchanged (kind is not prefixed): + +| Source | Companion name | +|--------|---------------| +| `ConfigMap/app-config` | `app-config` | +| `Secret/app-secret` | `app-secret` | + +These names are asserted directly in the test steps. diff --git a/test/e2e/referenced-data-mounts/chainsaw-test.yaml b/test/e2e/referenced-data-mounts/chainsaw-test.yaml new file mode 100644 index 00000000..bcef1234 --- /dev/null +++ b/test/e2e/referenced-data-mounts/chainsaw-test.yaml @@ -0,0 +1,352 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: referenced-data-mounts +spec: + description: | + Validates the FEDERATED DELIVERY path for referenced ConfigMap and Secret data. + + This test exercises the cross-plane chain introduced by the referenced-data + feature (feat/configmap-secret-mounts-federated): + + [Hop 1] Source ConfigMap and Secret are created in the project namespace + (control-plane, Chainsaw default cluster). + + [Hop 2] WorkloadDeployment is created; ReferencedDataController materialises + companion objects in ns-{project-uid} on the Karmada hub (downstream) + and stamps the expected-referenced-data annotation on the WD. + Companions are NOT written to the control-plane cluster. + + [Hop 3] Karmada hub holds the companion ConfigMap + Secret in ns-{project-uid}, + propagated by the always-on label selector in the city-dfw + PropagationPolicy (which includes ConfigMap and Secret selectors). + + [Hop 4] Karmada propagates the WD + companions to pop-dfw. + Companions appear in ns-{project-uid} on the cell alongside the WD. + + [Hop 5] WorkloadDeploymentReconciler on pop-dfw creates Instance test-refdata-wd-0 + with the ReferencedData scheduling gate (feature flag on). + InstanceReconciler clears the gate once companions are present and sets + ReferencedDataReady=True on the Instance. + + Scope: cross-plane DELIVERY up to the Instance gate-cleared state. + Actual env-var and file mounting is the provider+kubelet layer and is NOT + asserted here. See README.md for full scope and prerequisites. + + Prerequisites: + - task e2e:up completed: brings up control-plane + Karmada + pop-dfw + pop-ord + and deploys the management and both cell operators in-cluster. + - the cell operator runs with featureFlags.enableReferencedDataGate: true; the + e2e cell deploy layer sets it (test/e2e/deploy/cell/config_patch.yaml), so + no separate operator-start step is needed. + - tmp/e2e/kubeconfigs/downstream.yaml exists (the Taskfile writes it as a copy + of the Karmada hub kubeconfig). + + template: true + + steps: + + # ─── Hop 1: create source data ───────────────────────────────────────────── + + - name: create-source-data + description: Create the source ConfigMap and Secret in the project namespace. + try: + - apply: + file: source-configmap.yaml + - apply: + file: source-secret.yaml + + # ─── Hop 2: create WD; assert companion materialisation on Karmada hub ───── + + - name: create-workload-deployment + description: Create the WorkloadDeployment referencing the source ConfigMap and Secret. + try: + - apply: + file: workload-deployment.yaml + + - name: assert-companion-configmap-on-hub + description: | + Assert the ReferencedDataController materialised companion app-config + in ns-{project-uid} on the Karmada hub (downstream cluster). + Companions are written to the hub namespace, NOT the control-plane namespace. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($companionNS) + name: app-config + labels: + compute.datumapis.com/referenced-data: "true" + + - name: assert-companion-secret-on-hub + description: | + Assert the ReferencedDataController materialised companion app-secret + in ns-{project-uid} on the Karmada hub (downstream cluster). + Companions are written to the hub namespace, NOT the control-plane namespace. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($companionNS) + name: app-secret + labels: + compute.datumapis.com/referenced-data: "true" + + - name: assert-wd-annotation-and-condition-on-control-plane + description: | + Assert the WD carries the expected-referenced-data annotation and has + ReferencedDataReady=True condition set by the ReferencedDataController. + try: + - script: + # Verify the annotation is present and non-empty. + content: | + ANNO=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get workloaddeployment test-refdata-wd \ + --namespace "$NAMESPACE" \ + -o jsonpath='{.metadata.annotations.compute\.datumapis\.com/expected-referenced-data}') + if [ -z "$ANNO" ] || [ "$ANNO" = "[]" ]; then + echo "ERROR: expected-referenced-data annotation is missing or empty: '$ANNO'" + exit 1 + fi + echo "annotation: $ANNO" + timeout: 60s + - assert: + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($namespace) + name: test-refdata-wd + (status.conditions[?type == 'ReferencedDataReady'] | [0]): + status: "True" + reason: "Ready" + + # ─── Hop 3: assert companions on the Karmada hub (downstream) ────────────── + + - name: assert-wd-and-companions-on-hub + description: | + Assert the Karmada hub (downstream cluster) holds the WD with the + expected-referenced-data annotation and the companion ConfigMap + Secret + in ns-{project-uid}. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + # WD federated to Karmada with the expected-referenced-data annotation forwarded + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-refdata-wd + - assert: + # Companion ConfigMap on Karmada hub + timeout: 30s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($downstreamNS) + name: app-config + labels: + compute.datumapis.com/referenced-data: "true" + - assert: + # Companion Secret on Karmada hub + timeout: 30s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($downstreamNS) + name: app-secret + labels: + compute.datumapis.com/referenced-data: "true" + + - name: assert-propagation-policy-has-companion-selectors + description: | + Assert the PropagationPolicy city-dfw on the hub includes ConfigMap and + Secret resource selectors so companions co-propagate with the WD. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-dfw + spec: + resourceSelectors: + - apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($downstreamNS) + labelSelector: + matchLabels: + topology.datum.net/city-code: dfw + - apiVersion: v1 + kind: ConfigMap + namespace: ($downstreamNS) + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + - apiVersion: v1 + kind: Secret + namespace: ($downstreamNS) + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + + # ─── Hop 4: assert companions propagated to pop-dfw cell ─────────────────── + + - name: assert-wd-and-companions-on-cell + description: | + Assert Karmada propagated the WD and companion ConfigMap + Secret to + pop-dfw. All three objects must appear in ns-{project-uid} on the cell. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-refdata-wd + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($downstreamNS) + name: app-config + labels: + compute.datumapis.com/referenced-data: "true" + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($downstreamNS) + name: app-secret + labels: + compute.datumapis.com/referenced-data: "true" + + # ─── Hop 5: assert Instance gate cleared on pop-dfw ──────────────────────── + + - name: assert-instance-exists-on-cell + description: Assert the WorkloadDeploymentReconciler created Instance test-refdata-wd-0. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-refdata-wd-0 + + - name: assert-referenced-data-gate-cleared + description: | + Assert the ReferencedData scheduling gate is removed from the Instance and + the ReferencedDataReady=True condition is set by the InstanceReconciler. + The gate is stamped when the Instance is created (feature flag on) and + cleared once the cell InstanceReconciler confirms all expected companions + are present in ns-{project-uid}. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-refdata-wd-0 + (status.conditions[?type == 'ReferencedDataReady'] | [0]): + status: "True" + reason: "Ready" + - script: + # Verify the ReferencedData gate is absent from the Instance spec. + timeout: 60s + content: | + DOWNSTREAMS_NS=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}') + GATES=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml \ + get instance test-refdata-wd-0 \ + --namespace "$DOWNSTREAMS_NS" \ + -o jsonpath='{.spec.schedulingGates[*].name}') + if echo "$GATES" | grep -qw "ReferencedData"; then + echo "ERROR: ReferencedData gate still present in schedulingGates: '$GATES'" + exit 1 + fi + echo "ReferencedData gate cleared. Remaining gates: '$GATES'" diff --git a/test/e2e/referenced-data-mounts/source-configmap.yaml b/test/e2e/referenced-data-mounts/source-configmap.yaml new file mode 100644 index 00000000..3664e9ae --- /dev/null +++ b/test/e2e/referenced-data-mounts/source-configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-config + # namespace injected by Chainsaw from ($namespace) +data: + app.properties: "environment=test" + log.level: "info" diff --git a/test/e2e/referenced-data-mounts/source-secret.yaml b/test/e2e/referenced-data-mounts/source-secret.yaml new file mode 100644 index 00000000..afde8426 --- /dev/null +++ b/test/e2e/referenced-data-mounts/source-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: app-secret + # namespace injected by Chainsaw from ($namespace) +type: Opaque +stringData: + db.password: "test-db-password" diff --git a/test/e2e/referenced-data-mounts/workload-deployment.yaml b/test/e2e/referenced-data-mounts/workload-deployment.yaml new file mode 100644 index 00000000..d849be0b --- /dev/null +++ b/test/e2e/referenced-data-mounts/workload-deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-refdata-wd + # namespace injected by Chainsaw from ($namespace) +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000002" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + sandbox: + containers: + - name: app + image: docker.io/library/busybox:stable + env: + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: app-secret + key: db.password + volumeAttachments: + - name: config-vol + mountPath: /etc/config + volumes: + - name: config-vol + configMap: + name: app-config + networkInterfaces: + - network: + name: test-network + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/workload-deployment-federation/chainsaw-test.yaml b/test/e2e/workload-deployment-federation/chainsaw-test.yaml new file mode 100644 index 00000000..8d589310 --- /dev/null +++ b/test/e2e/workload-deployment-federation/chainsaw-test.yaml @@ -0,0 +1,98 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: workload-deployment-federation +spec: + description: | + Verifies that the WorkloadDeploymentFederator replicates a WorkloadDeployment + from the project namespace (control-plane cluster) to the Karmada API server + with the correct city-code label and PropagationPolicy. + + The federator follows the ns- convention for Karmada namespaces, + matching the MappedNamespaceResourceStrategy used by NSO. The test derives + the expected Karmada namespace dynamically from the Chainsaw test namespace UID. + + Verified: + - WorkloadDeployment exists in Karmada at ns- + - Karmada copy carries label topology.datum.net/city-code: dfw + - PropagationPolicy city-dfw exists in the Karmada namespace, + selecting WDs by city-code and routing them to matching POP-cell clusters. + + template: true + + steps: + - name: derive-ns-and-create-wd + description: Derive Karmada namespace and create the WorkloadDeployment. + try: + - apply: + file: workload-deployment.yaml + + - name: assert-wd-in-downstream + description: Assert WorkloadDeployment federated to Karmada with city-code label. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-federation-wd + labels: + topology.datum.net/city-code: dfw + + - name: assert-propagation-policy-in-downstream + description: Assert PropagationPolicy created for city-dfw. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-dfw + spec: + # The federator emits three resource selectors, not one: the + # WorkloadDeployment plus the always-on ConfigMap and Secret + # referenced-data selectors that co-propagate companions to the cell. + # Chainsaw matches list length exactly, so all three must be asserted. + resourceSelectors: + - apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + labelSelector: + matchLabels: + topology.datum.net/city-code: dfw + - apiVersion: v1 + kind: ConfigMap + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + - apiVersion: v1 + kind: Secret + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + placement: + clusterAffinity: + labelSelector: + matchLabels: + topology.datum.net/city-code: dfw diff --git a/test/e2e/workload-deployment-federation/workload-deployment.yaml b/test/e2e/workload-deployment-federation/workload-deployment.yaml new file mode 100644 index 00000000..0cd2347a --- /dev/null +++ b/test/e2e/workload-deployment-federation/workload-deployment.yaml @@ -0,0 +1,22 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-federation-wd + # namespace is injected by Chainsaw from ($namespace) +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + + scaleSettings: + minReplicas: 1