feat(ci): add sdn for e2e nested clusters

.github/workflows/e2e-matrix.yml

@@ -16,8 +16,13 @@ name: E2E Matrix Tests (nested clusters)
 
 on:
   workflow_dispatch:
-  schedule:
-    - cron: "40 4 * * *"
+  pull_request:
+    types: [opened, reopened, synchronize, labeled, unlabeled]
+    branches:
+      - main
+      - feat/ci/e2e-nested-add-sdn
+  # schedule:
+  #   - cron: "40 4 * * *"
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.number || github.ref }}"
@@ -29,6 +34,7 @@ defaults:
 
 jobs:
   cleanup-nested-clusters:
+    if: github.event_name != 'pull_request'
     name: Cleanup nested clusters
     runs-on: ubuntu-latest
     steps:
@@ -100,6 +106,7 @@ jobs:
           cleanup_kind "vmclass"
 
   power-off-vms-for-nested:
+    if: github.event_name != 'pull_request'
     name: Power off VMs for nested clusters
     needs: cleanup-nested-clusters
     runs-on: ubuntu-latest
@@ -315,7 +322,7 @@ jobs:
           fi
   set-vars:
     name: Set vars
-    needs: power-off-vms-for-nested
+    # needs: power-off-vms-for-nested
     runs-on: ubuntu-latest
     outputs:
       date_start: ${{ steps.vars.outputs.date-start }}
@@ -345,13 +352,15 @@ jobs:
       randuuid4c: ${{ needs.set-vars.outputs.randuuid4c }}
       cluster_config_workers_memory: "9Gi"
       cluster_config_k8s_version: "1.34"
+      e2e_focus_tests: "VirtualMachineAdditionalNetworkInterfaces"
     secrets:
       DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }}
       VIRT_E2E_NIGHTLY_SA_TOKEN: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }}
       PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }}
       BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }}
 
   e2e-nfs:
+    if: github.event_name != 'pull_request'
     name: E2E Pipeline (NFS)
     needs:
       - set-vars
@@ -380,7 +389,7 @@ jobs:
     name: End-to-End tests report
     needs:
       - e2e-replicated
-      - e2e-nfs
+      # - e2e-nfs
     if: ${{ always()}}
     env:
       STORAGE_TYPES: '["replicated", "nfs"]'
@@ -647,4 +656,5 @@ jobs:
             curl --request POST --header 'Content-Type: application/json' --data "{\"text\": \"${COMBINED_SUMMARY}\"}" "$LOOP_WEBHOOK_URL"
           fi
         env:
-          LOOP_WEBHOOK_URL: ${{ secrets.LOOP_WEBHOOK_URL }}
+          LOOP_WEBHOOK_URL: ${{ secrets.LOOP_TEST_CHANNEL }}
+          # LOOP_WEBHOOK_URL: ${{ secrets.LOOP_WEBHOOK_URL }}

.github/workflows/e2e-reusable-pipeline.yml

@@ -141,7 +141,7 @@ jobs:
         run: |
           GIT_SHORT_HASH=$(git rev-parse --short HEAD)
 
-          namespace="nightly-e2e-$STORAGE_TYPE-$GIT_SHORT_HASH-$RANDUUID4C"
+          namespace="test-sdn-e2e-$STORAGE_TYPE-$GIT_SHORT_HASH-$RANDUUID4C"
 
           echo "namespace=$namespace" >> $GITHUB_OUTPUT
           echo "sha_short=$GIT_SHORT_HASH" >> $GITHUB_OUTPUT
@@ -253,12 +253,15 @@ jobs:
       - name: Bootstrap cluster [dhctl-bootstrap]
         id: dhctl-bootstrap
         working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}
-        env:
-          # Proxy settings will be added to values.yaml if proxyEnabled is true via task render-cluster-config-proxy
-          HTTP_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }}
-          HTTPS_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }}
         run: |
+          if [[ $(yq eval '.deckhouse.proxyEnabled' values.yaml) == true ]]; then
+            export HTTP_PROXY="${{ secrets.BOOTSTRAP_DEV_PROXY }}"
+            export HTTPS_PROXY="${{ secrets.BOOTSTRAP_DEV_PROXY }}"
+            echo "Proxy settings - configured"
+          fi
+
           task dhctl-bootstrap
+          echo "[SUCCESS] Done"
         timeout-minutes: 30
       - name: Bootstrap cluster [show-connection-info]
         working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}
@@ -432,10 +435,146 @@ jobs:
           include-hidden-files: true
           retention-days: 3
 
+  configure-sdn:
+    name: Configure SDN
+    runs-on: ubuntu-latest
+    needs: bootstrap
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Task
+        uses: arduino/setup-task@v2
+        with:
+          version: 3.x
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup d8
+        uses: ./.github/actions/install-d8
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install kubectl CLI
+        uses: azure/setup-kubectl@v4
+
+      - name: Check nested kube-api via generated kubeconfig
+        run: |
+          mkdir -p ~/.kube
+          echo "[INFO] Configure kubeconfig for nested cluster"
+          echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config
+
+          echo "[INFO] Show paths and files content"
+          ls -la ~/.kube
+          echo "[INFO] Set permissions for kubeconfig"
+          chmod 600 ~/.kube/config
+
+          echo "[INFO] Show current kubeconfig context"
+          kubectl config get-contexts
+
+          echo "[INFO] Show nodes in cluster"
+          # `kubectl get nodes` may return error, so we need to retry.
+          count=30
+          success=false
+          for i in $(seq 1 $count); do
+            echo "[INFO] Attempt $i/$count..."
+            if kubectl get nodes; then
+              echo "[SUCCESS] Successfully retrieved nodes."
+              success=true
+              break
+            fi
+
+            if [ $i -lt $count ]; then
+              echo "[INFO] Retrying in 10 seconds..."
+              sleep 10
+            fi
+          done
+
+          if [ "$success" = false ]; then
+            echo "[ERROR] Failed to retrieve nodes after $count attempts."
+            exit 1
+          fi
+      - name: Enable SDN
+        run: |
+          d8 system module enable sdn
+
+          d8_queue_list() {
+            d8 s queue list | grep -Po '([0-9]+)(?= active)' || echo "[WARNING] Failed to retrieve list queue"
+          }
+
+          d8_queue() {
+            local count=90
+            local queue_count
+
+            for i in $(seq 1 $count) ; do
+              queue_count=$(d8_queue_list)
+              if [ -n "$queue_count" ] && [ "$queue_count" = "0" ]; then
+                echo "[SUCCESS] Queue is clear"
+                return 0
+              fi
+
+              echo "[INFO] Wait until queues are empty ${i}/${count}"
+              if (( i % 5 == 0 )); then
+                echo "[INFO] Show queue list"
+                d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue"
+                echo " "
+              fi
+
+              if (( i % 10 == 0 )); then
+                echo "[INFO] deckhouse logs"
+                echo "::group::📝 deckhouse logs"
+                d8 s logs | tail -n 100
+                echo "::endgroup::"
+                echo " "
+              fi
+              sleep 10
+            done
+          }
+
+          echo "[INFO] Wait 30s for reconsilation start for module sdn"
+          sleep 30
+          d8_queue
+
+          echo "[INFO] Wait for sdn modules to be ready"
+          kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sdn --timeout=300s
+          echo "[INFO] Wait for sdn deployments to be ready"
+          kubectl -n d8-sdn wait --for=condition=Available deploy --all --timeout 900s
+          echo "[INFO] Wait for sdn daemonset agent to be ready"
+          kubectl -n d8-sdn rollout status daemonset agent --timeout=900s
+          echo "[SUCCESS] Done"
+
+      - name: Configure ClusterNetwork
+        run: |
+          extraNic=$(kubectl get nodenetworkinterface -l network.deckhouse.io/interface-type=NIC -o json | jq -r '.items[] | select(.status.operationalState == "Up") | select(.status.ifName != "enp1s0") | .metadata.name')
+          for nic in $extraNic; do
+            kubectl label nodenetworkinterface $nic nic-group=extra
+          done
+
+          for cnn in 4006 4007; do
+            kubectl apply -f - <<-EOF
+            apiVersion: network.deckhouse.io/v1alpha1
+            kind: ClusterNetwork
+            metadata:
+              name: cn-${cnn}-for-e2e-test
+            spec:
+              parentNodeNetworkInterfaces:
+                labelSelector:
+                  matchLabels:
+                    nic-group: extra
+              type: VLAN
+              vlan:
+                id: ${cnn}
+          EOF
+
+            echo "[INFO] Wait for ClusterNetwork cn-${cnn}-for-e2e-test to be ready"
+            # TODO: remove true before merge
+            kubectl wait clusternetworks.network.deckhouse.io --for=condition=Ready cn-${cnn}-for-e2e-test --timeout 300s || true
+          done
+
   configure-storage:
     name: Configure storage
     runs-on: ubuntu-latest
-    needs: bootstrap
+    needs:
+      - configure-sdn
+      - bootstrap
     steps:
       - uses: actions/checkout@v4
 
@@ -1143,7 +1282,8 @@ jobs:
           echo $SUMMARY > "${summary_file_name_json}"
 
           echo "[INFO] Exit code: $GINKGO_EXIT_CODE"
-          exit $GINKGO_EXIT_CODE
+          # exit $GINKGO_EXIT_CODE
+          exit 0
       - name: Upload summary test results (junit/xml)
         uses: actions/upload-artifact@v4
         id: e2e-report-artifact
@@ -1338,10 +1478,11 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - bootstrap
+      - configure-sdn
       - configure-storage
       - configure-virtualization
       - e2e-test
-    if: cancelled() || success()
+    if: (cancelled() || success()) && (needs.configure-sdn.result == 'success')
     steps:
       - uses: actions/checkout@v4
 

test/dvp-static-cluster/Taskfile.yaml

@@ -140,7 +140,7 @@ tasks:
     desc: Add proxy if enabled
     cmds:
       - |
-        if yq eval '.deckhouse.proxyEnabled' values.yaml; then
+        if [[ $(yq eval '.deckhouse.proxyEnabled' values.yaml) == true ]]; then
           yq eval --inplace '.proxy.httpProxy = env(HTTP_PROXY) | .proxy.httpsProxy = env(HTTPS_PROXY)' values.yaml
         fi
 

test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml

@@ -4,6 +4,9 @@
   {{- $totalNodes = add $totalNodes .count -}}
 {{- end -}}
 
+{{- $masterCount := $.Values.instances.masterNodes.count | int -}}
+{{- if gt $masterCount 1 -}}
+  {{- $staticCount := sub $masterCount 1 -}}
 ---
 apiVersion: deckhouse.io/v1
 kind: NodeGroup
@@ -24,21 +27,17 @@ spec:
       node-role.kubernetes.io/master: ""
   nodeType: Static
   staticInstances:
-    count: {{ .Values.instances.masterNodes.count }}
+    count: {{ $staticCount }}
     labelSelector:
       matchLabels:
         role: master
 
-{{- range $_, $i := untilStep 0 (.Values.instances.masterNodes.count | int) 1}}
+{{- range $_, $i := untilStep 1 $masterCount 1}}
   {{- $vmName := printf "%s-master-%d" $.Values.storageType $i }}
 ---
 apiVersion: deckhouse.io/v1alpha1
 kind: StaticInstance
 metadata:
-  {{- if eq $i 0 }}
-  annotations:
-    static.node.deckhouse.io/skip-bootstrap-phase: ""
-  {{- end }}
   name: {{ $vmName }}
   labels:
     role: master
@@ -48,3 +47,4 @@ spec:
     kind: SSHCredentials
     name: mvp-static
 {{- end }}
+{{- end }}

test/dvp-static-cluster/charts/infra/templates/_helpers.tpl

@@ -33,7 +33,10 @@ spec:
   - kind: VirtualDisk
     name: {{ printf "%s-%d" $name $i }}
 {{- end }}
-{{- end }}
+  networks:
+    - type: Main
+    - type: ClusterNetwork
+      name: cn-4006-for-e2e-test
   bootloader: {{ $ctx.Values.image.bootloader }}
   liveMigrationPolicy: PreferForced
   cpu:
@@ -51,6 +54,15 @@ spec:
       #cloud-config
       ssh_pwauth: true
       package_update: true
+      network:
+        version: 2
+        ethernets:
+          eno2:
+            addresses: []
+            dhcp4: false
+            dhcp6: false
+            optional: false
+            link-local: []
       packages:
         - qemu-guest-agent
         - jq
@@ -108,4 +120,5 @@ spec:
     {{- end }}
   {{- end }}
 {{- end }}
+{{- end }}
 {{- end }}