diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.yml index 4c3a6160da..2d136d5ee8 100644 --- a/.github/workflows/e2e-matrix.yml +++ b/.github/workflows/e2e-matrix.yml @@ -16,8 +16,13 @@ name: E2E Matrix Tests (nested clusters) on: workflow_dispatch: - schedule: - - cron: "40 4 * * *" + pull_request: + types: [opened, reopened, synchronize, labeled, unlabeled] + branches: + - main + - feat/ci/e2e-nested-add-sdn + # schedule: + # - cron: "40 4 * * *" concurrency: group: "${{ github.workflow }}-${{ github.event.number || github.ref }}" @@ -29,6 +34,7 @@ defaults: jobs: cleanup-nested-clusters: + if: github.event_name != 'pull_request' name: Cleanup nested clusters runs-on: ubuntu-latest steps: @@ -100,6 +106,7 @@ jobs: cleanup_kind "vmclass" power-off-vms-for-nested: + if: github.event_name != 'pull_request' name: Power off VMs for nested clusters needs: cleanup-nested-clusters runs-on: ubuntu-latest @@ -315,7 +322,7 @@ jobs: fi set-vars: name: Set vars - needs: power-off-vms-for-nested + # needs: power-off-vms-for-nested runs-on: ubuntu-latest outputs: date_start: ${{ steps.vars.outputs.date-start }} @@ -345,6 +352,7 @@ jobs: randuuid4c: ${{ needs.set-vars.outputs.randuuid4c }} cluster_config_workers_memory: "9Gi" cluster_config_k8s_version: "1.34" + e2e_focus_tests: "VirtualMachineAdditionalNetworkInterfaces" secrets: DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }} VIRT_E2E_NIGHTLY_SA_TOKEN: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} @@ -352,6 +360,7 @@ jobs: BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} e2e-nfs: + if: github.event_name != 'pull_request' name: E2E Pipeline (NFS) needs: - set-vars @@ -380,7 +389,7 @@ jobs: name: End-to-End tests report needs: - e2e-replicated - - e2e-nfs + # - e2e-nfs if: ${{ always()}} env: STORAGE_TYPES: '["replicated", "nfs"]' @@ -647,4 +656,5 @@ jobs: curl --request POST --header 'Content-Type: application/json' --data "{\"text\": \"${COMBINED_SUMMARY}\"}" "$LOOP_WEBHOOK_URL" fi env: - LOOP_WEBHOOK_URL: ${{ secrets.LOOP_WEBHOOK_URL }} + LOOP_WEBHOOK_URL: ${{ secrets.LOOP_TEST_CHANNEL }} + # LOOP_WEBHOOK_URL: ${{ secrets.LOOP_WEBHOOK_URL }} diff --git a/.github/workflows/e2e-reusable-pipeline.yml b/.github/workflows/e2e-reusable-pipeline.yml index 17cb9b31b5..a5f4912b5e 100644 --- a/.github/workflows/e2e-reusable-pipeline.yml +++ b/.github/workflows/e2e-reusable-pipeline.yml @@ -141,7 +141,7 @@ jobs: run: | GIT_SHORT_HASH=$(git rev-parse --short HEAD) - namespace="nightly-e2e-$STORAGE_TYPE-$GIT_SHORT_HASH-$RANDUUID4C" + namespace="test-sdn-e2e-$STORAGE_TYPE-$GIT_SHORT_HASH-$RANDUUID4C" echo "namespace=$namespace" >> $GITHUB_OUTPUT echo "sha_short=$GIT_SHORT_HASH" >> $GITHUB_OUTPUT @@ -253,12 +253,15 @@ jobs: - name: Bootstrap cluster [dhctl-bootstrap] id: dhctl-bootstrap working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} - env: - # Proxy settings will be added to values.yaml if proxyEnabled is true via task render-cluster-config-proxy - HTTP_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} - HTTPS_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} run: | + if [[ $(yq eval '.deckhouse.proxyEnabled' values.yaml) == true ]]; then + export HTTP_PROXY="${{ secrets.BOOTSTRAP_DEV_PROXY }}" + export HTTPS_PROXY="${{ secrets.BOOTSTRAP_DEV_PROXY }}" + echo "Proxy settings - configured" + fi + task dhctl-bootstrap + echo "[SUCCESS] Done" timeout-minutes: 30 - name: Bootstrap cluster [show-connection-info] working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} @@ -432,10 +435,109 @@ jobs: include-hidden-files: true retention-days: 3 + configure-sdn: + name: Configure SDN + runs-on: ubuntu-latest + needs: bootstrap + steps: + - uses: actions/checkout@v4 + + - name: Install Task + uses: arduino/setup-task@v2 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Install kubectl CLI + uses: azure/setup-kubectl@v4 + + - name: Check nested kube-api via generated kubeconfig + run: | + mkdir -p ~/.kube + echo "[INFO] Configure kubeconfig for nested cluster" + echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config + + echo "[INFO] Show paths and files content" + ls -la ~/.kube + echo "[INFO] Set permissions for kubeconfig" + chmod 600 ~/.kube/config + + echo "[INFO] Show current kubeconfig context" + kubectl config get-contexts + + echo "[INFO] Show nodes in cluster" + # `kubectl get nodes` may return error, so we need to retry. + count=30 + success=false + for i in $(seq 1 $count); do + echo "[INFO] Attempt $i/$count..." + if kubectl get nodes; then + echo "[SUCCESS] Successfully retrieved nodes." + success=true + break + fi + + if [ $i -lt $count ]; then + echo "[INFO] Retrying in 10 seconds..." + sleep 10 + fi + done + + if [ "$success" = false ]; then + echo "[ERROR] Failed to retrieve nodes after $count attempts." + exit 1 + fi + - name: Enable SDN + run: | + echo "[INFO] Enable SDN" + d8 system module enable sdn + echo "[INFO] Wait for sdn modules to be ready, timeout: 300s" + kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sdn --timeout=300s + echo "[INFO] Wait for sdn deployments to be ready, timeout: 300s" + kubectl -n d8-sdn wait --for=condition=Available deploy --all --timeout 300s + echo "[INFO] Wait for sdn daemonset agent to be ready, timeout: 300s" + kubectl -n d8-sdn rollout status daemonset agent --timeout=300s + echo "[SUCCESS] Done" + + - name: Configure ClusterNetwork + run: | + extraNic=$(kubectl get nodenetworkinterface -l network.deckhouse.io/interface-type=NIC -o json | jq -r '.items[] | select(.status.operationalState == "Up") | select(.status.ifName != "enp1s0") | .metadata.name') + for nic in $extraNic; do + kubectl label nodenetworkinterface $nic nic-group=extra + done + + for cnn in 4006 4007; do + kubectl apply -f - <<-EOF + apiVersion: network.deckhouse.io/v1alpha1 + kind: ClusterNetwork + metadata: + name: cn-${cnn}-for-e2e-test + spec: + parentNodeNetworkInterfaces: + labelSelector: + matchLabels: + nic-group: extra + type: VLAN + vlan: + id: ${cnn} + EOF + + echo "[INFO] Wait for ClusterNetwork cn-${cnn}-for-e2e-test to be ready" + # TODO: remove true before merge + kubectl wait clusternetworks.network.deckhouse.io --for=condition=Ready cn-${cnn}-for-e2e-test --timeout 30s || true + done + configure-storage: name: Configure storage runs-on: ubuntu-latest - needs: bootstrap + needs: + - configure-sdn + - bootstrap steps: - uses: actions/checkout@v4 @@ -1076,74 +1178,75 @@ jobs: STORAGE_CLASS_NAME: ${{ inputs.nested_storageclass_name }} working-directory: ./test/e2e/ run: | - GINKGO_RESULT=$(mktemp -p $RUNNER_TEMP) - DATE=$(date +"%Y-%m-%d") - START_TIME=$(date +"%H:%M:%S") - summary_file_name_junit="e2e_summary_${CSI}_${DATE}.xml" - summary_file_name_json="e2e_summary_${CSI}_${DATE}.json" - - cp -a legacy/testdata /tmp/testdata - - set +e - FOCUS="${{ inputs.e2e_focus_tests }}" - if [ -n "$FOCUS" ]; then - go tool ginkgo \ - --focus="$FOCUS" \ - -v --race --timeout=$TIMEOUT \ - --junit-report=$summary_file_name_junit | tee $GINKGO_RESULT - else - go tool ginkgo \ - -v --race --timeout=$TIMEOUT \ - --junit-report=$summary_file_name_junit | tee $GINKGO_RESULT - fi - GINKGO_EXIT_CODE=$? - set -e - - RESULT=$(sed -e "s/\x1b\[[0-9;]*m//g" $GINKGO_RESULT | grep --color=never -E "FAIL!|SUCCESS!") - if [[ $RESULT == FAIL!* ]]; then - RESULT_STATUS=":x: FAIL!" - elif [[ $RESULT == SUCCESS!* ]]; then - RESULT_STATUS=":white_check_mark: SUCCESS!" - else - RESULT_STATUS=":question: UNKNOWN" - fi - - PASSED=$(echo "$RESULT" | grep -oP "\d+(?= Passed)") - FAILED=$(echo "$RESULT" | grep -oP "\d+(?= Failed)") - PENDING=$(echo "$RESULT" | grep -oP "\d+(?= Pending)") - SKIPPED=$(echo "$RESULT" | grep -oP "\d+(?= Skipped)") - - SUMMARY=$(jq -n \ - --arg csi "$CSI" \ - --arg date "$DATE" \ - --arg startTime "$START_TIME" \ - --arg branch "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" \ - --arg status "$RESULT_STATUS" \ - --argjson passed "$PASSED" \ - --argjson failed "$FAILED" \ - --argjson pending "$PENDING" \ - --argjson skipped "$SKIPPED" \ - --arg link "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ - '{ - CSI: $csi, - Date: $date, - StartTime: $startTime, - Branch: $branch, - Status: $status, - Passed: $passed, - Failed: $failed, - Pending: $pending, - Skipped: $skipped, - Link: $link - }' - ) - - echo "$SUMMARY" - echo "summary=$(echo "$SUMMARY" | jq -c .)" >> $GITHUB_OUTPUT - echo $SUMMARY > "${summary_file_name_json}" - - echo "[INFO] Exit code: $GINKGO_EXIT_CODE" - exit $GINKGO_EXIT_CODE + # GINKGO_RESULT=$(mktemp -p $RUNNER_TEMP) + # DATE=$(date +"%Y-%m-%d") + # START_TIME=$(date +"%H:%M:%S") + # summary_file_name_junit="e2e_summary_${CSI}_${DATE}.xml" + # summary_file_name_json="e2e_summary_${CSI}_${DATE}.json" + + # cp -a legacy/testdata /tmp/testdata + + # set +e + # FOCUS="${{ inputs.e2e_focus_tests }}" + # if [ -n "$FOCUS" ]; then + # go tool ginkgo \ + # --focus="$FOCUS" \ + # -v --race --timeout=$TIMEOUT \ + # --junit-report=$summary_file_name_junit | tee $GINKGO_RESULT + # else + # go tool ginkgo \ + # -v --race --timeout=$TIMEOUT \ + # --junit-report=$summary_file_name_junit | tee $GINKGO_RESULT + # fi + # GINKGO_EXIT_CODE=$? + # set -e + + # RESULT=$(sed -e "s/\x1b\[[0-9;]*m//g" $GINKGO_RESULT | grep --color=never -E "FAIL!|SUCCESS!") + # if [[ $RESULT == FAIL!* ]]; then + # RESULT_STATUS=":x: FAIL!" + # elif [[ $RESULT == SUCCESS!* ]]; then + # RESULT_STATUS=":white_check_mark: SUCCESS!" + # else + # RESULT_STATUS=":question: UNKNOWN" + # fi + + # PASSED=$(echo "$RESULT" | grep -oP "\d+(?= Passed)") + # FAILED=$(echo "$RESULT" | grep -oP "\d+(?= Failed)") + # PENDING=$(echo "$RESULT" | grep -oP "\d+(?= Pending)") + # SKIPPED=$(echo "$RESULT" | grep -oP "\d+(?= Skipped)") + + # SUMMARY=$(jq -n \ + # --arg csi "$CSI" \ + # --arg date "$DATE" \ + # --arg startTime "$START_TIME" \ + # --arg branch "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" \ + # --arg status "$RESULT_STATUS" \ + # --argjson passed "$PASSED" \ + # --argjson failed "$FAILED" \ + # --argjson pending "$PENDING" \ + # --argjson skipped "$SKIPPED" \ + # --arg link "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + # '{ + # CSI: $csi, + # Date: $date, + # StartTime: $startTime, + # Branch: $branch, + # Status: $status, + # Passed: $passed, + # Failed: $failed, + # Pending: $pending, + # Skipped: $skipped, + # Link: $link + # }' + # ) + + # echo "$SUMMARY" + # echo "summary=$(echo "$SUMMARY" | jq -c .)" >> $GITHUB_OUTPUT + # echo $SUMMARY > "${summary_file_name_json}" + + # echo "[INFO] Exit code: $GINKGO_EXIT_CODE" + # exit $GINKGO_EXIT_CODE + exit 1 - name: Upload summary test results (junit/xml) uses: actions/upload-artifact@v4 id: e2e-report-artifact @@ -1338,10 +1441,11 @@ jobs: runs-on: ubuntu-latest needs: - bootstrap + - configure-sdn - configure-storage - configure-virtualization - e2e-test - if: cancelled() || success() + if: (cancelled() || success()) && (needs.configure-sdn.result == 'success') steps: - uses: actions/checkout@v4 diff --git a/test/dvp-static-cluster/Taskfile.yaml b/test/dvp-static-cluster/Taskfile.yaml index f239956672..11759546e3 100644 --- a/test/dvp-static-cluster/Taskfile.yaml +++ b/test/dvp-static-cluster/Taskfile.yaml @@ -140,7 +140,7 @@ tasks: desc: Add proxy if enabled cmds: - | - if yq eval '.deckhouse.proxyEnabled' values.yaml; then + if [[ $(yq eval '.deckhouse.proxyEnabled' values.yaml) == true ]]; then yq eval --inplace '.proxy.httpProxy = env(HTTP_PROXY) | .proxy.httpsProxy = env(HTTPS_PROXY)' values.yaml fi diff --git a/test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml b/test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml index 191afcd96e..7a90f6c189 100644 --- a/test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml +++ b/test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml @@ -4,6 +4,9 @@ {{- $totalNodes = add $totalNodes .count -}} {{- end -}} +{{- $masterCount := $.Values.instances.masterNodes.count | int -}} +{{- if gt $masterCount 1 -}} + {{- $staticCount := sub $masterCount 1 -}} --- apiVersion: deckhouse.io/v1 kind: NodeGroup @@ -24,21 +27,17 @@ spec: node-role.kubernetes.io/master: "" nodeType: Static staticInstances: - count: {{ .Values.instances.masterNodes.count }} + count: {{ $staticCount }} labelSelector: matchLabels: role: master -{{- range $_, $i := untilStep 0 (.Values.instances.masterNodes.count | int) 1}} +{{- range $_, $i := untilStep 1 $masterCount 1}} {{- $vmName := printf "%s-master-%d" $.Values.storageType $i }} --- apiVersion: deckhouse.io/v1alpha1 kind: StaticInstance metadata: - {{- if eq $i 0 }} - annotations: - static.node.deckhouse.io/skip-bootstrap-phase: "" - {{- end }} name: {{ $vmName }} labels: role: master @@ -48,3 +47,4 @@ spec: kind: SSHCredentials name: mvp-static {{- end }} +{{- end }} diff --git a/test/dvp-static-cluster/charts/infra/templates/_helpers.tpl b/test/dvp-static-cluster/charts/infra/templates/_helpers.tpl index 4a5e99da43..34de58e72f 100644 --- a/test/dvp-static-cluster/charts/infra/templates/_helpers.tpl +++ b/test/dvp-static-cluster/charts/infra/templates/_helpers.tpl @@ -33,7 +33,10 @@ spec: - kind: VirtualDisk name: {{ printf "%s-%d" $name $i }} {{- end }} -{{- end }} + networks: + - type: Main + - type: ClusterNetwork + name: cn-4006-for-e2e-test bootloader: {{ $ctx.Values.image.bootloader }} liveMigrationPolicy: PreferForced cpu: @@ -51,6 +54,15 @@ spec: #cloud-config ssh_pwauth: true package_update: true + network: + version: 2 + ethernets: + eno2: + dhcp4: false + dhcp6: false + addresses: [] + link-local: [ipv6] + optional: false packages: - qemu-guest-agent - jq @@ -108,4 +120,5 @@ spec: {{- end }} {{- end }} {{- end }} +{{- end }} {{- end }} \ No newline at end of file