USHIFT-6810: Add C2CC upgrade test for RHEL 9.8 to RHEL 10.2

test/bin/c2cc_common.sh

@@ -172,6 +172,16 @@ c2cc_run_tests() {
         ip_family_var="--variable IP_FAMILY:${ip_family}"
     fi
 
+    local target_ref_var=""
+    if [ -n "${C2CC_TARGET_REF:-}" ]; then
+        target_ref_var="--variable TARGET_REF:${C2CC_TARGET_REF}"
+    fi
+
+    local bootc_registry_var=""
+    if [ -n "${MIRROR_REGISTRY_URL:-}" ]; then
+        bootc_registry_var="--variable BOOTC_REGISTRY:${MIRROR_REGISTRY_URL}"
+    fi
+
     local host2_ip host3_ip
     host2_ip=$(get_host_ip host2) || return 1
     host3_ip=$(get_host_ip host3) || return 1
@@ -207,6 +217,8 @@ c2cc_run_tests() {
         --variable "KUBECONFIG_C:${kubeconfig_c}" \
         ${foreign_cidr_var} \
         ${ip_family_var} \
+        ${target_ref_var} \
+        ${bootc_registry_var} \
         "${suites_dir}"
 }
 

test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile

@@ -27,4 +27,3 @@ RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \
     rm -vf /etc/yum.repos.d/microshift-*.repo && \
     rm -rvf $USHIFT_RPM_REPO_PATH && \
     dnf clean all
-

test/image-blueprints-bootc/el10/layer2-presubmit/group1/rhel102-bootc-source.containerfile

@@ -43,3 +43,10 @@ RUN firewall-offline-cmd --zone=public --add-port=22/tcp && \
 # when upgrading from older ostree commits to bootc container layers
 RUN mkdir -p /usr/lib/systemd/system/ovsdb-server.service.d
 COPY --chmod=644 ./bootc-images/microshift-ovsdb-ownership.conf /usr/lib/systemd/system/ovsdb-server.service.d/microshift-ovsdb-ownership.conf
+
+# Fix SSH host key permissions for cross-version upgrades (RHEL 9 uses 0640
+# with ssh_keys group, RHEL 10 sshd requires 0600)
+# Similar workaround is used in microshift-test-agent.sh
+RUN mkdir -p /usr/lib/systemd/system/sshd.service.d && \
+    printf '[Service]\nExecStartPre=/bin/bash -c "chmod 600 /etc/ssh/ssh_host_*_key 2>/dev/null || true"\n' \
+    > /usr/lib/systemd/system/sshd.service.d/fix-hostkey-perms.conf

test/image-blueprints-bootc/templates/rpm-repo-config.sh.template

@@ -12,6 +12,16 @@ function enable_eus_repositories() {
         "rhel-${vmajor}-for-$(uname -m)-appstream-eus-rpms"
 }
 
+function enable_ga_repositories() {
+    local -r vmajor="$(awk -F. '{print $1}' <<< "${VERSION_ID}")"
+
+    dnf config-manager --set-disabled '*'
+    dnf config-manager --set-enabled \
+        "rhel-${vmajor}-for-$(uname -m)-baseos-rpms" \
+        "rhel-${vmajor}-for-$(uname -m)-appstream-rpms"
+}
+
+
 # Lock the OS release version to prevent the installation of packages belonging
 # to a newer OS release
 source /etc/os-release
@@ -24,6 +34,10 @@ while [ $# -gt 0 ] ; do
         enable_eus_repositories
         shift
         ;;
+    --enable-ga)
+        enable_ga_repositories
+        shift
+        ;;
     --disable-all)
         dnf config-manager --set-disabled '*'
         shift

test/resources/c2cc.resource

@@ -77,7 +77,7 @@ Teardown All Remote Clusters
     END
     VAR    @{C2CC_REMOTE_ALIASES}=    @{EMPTY}    scope=SUITE
     ${local_conn}=    Get From Dictionary    ${C2CC_SSH_IDS}    cluster-a
-    SSHLibrary.Switch Connection    ${local_conn}
+    ${status}=    Run Keyword And Return Status    SSHLibrary.Switch Connection    ${local_conn}
 
 Command On Cluster
     [Documentation]    Run a shell command on the specified cluster via SSH.
@@ -384,6 +384,26 @@ DNS Lookup Should Succeed
     ...    oc exec curl-pod -n ${NAMESPACES}[${alias}] -- getent hosts ${fqdn}
     Should Not Be Empty    ${stdout}
 
+Verify RemoteCluster State
+    [Documentation]    Check that all RemoteCluster CRs on this cluster have the expected state.
+    [Arguments]    ${alias}    ${expected_state}
+    ${stdout}=    Oc On Cluster    ${alias}
+    ...    oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.state}'
+    Should Not Be Empty    ${stdout}
+    @{states}=    Split String    ${stdout}
+    ${count}=    Get Length    ${states}
+    Should Be Equal As Integers    ${count}    2    Expected 2 RemoteCluster states, got ${count}
+    FOR    ${state}    IN    @{states}
+        Should Be Equal As Strings    ${state}    ${expected_state}
+    END
+
+Verify All RemoteClusters Healthy
+    [Documentation]    Wait for all RemoteCluster CRs on all clusters to report Healthy.
+    FOR    ${alias}    IN    cluster-a    cluster-b    cluster-c
+        Wait Until Keyword Succeeds    3m    10s
+        ...    Verify RemoteCluster State    ${alias}    Healthy
+    END
+
 Curl DNS From Cluster
     [Documentation]    Curl a service by DNS name from curl-pod on the given cluster.
     [Arguments]    ${alias}    ${fqdn}    ${port}

test/scenarios-bootc/c2cc/el98-src@el102-src@c2cc-upgrade-ok.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Sourced from scenario.sh and uses functions defined there.
+
+# shellcheck source=test/bin/c2cc_common.sh
+source "${SCRIPTDIR}/c2cc_common.sh"
+
+export TEST_RANDOMIZATION=none
+export TEST_EXECUTION_TIMEOUT=60m
+
+C2CC_TARGET_REF=rhel102-bootc-source
+
+scenario_create_vms() {
+    c2cc_create_vms rhel98-bootc-source rhel98-bootc
+}
+
+scenario_remove_vms() {
+    c2cc_remove_vms
+}
+
+scenario_run_tests() {
+    # shellcheck disable=SC2119
+    configure_c2cc_hosts
+    c2cc_run_tests "suites/upgrade/upgrade-c2cc.robot"
+}

test/scenarios-bootc/el10/presubmits/el98-src@el102-src@c2cc-upgrade-ok.sh

@@ -0,0 +1 @@
+../../c2cc/el98-src@el102-src@c2cc-upgrade-ok.sh

test/suites/c2cc/probe.robot

@@ -151,19 +151,6 @@ Verify Probe Service ClusterIP
     ...    oc get service ${PROBE_DEPLOYMENT} -n ${C2CC_NAMESPACE} -o jsonpath='{.spec.clusterIP}'
     Should Be Equal As Strings    ${actual_ip}    ${expected_ip}    strip_spaces=True
 
-Verify RemoteCluster State
-    [Documentation]    Check that all RemoteCluster CRs on this cluster have the expected state.
-    [Arguments]    ${alias}    ${expected_state}
-    ${stdout}=    Oc On Cluster    ${alias}
-    ...    oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.state}'
-    Should Not Be Empty    ${stdout}
-    @{states}=    Split String    ${stdout}
-    ${count}=    Get Length    ${states}
-    Should Be Equal As Integers    ${count}    2    Expected 2 RemoteCluster states, got ${count}
-    FOR    ${state}    IN    @{states}
-        Should Be Equal As Strings    ${state}    ${expected_state}
-    END
-
 Verify RemoteCluster State By Name
     [Documentation]    Check that a specific RemoteCluster CR has the expected state.
     [Arguments]    ${alias}    ${cr_name}    ${expected_state}
@@ -189,10 +176,7 @@ RemoteCluster CR Name From IP
 
 Ensure All Clusters Healthy
     [Documentation]    Pre-condition: all clusters must be Healthy before fault injection.
-    FOR    ${alias}    IN    cluster-a    cluster-b    cluster-c
-        Wait Until Keyword Succeeds    3m    10s
-        ...    Verify RemoteCluster State    ${alias}    Healthy
-    END
+    Verify All RemoteClusters Healthy
 
 Apply Probe Deny Policy
     [Documentation]    Apply a NetworkPolicy that denies all ingress to the probe pod.

test/suites/upgrade/upgrade-c2cc.robot

@@ -0,0 +1,169 @@
+*** Settings ***
+Documentation       Tests RHEL 9.8 to RHEL 10.2 upgrade with C2CC enabled across 3 clusters.
+...                 Upgrades each cluster one by one and verifies C2CC connectivity
+...                 survives at each stage.
+
+Resource            ../../resources/common.resource
+Resource            ../../resources/c2cc.resource
+Resource            ../../resources/microshift-host.resource
+Library             Collections
+Library             SSHLibrary
+
+Suite Setup         Setup
+Suite Teardown      Teardown
+
+Test Tags           c2cc    ostree
+
+
+*** Variables ***
+${TARGET_REF}               ${EMPTY}
+${BOOTC_REGISTRY}           ${EMPTY}
+${CLUSTER_A_POD_CIDR}       ${EMPTY}
+${CLUSTER_A_SVC_CIDR}       ${EMPTY}
+${CLUSTER_A_DOMAIN}         ${EMPTY}
+${CLUSTER_B_POD_CIDR}       ${EMPTY}
+${CLUSTER_B_SVC_CIDR}       ${EMPTY}
+${CLUSTER_B_DOMAIN}         ${EMPTY}
+${KUBECONFIG_B}             ${EMPTY}
+${CLUSTER_C_POD_CIDR}       ${EMPTY}
+${CLUSTER_C_SVC_CIDR}       ${EMPTY}
+${CLUSTER_C_DOMAIN}         ${EMPTY}
+${KUBECONFIG_C}             ${EMPTY}
+
+
+*** Test Cases ***
+Upgrade C2CC Clusters From RHEL9 To RHEL10
+    [Documentation]    Upgrades 3 C2CC-connected clusters one by one from RHEL 9.8
+    ...    to RHEL 10.2 and verifies health and C2CC connectivity after each upgrade.
+
+    Verify All Clusters Healthy
+    Verify All RemoteClusters Healthy
+    Deploy Test Workloads
+    Verify Full C2CC Connectivity
+
+    FOR    ${alias}    IN    cluster-a    cluster-b    cluster-c
+        Log To Console    Upgrading ${alias} to ${TARGET_REF}
+        Upgrade Cluster    ${alias}
+        Verify All Clusters Healthy
+        Verify All RemoteClusters Healthy
+        Wait For Test Pods
+        Wait For Service Endpoints
+        Verify Full C2CC Connectivity
+    END
+
+    [Teardown]    Cleanup Test Workloads
+
+
+*** Keywords ***
+Setup
+    [Documentation]    Register all three clusters for SSH and oc access
+    ...    and store connection details for reconnection after reboots.
+    Check Required Env Variables
+    Should Not Be Empty    ${TARGET_REF}    TARGET_REF variable is required
+    Login MicroShift Host
+    Setup Kubeconfig
+    Logout MicroShift Host
+
+    Register Remote Cluster    cluster-a    ${USHIFT_HOST}    ${SSH_PORT}    ${KUBECONFIG}
+    Register Remote Cluster    cluster-b    ${HOST2_IP}    ${HOST2_SSH_PORT}    ${KUBECONFIG_B}
+    Register Remote Cluster    cluster-c    ${HOST3_IP}    ${HOST3_SSH_PORT}    ${KUBECONFIG_C}
+
+Teardown
+    [Documentation]    Close all connections and clean up.
+    Teardown All Remote Clusters
+    Remove Kubeconfig
+
+Verify All Clusters Healthy
+    [Documentation]    Verify all clusters are running and API server is reachable.
+    FOR    ${alias}    IN    cluster-a    cluster-b    cluster-c
+        ${stdout}=    Oc On Cluster    ${alias}    oc get --raw='/readyz'
+        Should Be Equal As Strings    ${stdout}    ok    strip_spaces=True
+    END
+
+Verify Full C2CC Connectivity
+    [Documentation]    Verify pod-to-pod and pod-to-service connectivity between all cluster pairs.
+    VAR    @{clusters}=    cluster-a    cluster-b    cluster-c
+    FOR    ${src}    IN    @{clusters}
+        FOR    ${dst}    IN    @{clusters}
+            IF    '${src}' != '${dst}'
+                Test Connectivity Between Clusters    ${src}    ${dst}    pod
+                Test Connectivity Between Clusters    ${src}    ${dst}    service
+            END
+        END
+    END
+
+Upgrade Cluster
+    [Documentation]    Upgrade a specific cluster to the target bootc image
+    ...    and verify it booted into the new deployment without rollback.
+    [Arguments]    ${alias}
+
+    ${initial_deploy_id}=    Get Deployment Id On Cluster    ${alias}
+
+    Command On Cluster
+    ...    ${alias}
+    ...    printf '[[registry]]\nlocation = "${BOOTC_REGISTRY}"\ninsecure = true\n' | sudo tee /etc/containers/registries.conf.d/999-microshift-insecure-registry.conf > /dev/null
+
+    Command On Cluster    ${alias}    bootc switch --quiet ${BOOTC_REGISTRY}/${TARGET_REF}
+
+    Command On Cluster    ${alias}
+    ...    rm -f /etc/containers/registries.conf.d/999-microshift-insecure-registry.conf
+
+    Reboot Cluster And Wait    ${alias}
+
+    ${current_deploy_id}=    Get Deployment Id On Cluster    ${alias}
+    Should Not Be Equal As Strings    ${current_deploy_id}    ${initial_deploy_id}
+    ...    msg=${alias} rolled back to initial deployment
+
+Get Deployment Id On Cluster
+    [Documentation]    Get the booted image digest from a specific cluster.
+    [Arguments]    ${alias}
+    ${stdout}=    Command On Cluster
+    ...    ${alias}
+    ...    bootc status --booted --json | python3 -c "import sys,json; print(json.load(sys.stdin)['status']['booted']['image']['imageDigest'])"
+    RETURN    ${stdout}
+
+Reboot Cluster And Wait
+    [Documentation]    Reboot a cluster and wait for it to come back with greenboot healthy.
+    [Arguments]    ${alias}
+
+    ${boot_id}=    Command On Cluster    ${alias}
+    ...    cat /proc/sys/kernel/random/boot_id    sudo_mode=False
+
+    Disruptive Command On Cluster    ${alias}    reboot
+
+    Wait Until Keyword Succeeds    10m    15s
+    ...    Cluster Rebooted And Healthy    ${alias}    ${boot_id}
+
+Cluster Rebooted And Healthy
+    [Documentation]    Verify cluster has rebooted and greenboot health check passed.
+    [Arguments]    ${alias}    ${old_boot_id}
+
+    ${old_conn_id}=    Get From Dictionary    ${C2CC_SSH_IDS}    ${alias}
+    ${status}=    Run Keyword And Return Status
+    ...    SSHLibrary.Switch Connection    ${old_conn_id}
+    IF    ${status}    SSHLibrary.Close Connection
+
+    ${host}    ${port}    ${kc}=    Get Cluster Connection Info    ${alias}
+    Remove Values From List    ${C2CC_REMOTE_ALIASES}    ${alias}
+    Register Remote Cluster    ${alias}    ${host}    ${port}    ${kc}
+
+    ${new_boot_id}=    Command On Cluster    ${alias}
+    ...    cat /proc/sys/kernel/random/boot_id    sudo_mode=False
+    Should Not Be Equal As Strings    ${old_boot_id}    ${new_boot_id}    strip_spaces=True
+
+    ${stdout}=    Command On Cluster    ${alias}
+    ...    systemctl show -p SubState greenboot-healthcheck.service --value
+    Should Be Equal As Strings    ${stdout}    exited    strip_spaces=True
+
+Get Cluster Connection Info
+    [Documentation]    Return host, port, and kubeconfig for a given cluster alias.
+    [Arguments]    ${alias}
+    IF    '${alias}' == 'cluster-a'
+        RETURN    ${USHIFT_HOST}    ${SSH_PORT}    ${KUBECONFIG}
+    ELSE IF    '${alias}' == 'cluster-b'
+        RETURN    ${HOST2_IP}    ${HOST2_SSH_PORT}    ${KUBECONFIG_B}
+    ELSE IF    '${alias}' == 'cluster-c'
+        RETURN    ${HOST3_IP}    ${HOST3_SSH_PORT}    ${KUBECONFIG_C}
+    ELSE
+        Fail    Unknown cluster alias: ${alias}
+    END