Skip to content

OCPCLOUD-3355: e2e testing automation: Implement Status Syncing for MAPI/CAPI migration #387

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sunzhaohua2 wants to merge 4 commits into
base: main
Choose a base branch
from
Open

OCPCLOUD-3355: e2e testing automation: Implement Status Syncing for MAPI/CAPI migration #387

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3e11483
machineset/machine status conversion e2e
sunzhaohua2 Oct 14, 2025
0f93401
add GinkgoHelper() to all helper functions
sunzhaohua2 Jan 7, 2026
8fd1b29
increase timeout
sunzhaohua2 Jan 23, 2026
bb475a5
Add skipWait helpers for parallel MachineSet creation
sunzhaohua2 Feb 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ unit: .localtestenv ## Run unit tests

.PHONY: e2e
e2e: ## Run e2e tests against active kubeconfig
./hack/test.sh "./e2e/..." 120m
./hack/test.sh "./e2e/..." 180m

run: ## Run the operator against the configured Kubernetes cluster
oc -n openshift-cluster-api patch lease cluster-capi-operator-leader -p '{"spec":{"acquireTime": null, "holderIdentity": null, "renewTime": null}}' --type=merge
Expand Down
29 changes: 29 additions & 0 deletions e2e/machine_migration_helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@ import (
clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest/komega"
yaml "sigs.k8s.io/yaml"
)

func createCAPIMachine(ctx context.Context, cl client.Client, machineName string) *clusterv1.Machine {
GinkgoHelper()
Expect(machineName).NotTo(BeEmpty(), "Machine name cannot be empty")

workerLabelSelector := metav1.LabelSelector{
Expand Down Expand Up @@ -96,6 +98,7 @@ func createCAPIMachine(ctx context.Context, cl client.Client, machineName string
}

func createMAPIMachineWithAuthority(ctx context.Context, cl client.Client, machineName string, authority mapiv1beta1.MachineAuthority) *mapiv1beta1.Machine {
GinkgoHelper()
Expect(machineName).NotTo(BeEmpty(), "Machine name cannot be empty")
workerLabelSelector := metav1.LabelSelector{
MatchLabels: map[string]string{
Expand Down Expand Up @@ -140,6 +143,7 @@ func createMAPIMachineWithAuthority(ctx context.Context, cl client.Client, machi

// verifyMachineRunning verifies that a machine reaches Running state using the machine object directly.
func verifyMachineRunning(cl client.Client, machine client.Object) {
GinkgoHelper()
Expect(machine).NotTo(BeNil(), "Machine parameter cannot be nil")
Expect(machine.GetName()).NotTo(BeEmpty(), "Machine name cannot be empty")
Eventually(func(g Gomega) string {
Expand All @@ -162,6 +166,7 @@ func verifyMachineRunning(cl client.Client, machine client.Object) {
}

func verifyMachineAuthoritative(mapiMachine *mapiv1beta1.Machine, authority mapiv1beta1.MachineAuthority) {
GinkgoHelper()
By(fmt.Sprintf("Verify the Machine authority is %s", authority))
Eventually(komega.Object(mapiMachine), capiframework.WaitMedium, capiframework.RetryMedium).Should(
HaveField("Status.AuthoritativeAPI", Equal(authority)),
Expand All @@ -170,6 +175,7 @@ func verifyMachineAuthoritative(mapiMachine *mapiv1beta1.Machine, authority mapi
}

func verifyMAPIMachineSynchronizedCondition(mapiMachine *mapiv1beta1.Machine, authority mapiv1beta1.MachineAuthority) {
GinkgoHelper()
By("Verify the MAPI Machine synchronized condition is True")
var expectedMessage string
switch authority {
Expand Down Expand Up @@ -202,6 +208,7 @@ func verifyMAPIMachineSynchronizedCondition(mapiMachine *mapiv1beta1.Machine, au
// verifyResourceRemoved verifies that a resource has been removed.
// This is a generic function that works with any client.Object type.
func verifyResourceRemoved(resource client.Object) {
GinkgoHelper()
Expect(resource).NotTo(BeNil(), "Resource parameter cannot be nil")
Expect(resource.GetName()).NotTo(BeEmpty(), "Resource name cannot be empty")
gvk := resource.GetObjectKind().GroupVersionKind()
Expand All @@ -214,6 +221,7 @@ func verifyResourceRemoved(resource client.Object) {
// verifyMachinePausedCondition verifies the Paused condition for either MAPI or CAPI machines.
// This unified function determines the machine type and expected pause state based on the authority.
func verifyMachinePausedCondition(machine client.Object, authority mapiv1beta1.MachineAuthority) {
GinkgoHelper()
Expect(machine).NotTo(BeNil(), "Machine parameter cannot be nil")
Expect(machine.GetName()).NotTo(BeEmpty(), "Machine name cannot be empty")
var conditionMatcher types.GomegaMatcher
Expand Down Expand Up @@ -298,12 +306,14 @@ func cleanupMachineResources(ctx context.Context, cl client.Client, capiMachines
}

func updateMachineAuthoritativeAPI(mapiMachine *mapiv1beta1.Machine, newAuthority mapiv1beta1.MachineAuthority) {
GinkgoHelper()
Eventually(komega.Update(mapiMachine, func() {
mapiMachine.Spec.AuthoritativeAPI = newAuthority
}), capiframework.WaitShort, capiframework.RetryShort).Should(Succeed(), "Failed to update MAPI Machine AuthoritativeAPI to %s", newAuthority)
}

func verifyMachineSynchronizedGeneration(cl client.Client, mapiMachine *mapiv1beta1.Machine, authority mapiv1beta1.MachineAuthority) {
GinkgoHelper()
Eventually(komega.Object(mapiMachine), capiframework.WaitMedium, capiframework.RetryMedium).Should(
HaveField("Status.SynchronizedGeneration", Not(BeZero())),
"MAPI Machine SynchronizedGeneration should not be zero",
Expand Down Expand Up @@ -333,3 +343,22 @@ func verifyMachineSynchronizedGeneration(cl client.Client, mapiMachine *mapiv1be
fmt.Sprintf("MAPI Machine SynchronizedGeneration should equal %s Machine Generation (%d)", authoritativeMachineType, expectedGeneration),
)
}

// verifyMAPIMachineProviderStatus verifies that a MAPI Machine's providerStatus matches the given Gomega matcher.
func verifyMAPIMachineProviderStatus(mapiMachine *mapiv1beta1.Machine, matcher types.GomegaMatcher) {
Copy link
Copy Markdown
Contributor

@nrb nrb Jan 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I haven't looked at the rest of the file, but we should probably start these functions with GinkgoHelper() See https://onsi.github.io/ginkgo/#mental-model-how-ginkgo-handles-failure, which explains how Ginkgo uses helpers when handling failed assertions.

Copy link
Copy Markdown
Contributor Author

@sunzhaohua2 sunzhaohua2 Jan 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you @nrb for helping review, I have added.

GinkgoHelper()
By(fmt.Sprintf("Verifying MAPI Machine %s providerStatus matches AWSMachine status", mapiMachine.Name))
Eventually(komega.Object(mapiMachine), capiframework.WaitMedium, capiframework.RetryMedium).Should(
WithTransform(getAWSProviderStatusFromMachine, matcher),
)
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

// getAWSProviderStatusFromMachine extracts and unmarshals the AWSMachineProviderStatus from a MAPI Machine.
func getAWSProviderStatusFromMachine(mapiMachine *mapiv1beta1.Machine) *mapiv1beta1.AWSMachineProviderStatus {
Copy link
Copy Markdown
Contributor

@nrb nrb Jan 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same comment for GinkgoHelper()

Expect(mapiMachine.Status.ProviderStatus.Raw).ToNot(BeNil())

providerStatus := &mapiv1beta1.AWSMachineProviderStatus{}
Expect(yaml.Unmarshal(mapiMachine.Status.ProviderStatus.Raw, providerStatus)).To(Succeed())

return providerStatus
}
Comment thread
sunzhaohua2 marked this conversation as resolved.
60 changes: 59 additions & 1 deletion e2e/machineset_migration_helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,14 @@ import (

// createCAPIMachineSet creates a CAPI MachineSet with an AWSMachineTemplate and waits for it to be ready.
func createCAPIMachineSet(ctx context.Context, cl client.Client, replicas int32, machineSetName string, instanceType string) *clusterv1.MachineSet {
return createCAPIMachineSetSkipWait(ctx, cl, replicas, machineSetName, instanceType, false)
}

// createCAPIMachineSetSkipWait creates a CAPI MachineSet with an AWSMachineTemplate.
// If skipWait is true, it skips waiting for the MachineSet to be ready.
// This allows for parallel creation of multiple MachineSets to reduce total test time.
func createCAPIMachineSetSkipWait(ctx context.Context, cl client.Client, replicas int32, machineSetName string, instanceType string, skipWait bool) *clusterv1.MachineSet {
GinkgoHelper()
By(fmt.Sprintf("Creating CAPI MachineSet %s with %d replicas", machineSetName, replicas))
_, mapiDefaultProviderSpec := getDefaultAWSMAPIProviderSpec()
createAWSClient(mapiDefaultProviderSpec.Placement.Region)
Expand All @@ -47,12 +55,22 @@ func createCAPIMachineSet(ctx context.Context, cl client.Client, replicas int32,
"worker-user-data",
))

capiframework.WaitForMachineSet(cl, machineSet.Name, machineSet.Namespace)
if !skipWait {
capiframework.WaitForMachineSet(cl, machineSet.Name, machineSet.Namespace)
}
return machineSet
}

// createMAPIMachineSetWithAuthoritativeAPI creates a MAPI MachineSet with specified authoritativeAPI and waits for the CAPI mirror to be created.
func createMAPIMachineSetWithAuthoritativeAPI(ctx context.Context, cl client.Client, replicas int, machineSetName string, machineSetAuthority mapiv1beta1.MachineAuthority, machineAuthority mapiv1beta1.MachineAuthority) *mapiv1beta1.MachineSet {
return createMAPIMachineSetWithAuthoritativeAPISkipWait(ctx, cl, replicas, machineSetName, machineSetAuthority, machineAuthority, false)
}

// createMAPIMachineSetWithAuthoritativeAPISkipWait creates a MAPI MachineSet with specified authoritativeAPI.
// If skipWait is true, it skips waiting for the CAPI mirror creation and Machine running state.
// This allows for parallel creation of multiple MachineSets to reduce total test time.
func createMAPIMachineSetWithAuthoritativeAPISkipWait(ctx context.Context, cl client.Client, replicas int, machineSetName string, machineSetAuthority mapiv1beta1.MachineAuthority, machineAuthority mapiv1beta1.MachineAuthority, skipWait bool) *mapiv1beta1.MachineSet {
GinkgoHelper()
By(fmt.Sprintf("Creating MAPI MachineSet with spec.authoritativeAPI: %s, spec.template.spec.authoritativeAPI: %s, replicas=%d", machineSetAuthority, machineAuthority, replicas))
machineSetParams := mapiframework.BuildMachineSetParams(ctx, cl, replicas)
machineSetParams.Name = machineSetName
Expand All @@ -64,6 +82,10 @@ func createMAPIMachineSetWithAuthoritativeAPI(ctx context.Context, cl client.Cli
mapiMachineSet, err := mapiframework.CreateMachineSet(cl, machineSetParams)
Expect(err).ToNot(HaveOccurred(), "MAPI MachineSet %s creation should succeed", machineSetName)

if skipWait {
return mapiMachineSet
}

capiMachineSet := &clusterv1.MachineSet{
ObjectMeta: metav1.ObjectMeta{
Name: machineSetName,
Expand All @@ -82,8 +104,31 @@ func createMAPIMachineSetWithAuthoritativeAPI(ctx context.Context, cl client.Cli
return mapiMachineSet
}

// waitForMAPIMachineSetReady waits for a MAPI MachineSet's CAPI mirror and Machines to be ready.
func waitForMAPIMachineSetReady(ctx context.Context, cl client.Client, machineSetName string, machineAuthority mapiv1beta1.MachineAuthority) {
GinkgoHelper()
By(fmt.Sprintf("Waiting for MachineSet %s to be ready", machineSetName))

capiMachineSet := &clusterv1.MachineSet{
ObjectMeta: metav1.ObjectMeta{
Name: machineSetName,
Namespace: capiframework.CAPINamespace,
},
}
Eventually(komega.Get(capiMachineSet), capiframework.WaitShort, capiframework.RetryShort).Should(
Succeed(), "Should have mirror CAPI MachineSet created within 1 minute")

switch machineAuthority {
case mapiv1beta1.MachineAuthorityMachineAPI:
mapiframework.WaitForMachineSet(ctx, cl, machineSetName)
case mapiv1beta1.MachineAuthorityClusterAPI:
capiframework.WaitForMachineSet(cl, machineSetName, capiframework.CAPINamespace)
}
}

// switchMachineSetAuthoritativeAPI updates the authoritativeAPI fields of a MAPI MachineSet.
func switchMachineSetAuthoritativeAPI(mapiMachineSet *mapiv1beta1.MachineSet, machineSetAuthority mapiv1beta1.MachineAuthority) {
GinkgoHelper()
By(fmt.Sprintf("Switching MachineSet %s AuthoritativeAPI to spec.authoritativeAPI: %s", mapiMachineSet.Name, machineSetAuthority))
Eventually(komega.Update(mapiMachineSet, func() {
mapiMachineSet.Spec.AuthoritativeAPI = machineSetAuthority
Expand All @@ -92,6 +137,7 @@ func switchMachineSetAuthoritativeAPI(mapiMachineSet *mapiv1beta1.MachineSet, ma

// switchMachineSetTemplateAuthoritativeAPI updates the authoritativeAPI fields of a MAPI MachineSet's template.
func switchMachineSetTemplateAuthoritativeAPI(mapiMachineSet *mapiv1beta1.MachineSet, machineAuthority mapiv1beta1.MachineAuthority) {
GinkgoHelper()
By(fmt.Sprintf("Switching MachineSet %s spec.template.spec.authoritativeAPI: %s", mapiMachineSet.Name, machineAuthority))
Eventually(komega.Update(mapiMachineSet, func() {
mapiMachineSet.Spec.Template.Spec.AuthoritativeAPI = machineAuthority
Expand All @@ -100,6 +146,7 @@ func switchMachineSetTemplateAuthoritativeAPI(mapiMachineSet *mapiv1beta1.Machin

// verifyMachineSetAuthoritative verifies that a MAPI MachineSet's status.authoritativeAPI matches the expected authority.
func verifyMachineSetAuthoritative(mapiMachineSet *mapiv1beta1.MachineSet, authority mapiv1beta1.MachineAuthority) {
GinkgoHelper()
By(fmt.Sprintf("Verifying the MachineSet authoritative is %s", authority))
Eventually(komega.Object(mapiMachineSet), capiframework.WaitMedium, capiframework.RetryMedium).Should(
HaveField("Status.AuthoritativeAPI", Equal(authority)),
Expand All @@ -109,6 +156,7 @@ func verifyMachineSetAuthoritative(mapiMachineSet *mapiv1beta1.MachineSet, autho

// verifyMachineSetPausedCondition verifies the Paused condition of a MachineSet (MAPI or CAPI) based on its authoritative API.
func verifyMachineSetPausedCondition(machineSet client.Object, authority mapiv1beta1.MachineAuthority) {
GinkgoHelper()
Expect(machineSet).NotTo(BeNil(), "MachineSet parameter cannot be nil")
Expect(machineSet.GetName()).NotTo(BeEmpty(), "MachineSet name cannot be empty")
var conditionMatcher types.GomegaMatcher
Expand Down Expand Up @@ -175,6 +223,7 @@ func verifyMachineSetPausedCondition(machineSet client.Object, authority mapiv1b

// verifyMachinesetReplicas verifies that a MachineSet (MAPI or CAPI) has the expected number of replicas in its status.
func verifyMachinesetReplicas(machineSet client.Object, replicas int) {
GinkgoHelper()
Expect(machineSet).NotTo(BeNil(), "Machine parameter cannot be nil")
Expect(machineSet.GetName()).NotTo(BeEmpty(), "Machine name cannot be empty")
switch ms := machineSet.(type) {
Expand All @@ -195,6 +244,7 @@ func verifyMachinesetReplicas(machineSet client.Object, replicas int) {

// verifyMAPIMachineSetSynchronizedCondition verifies that a MAPI MachineSet has the Synchronized condition set to True with the correct message.
func verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet *mapiv1beta1.MachineSet, authority mapiv1beta1.MachineAuthority) {
GinkgoHelper()
By("Verifying the MAPI MachineSet Synchronized condition is True")
var expectedMessage string

Expand Down Expand Up @@ -227,6 +277,7 @@ func verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet *mapiv1beta1.Machi

// verifyMAPIMachineSetProviderSpec verifies that a MAPI MachineSet's providerSpec matches the given Gomega matcher.
func verifyMAPIMachineSetProviderSpec(mapiMachineSet *mapiv1beta1.MachineSet, matcher types.GomegaMatcher) {
GinkgoHelper()
By(fmt.Sprintf("Verifying MAPI MachineSet %s ProviderSpec", mapiMachineSet.Name))
Eventually(komega.Object(mapiMachineSet), capiframework.WaitMedium, capiframework.RetryShort).Should(
WithTransform(getAWSProviderSpecFromMachineSet, matcher),
Expand All @@ -235,6 +286,7 @@ func verifyMAPIMachineSetProviderSpec(mapiMachineSet *mapiv1beta1.MachineSet, ma

// getAWSProviderSpecFromMachineSet extracts and unmarshals the AWSMachineProviderConfig from a MAPI MachineSet.
func getAWSProviderSpecFromMachineSet(mapiMachineSet *mapiv1beta1.MachineSet) *mapiv1beta1.AWSMachineProviderConfig {
GinkgoHelper()
Expect(mapiMachineSet.Spec.Template.Spec.ProviderSpec.Value).ToNot(BeNil())

providerSpec := &mapiv1beta1.AWSMachineProviderConfig{}
Expand All @@ -245,6 +297,7 @@ func getAWSProviderSpecFromMachineSet(mapiMachineSet *mapiv1beta1.MachineSet) *m

// updateAWSMachineSetProviderSpec updates a MAPI MachineSet's AWS providerSpec using the provided update function.
func updateAWSMachineSetProviderSpec(ctx context.Context, cl client.Client, mapiMachineSet *mapiv1beta1.MachineSet, updateFunc func(*mapiv1beta1.AWSMachineProviderConfig)) error {
GinkgoHelper()
By(fmt.Sprintf("Updating MachineSet %s providerSpec", mapiMachineSet.Name))
providerSpec := getAWSProviderSpecFromMachineSet(mapiMachineSet)

Expand All @@ -263,6 +316,7 @@ func updateAWSMachineSetProviderSpec(ctx context.Context, cl client.Client, mapi

// waitForMAPIMachineSetMirrors waits for the corresponding CAPI MachineSet and AWSMachineTemplate mirrors to be created for a MAPI MachineSet.
func waitForMAPIMachineSetMirrors(cl client.Client, machineSetNameMAPI string) (*clusterv1.MachineSet, *awsv1.AWSMachineTemplate) {
GinkgoHelper()
By(fmt.Sprintf("Verifying there is a CAPI MachineSet mirror and AWSMachineTemplate for MAPI MachineSet %s", machineSetNameMAPI))
var err error
var capiMachineSet *clusterv1.MachineSet
Expand All @@ -286,6 +340,7 @@ func waitForMAPIMachineSetMirrors(cl client.Client, machineSetNameMAPI string) (

// waitForCAPIMachineSetMirror waits for a CAPI MachineSet mirror to be created for a MAPI MachineSet.
func waitForCAPIMachineSetMirror(cl client.Client, machineName string) *clusterv1.MachineSet {
GinkgoHelper()
By(fmt.Sprintf("Verifying there is a CAPI MachineSet mirror for MAPI MachineSet %s", machineName))
var capiMachineSet *clusterv1.MachineSet
Eventually(func() error {
Expand All @@ -300,6 +355,7 @@ func waitForCAPIMachineSetMirror(cl client.Client, machineName string) *clusterv

// waitForAWSMachineTemplate waits for an AWSMachineTemplate with the specified name prefix to be created.
func waitForAWSMachineTemplate(cl client.Client, prefix string) *awsv1.AWSMachineTemplate {
GinkgoHelper()
By(fmt.Sprintf("Verifying there is an AWSMachineTemplate with prefix %s", prefix))
var awsMachineTemplate *awsv1.AWSMachineTemplate
Eventually(func() error {
Expand All @@ -316,6 +372,7 @@ func waitForAWSMachineTemplate(cl client.Client, prefix string) *awsv1.AWSMachin

// createAWSMachineTemplate creates a new AWSMachineTemplate with an optional update function to modify the spec.
func createAWSMachineTemplate(ctx context.Context, cl client.Client, originalName string, updateFunc func(*awsv1.AWSMachineSpec)) *awsv1.AWSMachineTemplate {
GinkgoHelper()
By("Creating a new awsMachineTemplate")
_, mapiDefaultProviderSpec := getDefaultAWSMAPIProviderSpec()
createAWSClient(mapiDefaultProviderSpec.Placement.Region)
Expand All @@ -335,6 +392,7 @@ func createAWSMachineTemplate(ctx context.Context, cl client.Client, originalNam

// updateCAPIMachineSetInfraTemplate updates a CAPI MachineSet's infrastructureRef to point to a new template.
func updateCAPIMachineSetInfraTemplate(capiMachineSet *clusterv1.MachineSet, newInfraTemplateName string) {
GinkgoHelper()
By(fmt.Sprintf("Updating CAPI MachineSet %s to point to new InfraTemplate %s", capiMachineSet.Name, newInfraTemplateName))
Eventually(komega.Update(capiMachineSet, func() {
capiMachineSet.Spec.Template.Spec.InfrastructureRef.Name = newInfraTemplateName
Expand Down
9 changes: 9 additions & 0 deletions e2e/migration_common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package e2e

import (
"fmt"
"time"

mapiv1beta1 "github.com/openshift/api/machine/v1beta1"
clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
)
Expand All @@ -13,3 +16,9 @@ const (
// CAPIPausedCondition represents the paused state for CAPI machines.
CAPIPausedCondition = clusterv1.PausedCondition
)

// UniqueName generates a unique name with the given prefix using nanosecond timestamp.
// This is useful for creating resources in parallel tests without naming conflicts.
func UniqueName(prefix string) string {
return fmt.Sprintf("%s-%d", prefix, time.Now().UnixNano())
}
Loading