diff --git a/xset/api/opslifecycle_types.go b/xset/api/opslifecycle_types.go deleted file mode 100644 index 3452001..0000000 --- a/xset/api/opslifecycle_types.go +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package api - -import ( - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type OperationType string - -var ( - OpsLifecycleTypeUpdate OperationType = "update" - OpsLifecycleTypeScaleIn OperationType = "scale-in" - OpsLifecycleTypeDelete OperationType = "delete" -) - -// LifecycleAdapter helps CRD Operators to easily access TargetOpsLifecycle -type LifecycleAdapter interface { - // GetID indicates ID of one TargetOpsLifecycle - GetID() string - - // GetType indicates type for an Operator - GetType() OperationType - - // AllowMultiType indicates whether multiple IDs which have the same Type are allowed - AllowMultiType() bool - - // WhenBegin will be executed when begin a lifecycle - WhenBegin(target client.Object) (needUpdate bool, err error) - - // WhenFinish will be executed when finish a lifecycle - WhenFinish(target client.Object) (needUpdate bool, err error) -} diff --git a/xset/api/resourcecontext_types.go b/xset/api/resourcecontext_types.go deleted file mode 100644 index 2bf18a0..0000000 --- a/xset/api/resourcecontext_types.go +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package api - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// +k8s:deepcopy-gen=file - -type ResourceContextObject client.Object - -// ResourceContextAdapter is used to adapt the resource context api -type ResourceContextAdapter interface { - ResourceContextMeta() metav1.TypeMeta - GetResourceContextSpec(object ResourceContextObject) *ResourceContextSpec - SetResourceContextSpec(spec *ResourceContextSpec, object ResourceContextObject) - GetContextKeys() map[ResourceContextKeyEnum]string - NewResourceContext() ResourceContextObject -} - -// ResourceContextKeyEnum defines the key of resource context -type ResourceContextKeyEnum int - -const EnumContextKeyNum = 8 -const ( - EnumOwnerContextKey ResourceContextKeyEnum = iota - EnumRevisionContextDataKey - EnumTargetDecorationRevisionKey - EnumJustCreateContextDataKey - EnumRecreateUpdateContextDataKey - EnumScaleInContextDataKey - EnumReplaceNewTargetIDContextDataKey - EnumReplaceOriginTargetIDContextDataKey -) - -// ResourceContextSpec defines the desired state of ResourceContext -type ResourceContextSpec struct { - Contexts []ContextDetail `json:"contexts,omitempty"` -} - -// ContextDetail defines the details of target -type ContextDetail struct { - ID int `json:"id"` - Data map[string]string `json:"data,omitempty"` -} - -// Contains is used to check whether the key-value pair in contained in Data. -func (cd *ContextDetail) Contains(key, value string) bool { - if cd.Data == nil { - return false - } - - return cd.Data[key] == value -} - -// Put is used to specify the value with the specified key in Data . -func (cd *ContextDetail) Put(key, value string) { - if cd.Data == nil { - cd.Data = map[string]string{} - } - - cd.Data[key] = value -} - -// Get is used to get the specified key from Data. -func (cd *ContextDetail) Get(key string) (string, bool) { - if cd.Data == nil { - return "", false - } - val, ok := cd.Data[key] - return val, ok -} - -// Remove is used to remove the specified key from Data . -func (cd *ContextDetail) Remove(key string) { - if cd.Data == nil { - return - } - - delete(cd.Data, key) -} diff --git a/xset/api/validation/resourcecontext.go b/xset/api/validation/resourcecontext.go deleted file mode 100644 index fbfa0c4..0000000 --- a/xset/api/validation/resourcecontext.go +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package validation - -import ( - "errors" - - "kusionstack.io/kube-utils/xset/api" -) - -// ValidateResourceContextAdapter validates the resource context adapter -func ValidateResourceContextAdapter(adapter api.ResourceContextAdapter) error { - if adapter == nil { - return errors.New("resource context adapter is nil") - } - return errors.Join( - validateMeta(adapter.ResourceContextMeta()), - validateResourceContextKey(adapter.GetContextKeys()), - ) -} - -func validateResourceContextKey(m map[api.ResourceContextKeyEnum]string) error { - if m == nil { - return errors.New("resource context keys is nil") - } - - for i := range api.EnumContextKeyNum { - if _, ok := m[api.ResourceContextKeyEnum(i)]; !ok { - return errors.New("resource context keys are not valid, please add enough context keys") - } - } - return nil -} diff --git a/xset/api/validation/shared.go b/xset/api/validation/shared.go deleted file mode 100644 index 6fa1eb4..0000000 --- a/xset/api/validation/shared.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package validation - -import ( - "fmt" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func validateMeta(t metav1.TypeMeta) error { - if len(t.String()) == 0 { - return fmt.Errorf("type meta {%s} {%s} is inValid", t.APIVersion, t.Kind) - } - return nil -} diff --git a/xset/api/validation/xset_controller.go b/xset/api/validation/xset_controller.go deleted file mode 100644 index 60179e7..0000000 --- a/xset/api/validation/xset_controller.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package validation - -import ( - "errors" - "fmt" - - "k8s.io/apimachinery/pkg/util/validation" - - "kusionstack.io/kube-utils/xset/api" -) - -// ValidateXSetController validates the XSetController -func ValidateXSetController(xSetController api.XSetController) error { - if xSetController == nil { - return errors.New("xSetController is nil") - } - if _, ok := xSetController.(api.XSetOperation); !ok { - return errors.New("XSetOperation is not implemented") - } - if _, ok := xSetController.(api.XOperation); !ok { - return errors.New("XOperation is not implemented") - } - return errors.Join( - validateMeta(xSetController.XSetMeta()), - validateMeta(xSetController.XMeta()), - validateFinalizerName(xSetController.FinalizerName()), - validateXSetLabelAnnotationManager(xSetController), - ) -} - -func validateFinalizerName(name string) error { - msg := validation.IsQualifiedName(name) - if len(msg) > 0 { - return errors.New(fmt.Sprintf("%v", msg)) - } - return nil -} - -func validateXSetLabelAnnotationManager(xSetController api.XSetController) error { - var manager api.XSetLabelAnnotationManager - if getter, ok := xSetController.(api.LabelAnnotationManagerGetter); ok { - manager = getter.GetLabelManagerAdapter() - } else { - manager = api.NewXSetLabelAnnotationManager() - } - if manager == nil { - return nil - } - - for i := range api.EnumXSetLabelAnnotationsNum { - if len(manager.Value(api.XSetLabelAnnotationEnum(i))) == 0 { - return errors.New("XSetLabelAnnotationManager label annotations are not valid, please add enough label annotations") - } - } - return nil -} diff --git a/xset/api/well_knowns.go b/xset/api/well_knowns.go deleted file mode 100644 index 1830b78..0000000 --- a/xset/api/well_knowns.go +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package api - -import ( - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type XSetLabelAnnotationEnum int - -const EnumXSetLabelAnnotationsNum = int(wellKnownCount) -const ( - // OperatingLabelPrefix indicates a target is under operation - // set by xset controller - OperatingLabelPrefix XSetLabelAnnotationEnum = iota - - // OperationTypeLabelPrefix indicates the type of operation - // set by xset controller - OperationTypeLabelPrefix - - // OperateLabelPrefix indicates a target could start operation - // set by related opsLifecycle controller. - // xset controller will start operation only after this label is set - OperateLabelPrefix - - // UndoOperationTypeLabelPrefix indicates a type of operation has been canceled. - // need to be handled by related opsLifecycle controller - UndoOperationTypeLabelPrefix - - // ServiceAvailableLabel indicates a target is available for service. - // set by related opsLifecycle controller. - ServiceAvailableLabel - - // PreparingDeleteLabel indicates a target is preparing to be deleted. - // set by xset controller, - // handle by related opsLifecycle controller if needed. - PreparingDeleteLabel - - // ControlledByXSetLabel indicates a target is controlled by xset. - // set by xset controller - ControlledByXSetLabel - - // XInstanceIdLabelKey is used to attach instance ID on x - XInstanceIdLabelKey - - // XSetUpdateIndicationLabelKey is used to indicate a target is updated by xset - XSetUpdateIndicationLabelKey - - // XDeletionIndicationLabelKey is used to indicate a target is deleted by xset - XDeletionIndicationLabelKey - - // XReplaceIndicationLabelKey is used to indicate a target is replaced by xset - XReplaceIndicationLabelKey - - // XReplacePairNewId is used to indicate the new created target on replace origin target - XReplacePairNewId - - // XReplacePairOriginName is used to indicate replace origin target name on the new created target - XReplacePairOriginName - - // XReplaceByReplaceUpdateLabelKey indicates a target is replaced by update by xset - XReplaceByReplaceUpdateLabelKey - - // XOrphanedIndicationLabelKey is used to indicate a target is orphaned by xset - XOrphanedIndicationLabelKey - - // XCreatingLabel indicates a target is creating by xset - XCreatingLabel - - // XCompletingLabel indicates a target is completing by xset - XCompletingLabel - - // XExcludeIndicationLabelKey is used to indicate a target is excluded by xset - XExcludeIndicationLabelKey - - // SubResourcePvcTemplateLabelKey is used to attach pvc template name to pvc resources - SubResourcePvcTemplateLabelKey - - // SubResourcePvcTemplateHashLabelKey is used to attach hash of pvc template to pvc subresource - SubResourcePvcTemplateHashLabelKey - - // wellKnownCount is the number of XSetLabelAnnotationEnum - wellKnownCount -) - -type XSetLabelAnnotationManager interface { - Get(labels map[string]string, labelType XSetLabelAnnotationEnum) (string, bool) - Set(obj client.Object, labelType XSetLabelAnnotationEnum, value string) - Delete(labels map[string]string, labelType XSetLabelAnnotationEnum) - Value(labelType XSetLabelAnnotationEnum) string -} - -var defaultXSetLabelAnnotationManager = map[XSetLabelAnnotationEnum]string{ - OperatingLabelPrefix: appsv1alpha1.PodOperatingLabelPrefix, - OperationTypeLabelPrefix: appsv1alpha1.PodOperationTypeLabelPrefix, - OperateLabelPrefix: appsv1alpha1.PodOperateLabelPrefix, - UndoOperationTypeLabelPrefix: appsv1alpha1.PodUndoOperationTypeLabelPrefix, - ServiceAvailableLabel: appsv1alpha1.PodServiceAvailableLabel, - PreparingDeleteLabel: appsv1alpha1.PodPreparingDeleteLabel, - - ControlledByXSetLabel: appsv1alpha1.ControlledByKusionStackLabelKey, - XInstanceIdLabelKey: appsv1alpha1.PodInstanceIDLabelKey, - XSetUpdateIndicationLabelKey: appsv1alpha1.CollaSetUpdateIndicateLabelKey, - XDeletionIndicationLabelKey: appsv1alpha1.PodDeletionIndicationLabelKey, - XReplaceIndicationLabelKey: appsv1alpha1.PodReplaceIndicationLabelKey, - XReplacePairNewId: appsv1alpha1.PodReplacePairNewId, - XReplacePairOriginName: appsv1alpha1.PodReplacePairOriginName, - XReplaceByReplaceUpdateLabelKey: appsv1alpha1.PodReplaceByReplaceUpdateLabelKey, - XOrphanedIndicationLabelKey: appsv1alpha1.PodOrphanedIndicateLabelKey, - XCreatingLabel: appsv1alpha1.PodCreatingLabel, - XCompletingLabel: appsv1alpha1.PodCompletingLabel, - XExcludeIndicationLabelKey: appsv1alpha1.PodExcludeIndicationLabelKey, - SubResourcePvcTemplateLabelKey: appsv1alpha1.PvcTemplateLabelKey, - SubResourcePvcTemplateHashLabelKey: appsv1alpha1.PvcTemplateHashLabelKey, -} - -func NewXSetLabelAnnotationManager() XSetLabelAnnotationManager { - return &xSetLabelAnnotationManager{ - labelManager: defaultXSetLabelAnnotationManager, - } -} - -type xSetLabelAnnotationManager struct { - labelManager map[XSetLabelAnnotationEnum]string -} - -func (m *xSetLabelAnnotationManager) Get(labels map[string]string, key XSetLabelAnnotationEnum) (string, bool) { - if labels == nil { - return "", false - } - labelKey := m.labelManager[key] - val, exist := labels[labelKey] - return val, exist -} - -func (m *xSetLabelAnnotationManager) Set(obj client.Object, key XSetLabelAnnotationEnum, val string) { - if obj.GetLabels() == nil { - obj.SetLabels(map[string]string{}) - } - labelKey := m.labelManager[key] - obj.GetLabels()[labelKey] = val -} - -func (m *xSetLabelAnnotationManager) Delete(labels map[string]string, key XSetLabelAnnotationEnum) { - if labels == nil { - return - } - labelKey := m.labelManager[key] - delete(labels, labelKey) -} - -func (m *xSetLabelAnnotationManager) Value(key XSetLabelAnnotationEnum) string { - return m.labelManager[key] -} - -func GetWellKnownLabelPrefixesWithID(m XSetLabelAnnotationManager) []string { - return []string{ - m.Value(OperatingLabelPrefix), - m.Value(OperationTypeLabelPrefix), - m.Value(UndoOperationTypeLabelPrefix), - m.Value(OperatingLabelPrefix), - } -} - -func GetXSetLabelAnnotationManager(xsetController XSetController) XSetLabelAnnotationManager { - if getter, ok := xsetController.(LabelAnnotationManagerGetter); ok { - return getter.GetLabelManagerAdapter() - } - return NewXSetLabelAnnotationManager() -} diff --git a/xset/api/xset_controller_types.go b/xset/api/xset_controller_types.go deleted file mode 100644 index 88384d9..0000000 --- a/xset/api/xset_controller_types.go +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package api - -import ( - "context" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" -) - -type XSetController interface { - ControllerName() string - FinalizerName() string - - XSetMeta() metav1.TypeMeta // todo gvk - XMeta() metav1.TypeMeta - NewXSetObject() XSetObject - NewXObject() client.Object - NewXObjectList() client.ObjectList - - // XSetOperation are implemented to access XSetSpec, XSetStatus, etc. - XSetOperation - // XOperation are implemented to access X object and status, etc. - XOperation -} - -type XSetObject client.Object - -type XSetOperation interface { - GetXSetSpec(object XSetObject) *XSetSpec - GetXSetPatch(object metav1.Object) ([]byte, error) - GetXSetStatus(object XSetObject) *XSetStatus - SetXSetStatus(object XSetObject, status *XSetStatus) - UpdateScaleStrategy(ctx context.Context, c client.Client, object XSetObject, scaleStrategy *ScaleStrategy) error - GetXSetTemplatePatcher(object metav1.Object) func(client.Object) error -} - -type XOperation interface { - GetXObjectFromRevision(revision *appsv1.ControllerRevision) (client.Object, error) - CheckScheduled(object client.Object) bool - CheckReadyTime(object client.Object) (bool, *metav1.Time) - CheckAvailable(object client.Object) bool - CheckInactive(object client.Object) bool - GetXOpsPriority(ctx context.Context, c client.Client, object client.Object) (*OpsPriority, error) -} - -// LifecycleAdapterGetter is used to get lifecycle adapters. -type LifecycleAdapterGetter interface { - GetScaleInOpsLifecycleAdapter() LifecycleAdapter - GetUpdateOpsLifecycleAdapter() LifecycleAdapter -} - -// ResourceContextAdapterGetter is used to get resource context adapter. -type ResourceContextAdapterGetter interface { - GetResourceContextAdapter() ResourceContextAdapter -} - -// LabelAnnotationManagerGetter is used to get label manager adapter. -type LabelAnnotationManagerGetter interface { - GetLabelManagerAdapter() XSetLabelAnnotationManager -} - -// SubResourcePvcAdapter is used to manage pvc subresource for X, which are declared on XSet, e.g., spec.volumeClaimTemplate. -// Once adapter is implemented, XSetController will automatically manage pvc: (1) create pvcs from GetXSetPvcTemplate for each -// X object and attach theses pvcs with same instance-id, (2) upgrade pvcs and recreate X object pvcs when PvcTemplateChanged, -// (3) retain pvcs when XSet is deleted or scaledIn according to RetainPvcWhenXSetDeleted and RetainPvcWhenXSetScaled. -type SubResourcePvcAdapter interface { - // RetainPvcWhenXSetDeleted returns true if pvc should be retained when XSet is deleted. - RetainPvcWhenXSetDeleted(object XSetObject) bool - // RetainPvcWhenXSetScaled returns true if pvc should be retained when XSet replicas is scaledIn. - RetainPvcWhenXSetScaled(object XSetObject) bool - // GetXSetPvcTemplate returns pvc template from XSet object. - GetXSetPvcTemplate(object XSetObject) []corev1.PersistentVolumeClaim - // GetXSpecVolumes returns spec.volumes from X object. - GetXSpecVolumes(object client.Object) []corev1.Volume - // GetXVolumeMounts returns containers volumeMounts from X (pod) object. - GetXVolumeMounts(object client.Object) []corev1.VolumeMount - // SetXSpecVolumes sets spec.volumes to X object. - SetXSpecVolumes(object client.Object, pvcs []corev1.Volume) -} - -// DecorationAdapter is used to manage decoration for XSet. Decoration should be a workload to manage patcher on X target. -// Once adapter is implemented, XSetController will (1) watch for decoration change, (2) patch effective decorations on -// X target when creating, (3) manage decoration update when decoration changed. -type DecorationAdapter interface { - // WatchDecoration allows controller to watch decoration change. - WatchDecoration(c controller.Controller) error - // GetDecorationGroupVersionKind returns decoration gvk. - GetDecorationGroupVersionKind() metav1.GroupVersionKind - // GetTargetCurrentDecorationRevisions returns decoration revision on target. - GetTargetCurrentDecorationRevisions(ctx context.Context, c client.Client, target client.Object) (string, error) - // GetTargetUpdatedDecorationRevisions returns decoration revision on target. - GetTargetUpdatedDecorationRevisions(ctx context.Context, c client.Client, target client.Object) (string, error) - // GetDecorationPatcherByRevisions returns patcher for decoration from revisions. - GetDecorationPatcherByRevisions(ctx context.Context, c client.Client, target client.Object, revision string) (func(client.Object) error, error) - // IsTargetDecorationChanged returns true if decoration on target is changed. - IsTargetDecorationChanged(currentRevision, updatedRevision string) (bool, error) -} diff --git a/xset/api/xset_types.go b/xset/api/xset_types.go deleted file mode 100644 index 40c5b9a..0000000 --- a/xset/api/xset_types.go +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package api - -// +k8s:deepcopy-gen=file - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -type XSetConditionType string - -const ( - XSetScale XSetConditionType = "Scale" - XSetUpdate XSetConditionType = "Update" -) - -type XSetSpec struct { - // Indicates that the scaling and updating is paused and will not be processed by the - // XSet controller. - // +optional - Paused bool `json:"paused,omitempty"` - // Replicas is the desired number of replicas of the given Template. - // These are replicas in the sense that they are instantiations of the - // same Template, but individual replicas also have a consistent identity. - // If unspecified, defaults to 0. - // +optional - Replicas *int32 `json:"replicas,omitempty"` - - // Selector is a label query over targets that should match the replica count. - // It must match the target template's labels. - Selector *metav1.LabelSelector `json:"selector,omitempty"` - - // UpdateStrategy indicates the XSetUpdateStrategy that will be - // employed to update Targets in the XSet when a revision is made to - // Template. - // +optional - UpdateStrategy UpdateStrategy `json:"updateStrategy,omitempty"` - - // ScaleStrategy indicates the strategy detail that will be used during targets scaling. - // +optional - ScaleStrategy ScaleStrategy `json:"scaleStrategy,omitempty"` - - // NamigPolicy indicates the strategy detail that will be used for replica naming - // +optional - NamingStrategy *NamingStrategy `json:"namingStrategy,omitempty"` - - // Indicate the number of histories to be conserved - // If unspecified, defaults to 20 - // +optional - HistoryLimit int32 `json:"historyLimit,omitempty"` -} - -type ByPartition struct { - // Partition controls the number of targets in old revisions. - // Defaults to nil (all targets will be updated) - // +optional - Partition *int32 `json:"partition,omitempty"` -} - -type ByLabel struct{} - -// RollingUpdateStrategy is used to communicate parameter for rolling update. -type RollingUpdateStrategy struct { - // ByPartition indicates the update progress is controlled by partition value. - // +optional - ByPartition *ByPartition `json:"byPartition,omitempty"` - - // ByLabel indicates the update progress is controlled by attaching target label. - // +optional - ByLabel *ByLabel `json:"byLabel,omitempty"` -} - -type UpdateStrategy struct { - // RollingUpdate is used to communicate parameters when Type is RollingUpdateStatefulSetStrategyType. - // +optional - RollingUpdate *RollingUpdateStrategy `json:"rollingUpdate,omitempty"` - - // UpdatePolicy indicates the policy by to update targets. - // +optional - UpdatePolicy UpdateStrategyType `json:"upgradePolicy,omitempty"` - - // OperationDelaySeconds indicates how many seconds it should delay before operating update. - // +optional - OperationDelaySeconds *int32 `json:"operationDelaySeconds,omitempty"` -} - -type ScaleStrategy struct { - // Context indicates the pool from which to allocate Target instance ID. - // XSets are allowed to share the same Context. - // It is not allowed to change. - // Context defaults to be XSet's name. - // +optional - Context string `json:"context,omitempty"` - - // TargetToExclude indicates the syncContext which will be orphaned by XSet. - // +optional - TargetToExclude []string `json:"targetToExclude,omitempty"` - - // TargetToInclude indicates the syncContext which will be adapted by XSet. - // +optional - TargetToInclude []string `json:"targetToInclude,omitempty"` - - // TargetToDelete indicates the syncContext which will be deleted by XSet. - // +optional - TargetToDelete []string `json:"targetToDelete,omitempty"` - - // OperationDelaySeconds indicates how many seconds it should delay before operating scale. - // +optional - OperationDelaySeconds *int32 `json:"operationDelaySeconds,omitempty"` -} - -// TargetNamingSuffixPolicy indicates how a new pod name suffix part is generated. -type TargetNamingSuffixPolicy string - -const ( - // TargetNamingSuffixPolicyPersistentSequence uses persistent sequential numbers as pod name suffix. - TargetNamingSuffixPolicyPersistentSequence TargetNamingSuffixPolicy = "PersistentSequence" - // TargetNamingSuffixPolicyRandom uses collaset name as pod generateName, which is the prefix - // of pod name. Kubernetes then adds a random string as suffix after the generateName. - // This is defaulting policy. - TargetNamingSuffixPolicyRandom TargetNamingSuffixPolicy = "Random" -) - -type NamingStrategy struct { - // TargetNamingSuffixPolicy is a string enumeration that determaines how pod name suffix will be generated. - // A collaset pod name contains two parts to be placed in a string formation %s-%s; the prefix is collaset - // name, and the suffix is determined by TargetNamingSuffixPolicy. - TargetNamingSuffixPolicy TargetNamingSuffixPolicy `json:"TargetNamingSuffixPolicy,omitempty"` -} - -// UpdateStrategyType is a string enumeration type that enumerates -// all possible ways we can update a Target when updating application -type UpdateStrategyType string - -const ( - // XSetRecreateTargetUpdateStrategyType indicates that XSet will always update Target by deleting and recreate it. - XSetRecreateTargetUpdateStrategyType UpdateStrategyType = "Recreate" - // XSetInPlaceIfPossibleTargetUpdateStrategyType indicates thath XSet will try to update Target by in-place update - // when it is possible. Recently, only Target image can be updated in-place. Any other Target spec change will make the - // policy fall back to XSetRecreateTargetUpdateStrategyType. - XSetInPlaceIfPossibleTargetUpdateStrategyType UpdateStrategyType = "InPlaceIfPossible" - // XSetInPlaceOnlyTargetUpdateStrategyType indicates that XSet will always update Target in-place, instead of - // recreating target. It will encounter an error on original Kubernetes cluster. - XSetInPlaceOnlyTargetUpdateStrategyType UpdateStrategyType = "InPlaceOnly" - // XSetReplaceTargetUpdateStrategyType indicates that XSet will always update Target by replace, it will - // create a new Target and delete the old target when the new one service available. - XSetReplaceTargetUpdateStrategyType UpdateStrategyType = "Replace" -) - -type XSetStatus struct { - // ObservedGeneration is the most recent generation observed for this XSet. It corresponds to the - // XSet's generation, which is updated on mutation by the API Server. - // +optional - ObservedGeneration int64 `json:"observedGeneration,omitempty"` - - // CurrentRevision, if not empty, indicates the version of the XSet. - // +optional - CurrentRevision string `json:"currentRevision,omitempty"` - - // UpdatedRevision, if not empty, indicates the version of the XSet currently updated. - // +optional - UpdatedRevision string `json:"updatedRevision,omitempty"` - - // Count of hash collisions for the XSet. The XSet controller - // uses this field as a collision avoidance mechanism when it needs to - // create the name for the newest ControllerRevision. - // +optional - CollisionCount *int32 `json:"collisionCount,omitempty"` - - // ReadyReplicas indicates the number of the target with ready condition - // +optional - ReadyReplicas int32 `json:"readyReplicas,omitempty"` - - // Replicas is the most recently observed number of replicas. - // +optional - Replicas int32 `json:"replicas,omitempty"` - - // the number of scheduled replicas for the replicas set. - // +optional - ScheduledReplicas int32 `json:"scheduledReplicas,omitempty"` - - // The number of targets in updated version. - // +optional - UpdatedReplicas int32 `json:"updatedReplicas,omitempty"` - - // OperatingReplicas indicates the number of targets during target ops lifecycle and not finish update-phase. - // +optional - OperatingReplicas int32 `json:"operatingReplicas,omitempty"` - - // UpdatedReadyReplicas indicates the number of the target with updated revision and ready condition - // +optional - UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"` - - // The number of available replicas for this replica set. - // +optional - AvailableReplicas int32 `json:"availableReplicas,omitempty"` - - // UpdatedAvailableReplicas indicates the number of available updated revision replicas for this replicas set. - // A target is updated available means the target is ready for updated revision and accessible - // +optional - UpdatedAvailableReplicas int32 `json:"updatedAvailableReplicas,omitempty"` - - // Represents the latest available observations of a XSet's current state. - // +optional - Conditions []metav1.Condition `json:"conditions,omitempty"` -} - -// OpsPriority is used to store the ops priority of a target -type OpsPriority struct { - // PriorityClass is the priority class of the target - PriorityClass int32 - // DeletionCost is the deletion cost of the target - DeletionCost int32 -} diff --git a/xset/api/zz_generated.deepcopy.go b/xset/api/zz_generated.deepcopy.go deleted file mode 100644 index 46192a5..0000000 --- a/xset/api/zz_generated.deepcopy.go +++ /dev/null @@ -1,199 +0,0 @@ -//go:build !ignore_autogenerated - -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Code generated by controller-gen. DO NOT EDIT. - -package api - -import ( - "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ByLabel) DeepCopyInto(out *ByLabel) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ByLabel. -func (in *ByLabel) DeepCopy() *ByLabel { - if in == nil { - return nil - } - out := new(ByLabel) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ByPartition) DeepCopyInto(out *ByPartition) { - *out = *in - if in.Partition != nil { - in, out := &in.Partition, &out.Partition - *out = new(int32) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ByPartition. -func (in *ByPartition) DeepCopy() *ByPartition { - if in == nil { - return nil - } - out := new(ByPartition) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RollingUpdateStrategy) DeepCopyInto(out *RollingUpdateStrategy) { - *out = *in - if in.ByPartition != nil { - in, out := &in.ByPartition, &out.ByPartition - *out = new(ByPartition) - (*in).DeepCopyInto(*out) - } - if in.ByLabel != nil { - in, out := &in.ByLabel, &out.ByLabel - *out = new(ByLabel) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpdateStrategy. -func (in *RollingUpdateStrategy) DeepCopy() *RollingUpdateStrategy { - if in == nil { - return nil - } - out := new(RollingUpdateStrategy) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ScaleStrategy) DeepCopyInto(out *ScaleStrategy) { - *out = *in - if in.TargetToExclude != nil { - in, out := &in.TargetToExclude, &out.TargetToExclude - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.TargetToInclude != nil { - in, out := &in.TargetToInclude, &out.TargetToInclude - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.TargetToDelete != nil { - in, out := &in.TargetToDelete, &out.TargetToDelete - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.OperationDelaySeconds != nil { - in, out := &in.OperationDelaySeconds, &out.OperationDelaySeconds - *out = new(int32) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScaleStrategy. -func (in *ScaleStrategy) DeepCopy() *ScaleStrategy { - if in == nil { - return nil - } - out := new(ScaleStrategy) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *UpdateStrategy) DeepCopyInto(out *UpdateStrategy) { - *out = *in - if in.RollingUpdate != nil { - in, out := &in.RollingUpdate, &out.RollingUpdate - *out = new(RollingUpdateStrategy) - (*in).DeepCopyInto(*out) - } - if in.OperationDelaySeconds != nil { - in, out := &in.OperationDelaySeconds, &out.OperationDelaySeconds - *out = new(int32) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateStrategy. -func (in *UpdateStrategy) DeepCopy() *UpdateStrategy { - if in == nil { - return nil - } - out := new(UpdateStrategy) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *XSetSpec) DeepCopyInto(out *XSetSpec) { - *out = *in - if in.Replicas != nil { - in, out := &in.Replicas, &out.Replicas - *out = new(int32) - **out = **in - } - if in.Selector != nil { - in, out := &in.Selector, &out.Selector - *out = new(v1.LabelSelector) - (*in).DeepCopyInto(*out) - } - in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy) - in.ScaleStrategy.DeepCopyInto(&out.ScaleStrategy) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new XSetSpec. -func (in *XSetSpec) DeepCopy() *XSetSpec { - if in == nil { - return nil - } - out := new(XSetSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *XSetStatus) DeepCopyInto(out *XSetStatus) { - *out = *in - if in.CollisionCount != nil { - in, out := &in.CollisionCount, &out.CollisionCount - *out = new(int32) - **out = **in - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new XSetStatus. -func (in *XSetStatus) DeepCopy() *XSetStatus { - if in == nil { - return nil - } - out := new(XSetStatus) - in.DeepCopyInto(out) - return out -} diff --git a/xset/opslifecycle/default_adapters.go b/xset/opslifecycle/default_adapters.go deleted file mode 100644 index 6f0fc4d..0000000 --- a/xset/opslifecycle/default_adapters.go +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package opslifecycle - -import ( - "strings" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - "kusionstack.io/kube-utils/xset/api" -) - -var _ api.LifecycleAdapter = &DefaultUpdateLifecycleAdapter{} - -type DefaultUpdateLifecycleAdapter struct { - LabelAnnoManager api.XSetLabelAnnotationManager - XSetType metav1.TypeMeta -} - -func (d *DefaultUpdateLifecycleAdapter) GetID() string { - return strings.ToLower(d.XSetType.Kind) -} - -func (d *DefaultUpdateLifecycleAdapter) GetType() api.OperationType { - return api.OpsLifecycleTypeUpdate -} - -func (d *DefaultUpdateLifecycleAdapter) AllowMultiType() bool { - return true -} - -func (d *DefaultUpdateLifecycleAdapter) WhenBegin(_ client.Object) (bool, error) { - return true, nil -} - -func (d *DefaultUpdateLifecycleAdapter) WhenFinish(target client.Object) (bool, error) { - // TODO inplace update post actions - return false, nil -} - -var _ api.LifecycleAdapter = &DefaultScaleInLifecycleAdapter{} - -type DefaultScaleInLifecycleAdapter struct { - LabelAnnoManager api.XSetLabelAnnotationManager - XSetType metav1.TypeMeta -} - -func (d *DefaultScaleInLifecycleAdapter) GetID() string { - return strings.ToLower(d.XSetType.Kind) -} - -func (d *DefaultScaleInLifecycleAdapter) GetType() api.OperationType { - return api.OpsLifecycleTypeScaleIn -} - -func (d *DefaultScaleInLifecycleAdapter) AllowMultiType() bool { - return true -} - -func (d *DefaultScaleInLifecycleAdapter) WhenBegin(target client.Object) (bool, error) { - return WhenBeginDelete(d.LabelAnnoManager, target) -} - -func (d *DefaultScaleInLifecycleAdapter) WhenFinish(_ client.Object) (bool, error) { - return false, nil -} - -func GetLifecycleAdapters(xsetController api.XSetController, labelAnnoMgr api.XSetLabelAnnotationManager, xsetTypeMeta metav1.TypeMeta) (api.LifecycleAdapter, api.LifecycleAdapter) { - if getter, ok := xsetController.(api.LifecycleAdapterGetter); ok { - return getter.GetUpdateOpsLifecycleAdapter(), getter.GetScaleInOpsLifecycleAdapter() - } - return &DefaultUpdateLifecycleAdapter{LabelAnnoManager: labelAnnoMgr, XSetType: xsetTypeMeta}, &DefaultScaleInLifecycleAdapter{LabelAnnoManager: labelAnnoMgr, XSetType: xsetTypeMeta} -} diff --git a/xset/opslifecycle/utils.go b/xset/opslifecycle/utils.go deleted file mode 100644 index 4c04f43..0000000 --- a/xset/opslifecycle/utils.go +++ /dev/null @@ -1,332 +0,0 @@ -/* - Copyright 2024-2025 The KusionStack Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package opslifecycle - -import ( - "context" - "fmt" - "strconv" - "strings" - "time" - - "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client" - - "kusionstack.io/kube-utils/xset/api" -) - -type UpdateFunc func(object client.Object) (bool, error) - -// IDToLabelsMap returns a map of target id to labels map and a map of operation type to number of targets. -func IDToLabelsMap(m api.XSetLabelAnnotationManager, target client.Object) (map[string]map[string]string, map[string]int, error) { - idToLabelsMap := map[string]map[string]string{} - typeToNumsMap := map[string]int{} - - ids := sets.String{} - labels := target.GetLabels() - for k := range labels { - if strings.HasPrefix(k, m.Value(api.OperatingLabelPrefix)) || - strings.HasPrefix(k, m.Value(api.OperateLabelPrefix)) { - s := strings.Split(k, "/") - if len(s) < 2 { - return nil, nil, fmt.Errorf("invalid label %s", k) - } - ids.Insert(s[1]) - } - } - - for id := range ids { - if operationType, ok := labels[fmt.Sprintf("%s/%s", m.Value(api.OperationTypeLabelPrefix), id)]; ok { - typeToNumsMap[operationType] += 1 - } - - for _, prefix := range api.GetWellKnownLabelPrefixesWithID(m) { - label := fmt.Sprintf("%s/%s", prefix, id) - value, ok := labels[label] - if !ok { - continue - } - - labelsMap, ok := idToLabelsMap[id] - if !ok { - labelsMap = make(map[string]string) - idToLabelsMap[id] = labelsMap - } - labelsMap[prefix] = value - } - } - return idToLabelsMap, typeToNumsMap, nil -} - -// NumOfLifecycleOnTarget returns the nums of lifecycles on target -func NumOfLifecycleOnTarget(m api.XSetLabelAnnotationManager, target client.Object) (int, error) { - if target == nil { - return 0, nil - } - newIDToLabelsMap, _, err := IDToLabelsMap(m, target) - return len(newIDToLabelsMap), err -} - -func WhenBeginDelete(m api.XSetLabelAnnotationManager, obj client.Object) (bool, error) { - return AddLabel(obj, m.Value(api.PreparingDeleteLabel), strconv.FormatInt(time.Now().UnixNano(), 10)), nil -} - -func AddLabel(po client.Object, k, v string) bool { - labels := po.GetLabels() - if labels == nil { - labels = map[string]string{} - po.SetLabels(labels) - } - if _, ok := labels[k]; !ok { - labels[k] = v - return true - } - return false -} - -// IsDuringOps decides whether the Target is during ops or not -// DuringOps means the Target's OpsLifecycle phase is in or after PreCheck phase and before Finish phase. -func IsDuringOps(m api.XSetLabelAnnotationManager, adapter api.LifecycleAdapter, obj client.Object) bool { - _, hasID := checkOperatingID(m, adapter, obj) - _, hasType := checkOperationType(m, adapter, obj) - - return hasID && hasType -} - -// Begin is used for an CRD Operator to begin a lifecycle -func Begin(m api.XSetLabelAnnotationManager, c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFuncs ...UpdateFunc) (updated bool, err error) { - if obj.GetLabels() == nil { - obj.SetLabels(map[string]string{}) - } - - operatingID, hasID := checkOperatingID(m, adapter, obj) - operationType, hasType := checkOperationType(m, adapter, obj) - var needUpdate bool - - // ensure operatingID and operationType - if hasID && hasType { - if operationType != adapter.GetType() { - err = fmt.Errorf("operatingID %s already has operationType %s", operatingID, operationType) - return false, err - } - } else { - // check another id/type = this.type - currentTypeIDs := queryByOperationType(m, adapter, obj) - if currentTypeIDs != nil && currentTypeIDs.Len() > 0 && !adapter.AllowMultiType() { - err = fmt.Errorf("operationType %s exists: %v", adapter.GetType(), currentTypeIDs) - return updated, err - } - - if !hasID { - needUpdate = true - setOperatingID(m, adapter, obj) - } - if !hasType { - needUpdate = true - setOperationType(m, adapter, obj) - } - } - - updated, err = DefaultUpdateAll(obj, append(updateFuncs, adapter.WhenBegin)...) - if err != nil { - return updated, err - } - - if needUpdate || updated { - err = c.Update(context.Background(), obj) - return err == nil, err - } - - return false, nil -} - -// BeginWithCleaningOld is used for an CRD Operator to begin a lifecycle with cleaning the old lifecycle -func BeginWithCleaningOld(m api.XSetLabelAnnotationManager, c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFunc ...UpdateFunc) (updated bool, err error) { - if targetInUpdateLifecycle, err := IsLifecycleOnTarget(m, adapter.GetID(), obj); err != nil { - return false, fmt.Errorf("fail to check %s TargetOpsLifecycle on Target %s/%s: %w", adapter.GetID(), obj.GetNamespace(), obj.GetName(), err) - } else if targetInUpdateLifecycle { - if err := Undo(m, c, adapter, obj); err != nil { - return false, err - } - } - return Begin(m, c, adapter, obj, updateFunc...) -} - -// AllowOps is used to check whether the TargetOpsLifecycle phase is in UPGRADE to do following operations. -func AllowOps(m api.XSetLabelAnnotationManager, adapter api.LifecycleAdapter, operationDelaySeconds int32, obj client.Object) (requeueAfter *time.Duration, allow bool) { - if !IsDuringOps(m, adapter, obj) { - return nil, false - } - - startedTimestampStr, started := checkOperate(m, adapter, obj) - if !started || operationDelaySeconds <= 0 { - return nil, started - } - - startedTimestamp, err := strconv.ParseInt(startedTimestampStr, 10, 64) - if err != nil { - return nil, started - } - - startedTime := time.Unix(0, startedTimestamp) - duration := time.Since(startedTime) - delay := time.Duration(operationDelaySeconds) * time.Second - if duration < delay { - du := delay - duration - return &du, started - } - - return nil, started -} - -// Finish is used for an CRD Operator to finish a lifecycle -func Finish(m api.XSetLabelAnnotationManager, c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFuncs ...UpdateFunc) (updated bool, err error) { - operatingID, hasID := checkOperatingID(m, adapter, obj) - operationType, hasType := checkOperationType(m, adapter, obj) - - if hasType && operationType != adapter.GetType() { - return false, fmt.Errorf("operatingID %s has invalid operationType %s", operatingID, operationType) - } - - var needUpdate bool - if hasID || hasType { - needUpdate = true - deleteOperatingID(m, adapter, obj) - } - - updated, err = DefaultUpdateAll(obj, append(updateFuncs, adapter.WhenFinish)...) - if err != nil { - return - } - if needUpdate || updated { - err = c.Update(context.Background(), obj) - return err == nil, err - } - - return false, err -} - -// Undo is used for an CRD Operator to undo a lifecycle -func Undo(m api.XSetLabelAnnotationManager, c client.Client, adapter api.LifecycleAdapter, obj client.Object) error { - setUndo(m, adapter, obj) - return c.Update(context.Background(), obj) -} - -func checkOperatingID(m api.XSetLabelAnnotationManager, adapter api.LifecycleAdapter, obj client.Object) (val string, ok bool) { - labelID := fmt.Sprintf("%s/%s", m.Value(api.OperatingLabelPrefix), adapter.GetID()) - _, ok = obj.GetLabels()[labelID] - return adapter.GetID(), ok -} - -func checkOperationType(m api.XSetLabelAnnotationManager, adapter api.LifecycleAdapter, obj client.Object) (val api.OperationType, ok bool) { - labelType := fmt.Sprintf("%s/%s", m.Value(api.OperationTypeLabelPrefix), adapter.GetID()) - labelVal := obj.GetLabels()[labelType] - val = api.OperationType(labelVal) - return val, val == adapter.GetType() -} - -func checkOperate(m api.XSetLabelAnnotationManager, adapter api.LifecycleAdapter, obj client.Object) (val string, ok bool) { - labelOperate := fmt.Sprintf("%s/%s", m.Value(api.OperateLabelPrefix), adapter.GetID()) - val, ok = obj.GetLabels()[labelOperate] - return -} - -func setOperatingID(m api.XSetLabelAnnotationManager, adapter api.LifecycleAdapter, obj client.Object) { - labelID := fmt.Sprintf("%s/%s", m.Value(api.OperatingLabelPrefix), adapter.GetID()) - obj.GetLabels()[labelID] = fmt.Sprintf("%d", time.Now().UnixNano()) - return -} - -func setOperationType(m api.XSetLabelAnnotationManager, adapter api.LifecycleAdapter, obj client.Object) { - labelType := fmt.Sprintf("%s/%s", m.Value(api.OperationTypeLabelPrefix), adapter.GetID()) - obj.GetLabels()[labelType] = string(adapter.GetType()) - return -} - -// setOperate only for test, expected to be called by adapter -func setOperate(m api.XSetLabelAnnotationManager, adapter api.LifecycleAdapter, obj client.Object) { - labelOperate := fmt.Sprintf("%s/%s", m.Value(api.OperateLabelPrefix), adapter.GetID()) - obj.GetLabels()[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) - return -} - -func setUndo(m api.XSetLabelAnnotationManager, adapter api.LifecycleAdapter, obj client.Object) { - labelUndo := fmt.Sprintf("%s/%s", m.Value(api.UndoOperationTypeLabelPrefix), adapter.GetID()) - obj.GetLabels()[labelUndo] = string(adapter.GetType()) -} - -func deleteOperatingID(m api.XSetLabelAnnotationManager, adapter api.LifecycleAdapter, obj client.Object) { - labelID := fmt.Sprintf("%s/%s", m.Value(api.OperatingLabelPrefix), adapter.GetID()) - delete(obj.GetLabels(), labelID) - return -} - -func queryByOperationType(m api.XSetLabelAnnotationManager, adapter api.LifecycleAdapter, obj client.Object) sets.String { - res := sets.String{} - valType := adapter.GetType() - - for k, v := range obj.GetLabels() { - if strings.HasPrefix(k, m.Value(api.OperationTypeLabelPrefix)) && v == string(valType) { - res.Insert(k) - } - } - - return res -} - -func IsLifecycleOnTarget(m api.XSetLabelAnnotationManager, operatingID string, target client.Object) (bool, error) { - if target == nil { - return false, fmt.Errorf("nil target") - } - - labels := target.GetLabels() - if labels == nil { - return false, nil - } - - if val, ok := labels[fmt.Sprintf("%s/%s", m.Value(api.OperatingLabelPrefix), operatingID)]; ok { - return val != "", nil - } - - return false, nil -} - -func CancelOpsLifecycle(m api.XSetLabelAnnotationManager, client client.Client, adapter api.LifecycleAdapter, target client.Object) error { - if target == nil { - return nil - } - - // only cancel when lifecycle exist on target - if exist, err := IsLifecycleOnTarget(m, adapter.GetID(), target); err != nil { - return fmt.Errorf("fail to check %s TargetOpsLifecycle on Target %s/%s: %w", adapter.GetID(), target.GetNamespace(), target.GetName(), err) - } else if !exist { - return nil - } - - return Undo(m, client, adapter, target) -} - -func DefaultUpdateAll(target client.Object, updateFuncs ...UpdateFunc) (updated bool, err error) { - for _, updateFunc := range updateFuncs { - ok, updateErr := updateFunc(target) - if updateErr != nil { - return updated, updateErr - } - updated = updated || ok - } - return updated, nil -} diff --git a/xset/opslifecycle/utils_test.go b/xset/opslifecycle/utils_test.go deleted file mode 100644 index c0db4ae..0000000 --- a/xset/opslifecycle/utils_test.go +++ /dev/null @@ -1,157 +0,0 @@ -/* - Copyright 2023-2025 The KusionStack Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package opslifecycle - -import ( - "context" - "fmt" - "reflect" - "testing" - - "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "kusionstack.io/kube-utils/xset/api" -) - -const ( - mockLabelKey = "mockLabel" - mockLabelValue = "mockLabelValue" - testNamespace = "default" - testName = "target-1" -) - -var ( - allowTypes = false - scheme = runtime.NewScheme() -) - -func init() { - corev1.AddToScheme(scheme) -} - -func TestLifecycle(t *testing.T) { - c := fake.NewClientBuilder().WithScheme(scheme).Build() - g := gomega.NewGomegaWithT(t) - - a := &mockAdapter{id: "id-1", operationType: "type-1"} - b := &mockAdapter{id: "id-2", operationType: "type-1"} - mgr := api.NewXSetLabelAnnotationManager() - - inputs := []struct { - hasOperating, hasConflictID bool - started bool - err error - allow bool - }{ - { - hasOperating: false, - started: false, - }, - { - hasOperating: true, - started: false, - }, - { - hasConflictID: true, - started: false, - err: fmt.Errorf("operationType %s exists: %v", a.GetType(), sets.NewString(fmt.Sprintf("%s/%s", mgr.Value(api.OperationTypeLabelPrefix), b.GetID()))), - }, - { - hasConflictID: true, - started: false, - allow: true, - err: nil, - }, - } - - for i, input := range inputs { - allowTypes = input.allow - - target := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: testNamespace, - Name: fmt.Sprintf("%s-%d", testName, i), - Labels: map[string]string{}, - Annotations: map[string]string{}, - }, - } - g.Expect(c.Create(context.TODO(), target)).Should(gomega.Succeed()) - - if input.hasOperating { - setOperatingID(mgr, a, target) - setOperationType(mgr, a, target) - a.WhenBegin(target) - } - - if input.hasConflictID { - setOperatingID(mgr, b, target) - setOperationType(mgr, b, target) - } - - _, err := Begin(mgr, c, a, target) - g.Expect(reflect.DeepEqual(err, input.err)).Should(gomega.BeTrue()) - if err != nil { - continue - } - g.Expect(target.Labels[mockLabelKey]).Should(gomega.BeEquivalentTo(mockLabelValue)) - - setOperate(mgr, a, target) - started, err := Begin(mgr, c, a, target) - g.Expect(err).ShouldNot(gomega.HaveOccurred()) - g.Expect(started).Should(gomega.BeTrue()) - g.Expect(IsDuringOps(mgr, a, target)).Should(gomega.BeTrue()) - - finished, err := Finish(mgr, c, a, target) - g.Expect(err).ShouldNot(gomega.HaveOccurred()) - g.Expect(finished).Should(gomega.BeTrue()) - g.Expect(target.Labels[mockLabelKey]).Should(gomega.BeEquivalentTo("")) - g.Expect(IsDuringOps(mgr, a, target)).Should(gomega.BeFalse()) - } -} - -type mockAdapter struct { - id string - operationType api.OperationType -} - -func (m *mockAdapter) GetID() string { - return m.id -} - -func (m *mockAdapter) GetType() api.OperationType { - return m.operationType -} - -func (m *mockAdapter) AllowMultiType() bool { - return allowTypes -} - -func (m *mockAdapter) WhenBegin(target client.Object) (bool, error) { - target.GetLabels()[mockLabelKey] = mockLabelValue - return true, nil -} - -func (m *mockAdapter) WhenFinish(target client.Object) (bool, error) { - delete(target.GetLabels(), mockLabelKey) - return true, nil -} diff --git a/xset/resourcecontexts/default_adapters.go b/xset/resourcecontexts/default_adapters.go deleted file mode 100644 index 240f6f1..0000000 --- a/xset/resourcecontexts/default_adapters.go +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2024 - 2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package resourcecontexts - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - - "kusionstack.io/kube-utils/xset/api" -) - -var _ api.ResourceContextAdapter = &DefaultResourceContextAdapter{} - -var defaultResourceContextKeys = map[api.ResourceContextKeyEnum]string{ - api.EnumOwnerContextKey: "Owner", - api.EnumRevisionContextDataKey: "Revision", - api.EnumTargetDecorationRevisionKey: "TargetDecorationRevisions", - api.EnumJustCreateContextDataKey: "TargetJustCreate", - api.EnumRecreateUpdateContextDataKey: "TargetRecreateUpdate", - api.EnumScaleInContextDataKey: "ScaleIn", - api.EnumReplaceNewTargetIDContextDataKey: "ReplaceNewTargetID", - api.EnumReplaceOriginTargetIDContextDataKey: "ReplaceOriginTargetID", -} - -type ResourceContextAdapterGetter struct{} - -func (r *ResourceContextAdapterGetter) GetResourceContextAdapter() api.ResourceContextAdapter { - return &DefaultResourceContextAdapter{} -} - -// DefaultResourceContextAdapter is the adapter to api apps.kusionstack.io.resourcecontexts -type DefaultResourceContextAdapter struct{} - -func (*DefaultResourceContextAdapter) ResourceContextMeta() metav1.TypeMeta { - return metav1.TypeMeta{APIVersion: appsv1alpha1.SchemeGroupVersion.String(), Kind: "ResourceContext"} -} - -func (*DefaultResourceContextAdapter) GetResourceContextSpec(object api.ResourceContextObject) *api.ResourceContextSpec { - rc := object.(*appsv1alpha1.ResourceContext) - var contexts []api.ContextDetail - for i := range rc.Spec.Contexts { - c := rc.Spec.Contexts[i] - contexts = append(contexts, api.ContextDetail{ - ID: c.ID, - Data: c.Data, - }) - } - return &api.ResourceContextSpec{ - Contexts: contexts, - } -} - -func (*DefaultResourceContextAdapter) SetResourceContextSpec(spec *api.ResourceContextSpec, object api.ResourceContextObject) { - rc := object.(*appsv1alpha1.ResourceContext) - var contexts []appsv1alpha1.ContextDetail - for i := range spec.Contexts { - c := spec.Contexts[i] - contexts = append(contexts, appsv1alpha1.ContextDetail{ - ID: c.ID, - Data: c.Data, - }) - } - rc.Spec.Contexts = contexts -} - -func (*DefaultResourceContextAdapter) GetContextKeys() map[api.ResourceContextKeyEnum]string { - return defaultResourceContextKeys -} - -func (*DefaultResourceContextAdapter) NewResourceContext() api.ResourceContextObject { - return &appsv1alpha1.ResourceContext{} -} - -func GetResourceContextAdapter(controller api.XSetController) api.ResourceContextAdapter { - if getter, ok := controller.(api.ResourceContextAdapterGetter); ok { - return getter.GetResourceContextAdapter() - } - return &DefaultResourceContextAdapter{} -} diff --git a/xset/resourcecontexts/resource_context.go b/xset/resourcecontexts/resource_context.go deleted file mode 100644 index daa1a04..0000000 --- a/xset/resourcecontexts/resource_context.go +++ /dev/null @@ -1,303 +0,0 @@ -/* -Copyright 2023-2025 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resourcecontexts - -import ( - "context" - "fmt" - "sort" - - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client" - - clientutil "kusionstack.io/kube-utils/client" - "kusionstack.io/kube-utils/controller/expectations" - "kusionstack.io/kube-utils/xset/api" -) - -type ResourceContextControl interface { - AllocateID(ctx context.Context, xsetObject api.XSetObject, defaultRevision string, replicas int) (map[int]*api.ContextDetail, error) - UpdateToTargetContext(ctx context.Context, xsetObject api.XSetObject, ownedIDs map[int]*api.ContextDetail) error - ExtractAvailableContexts(diff int, ownedIDs map[int]*api.ContextDetail, targetInstanceIDSet sets.Int) []*api.ContextDetail - Get(detail *api.ContextDetail, enum api.ResourceContextKeyEnum) (string, bool) - Contains(detail *api.ContextDetail, enum api.ResourceContextKeyEnum, value string) bool - Put(detail *api.ContextDetail, enum api.ResourceContextKeyEnum, value string) - Remove(detail *api.ContextDetail, enum api.ResourceContextKeyEnum) -} - -type RealResourceContextControl struct { - client.Client - xsetController api.XSetController - resourceContextAdapter api.ResourceContextAdapter - resourceContextKeys map[api.ResourceContextKeyEnum]string - resourceContextGVK schema.GroupVersionKind - cacheExpectations expectations.CacheExpectationsInterface -} - -func NewRealResourceContextControl( - c client.Client, - xsetController api.XSetController, - resourceContextAdapter api.ResourceContextAdapter, - resourceContextGVK schema.GroupVersionKind, - cacheExpectations expectations.CacheExpectationsInterface, -) ResourceContextControl { - resourceContextKeys := resourceContextAdapter.GetContextKeys() - if resourceContextKeys == nil { - resourceContextKeys = defaultResourceContextKeys - } - - return &RealResourceContextControl{ - Client: c, - xsetController: xsetController, - resourceContextAdapter: resourceContextAdapter, - resourceContextKeys: resourceContextKeys, - resourceContextGVK: resourceContextGVK, - cacheExpectations: cacheExpectations, - } -} - -func (r *RealResourceContextControl) AllocateID( - ctx context.Context, - xsetObject api.XSetObject, - defaultRevision string, - replicas int, -) (map[int]*api.ContextDetail, error) { - contextName := getContextName(r.xsetController, xsetObject) - targetContext := r.resourceContextAdapter.NewResourceContext() - notFound := false - if err := r.Client.Get(ctx, types.NamespacedName{Namespace: xsetObject.GetNamespace(), Name: contextName}, targetContext); err != nil { - if !errors.IsNotFound(err) { - return nil, fmt.Errorf("fail to find ResourceContext %s/%s for owner %s: %s", xsetObject.GetNamespace(), contextName, xsetObject.GetName(), err.Error()) - } - - notFound = true - targetContext.SetNamespace(xsetObject.GetNamespace()) - targetContext.SetName(contextName) - } - - xsetSpec := r.xsetController.GetXSetSpec(xsetObject) - // store all the IDs crossing Multiple workload - existingIDs := map[int]*api.ContextDetail{} - // only store the IDs belonging to this owner - ownedIDs := map[int]*api.ContextDetail{} - resourceContextSpec := r.resourceContextAdapter.GetResourceContextSpec(targetContext) - for i := range resourceContextSpec.Contexts { - detail := &resourceContextSpec.Contexts[i] - if r.Contains(detail, api.EnumOwnerContextKey, xsetObject.GetName()) { - ownedIDs[detail.ID] = detail - existingIDs[detail.ID] = detail - } else if xsetSpec.ScaleStrategy.Context != "" { - // add other collaset targetContexts only if context pool enabled - existingIDs[detail.ID] = detail - } - } - - // if owner has enough ID, return - if len(ownedIDs) >= replicas { - return ownedIDs, nil - } - - // find new IDs for owner - candidateID := 0 - for len(ownedIDs) < replicas { - // find one new ID - for { - if _, exist := existingIDs[candidateID]; exist { - candidateID++ - continue - } - - break - } - - detail := &api.ContextDetail{ - ID: candidateID, - // TODO choose just create targets' revision according to scaleStrategy - Data: map[string]string{ - r.resourceContextKeys[api.EnumOwnerContextKey]: xsetObject.GetName(), - r.resourceContextKeys[api.EnumRevisionContextDataKey]: defaultRevision, - r.resourceContextKeys[api.EnumJustCreateContextDataKey]: "true", - }, - } - existingIDs[candidateID] = detail - ownedIDs[candidateID] = detail - } - - if notFound { - return ownedIDs, r.doCreateTargetContext(ctx, xsetObject, ownedIDs) - } - - return ownedIDs, r.doUpdateTargetContext(ctx, xsetObject, ownedIDs, targetContext) -} - -func (r *RealResourceContextControl) UpdateToTargetContext( - ctx context.Context, - xSetObject api.XSetObject, - ownedIDs map[int]*api.ContextDetail, -) error { - contextName := getContextName(r.xsetController, xSetObject) - targetContext := r.resourceContextAdapter.NewResourceContext() - if err := r.Client.Get(ctx, types.NamespacedName{Namespace: xSetObject.GetNamespace(), Name: contextName}, targetContext); err != nil { - if !errors.IsNotFound(err) { - return fmt.Errorf("fail to find ResourceContext %s/%s: %w", xSetObject.GetNamespace(), contextName, err) - } - - if len(ownedIDs) == 0 { - return nil - } - - if err := r.doCreateTargetContext(ctx, xSetObject, ownedIDs); err != nil { - return fmt.Errorf("fail to create ResourceContext %s/%s after not found: %w", xSetObject.GetNamespace(), contextName, err) - } - } - - return r.doUpdateTargetContext(ctx, xSetObject, ownedIDs, targetContext) -} - -func (r *RealResourceContextControl) ExtractAvailableContexts(diff int, ownedIDs map[int]*api.ContextDetail, targetInstanceIDSet sets.Int) []*api.ContextDetail { - var availableContexts []*api.ContextDetail - if diff <= 0 { - return availableContexts - } - - idx := 0 - for id := range ownedIDs { - if _, inUsed := targetInstanceIDSet[id]; inUsed { - continue - } - - availableContexts = append(availableContexts, ownedIDs[id]) - idx++ - if idx == diff { - break - } - } - - return availableContexts -} - -func (r *RealResourceContextControl) Get(detail *api.ContextDetail, enum api.ResourceContextKeyEnum) (string, bool) { - return detail.Get(r.resourceContextKeys[enum]) -} - -func (r *RealResourceContextControl) Contains(detail *api.ContextDetail, enum api.ResourceContextKeyEnum, value string) bool { - return detail.Contains(r.resourceContextKeys[enum], value) -} - -func (r *RealResourceContextControl) Put(detail *api.ContextDetail, enum api.ResourceContextKeyEnum, value string) { - detail.Put(r.resourceContextKeys[enum], value) -} - -func (r *RealResourceContextControl) Remove(detail *api.ContextDetail, enum api.ResourceContextKeyEnum) { - detail.Remove(r.resourceContextKeys[enum]) -} - -func (r *RealResourceContextControl) doCreateTargetContext( - ctx context.Context, - xSetObject api.XSetObject, - ownerIDs map[int]*api.ContextDetail, -) error { - contextName := getContextName(r.xsetController, xSetObject) - targetContext := r.resourceContextAdapter.NewResourceContext() - targetContext.SetNamespace(xSetObject.GetNamespace()) - targetContext.SetName(contextName) - - spec := &api.ResourceContextSpec{} - for i := range ownerIDs { - spec.Contexts = append(spec.Contexts, *ownerIDs[i]) - } - r.resourceContextAdapter.SetResourceContextSpec(spec, targetContext) - if err := r.Client.Create(ctx, targetContext); err != nil { - return err - } - return r.cacheExpectations.ExpectCreation(clientutil.ObjectKeyString(xSetObject), r.resourceContextGVK, targetContext.GetNamespace(), targetContext.GetName()) -} - -func (r *RealResourceContextControl) doUpdateTargetContext( - ctx context.Context, - xsetObject client.Object, - ownedIDs map[int]*api.ContextDetail, - targetContext api.ResourceContextObject, -) error { - // store all IDs crossing all workload - existingIDs := map[int]*api.ContextDetail{} - - // add other collaset targetContexts only if context pool enabled - xsetSpec := r.xsetController.GetXSetSpec(xsetObject) - resourceContextSpec := r.resourceContextAdapter.GetResourceContextSpec(targetContext) - ownerContextKey := r.resourceContextKeys[api.EnumOwnerContextKey] - if xsetSpec.ScaleStrategy.Context != "" { - for i := range resourceContextSpec.Contexts { - detail := resourceContextSpec.Contexts[i] - if detail.Contains(ownerContextKey, xsetObject.GetName()) { - continue - } - existingIDs[detail.ID] = &detail - } - } - - for _, contextDetail := range ownedIDs { - existingIDs[contextDetail.ID] = contextDetail - } - - // delete TargetContext if it is empty - if len(existingIDs) == 0 { - err := r.Client.Delete(ctx, targetContext) - if err != nil { - return err - } - return r.cacheExpectations.ExpectDeletion(clientutil.ObjectKeyString(xsetObject), r.resourceContextGVK, targetContext.GetNamespace(), targetContext.GetName()) - } - - resourceContextSpec.Contexts = make([]api.ContextDetail, len(existingIDs)) - idx := 0 - for _, contextDetail := range existingIDs { - resourceContextSpec.Contexts[idx] = *contextDetail - idx++ - } - - // keep context detail in order by ID - sort.Sort(ContextDetailsByOrder(resourceContextSpec.Contexts)) - r.resourceContextAdapter.SetResourceContextSpec(resourceContextSpec, targetContext) - err := r.Client.Update(ctx, targetContext) - if err != nil { - return err - } - return r.cacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(xsetObject), r.resourceContextGVK, targetContext.GetNamespace(), targetContext.GetName(), targetContext.GetResourceVersion()) -} - -func getContextName(xsetControl api.XSetController, instance api.XSetObject) string { - spec := xsetControl.GetXSetSpec(instance) - if spec.ScaleStrategy.Context != "" { - return spec.ScaleStrategy.Context - } - - return instance.GetName() -} - -type ContextDetailsByOrder []api.ContextDetail - -func (s ContextDetailsByOrder) Len() int { return len(s) } -func (s ContextDetailsByOrder) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func (s ContextDetailsByOrder) Less(i, j int) bool { - l, r := s[i], s[j] - return l.ID < r.ID -} diff --git a/xset/revisionowner/revision_owner.go b/xset/revisionowner/revision_owner.go deleted file mode 100644 index bfed345..0000000 --- a/xset/revisionowner/revision_owner.go +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package revisionowner - -import ( - "context" - - appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/sets" - - "kusionstack.io/kube-utils/controller/history" - "kusionstack.io/kube-utils/xset/api" - "kusionstack.io/kube-utils/xset/xcontrol" -) - -var _ history.RevisionOwner = &revisionOwner{} - -type revisionOwner struct { - api.XSetController - - xcontrol.TargetControl -} - -func NewRevisionOwner(xsetController api.XSetController, xcontrol xcontrol.TargetControl) *revisionOwner { - return &revisionOwner{ - XSetController: xsetController, - TargetControl: xcontrol, - } -} - -func (r *revisionOwner) GetGroupVersionKind() schema.GroupVersionKind { - meta := r.XSetController.XSetMeta() - return meta.GroupVersionKind() -} - -func (r *revisionOwner) GetMatchLabels(parent metav1.Object) map[string]string { - obj := parent.(api.XSetObject) - xsetSpec := r.XSetController.GetXSetSpec(obj) - // that's ok to return nil, manager will use nothingSelector instead - return xsetSpec.Selector.MatchLabels -} - -func (r *revisionOwner) GetInUsedRevisions(parent metav1.Object) (sets.String, error) { - xSetObject, _ := parent.(api.XSetObject) - spec := r.XSetController.GetXSetSpec(xSetObject) - status := r.XSetController.GetXSetStatus(xSetObject) - - res := sets.String{} - - res.Insert(status.UpdatedRevision) - res.Insert(status.CurrentRevision) - - targets, _, err := r.TargetControl.GetFilteredTargets(context.TODO(), spec.Selector, xSetObject) - if err != nil { - return nil, err - } - for _, target := range targets { - if target.GetLabels() != nil { - currentRevisionName, exist := target.GetLabels()[appsv1.ControllerRevisionHashLabelKey] - if exist { - res.Insert(currentRevisionName) - } - } - } - return res, nil -} - -func (r *revisionOwner) GetCollisionCount(obj metav1.Object) *int32 { - xset := obj.(api.XSetObject) - return r.XSetController.GetXSetStatus(xset).CollisionCount -} - -func (r *revisionOwner) GetHistoryLimit(obj metav1.Object) int32 { - xset := obj.(api.XSetObject) - return r.XSetController.GetXSetSpec(xset).HistoryLimit -} - -func (r *revisionOwner) GetPatch(obj metav1.Object) ([]byte, error) { - return r.getXSetPatch(obj) -} - -func (r *revisionOwner) GetCurrentRevision(obj metav1.Object) string { - xset := obj.(api.XSetObject) - return r.XSetController.GetXSetStatus(xset).CurrentRevision -} - -func (r *revisionOwner) getXSetPatch(obj metav1.Object) ([]byte, error) { - return r.XSetController.GetXSetPatch(obj) -} diff --git a/xset/subresources/getter.go b/xset/subresources/getter.go deleted file mode 100644 index 3c9349f..0000000 --- a/xset/subresources/getter.go +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package subresources - -import "kusionstack.io/kube-utils/xset/api" - -func GetSubresourcePvcAdapter(control api.XSetController) (adapter api.SubResourcePvcAdapter, enabled bool) { - adapter, enabled = control.(api.SubResourcePvcAdapter) - return adapter, enabled -} diff --git a/xset/subresources/pvc_control.go b/xset/subresources/pvc_control.go deleted file mode 100644 index eca2021..0000000 --- a/xset/subresources/pvc_control.go +++ /dev/null @@ -1,557 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package subresources - -import ( - "context" - "encoding/json" - "fmt" - "hash/fnv" - "strings" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/rand" - "k8s.io/apimachinery/pkg/util/sets" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" - - kubeutilclient "kusionstack.io/kube-utils/client" - "kusionstack.io/kube-utils/controller/expectations" - "kusionstack.io/kube-utils/controller/mixin" - refmanagerutil "kusionstack.io/kube-utils/controller/refmanager" - "kusionstack.io/kube-utils/xset/api" -) - -const ( - FieldIndexOwnerRefUID = "ownerRefUID" -) - -var PVCGvk = corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim") - -type PvcControl interface { - GetFilteredPvcs(context.Context, api.XSetObject) ([]*corev1.PersistentVolumeClaim, error) - CreateTargetPvcs(context.Context, api.XSetObject, client.Object, []*corev1.PersistentVolumeClaim) error - DeleteTargetPvcs(context.Context, api.XSetObject, client.Object, []*corev1.PersistentVolumeClaim) error - DeleteTargetUnusedPvcs(context.Context, api.XSetObject, client.Object, []*corev1.PersistentVolumeClaim) error - OrphanPvc(context.Context, api.XSetObject, *corev1.PersistentVolumeClaim) error - AdoptPvc(context.Context, api.XSetObject, *corev1.PersistentVolumeClaim) error - AdoptPvcsLeftByRetainPolicy(context.Context, api.XSetObject) ([]*corev1.PersistentVolumeClaim, error) - IsTargetPvcTmpChanged(api.XSetObject, client.Object, []*corev1.PersistentVolumeClaim) (bool, error) - RetainPvcWhenXSetDeleted(xset api.XSetObject) bool - RetainPvcWhenXSetScaled(xset api.XSetObject) bool -} - -type RealPvcControl struct { - client client.Client - scheme *runtime.Scheme - pvcAdapter api.SubResourcePvcAdapter - expectations *expectations.CacheExpectations - xsetLabelAnnoMgr api.XSetLabelAnnotationManager - xsetController api.XSetController -} - -func NewRealPvcControl(mixin *mixin.ReconcilerMixin, expectations *expectations.CacheExpectations, xsetLabelAnnoMgr api.XSetLabelAnnotationManager, xsetController api.XSetController) (PvcControl, error) { - // requires implementation of SubResourcePvcAdapter - pvcAdapter, ok := GetSubresourcePvcAdapter(xsetController) - if !ok { - return nil, nil - } - // here we go, set up cache and return real pvc control - if err := setUpCache(mixin.Cache, xsetController); err != nil { - return nil, err - } - return &RealPvcControl{ - client: mixin.Client, - scheme: mixin.Scheme, - pvcAdapter: pvcAdapter, - expectations: expectations, - xsetLabelAnnoMgr: xsetLabelAnnoMgr, - xsetController: xsetController, - }, nil -} - -func (pc *RealPvcControl) GetFilteredPvcs(ctx context.Context, xset api.XSetObject) ([]*corev1.PersistentVolumeClaim, error) { - // list pvcs using ownerReference - var filteredPvcs []*corev1.PersistentVolumeClaim - ownedPvcList := &corev1.PersistentVolumeClaimList{} - if err := pc.client.List(ctx, ownedPvcList, &client.ListOptions{ - Namespace: xset.GetNamespace(), - FieldSelector: fields.OneTermEqualSelector(FieldIndexOwnerRefUID, string(xset.GetUID())), - }); err != nil { - return nil, err - } - - for i := range ownedPvcList.Items { - pvc := &ownedPvcList.Items[i] - if pvc.DeletionTimestamp == nil { - filteredPvcs = append(filteredPvcs, pvc) - } - } - return filteredPvcs, nil -} - -func (pc *RealPvcControl) CreateTargetPvcs(ctx context.Context, xset api.XSetObject, x client.Object, existingPvcs []*corev1.PersistentVolumeClaim) error { - id, exist := pc.xsetLabelAnnoMgr.Get(x.GetLabels(), api.XInstanceIdLabelKey) - if !exist { - return nil - } - - // provision pvcs related to pod using pvc template, and reuse - // pvcs if "instance-id" and "pvc-template-hash" label matched - pvcsMap, err := pc.provisionUpdatedPvc(ctx, id, xset, existingPvcs) - if err != nil { - return err - } - - newVolumes := make([]corev1.Volume, 0, len(pvcsMap)) - // mount updated pvcs to target - for name, pvc := range pvcsMap { - volume := corev1.Volume{ - Name: name, - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc.Name, - ReadOnly: false, - }, - }, - } - newVolumes = append(newVolumes, volume) - } - - // append legacy pvcs - currentVolumes := pc.pvcAdapter.GetXSpecVolumes(x) - for i := range currentVolumes { - currentVolume := currentVolumes[i] - if _, ok := pvcsMap[currentVolume.Name]; !ok { - newVolumes = append(newVolumes, currentVolume) - } - } - pc.pvcAdapter.SetXSpecVolumes(x, newVolumes) - return nil -} - -func (pc *RealPvcControl) provisionUpdatedPvc(ctx context.Context, id string, xset api.XSetObject, existingPvcs []*corev1.PersistentVolumeClaim) (map[string]*corev1.PersistentVolumeClaim, error) { - updatedPvcs, _, err := pc.classifyTargetPvcs(id, xset, existingPvcs) - if err != nil { - return nil, err - } - - templates := pc.pvcAdapter.GetXSetPvcTemplate(xset) - for i := range templates { - pvcTmp := templates[i] - // reuse pvc - if _, exist := updatedPvcs[pvcTmp.Name]; exist { - continue - } - // create new pvc - claim, err := pc.buildPvcWithHash(id, xset, &pvcTmp) - if err != nil { - return nil, err - } - - if err := pc.client.Create(ctx, claim); err != nil { - return nil, fmt.Errorf("fail to create pvc for id %s: %w", id, err) - } - - if err := pc.expectations.ExpectCreation( - kubeutilclient.ObjectKeyString(xset), - PVCGvk, - claim.Namespace, - claim.Name, - ); err != nil { - return nil, err - } - - updatedPvcs[pvcTmp.Name] = claim - } - return updatedPvcs, nil -} - -func (pc *RealPvcControl) DeleteTargetPvcs(ctx context.Context, xset api.XSetObject, x client.Object, pvcs []*corev1.PersistentVolumeClaim) error { - for _, pvc := range pvcs { - if pvc.Labels == nil || x.GetLabels() == nil { - continue - } - - // only delete pvcs used by target - pvcId, _ := pc.xsetLabelAnnoMgr.Get(pvc.Labels, api.XInstanceIdLabelKey) - targetId, _ := pc.xsetLabelAnnoMgr.Get(x.GetLabels(), api.XInstanceIdLabelKey) - if pvcId != targetId { - continue - } - - if err := deletePvcWithExpectations(ctx, pc.client, xset, pc.expectations, pvc); err != nil { - return err - } - } - return nil -} - -func (pc *RealPvcControl) DeleteTargetUnusedPvcs(ctx context.Context, xset api.XSetObject, x client.Object, existingPvcs []*corev1.PersistentVolumeClaim) error { - if x.GetLabels() == nil { - return nil - } - id, exist := pc.xsetLabelAnnoMgr.Get(x.GetLabels(), api.XInstanceIdLabelKey) - if !exist { - return nil - } - - newPvcs, oldPvcs, err := pc.classifyTargetPvcs(id, xset, existingPvcs) - if err != nil { - return err - } - - volumeMounts := pc.pvcAdapter.GetXVolumeMounts(x) - mountedVolumeTmps := sets.String{} - for i := range volumeMounts { - mountedVolumeTmps.Insert(volumeMounts[i].Name) - } - - // delete pvc which is not claimed in templates - if err := pc.deleteUnclaimedPvcs(ctx, xset, oldPvcs, mountedVolumeTmps); err != nil { - return err - } - // delete old pvc if new pvc is provisioned and not RetainPVCWhenXSetScaled - if !pc.pvcAdapter.RetainPvcWhenXSetScaled(xset) { - return pc.deleteOldPvcs(ctx, xset, newPvcs, oldPvcs) - } - return nil -} - -func (pc *RealPvcControl) AdoptPvc(ctx context.Context, xset api.XSetObject, pvc *corev1.PersistentVolumeClaim) error { - xsetSpec := pc.xsetController.GetXSetSpec(xset) - if xsetSpec.Selector.MatchLabels == nil { - return nil - } - refWriter := refmanagerutil.NewOwnerRefWriter(pc.client) - matcher, err := refmanagerutil.LabelSelectorAsMatch(xsetSpec.Selector) - if err != nil { - return fmt.Errorf("fail to create labelSelector matcher: %s", err.Error()) - } - refManager := refmanagerutil.NewObjectControllerRefManager(refWriter, xset, xset.GetObjectKind().GroupVersionKind(), matcher) - - if _, err := refManager.Claim(ctx, pvc); err != nil { - return fmt.Errorf("failed to adopt pvc: %s", err.Error()) - } - return nil -} - -func (pc *RealPvcControl) OrphanPvc(ctx context.Context, xset api.XSetObject, pvc *corev1.PersistentVolumeClaim) error { - xsetSpec := pc.xsetController.GetXSetSpec(xset) - if xsetSpec.Selector.MatchLabels == nil { - return nil - } - if pvc.Labels == nil { - pvc.Labels = make(map[string]string) - } - if pvc.Annotations == nil { - pvc.Annotations = make(map[string]string) - } - - refWriter := refmanagerutil.NewOwnerRefWriter(pc.client) - if err := refWriter.Release(ctx, xset, pvc); err != nil { - return fmt.Errorf("failed to orphan target: %s", err.Error()) - } - return nil -} - -func (pc *RealPvcControl) AdoptPvcsLeftByRetainPolicy(ctx context.Context, xset api.XSetObject) ([]*corev1.PersistentVolumeClaim, error) { - xsetSpec := pc.xsetController.GetXSetSpec(xset) - ownerSelector := xsetSpec.Selector.DeepCopy() - if ownerSelector.MatchLabels == nil { - ownerSelector.MatchLabels = map[string]string{} - } - ownerSelector.MatchLabels[pc.xsetLabelAnnoMgr.Value(api.ControlledByXSetLabel)] = "true" - ownerSelector.MatchExpressions = append(ownerSelector.MatchExpressions, metav1.LabelSelectorRequirement{ // nolint - Key: pc.xsetLabelAnnoMgr.Value(api.XOrphanedIndicationLabelKey), // should not be excluded pvcs - Operator: metav1.LabelSelectorOpDoesNotExist, - }) - ownerSelector.MatchExpressions = append(ownerSelector.MatchExpressions, metav1.LabelSelectorRequirement{ - Key: pc.xsetLabelAnnoMgr.Value(api.XInstanceIdLabelKey), // instance-id label should exist - Operator: metav1.LabelSelectorOpExists, - }) - ownerSelector.MatchExpressions = append(ownerSelector.MatchExpressions, metav1.LabelSelectorRequirement{ - Key: pc.xsetLabelAnnoMgr.Value(api.SubResourcePvcTemplateHashLabelKey), // pvc-hash label should exist - Operator: metav1.LabelSelectorOpExists, - }) - - selector, err := metav1.LabelSelectorAsSelector(ownerSelector) - if err != nil { - return nil, err - } - - orphanedPvcList := &corev1.PersistentVolumeClaimList{} - if err := pc.client.List(ctx, orphanedPvcList, &client.ListOptions{Namespace: xset.GetNamespace(), LabelSelector: selector}); err != nil { - return nil, err - } - - // adopt orphaned pvcs - var claims []*corev1.PersistentVolumeClaim - for i := range orphanedPvcList.Items { - pvc := orphanedPvcList.Items[i] - if pvc.OwnerReferences != nil && len(pvc.OwnerReferences) > 0 { - continue - } - if pvc.Labels == nil { - pvc.Labels = make(map[string]string) - } - if pvc.Annotations == nil { - pvc.Annotations = make(map[string]string) - } - - claims = append(claims, &pvc) - } - for i := range claims { - if err := pc.AdoptPvc(ctx, xset, claims[i]); err != nil { - return nil, err - } - } - return claims, nil -} - -func (pc *RealPvcControl) IsTargetPvcTmpChanged(xset api.XSetObject, x client.Object, existingPvcs []*corev1.PersistentVolumeClaim) (bool, error) { - pvcTemplates := pc.pvcAdapter.GetXSetPvcTemplate(xset) - xSpecVolumes := pc.pvcAdapter.GetXSpecVolumes(x) - // get pvc template hash values - newHashMapping, err := PvcTmpHashMapping(pvcTemplates) - if err != nil { - return false, err - } - - // get existing x pvcs hash values - existingPvcHash := map[string]string{} - for _, pvc := range existingPvcs { - if pvc.Labels == nil || x.GetLabels() == nil { - continue - } - pvcId, _ := pc.xsetLabelAnnoMgr.Get(pvc.Labels, api.XInstanceIdLabelKey) - targetId, _ := pc.xsetLabelAnnoMgr.Get(x.GetLabels(), api.XInstanceIdLabelKey) - if pvcId != targetId { - continue - } - if _, exist := pc.xsetLabelAnnoMgr.Get(pvc.Labels, api.SubResourcePvcTemplateHashLabelKey); !exist { - continue - } - existingPvcHash[pvc.Name] = pvc.Labels[appsv1alpha1.PvcTemplateHashLabelKey] - } - - // check mounted pvcs changed - for i := range xSpecVolumes { - volume := xSpecVolumes[i] - if volume.PersistentVolumeClaim == nil || volume.PersistentVolumeClaim.ClaimName == "" { - continue - } - pvcName := volume.PersistentVolumeClaim.ClaimName - TmpName := volume.Name - if newHashMapping[TmpName] != existingPvcHash[pvcName] { - return true, nil - } - } - return false, nil -} - -func (pc *RealPvcControl) RetainPvcWhenXSetDeleted(xset api.XSetObject) bool { - return pc.pvcAdapter.RetainPvcWhenXSetDeleted(xset) -} - -func (pc *RealPvcControl) RetainPvcWhenXSetScaled(xset api.XSetObject) bool { - return pc.pvcAdapter.RetainPvcWhenXSetScaled(xset) -} - -func (pc *RealPvcControl) deleteUnclaimedPvcs(ctx context.Context, xset api.XSetObject, oldPvcs map[string]*corev1.PersistentVolumeClaim, mountedPvcNames sets.String) error { - inUsedPvcNames := sets.String{} - templates := pc.pvcAdapter.GetXSetPvcTemplate(xset) - for i := range templates { - inUsedPvcNames.Insert(templates[i].Name) - } - for pvcTmpName, pvc := range oldPvcs { - // if pvc is still mounted on target, keep it - if mountedPvcNames.Has(pvcTmpName) { - continue - } - - // is pvc is claimed in pvc templates, keep it - if inUsedPvcNames.Has(pvcTmpName) { - continue - } - - if err := deletePvcWithExpectations(ctx, pc.client, xset, pc.expectations, pvc); err != nil { - return err - } - } - return nil -} - -func (pc *RealPvcControl) deleteOldPvcs(ctx context.Context, xset api.XSetObject, newPvcs, oldPvcs map[string]*corev1.PersistentVolumeClaim) error { - for pvcTmpName, pvc := range oldPvcs { - if _, newPvcExist := newPvcs[pvcTmpName]; !newPvcExist { - continue - } - if err := deletePvcWithExpectations(ctx, pc.client, xset, pc.expectations, pvc); err != nil { - return err - } - } - return nil -} - -func (pc *RealPvcControl) buildPvcWithHash(id string, xset api.XSetObject, pvcTmp *corev1.PersistentVolumeClaim) (*corev1.PersistentVolumeClaim, error) { - claim := pvcTmp.DeepCopy() - claim.Name = "" - claim.GenerateName = fmt.Sprintf("%s-%s-", xset.GetName(), pvcTmp.Name) - claim.Namespace = xset.GetNamespace() - xsetMeta := pc.xsetController.XSetMeta() - xsetGvk := xsetMeta.GroupVersionKind() - claim.OwnerReferences = append(claim.OwnerReferences, - *metav1.NewControllerRef(xset, xsetGvk)) - - if claim.Labels == nil { - claim.Labels = map[string]string{} - } - xsetSpec := pc.xsetController.GetXSetSpec(xset) - for k, v := range xsetSpec.Selector.MatchLabels { - claim.Labels[k] = v - } - pc.xsetLabelAnnoMgr.Set(claim, api.ControlledByXSetLabel, "true") - - hash, err := PvcTmpHash(pvcTmp) - if err != nil { - return nil, err - } - pc.xsetLabelAnnoMgr.Set(claim, api.SubResourcePvcTemplateHashLabelKey, hash) - pc.xsetLabelAnnoMgr.Set(claim, api.XInstanceIdLabelKey, id) - pc.xsetLabelAnnoMgr.Set(claim, api.SubResourcePvcTemplateLabelKey, pvcTmp.Name) - return claim, nil -} - -// classify pvcs into old and new ones -func (pc *RealPvcControl) classifyTargetPvcs(id string, xset api.XSetObject, existingPvcs []*corev1.PersistentVolumeClaim) (map[string]*corev1.PersistentVolumeClaim, map[string]*corev1.PersistentVolumeClaim, error) { - newPvcs := map[string]*corev1.PersistentVolumeClaim{} - oldPvcs := map[string]*corev1.PersistentVolumeClaim{} - - newPvcTemplates := pc.pvcAdapter.GetXSetPvcTemplate(xset) - newTmpHash, err := PvcTmpHashMapping(newPvcTemplates) - if err != nil { - return newPvcs, oldPvcs, err - } - - for _, pvc := range existingPvcs { - if pvc.DeletionTimestamp != nil { - continue - } - - if pvc.Labels == nil { - continue - } - - if val, exist := pc.xsetLabelAnnoMgr.Get(pvc.Labels, api.XInstanceIdLabelKey); !exist { - continue - } else if val != id { - continue - } - - if _, exist := pc.xsetLabelAnnoMgr.Get(pvc.Labels, api.SubResourcePvcTemplateHashLabelKey); !exist { - continue - } - hash, _ := pc.xsetLabelAnnoMgr.Get(pvc.Labels, api.SubResourcePvcTemplateHashLabelKey) - pvcTmpName, err := pc.extractPvcTmpName(xset, pvc) - if err != nil { - return nil, nil, err - } - - // classify into updated and old pvcs - if newTmpHash[pvcTmpName] == hash { - newPvcs[pvcTmpName] = pvc - } else { - oldPvcs[pvcTmpName] = pvc - } - } - - return newPvcs, oldPvcs, nil -} - -func (pc *RealPvcControl) extractPvcTmpName(xset api.XSetObject, pvc *corev1.PersistentVolumeClaim) (string, error) { - if pvcTmpName, exist := pc.xsetLabelAnnoMgr.Get(pvc.Labels, api.SubResourcePvcTemplateLabelKey); exist { - return pvcTmpName, nil - } - lastDashIndex := strings.LastIndex(pvc.Name, "-") - if lastDashIndex == -1 { - return "", fmt.Errorf("pvc %s has no postfix", pvc.Name) - } - - rest := pvc.Name[:lastDashIndex] - if !strings.HasPrefix(rest, xset.GetName()+"-") { - return "", fmt.Errorf("malformed pvc name %s, expected a part of CollaSet name %s", pvc.Name, xset.GetName()) - } - - return strings.TrimPrefix(rest, xset.GetName()+"-"), nil -} - -func PvcTmpHash(pvc *corev1.PersistentVolumeClaim) (string, error) { - bytes, err := json.Marshal(pvc) - if err != nil { - return "", fmt.Errorf("fail to marshal pvc template: %w", err) - } - - hf := fnv.New32() - if _, err = hf.Write(bytes); err != nil { - return "", fmt.Errorf("fail to calculate pvc template hash: %w", err) - } - - return rand.SafeEncodeString(fmt.Sprint(hf.Sum32())), nil -} - -func PvcTmpHashMapping(pvcTmps []corev1.PersistentVolumeClaim) (map[string]string, error) { - pvcHashMapping := map[string]string{} - for i := range pvcTmps { - pvcTmp := pvcTmps[i] - hash, err := PvcTmpHash(&pvcTmp) - if err != nil { - return nil, err - } - pvcHashMapping[pvcTmp.Name] = hash - } - return pvcHashMapping, nil -} - -func deletePvcWithExpectations(ctx context.Context, client client.Client, xset api.XSetObject, expectations *expectations.CacheExpectations, pvc *corev1.PersistentVolumeClaim) error { - if err := client.Delete(ctx, pvc); err != nil { - return err - } - - // expect deletion - if err := expectations.ExpectDeletion(kubeutilclient.ObjectKeyString(xset), PVCGvk, pvc.GetNamespace(), pvc.GetName()); err != nil { - return err - } - return nil -} - -func setUpCache(cache cache.Cache, controller api.XSetController) error { - if err := cache.IndexField(context.TODO(), &corev1.PersistentVolumeClaim{}, FieldIndexOwnerRefUID, func(object client.Object) []string { - ownerRef := metav1.GetControllerOf(object) - if ownerRef == nil || ownerRef.Kind != controller.XSetMeta().Kind { - return nil - } - return []string{string(ownerRef.UID)} - }); err != nil { - return fmt.Errorf("failed to index by field for pvc->xset %s: %s", FieldIndexOwnerRefUID, err.Error()) - } - return nil -} diff --git a/xset/synccontrols/inexclude.go b/xset/synccontrols/inexclude.go deleted file mode 100644 index 70b5d6e..0000000 --- a/xset/synccontrols/inexclude.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package synccontrols - -import ( - "fmt" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "kusionstack.io/kube-utils/xset/api" -) - -// AllowResourceExclude checks if pod or pvc is allowed to exclude -func AllowResourceExclude(obj metav1.Object, ownerName, ownerKind string, manager api.XSetLabelAnnotationManager) (bool, string) { - labels := obj.GetLabels() - // not controlled by ks manager - if labels == nil { - return false, "object's label is empty" - } else if val, exist := manager.Get(labels, api.ControlledByXSetLabel); !exist || val != "true" { - return false, "object is not controlled by kusionstack system" - } - - // not controlled by current xset - if controller := metav1.GetControllerOf(obj); controller == nil || controller.Name != ownerName || controller.Kind != ownerKind { - return false, "object is not owned by any one, not allowed to exclude" - } - return true, "" -} - -// AllowResourceInclude checks if pod or pvc is allowed to include -func AllowResourceInclude(obj metav1.Object, ownerName, ownerKind string, manager api.XSetLabelAnnotationManager) (bool, string) { - labels := obj.GetLabels() - ownerRefs := obj.GetOwnerReferences() - - // not controlled by ks manager - if labels == nil { - return false, "object's label is empty" - } else if val, exist := manager.Get(labels, api.ControlledByXSetLabel); !exist || val != "true" { - return false, "object is not controlled by kusionstack system" - } - - if ownerRefs != nil { - if controller := metav1.GetControllerOf(obj); controller != nil { - // controlled by others - if controller.Name != ownerName || controller.Kind != ownerKind { - return false, fmt.Sprintf("object's ownerReference controller is not %s/%s", ownerKind, ownerName) - } - } - } - return true, "" -} diff --git a/xset/synccontrols/inexclude_test.go b/xset/synccontrols/inexclude_test.go deleted file mode 100644 index c4803ad..0000000 --- a/xset/synccontrols/inexclude_test.go +++ /dev/null @@ -1,302 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package synccontrols - -import ( - "testing" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - - "kusionstack.io/kube-utils/xset/api" -) - -func TestAllowResourceExclude(t *testing.T) { - ownerName, ownerKind := "test", "CollaSet" - tests := []struct { - name string - obj *corev1.Pod - allow bool - reason string - }{ - { - name: "label is nil", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: nil, - }, - }, - allow: false, - reason: "object's label is empty", - }, - { - name: "KusionStack control label not satisfied", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "false", - }, - }, - }, - allow: false, - reason: "object is not controlled by kusionstack system", - }, - { - name: "controller is nil", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: make([]metav1.OwnerReference, 0), - }, - }, - allow: false, - reason: "object is not owned by any one, not allowed to exclude", - }, - { - name: "controller name not equals to ownerName", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: []metav1.OwnerReference{ - { - Name: "test1", - Kind: ownerKind, - Controller: pointer.Bool(true), - }, - }, - }, - }, - allow: false, - reason: "object is not owned by any one, not allowed to exclude", - }, - { - name: "controller kind not equals to ownerName", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: []metav1.OwnerReference{ - { - Name: ownerName, - Kind: "kind2", - Controller: pointer.Bool(true), - }, - }, - }, - }, - allow: false, - reason: "object is not owned by any one, not allowed to exclude", - }, - { - name: "allowed case", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: []metav1.OwnerReference{ - { - Name: ownerName, - Kind: ownerKind, - Controller: pointer.Bool(true), - }, - }, - }, - }, - allow: true, - reason: "", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, got1 := AllowResourceExclude(tt.obj, ownerName, ownerKind, api.NewXSetLabelAnnotationManager()) - if got != tt.allow { - t.Errorf("AllowResourceExclude() got = %v, want %v", got, tt.allow) - } - if got1 != tt.reason { - t.Errorf("AllowResourceExclude() got1 = %v, want %v", got1, tt.reason) - } - }) - } -} - -func TestAllowResourceInclude(t *testing.T) { - ownerName := "test" - ownerKind := "CollaSet" - tests := []struct { - name string - obj *corev1.Pod - allow bool - reason string - }{ - { - name: "label is nil", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: nil, - }, - }, - allow: false, - reason: "object's label is empty", - }, - { - name: "KusionStack control label not satisfied", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "false", - }, - }, - }, - allow: false, - reason: "object is not controlled by kusionstack system", - }, - { - name: "controller is nil", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: make([]metav1.OwnerReference, 0), - }, - }, - allow: true, - reason: "", - }, - { - name: "controller name not equals to ownerName", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: []metav1.OwnerReference{ - { - Name: "test1", - Kind: ownerKind, - Controller: pointer.Bool(true), - }, - }, - }, - }, - allow: false, - reason: "object's ownerReference controller is not CollaSet/test", - }, - { - name: "controller kind not equals to ownerName", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: []metav1.OwnerReference{ - { - Name: "test", - Kind: "kind2", - Controller: pointer.Bool(true), - }, - }, - }, - }, - allow: false, - reason: "object's ownerReference controller is not CollaSet/test", - }, - { - name: "controller kind not equals to ownerName", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - OwnerReferences: []metav1.OwnerReference{ - { - Name: ownerName, - Kind: ownerKind, - Controller: pointer.Bool(true), - }, - }, - }, - }, - allow: true, - reason: "", - }, - { - name: "allowed case1", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - appsv1alpha1.PodOrphanedIndicateLabelKey: "true", - }, - OwnerReferences: []metav1.OwnerReference{ - { - Name: ownerName, - Kind: ownerKind, - Controller: pointer.Bool(true), - }, - }, - }, - }, - allow: true, - reason: "", - }, - { - name: "allowed case2", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - }, - }, - }, - allow: true, - reason: "", - }, - { - name: "allowed case3", - obj: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - appsv1alpha1.ControlledByKusionStackLabelKey: "true", - appsv1alpha1.PodOrphanedIndicateLabelKey: "true", - }, - }, - }, - allow: true, - reason: "", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, got1 := AllowResourceInclude(tt.obj, ownerName, ownerKind, api.NewXSetLabelAnnotationManager()) - if got != tt.allow { - t.Errorf("AllowResourceExclude() got = %v, want %v", got, tt.allow) - } - if got1 != tt.reason { - t.Errorf("AllowResourceExclude() got1 = %v, want %v", got1, tt.reason) - } - }) - } -} diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go deleted file mode 100644 index 3934764..0000000 --- a/xset/synccontrols/sync_control.go +++ /dev/null @@ -1,1093 +0,0 @@ -/* -Copyright 2023-2025 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package synccontrols - -import ( - "context" - "errors" - "fmt" - "strconv" - "strings" - "sync/atomic" - "time" - - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/util/retry" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - - clientutil "kusionstack.io/kube-utils/client" - "kusionstack.io/kube-utils/controller/expectations" - "kusionstack.io/kube-utils/controller/mixin" - controllerutils "kusionstack.io/kube-utils/controller/utils" - "kusionstack.io/kube-utils/xset/api" - "kusionstack.io/kube-utils/xset/opslifecycle" - "kusionstack.io/kube-utils/xset/resourcecontexts" - "kusionstack.io/kube-utils/xset/subresources" - "kusionstack.io/kube-utils/xset/xcontrol" -) - -type SyncControl interface { - SyncTargets(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) (bool, error) - - Replace(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) error - - Scale(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) - - Update(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) - - CalculateStatus(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) *api.XSetStatus - - BatchDeleteTargetsByLabel(ctx context.Context, targetControl xcontrol.TargetControl, needDeleteTargets []client.Object) error -} - -func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, - xsetController api.XSetController, - xControl xcontrol.TargetControl, - pvcControl subresources.PvcControl, - xsetLabelAnnoManager api.XSetLabelAnnotationManager, - resourceContexts resourcecontexts.ResourceContextControl, - cacheExpectations expectations.CacheExpectationsInterface, -) SyncControl { - xMeta := xsetController.XMeta() - targetGVK := xMeta.GroupVersionKind() - xsetMeta := xsetController.XSetMeta() - xsetGVK := xsetMeta.GroupVersionKind() - updateLifecycleAdapter, scaleInOpsLifecycleAdapter := opslifecycle.GetLifecycleAdapters(xsetController, xsetLabelAnnoManager, xsetMeta) - - updateConfig := &UpdateConfig{ - XsetController: xsetController, - XsetLabelAnnoMgr: xsetLabelAnnoManager, - Client: reconcileMixIn.Client, - TargetControl: xControl, - ResourceContextControl: resourceContexts, - Recorder: reconcileMixIn.Recorder, - - scaleInLifecycleAdapter: scaleInOpsLifecycleAdapter, - updateLifecycleAdapter: updateLifecycleAdapter, - CacheExpectations: cacheExpectations, - TargetGVK: targetGVK, - } - return &RealSyncControl{ - ReconcilerMixin: *reconcileMixIn, - xsetController: xsetController, - xsetLabelAnnoMgr: xsetLabelAnnoManager, - resourceContextControl: resourceContexts, - xControl: xControl, - pvcControl: pvcControl, - - updateConfig: updateConfig, - cacheExpectations: cacheExpectations, - xsetGVK: xsetGVK, - targetGVK: targetGVK, - - scaleInLifecycleAdapter: scaleInOpsLifecycleAdapter, - updateLifecycleAdapter: updateLifecycleAdapter, - } -} - -var _ SyncControl = &RealSyncControl{} - -type RealSyncControl struct { - mixin.ReconcilerMixin - xControl xcontrol.TargetControl - pvcControl subresources.PvcControl - xsetController api.XSetController - xsetLabelAnnoMgr api.XSetLabelAnnotationManager - resourceContextControl resourcecontexts.ResourceContextControl - - updateConfig *UpdateConfig - scaleInLifecycleAdapter api.LifecycleAdapter - updateLifecycleAdapter api.LifecycleAdapter - - cacheExpectations expectations.CacheExpectationsInterface - xsetGVK schema.GroupVersionKind - targetGVK schema.GroupVersionKind -} - -// SyncTargets is used to parse targetWrappers and reclaim Target instance ID -func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) ( - bool, error, -) { - xspec := r.xsetController.GetXSetSpec(instance) - if xspec == nil { - return false, fmt.Errorf("fail to get XSetSpec") - } - - filteredTargets, allTargets, err := r.xControl.GetFilteredTargets(ctx, xspec.Selector, instance) - if err != nil { - return false, fmt.Errorf("fail to get filtered Targets: %w", err) - } - - if IsTargetNamingSuffixPolicyPersistentSequence(xspec) { - // for naming with persistent sequences suffix, targets with same name should not exist at same time - syncContext.FilteredTarget = allTargets - } else { - // for naming with random suffix, targets with random names can be created at same time - syncContext.FilteredTarget = filteredTargets - } - - // sync subresource - // 1. list pvcs using ownerReference - // 2. adopt and retain orphaned pvcs according to PVC retention policy - if _, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - var existingPvcs, adoptedPvcs []*corev1.PersistentVolumeClaim - if existingPvcs, err = r.pvcControl.GetFilteredPvcs(ctx, instance); err != nil { - return false, fmt.Errorf("fail to get filtered subresource PVCs: %w", err) - } - if adoptedPvcs, err = r.pvcControl.AdoptPvcsLeftByRetainPolicy(ctx, instance); err != nil { - return false, fmt.Errorf("fail to adopt orphaned left by whenDelete retention policy PVCs: %w", err) - } - syncContext.ExistingPvcs = append(syncContext.ExistingPvcs, existingPvcs...) - syncContext.ExistingPvcs = append(syncContext.ExistingPvcs, adoptedPvcs...) - } - - // sync include exclude targets - toExcludeTargetNames, toIncludeTargetNames, err := r.dealIncludeExcludeTargets(ctx, instance, syncContext.FilteredTarget) - if err != nil { - return false, fmt.Errorf("fail to deal with include exclude targets: %s", err.Error()) - } - - // get owned IDs - var ownedIDs map[int]*api.ContextDetail - if err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - ownedIDs, err = r.resourceContextControl.AllocateID(ctx, instance, syncContext.UpdatedRevision.GetName(), int(ptr.Deref(xspec.Replicas, 0))) - syncContext.OwnedIds = ownedIDs - return err - }); err != nil { - return false, fmt.Errorf("fail to allocate %d IDs using context when sync Targets: %w", xspec.Replicas, err) - } - - // stateless case - var targetWrappers []*TargetWrapper - syncContext.CurrentIDs = sets.Int{} - idToReclaim := sets.Int{} - toDeleteTargetNames := sets.NewString(xspec.ScaleStrategy.TargetToDelete...) - - for i := range syncContext.FilteredTarget { - target := syncContext.FilteredTarget[i] - xName := target.GetName() - id, _ := GetInstanceID(r.xsetLabelAnnoMgr, target) - toDelete := toDeleteTargetNames.Has(xName) - toExclude := toExcludeTargetNames.Has(xName) - - // priority: toDelete > toReplace > toExclude - if toDelete { - toDeleteTargetNames.Delete(xName) - } - if toExclude { - if targetDuringReplace(r.xsetLabelAnnoMgr, target) || toDelete { - // skip exclude until replace and toDelete done - toExcludeTargetNames.Delete(xName) - } else { - // exclude target and delete its targetContext - idToReclaim.Insert(id) - } - } - - // for naming with persistent sequences suffix, targets with same name should not exist at same time - if target.GetDeletionTimestamp() != nil && !IsTargetNamingSuffixPolicyPersistentSequence(xspec) { - // 1. Reclaim ID from Target which is scaling in and terminating. - if contextDetail, exist := ownedIDs[id]; exist && r.resourceContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") { - idToReclaim.Insert(id) - } - - _, replaceIndicate := r.xsetLabelAnnoMgr.Get(target.GetLabels(), api.XReplaceIndicationLabelKey) - // 2. filter out Targets which are terminating and not replace indicate - if !replaceIndicate { - continue - } - } - - // delete unused pvcs - if _, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - err = r.pvcControl.DeleteTargetUnusedPvcs(ctx, instance, target, syncContext.ExistingPvcs) - if err != nil { - return false, fmt.Errorf("fail to delete unused pvcs %w", err) - } - } - - // sync decoration revisions - var decorationInfo DecorationInfo - if decorationAdapter, enabled := r.xsetController.(api.DecorationAdapter); enabled { - if decorationInfo.DecorationCurrentRevisions, err = decorationAdapter.GetTargetCurrentDecorationRevisions(ctx, r.Client, target); err != nil { - return false, err - } - if decorationInfo.DecorationUpdatedRevisions, err = decorationAdapter.GetTargetUpdatedDecorationRevisions(ctx, r.Client, target); err != nil { - return false, err - } - if decorationInfo.DecorationChanged, err = decorationAdapter.IsTargetDecorationChanged(decorationInfo.DecorationCurrentRevisions, decorationInfo.DecorationUpdatedRevisions); err != nil { - return false, err - } - } - - // sync target ops priority - var opsPriority *api.OpsPriority - if opsPriority, err = r.xsetController.GetXOpsPriority(ctx, r.Client, target); err != nil { - return false, err - } - - targetWrappers = append(targetWrappers, &TargetWrapper{ - Object: target, - ID: id, - ContextDetail: ownedIDs[id], - PlaceHolder: false, - - ToDelete: toDelete, - ToExclude: toExclude, - - IsDuringScaleInOps: opslifecycle.IsDuringOps(r.updateConfig.XsetLabelAnnoMgr, r.scaleInLifecycleAdapter, target), - IsDuringUpdateOps: opslifecycle.IsDuringOps(r.updateConfig.XsetLabelAnnoMgr, r.updateLifecycleAdapter, target), - - DecorationInfo: decorationInfo, - OpsPriority: opsPriority, - }) - - if id >= 0 { - syncContext.CurrentIDs.Insert(id) - } - } - - // do include exclude targets, and skip doSync() if succeeded - var inExSucceed bool - if len(toExcludeTargetNames) > 0 || len(toIncludeTargetNames) > 0 { - var availableContexts []*api.ContextDetail - var getErr error - availableContexts, ownedIDs, getErr = r.getAvailableTargetIDs(ctx, len(toIncludeTargetNames), instance, syncContext) - if getErr != nil { - return false, getErr - } - if err = r.doIncludeExcludeTargets(ctx, instance, toExcludeTargetNames.List(), toIncludeTargetNames.List(), availableContexts); err != nil { - r.Recorder.Eventf(instance, corev1.EventTypeWarning, "ExcludeIncludeFailed", "%s syncTargets include exclude with error: %s", r.xsetGVK.Kind, err.Error()) - return false, err - } - inExSucceed = true - } - - // reclaim Target ID which is (1) during ScalingIn, (2) ExcludeTargets - err = r.reclaimOwnedIDs(ctx, false, instance, idToReclaim, ownedIDs, syncContext.CurrentIDs) - if err != nil { - r.Recorder.Eventf(instance, corev1.EventTypeWarning, "ReclaimOwnedIDs", "reclaim target contexts with error: %s", err.Error()) - return false, err - } - - // reclaim scaleStrategy for delete, exclude, include - err = r.reclaimScaleStrategy(ctx, toDeleteTargetNames, toExcludeTargetNames, toIncludeTargetNames, instance) - if err != nil { - r.Recorder.Eventf(instance, corev1.EventTypeWarning, "ReclaimScaleStrategy", "reclaim scaleStrategy with error: %s", err.Error()) - return false, err - } - - syncContext.TargetWrappers = targetWrappers - syncContext.OwnedIds = ownedIDs - - return inExSucceed, nil -} - -// dealIncludeExcludeTargets returns targets which are allowed to exclude and include -func (r *RealSyncControl) dealIncludeExcludeTargets(ctx context.Context, xsetObject api.XSetObject, targets []client.Object) (sets.String, sets.String, error) { - spec := r.xsetController.GetXSetSpec(xsetObject) - ownedTargets := sets.String{} - excludeTargetNames := sets.String{} - includeTargetNames := sets.String{} - - for _, target := range targets { - ownedTargets.Insert(target.GetName()) - if _, exist := r.xsetLabelAnnoMgr.Get(target.GetLabels(), api.XExcludeIndicationLabelKey); exist { - excludeTargetNames.Insert(target.GetName()) - } - } - - tmpUnOwnedExcludeTargets := sets.String{} - for _, targetName := range spec.ScaleStrategy.TargetToExclude { - if ownedTargets.Has(targetName) { - excludeTargetNames.Insert(targetName) - } else { - tmpUnOwnedExcludeTargets.Insert(targetName) - } - } - - intersection := sets.String{} - for _, targetName := range spec.ScaleStrategy.TargetToInclude { - if excludeTargetNames.Has(targetName) { - intersection.Insert(targetName) - excludeTargetNames.Delete(targetName) - } else if tmpUnOwnedExcludeTargets.Has(targetName) { - intersection.Insert(targetName) - } else if !ownedTargets.Has(targetName) { - includeTargetNames.Insert(targetName) - } - } - - if len(intersection) > 0 { - r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "DupExIncludedTarget", "duplicated targets %s in both excluding and including sets", strings.Join(intersection.List(), ", ")) - } - - toExcludeTargets, notAllowedExcludeTargets, exErr := r.allowIncludeExcludeTargets(ctx, xsetObject, excludeTargetNames.List(), AllowResourceExclude, r.xsetLabelAnnoMgr) - toIncludeTargets, notAllowedIncludeTargets, inErr := r.allowIncludeExcludeTargets(ctx, xsetObject, includeTargetNames.List(), AllowResourceInclude, r.xsetLabelAnnoMgr) - if notAllowedExcludeTargets.Len() > 0 { - r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ExcludeNotAllowed", fmt.Sprintf("targets [%v] are not allowed to exclude, please find out the reason from target's event", notAllowedExcludeTargets.List())) - } - if notAllowedIncludeTargets.Len() > 0 { - r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "IncludeNotAllowed", fmt.Sprintf("targets [%v] are not allowed to include, please find out the reason from target's event", notAllowedIncludeTargets.List())) - } - return toExcludeTargets, toIncludeTargets, errors.Join(exErr, inErr) -} - -// checkAllowFunc refers to AllowResourceExclude and AllowResourceInclude -type checkAllowFunc func(obj metav1.Object, ownerName, ownerKind string, labelMgr api.XSetLabelAnnotationManager) (bool, string) - -// allowIncludeExcludeTargets try to classify targetNames to allowedTargets and notAllowedTargets, using checkAllowFunc func -func (r *RealSyncControl) allowIncludeExcludeTargets(ctx context.Context, xset api.XSetObject, targetNames []string, fn checkAllowFunc, labelMgr api.XSetLabelAnnotationManager) (allowTargets, notAllowTargets sets.String, err error) { - allowTargets = sets.String{} - notAllowTargets = sets.String{} - for i := range targetNames { - target := r.xsetController.NewXObject() - targetName := targetNames[i] - err = r.Client.Get(ctx, types.NamespacedName{Namespace: xset.GetNamespace(), Name: targetName}, target) - if apierrors.IsNotFound(err) { - notAllowTargets.Insert(targetNames[i]) - continue - } else if err != nil { - r.Recorder.Eventf(xset, corev1.EventTypeWarning, "ExcludeIncludeFailed", fmt.Sprintf("failed to find target %s: %s", targetNames[i], err.Error())) - return allowTargets, notAllowTargets, err - } - - // check allowance for target - if allowed, reason := fn(target, xset.GetName(), xset.GetObjectKind().GroupVersionKind().Kind, labelMgr); !allowed { - r.Recorder.Eventf(target, corev1.EventTypeWarning, "ExcludeIncludeNotAllowed", - fmt.Sprintf("target is not allowed to exclude/include from/to %s %s/%s: %s", r.xsetGVK.Kind, xset.GetNamespace(), xset.GetName(), reason)) - notAllowTargets.Insert(targetName) - continue - } - - // check allowance for subresource - pvcsAllowed := true - if adapter, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - volumes := adapter.GetXSpecVolumes(target) - for i := range volumes { - volume := volumes[i] - if volume.PersistentVolumeClaim == nil { - continue - } - pvc := &corev1.PersistentVolumeClaim{} - err = r.Client.Get(ctx, types.NamespacedName{Namespace: target.GetNamespace(), Name: volume.PersistentVolumeClaim.ClaimName}, pvc) - // if pvc not found, ignore it. In case of pvc is filtered by controller-mesh - if apierrors.IsNotFound(err) { - continue - } else if err != nil { - r.Recorder.Eventf(target, corev1.EventTypeWarning, "ExcludeIncludeNotAllowed", fmt.Sprintf("failed to check allowed to exclude/include from/to xset %s/%s: %s", xset.GetNamespace(), xset.GetName(), err.Error())) - pvcsAllowed = false - } - if allowed, reason := fn(pvc, xset.GetName(), xset.GetObjectKind().GroupVersionKind().Kind, labelMgr); !allowed { - r.Recorder.Eventf(target, corev1.EventTypeWarning, "ExcludeIncludeNotAllowed", fmt.Sprintf("failed to check allowed to exclude/include from/to xset %s/%s: %s", xset.GetNamespace(), xset.GetName(), reason)) - pvcsAllowed = false - } - } - } - if pvcsAllowed { - allowTargets.Insert(targetName) - } else { - notAllowTargets.Insert(targetName) - } - } - return allowTargets, notAllowTargets, nil -} - -// Replace is used to replace replace-indicate targets -func (r *RealSyncControl) Replace(ctx context.Context, xsetObject api.XSetObject, syncContext *SyncContext) error { - var err error - var needUpdateContext bool - var idToReclaim sets.Int - - defer func() { - syncContext.activeTargets = FilterOutActiveTargetWrappers(syncContext.TargetWrappers) - syncContext.replacingMap = classifyTargetReplacingMapping(r.xsetLabelAnnoMgr, syncContext.activeTargets) - }() - - needReplaceOriginTargets, needCleanLabelTargets, targetsNeedCleanLabels, needDeleteTargets := r.dealReplaceTargets(ctx, syncContext.TargetWrappers) - - // delete origin targets for replace - err = r.BatchDeleteTargetsByLabel(ctx, r.xControl, needDeleteTargets) - if err != nil { - r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ReplaceTarget", "delete targets by label with error: %s", err.Error()) - return err - } - - // clean labels for replace targets - needUpdateContext, idToReclaim, err = r.cleanReplaceTargetLabels(ctx, needCleanLabelTargets, targetsNeedCleanLabels, syncContext.OwnedIds, syncContext.CurrentIDs) - if err != nil { - r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ReplaceTarget", fmt.Sprintf("clean targets replace pair origin name label with error: %s", err.Error())) - return err - } - - // create new targets for need replace targets - if len(needReplaceOriginTargets) > 0 { - var availableContexts []*api.ContextDetail - var getErr error - availableContexts, syncContext.OwnedIds, getErr = r.getAvailableTargetIDs(ctx, len(needReplaceOriginTargets), xsetObject, syncContext) - if getErr != nil { - return getErr - } - successCount, err := r.replaceOriginTargets(ctx, xsetObject, syncContext, needReplaceOriginTargets, syncContext.OwnedIds, availableContexts) - needUpdateContext = needUpdateContext || successCount > 0 - if err != nil { - r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ReplaceTarget", "deal replace targets with error: %s", err.Error()) - return err - } - } - - // reclaim Target ID which is ReplaceOriginTarget - if err := r.reclaimOwnedIDs(ctx, needUpdateContext, xsetObject, idToReclaim, syncContext.OwnedIds, syncContext.CurrentIDs); err != nil { - r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ReclaimOwnedIDs", "reclaim target contexts with error: %s", err.Error()) - return err - } - - // create targetWrappers for non-exist targets - for id, contextDetail := range syncContext.OwnedIds { - if _, inUsed := syncContext.CurrentIDs[id]; inUsed { - continue - } - syncContext.TargetWrappers = append(syncContext.TargetWrappers, &TargetWrapper{ - ID: id, - Object: nil, - ContextDetail: contextDetail, - PlaceHolder: true, - }) - } - - return nil -} - -func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) { - spec := r.xsetController.GetXSetSpec(xsetObject) - logger := logr.FromContext(ctx) - var recordedRequeueAfter *time.Duration - - diff := int(ptr.Deref(spec.Replicas, 0)) - len(syncContext.replacingMap) - scaling := false - - if diff >= 0 { - // trigger delete targets indicated in ScaleStrategy.TargetToDelete by label - for _, targetWrapper := range syncContext.activeTargets { - if targetWrapper.ToDelete { - err := r.BatchDeleteTargetsByLabel(ctx, r.xControl, []client.Object{targetWrapper.Object}) - if err != nil { - return false, recordedRequeueAfter, err - } - } - } - - // scale out targets and return if diff > 0 - if diff > 0 { - // collect instance ID in used from owned Targets - targetInstanceIDSet := sets.Int{} - for _, target := range syncContext.activeTargets { - targetInstanceIDSet[target.ID] = struct{}{} - } - - // find IDs and their contexts which have not been used by owned Targets - var availableContexts []*api.ContextDetail - var getErr error - availableContexts, syncContext.OwnedIds, getErr = r.getAvailableTargetIDs(ctx, diff, xsetObject, syncContext) - if getErr != nil { - return false, recordedRequeueAfter, getErr - } - - needUpdateContext := atomic.Bool{} - succCount, err := controllerutils.SlowStartBatch(len(availableContexts), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) (err error) { - availableIDContext := availableContexts[i] - defer func() { - if r.decideContextRevision(availableIDContext, syncContext.UpdatedRevision, err == nil) { - needUpdateContext.Store(true) - } - }() - // use revision recorded in Context - revision := syncContext.UpdatedRevision - if revisionName, exist := r.resourceContextControl.Get(availableIDContext, api.EnumRevisionContextDataKey); exist && revisionName != "" { - for i := range syncContext.Revisions { - if syncContext.Revisions[i].GetName() == revisionName { - revision = syncContext.Revisions[i] - break - } - } - } - // scale out new Targets with updatedRevision - // TODO use cache - target, err := NewTargetFrom(r.xsetController, r.xsetLabelAnnoMgr, xsetObject, revision, availableIDContext.ID, - func(object client.Object) error { - if _, exist := r.resourceContextControl.Get(availableIDContext, api.EnumJustCreateContextDataKey); exist { - r.xsetLabelAnnoMgr.Set(object, api.XCreatingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) - } else { - r.xsetLabelAnnoMgr.Set(object, api.XCompletingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) - } - - // decoration for target template - if decorationAdapter, ok := r.xsetController.(api.DecorationAdapter); ok { - revisionsInfo, ok := r.resourceContextControl.Get(availableIDContext, api.EnumTargetDecorationRevisionKey) - if !ok { - // get updated decoration revisions from target and write to resource context - if revisionsInfo, err = decorationAdapter.GetTargetUpdatedDecorationRevisions(ctx, r.Client, object); err != nil { - return err - } - r.resourceContextControl.Put(availableIDContext, api.EnumTargetDecorationRevisionKey, revisionsInfo) - needUpdateContext.Store(true) - } - // get patcher from decoration revisions and patch target - if fn, err := decorationAdapter.GetDecorationPatcherByRevisions(ctx, r.Client, object, revisionsInfo); err != nil { - return err - } else { - return fn(object) - } - } - return nil - }, - r.xsetController.GetXSetTemplatePatcher(xsetObject), - ) - if err != nil { - return fmt.Errorf("fail to new Target from revision %s: %w", revision.GetName(), err) - } - // create pvcs for targets (pod) - if _, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - err = r.pvcControl.CreateTargetPvcs(ctx, xsetObject, target, syncContext.ExistingPvcs) - if err != nil { - return fmt.Errorf("fail to create PVCs for target %s: %w", target.GetName(), err) - } - } - newTarget := target.DeepCopyObject().(client.Object) - logger.Info("try to create Target with revision of "+r.xsetGVK.Kind, "revision", revision.GetName()) - if target, err = r.xControl.CreateTarget(ctx, newTarget); err != nil { - return err - } - // add an expectation for this target creation, before next reconciling - return r.cacheExpectations.ExpectCreation(clientutil.ObjectKeyString(xsetObject), r.targetGVK, target.GetNamespace(), target.GetName()) - }) - if needUpdateContext.Load() { - logger.Info("try to update ResourceContext for XSet after scaling out", "Context", syncContext.OwnedIds) - if updateContextErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.resourceContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) - }); updateContextErr != nil { - err = errors.Join(updateContextErr, err) - } - } - if err != nil { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, err, "ScaleOutFailed", err.Error()) - return succCount > 0, recordedRequeueAfter, err - } - r.Recorder.Eventf(xsetObject, corev1.EventTypeNormal, "ScaleOut", "scale out %d Target(s)", succCount) - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleOut", "") - return succCount > 0, recordedRequeueAfter, err - } - } - - if diff <= 0 { - // chose the targets to scale in - targetsToScaleIn := r.getTargetsToDelete(xsetObject, syncContext.activeTargets, syncContext.replacingMap, diff*-1) - // filter out Targets need to trigger TargetOpsLifecycle - wrapperCh := make(chan *TargetWrapper, len(targetsToScaleIn)) - for i := range targetsToScaleIn { - if targetsToScaleIn[i].IsDuringScaleInOps { - continue - } - wrapperCh <- targetsToScaleIn[i] - } - - // trigger Targets to enter TargetOpsLifecycle - succCount, err := controllerutils.SlowStartBatch(len(wrapperCh), controllerutils.SlowStartInitialBatchSize, false, func(_ int, err error) error { - wrapper := <-wrapperCh - object := wrapper.Object - - // trigger TargetOpsLifecycle with scaleIn OperationType - logger.V(1).Info("try to begin TargetOpsLifecycle for scaling in Target in XSet", "wrapper", ObjectKeyString(object)) - if updated, err := opslifecycle.Begin(r.updateConfig.XsetLabelAnnoMgr, r.Client, r.scaleInLifecycleAdapter, object); err != nil { - return fmt.Errorf("fail to begin TargetOpsLifecycle for Scaling in Target %s/%s: %w", object.GetNamespace(), object.GetName(), err) - } else if updated { - wrapper.IsDuringScaleInOps = true - r.Recorder.Eventf(object, corev1.EventTypeNormal, "BeginScaleInLifecycle", "succeed to begin TargetOpsLifecycle for scaling in") - // add an expectation for this wrapper creation, before next reconciling - if err := r.cacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(xsetObject), r.targetGVK, object.GetNamespace(), object.GetName(), object.GetResourceVersion()); err != nil { - return err - } - } - - return nil - }) - scaling = succCount > 0 - - if err != nil { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, err, "ScaleInFailed", err.Error()) - return scaling, recordedRequeueAfter, err - } else { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleIn", "") - } - - needUpdateContext := false - for i, targetWrapper := range targetsToScaleIn { - requeueAfter, allowed := opslifecycle.AllowOps(r.updateConfig.XsetLabelAnnoMgr, r.scaleInLifecycleAdapter, ptr.Deref(spec.ScaleStrategy.OperationDelaySeconds, 0), targetWrapper.Object) - if !allowed && targetWrapper.Object.GetDeletionTimestamp() == nil { - r.Recorder.Eventf(targetWrapper.Object, corev1.EventTypeNormal, "TargetScaleInLifecycle", "Target is not allowed to scale in") - continue - } - - if requeueAfter != nil { - r.Recorder.Eventf(targetWrapper.Object, corev1.EventTypeNormal, "TargetScaleInLifecycle", "delay Target scale in for %d seconds", requeueAfter.Seconds()) - if recordedRequeueAfter == nil || *requeueAfter < *recordedRequeueAfter { - recordedRequeueAfter = requeueAfter - } - - continue - } - - // if Target is allowed to operate or Target has already been deleted, promte to delete Target - if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && !r.resourceContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") { - needUpdateContext = true - r.resourceContextControl.Put(contextDetail, api.EnumScaleInContextDataKey, "true") - } - - if targetWrapper.GetDeletionTimestamp() != nil { - continue - } - - wrapperCh <- targetsToScaleIn[i] - } - - // mark these Targets to scalingIn - if needUpdateContext { - logger.Info("try to update ResourceContext for XSet when scaling in Target") - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.resourceContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) - }); err != nil { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, err, "ScaleInFailed", fmt.Sprintf("failed to update Context for scaling in: %s", err)) - return scaling, recordedRequeueAfter, err - } else { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleIn", "") - } - } - - // do delete Target resource - succCount, err = controllerutils.SlowStartBatch(len(wrapperCh), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { - target := <-wrapperCh - logger.Info("try to scale in Target", "target", ObjectKeyString(target)) - if err := r.xControl.DeleteTarget(ctx, target.Object); err != nil { - return fmt.Errorf("fail to delete Target %s/%s when scaling in: %w", target.GetNamespace(), target.GetName(), err) - } - - r.Recorder.Eventf(xsetObject, corev1.EventTypeNormal, "TargetDeleted", "succeed to scale in Target %s/%s", target.GetNamespace(), target.GetName()) - if err := r.cacheExpectations.ExpectDeletion(clientutil.ObjectKeyString(xsetObject), r.targetGVK, target.GetNamespace(), target.GetName()); err != nil { - return err - } - - // delete pvcs if target is in update replace, or retention policy is "Deleted" - if _, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - _, replaceOrigin := r.xsetLabelAnnoMgr.Get(target.Object.GetLabels(), api.XReplacePairOriginName) - _, replaceNew := r.xsetLabelAnnoMgr.Get(target.Object.GetLabels(), api.XReplacePairNewId) - if replaceOrigin || replaceNew || !r.pvcControl.RetainPvcWhenXSetScaled(xsetObject) { - return r.pvcControl.DeleteTargetPvcs(ctx, xsetObject, target.Object, syncContext.ExistingPvcs) - } - } - return nil - }) - scaling = scaling || succCount > 0 - - if succCount > 0 { - r.Recorder.Eventf(xsetObject, corev1.EventTypeNormal, "ScaleIn", "scale in %d Target(s)", succCount) - } - if err != nil { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, err, "ScaleInFailed", fmt.Sprintf("fail to delete Target for scaling in: %s", err)) - return scaling, recordedRequeueAfter, err - } else { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleIn", "") - } - } - - // reset ContextDetail.ScalingIn, if there are Targets had its TargetOpsLifecycle reverted - needUpdateTargetContext := false - for _, targetWrapper := range syncContext.activeTargets { - if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && - r.resourceContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") && !targetWrapper.IsDuringScaleInOps { - needUpdateTargetContext = true - r.resourceContextControl.Remove(contextDetail, api.EnumScaleInContextDataKey) - } - } - - if needUpdateTargetContext { - logger.V(1).Info("try to update ResourceContext for XSet after scaling") - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.resourceContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) - }); err != nil { - return scaling, recordedRequeueAfter, fmt.Errorf("fail to reset ResourceContext: %w", err) - } - } - - if !scaling { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "Scaled", "") - } - - return scaling, recordedRequeueAfter, nil -} - -func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) { - logger := logr.FromContext(ctx) - var err error - var recordedRequeueAfter *time.Duration - - // 1. scan and analysis targets update info for active targets and PlaceHolder targets - targetUpdateInfos, err := r.attachTargetUpdateInfo(ctx, xsetObject, syncContext) - if err != nil { - return false, recordedRequeueAfter, err - } - - // 2. decide Target update candidates - candidates := r.decideTargetToUpdate(r.xsetController, xsetObject, targetUpdateInfos) - targetToUpdate := filterOutPlaceHolderUpdateInfos(candidates) - targetCh := make(chan *TargetUpdateInfo, len(targetToUpdate)) - updater := r.newTargetUpdater(xsetObject) - updating := false - - // 3. filter already updated revision, - for i, targetInfo := range targetToUpdate { - // TODO check decoration and pvc template changed - if targetInfo.IsUpdatedRevision && !targetInfo.PvcTmpHashChanged && !targetInfo.DecorationChanged { - continue - } - - // 3.1 fulfillTargetUpdateInfo to all not updatedRevision target - if targetInfo.CurrentRevision.GetName() != UnknownRevision { - if err = updater.FulfillTargetUpdatedInfo(ctx, syncContext.UpdatedRevision, targetInfo); err != nil { - logger.Error(err, fmt.Sprintf("fail to analyze target %s/%s in-place update support", targetInfo.GetNamespace(), targetInfo.GetName())) - continue - } - } - - if targetInfo.GetDeletionTimestamp() != nil { - continue - } - - if targetInfo.IsDuringScaleInOps || targetInfo.IsDuringUpdateOps { - continue - } - - targetCh <- targetToUpdate[i] - } - - // 4. begin target update lifecycle - updating, err = updater.BeginUpdateTarget(ctx, syncContext, targetCh) - if err != nil { - return updating, recordedRequeueAfter, err - } - - // 5. (1) filter out targets not allow to ops now, such as OperationDelaySeconds strategy; (2) update PlaceHolder Targets resourceContext revision - recordedRequeueAfter, err = updater.FilterAllowOpsTargets(ctx, candidates, syncContext.OwnedIds, syncContext, targetCh) - if err != nil { - AddOrUpdateCondition(syncContext.NewStatus, - api.XSetUpdate, err, "UpdateFailed", - fmt.Sprintf("fail to update Context for updating: %s", err)) - return updating, recordedRequeueAfter, err - } else { - AddOrUpdateCondition(syncContext.NewStatus, - api.XSetUpdate, nil, "Updated", "") - } - - // 6. update Target - succCount, err := controllerutils.SlowStartBatch(len(targetCh), controllerutils.SlowStartInitialBatchSize, false, func(_ int, _ error) error { - targetInfo := <-targetCh - logger.Info("before target update operation", - "target", ObjectKeyString(targetInfo.Object), - "revision.from", targetInfo.CurrentRevision.GetName(), - "revision.to", syncContext.UpdatedRevision.GetName(), - "inPlaceUpdate", targetInfo.InPlaceUpdateSupport, - "onlyMetadataChanged", targetInfo.OnlyMetadataChanged, - ) - - spec := r.xsetController.GetXSetSpec(xsetObject) - if targetInfo.IsInReplace && spec.UpdateStrategy.UpdatePolicy != api.XSetReplaceTargetUpdateStrategyType { - // a replacing target should be replaced by an updated revision target when encountering upgrade - if err := updateReplaceOriginTarget(ctx, r.Client, r.Recorder, r.xsetLabelAnnoMgr, targetInfo, targetInfo.ReplacePairNewTargetInfo); err != nil { - return err - } - } else { - if err := updater.UpgradeTarget(ctx, targetInfo); err != nil { - return err - } - } - - return nil - }) - - updating = updating || succCount > 0 - if err != nil { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetUpdate, err, "UpdateFailed", err.Error()) - return updating, recordedRequeueAfter, err - } else { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetUpdate, nil, "Updated", "") - } - - targetToUpdateSet := sets.String{} - for i := range targetToUpdate { - targetToUpdateSet.Insert(targetToUpdate[i].GetName()) - } - // 7. try to finish all Targets'TargetOpsLifecycle if its update is finished. - succCount, err = controllerutils.SlowStartBatch(len(targetUpdateInfos), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { - targetInfo := targetUpdateInfos[i] - - if !(targetInfo.IsDuringUpdateOps || targetInfo.IsInReplaceUpdate) || targetInfo.PlaceHolder || targetInfo.GetDeletionTimestamp() != nil { - return nil - } - - var finishByCancelUpdate bool - var updateFinished bool - var msg string - var err error - - if !targetToUpdateSet.Has(targetInfo.GetName()) { - // target is out of scope (partition or by label) and not start update yet, finish update by cancel - finishByCancelUpdate = !targetInfo.IsAllowUpdateOps - logger.Info("out of update scope", "target", ObjectKeyString(targetInfo.Object), "finishByCancelUpdate", finishByCancelUpdate) - } else if !targetInfo.IsAllowUpdateOps { - // target is in update scope, but is not start update yet, if pod is updatedRevision, just finish update by cancel - finishByCancelUpdate = targetInfo.IsUpdatedRevision - } else { - // target is in update scope and allowed to update, check and finish update gracefully - if updateFinished, msg, err = updater.GetTargetUpdateFinishStatus(ctx, targetInfo); err != nil { - return fmt.Errorf("failed to get target %s/%s update finished: %w", targetInfo.GetNamespace(), targetInfo.GetName(), err) - } else if !updateFinished { - r.Recorder.Eventf(targetInfo.Object, - corev1.EventTypeNormal, - "WaitingUpdateReady", - "waiting for target %s/%s to update finished: %s", - targetInfo.GetNamespace(), targetInfo.GetName(), msg) - } - } - - if updateFinished || finishByCancelUpdate { - if err := updater.FinishUpdateTarget(ctx, targetInfo, finishByCancelUpdate); err != nil { - return fmt.Errorf("failed to finish target %s/%s update: %w", targetInfo.GetNamespace(), targetInfo.GetName(), err) - } - r.Recorder.Eventf(targetInfo.Object, - corev1.EventTypeNormal, - "UpdateTargetFinished", - "target %s/%s is finished for upgrade to revision %s", - targetInfo.GetNamespace(), targetInfo.GetName(), targetInfo.UpdateRevision.GetName()) - } - - return nil - }) - - return updating || succCount > 0, recordedRequeueAfter, err -} - -func (r *RealSyncControl) CalculateStatus(_ context.Context, instance api.XSetObject, syncContext *SyncContext) *api.XSetStatus { - newStatus := syncContext.NewStatus - newStatus.ObservedGeneration = instance.GetGeneration() - - var readyReplicas, scheduledReplicas, replicas, updatedReplicas, operatingReplicas, updatedReadyReplicas, availableReplicas, updatedAvailableReplicas int32 - - activeTargets := FilterOutActiveTargetWrappers(syncContext.TargetWrappers) - for _, targetWrapper := range activeTargets { - // for naming with persistent sequences suffix, terminating targets can be shown in status - if targetWrapper.GetDeletionTimestamp() != nil && !IsTargetNamingSuffixPolicyPersistentSequence(r.xsetController.GetXSetSpec(instance)) { - continue - } - - replicas++ - - isUpdated := false - if isUpdated = IsTargetUpdatedRevision(targetWrapper.Object, syncContext.UpdatedRevision.Name); isUpdated { - updatedReplicas++ - } - - if opslifecycle.IsDuringOps(r.updateConfig.XsetLabelAnnoMgr, r.scaleInLifecycleAdapter, targetWrapper) || - opslifecycle.IsDuringOps(r.updateConfig.XsetLabelAnnoMgr, r.updateLifecycleAdapter, targetWrapper) { - operatingReplicas++ - } - - if ready, _ := r.xsetController.CheckReadyTime(targetWrapper.Object); ready { - readyReplicas++ - if isUpdated { - updatedReadyReplicas++ - } - } - - if r.xsetController.CheckAvailable(targetWrapper.Object) { - availableReplicas++ - if isUpdated { - updatedAvailableReplicas++ - } - } - - if r.xsetController.CheckScheduled(targetWrapper.Object) { - scheduledReplicas++ - } - } - - newStatus.ReadyReplicas = readyReplicas - newStatus.Replicas = replicas - newStatus.UpdatedReplicas = updatedReplicas - newStatus.OperatingReplicas = operatingReplicas - newStatus.UpdatedReadyReplicas = updatedReadyReplicas - newStatus.ScheduledReplicas = scheduledReplicas - newStatus.AvailableReplicas = availableReplicas - newStatus.UpdatedAvailableReplicas = updatedAvailableReplicas - - spec := r.xsetController.GetXSetSpec(instance) - if (spec.Replicas == nil && newStatus.UpdatedReadyReplicas >= 0) || - newStatus.UpdatedReadyReplicas >= *spec.Replicas { - newStatus.CurrentRevision = syncContext.UpdatedRevision.Name - } - - return newStatus -} - -// getAvailableTargetIDs try to extract and re-allocate want available IDs. -func (r *RealSyncControl) getAvailableTargetIDs( - ctx context.Context, - want int, - instance api.XSetObject, - syncContext *SyncContext, -) ([]*api.ContextDetail, map[int]*api.ContextDetail, error) { - ownedIDs := syncContext.OwnedIds - currentIDs := syncContext.CurrentIDs - - availableContexts := r.resourceContextControl.ExtractAvailableContexts(want, ownedIDs, currentIDs) - if len(availableContexts) >= want { - return availableContexts, ownedIDs, nil - } - - diff := want - len(availableContexts) - - var newOwnedIDs map[int]*api.ContextDetail - var err error - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - newOwnedIDs, err = r.resourceContextControl.AllocateID(ctx, instance, syncContext.UpdatedRevision.GetName(), len(ownedIDs)+diff) - return err - }); err != nil { - return nil, ownedIDs, fmt.Errorf("fail to allocate IDs using context when include Targets: %w", err) - } - - return r.resourceContextControl.ExtractAvailableContexts(want, newOwnedIDs, currentIDs), newOwnedIDs, nil -} - -// reclaimOwnedIDs delete and reclaim unused IDs -func (r *RealSyncControl) reclaimOwnedIDs( - ctx context.Context, - needUpdateContext bool, - xset api.XSetObject, - idToReclaim sets.Int, - ownedIDs map[int]*api.ContextDetail, - currentIDs sets.Int, -) error { - // TODO stateful case - // 1) only reclaim non-existing Targets' ID. Do not reclaim terminating Targets' ID until these Targets and PVC have been deleted from ETCD - // 2) do not filter out these terminating Targets - for id, contextDetail := range ownedIDs { - if _, exist := currentIDs[id]; exist { - continue - } - if r.resourceContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") { - idToReclaim.Insert(id) - } - } - - for _, id := range idToReclaim.List() { - needUpdateContext = true - delete(ownedIDs, id) - } - - // TODO clean replace-pair-keys or dirty targetContext - // 1) replace pair target are not exists - // 2) target exists but is not replaceIndicated - - if needUpdateContext { - logger := r.Logger.WithValues(r.xsetGVK.Kind, ObjectKeyString(xset)) - logger.V(1).Info("try to update ResourceContext for XSet when sync") - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.resourceContextControl.UpdateToTargetContext(ctx, xset, ownedIDs) - }); err != nil { - return fmt.Errorf("fail to update ResourceContextControl when reclaiming IDs: %w", err) - } - } - return nil -} - -// FilterOutActiveTargetWrappers filter out non placeholder targets -func FilterOutActiveTargetWrappers(targets []*TargetWrapper) []*TargetWrapper { - var filteredTargetWrappers []*TargetWrapper - for i, target := range targets { - if target.PlaceHolder { - continue - } - filteredTargetWrappers = append(filteredTargetWrappers, targets[i]) - } - return filteredTargetWrappers -} - -func targetDuringReplace(labelMgr api.XSetLabelAnnotationManager, target client.Object) bool { - labels := target.GetLabels() - if labels == nil { - return false - } - _, replaceIndicate := labelMgr.Get(labels, api.XReplaceIndicationLabelKey) - _, replaceOriginTarget := labelMgr.Get(labels, api.XReplacePairOriginName) - _, replaceNewTarget := labelMgr.Get(labels, api.XReplacePairNewId) - return replaceIndicate || replaceOriginTarget || replaceNewTarget -} - -// BatchDeleteTargetsByLabel try to trigger target deletion by to-delete label -func (r *RealSyncControl) BatchDeleteTargetsByLabel(ctx context.Context, targetControl xcontrol.TargetControl, needDeleteTargets []client.Object) error { - _, err := controllerutils.SlowStartBatch(len(needDeleteTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { - target := needDeleteTargets[i] - if _, exist := r.xsetLabelAnnoMgr.Get(target.GetLabels(), api.XDeletionIndicationLabelKey); !exist { - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%d"}}}`, r.xsetLabelAnnoMgr.Value(api.XDeletionIndicationLabelKey), time.Now().UnixNano()))) // nolint - if err := targetControl.PatchTarget(ctx, target, patch); err != nil { - return fmt.Errorf("failed to delete target when syncTargets %s/%s/%w", target.GetNamespace(), target.GetName(), err) - } - } - return nil - }) - return err -} - -// decideContextRevision decides revision for 3 target create types: (1) just create, (2) upgrade by recreate, (3) delete and recreate -func (r *RealSyncControl) decideContextRevision(contextDetail *api.ContextDetail, updatedRevision *appsv1.ControllerRevision, createSucceeded bool) bool { - needUpdateContext := false - if !createSucceeded { - if r.resourceContextControl.Contains(contextDetail, api.EnumJustCreateContextDataKey, "true") { - // TODO choose just create targets' revision according to scaleStrategy - r.resourceContextControl.Put(contextDetail, api.EnumRevisionContextDataKey, updatedRevision.GetName()) - r.resourceContextControl.Remove(contextDetail, api.EnumTargetDecorationRevisionKey) - needUpdateContext = true - } else if r.resourceContextControl.Contains(contextDetail, api.EnumRecreateUpdateContextDataKey, "true") { - r.resourceContextControl.Put(contextDetail, api.EnumRevisionContextDataKey, updatedRevision.GetName()) - r.resourceContextControl.Remove(contextDetail, api.EnumTargetDecorationRevisionKey) - needUpdateContext = true - } - // if target is delete and recreate, never change revisionKey - } else { - // TODO delete ID if create succeeded - r.resourceContextControl.Remove(contextDetail, api.EnumJustCreateContextDataKey) - r.resourceContextControl.Remove(contextDetail, api.EnumRecreateUpdateContextDataKey) - needUpdateContext = true - } - return needUpdateContext -} diff --git a/xset/synccontrols/types.go b/xset/synccontrols/types.go deleted file mode 100644 index 35580e6..0000000 --- a/xset/synccontrols/types.go +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package synccontrols - -import ( - "time" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client" - - "kusionstack.io/kube-utils/xset/api" -) - -type SyncContext struct { - Revisions []*appsv1.ControllerRevision - CurrentRevision *appsv1.ControllerRevision - UpdatedRevision *appsv1.ControllerRevision - ExistingSubResource []client.Object - - FilteredTarget []client.Object - TargetWrappers []*TargetWrapper - activeTargets []*TargetWrapper - replacingMap map[string]*TargetWrapper - - CurrentIDs sets.Int - OwnedIds map[int]*api.ContextDetail - - SubResources - - NewStatus *api.XSetStatus -} - -type SubResources struct { - ExistingPvcs []*corev1.PersistentVolumeClaim -} - -type TargetWrapper struct { - // parameters must be set during creation - client.Object - ID int - ContextDetail *api.ContextDetail - PlaceHolder bool - - ToDelete bool - ToExclude bool - - IsDuringScaleInOps bool - IsDuringUpdateOps bool - - DecorationInfo - - OpsPriority *api.OpsPriority -} - -type DecorationInfo struct { - // indicate if the decoration changed - DecorationChanged bool - // updated revisions of decoration - DecorationUpdatedRevisions string - // current revisions of decoration - DecorationCurrentRevisions string -} - -type TargetUpdateInfo struct { - *TargetWrapper - - UpdatedTarget client.Object - - InPlaceUpdateSupport bool - OnlyMetadataChanged bool - - // indicate if this target has up-to-date revision from its owner, like XSet - IsUpdatedRevision bool - // carry the target's current revision - CurrentRevision *appsv1.ControllerRevision - // carry the desired update revision - UpdateRevision *appsv1.ControllerRevision - - SubResourcesChanged - - // indicates operate is allowed for TargetOpsLifecycle. - IsAllowUpdateOps bool - // requeue after for operationDelaySeconds - RequeueForOperationDelay *time.Duration - - // for replace update - // judge target in replace updating - IsInReplace bool - IsInReplaceUpdate bool - - // replace new created target - ReplacePairNewTargetInfo *TargetUpdateInfo - - // replace origin target - ReplacePairOriginTargetName string -} - -type SubResourcesChanged struct { - // indicate if the pvc template changed - PvcTmpHashChanged bool -} diff --git a/xset/synccontrols/x_replace.go b/xset/synccontrols/x_replace.go deleted file mode 100644 index 119f43c..0000000 --- a/xset/synccontrols/x_replace.go +++ /dev/null @@ -1,458 +0,0 @@ -/* -Copyright 2024-2025 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package synccontrols - -import ( - "context" - "encoding/json" - "fmt" - "strconv" - "strings" - "time" - - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" - - clientutil "kusionstack.io/kube-utils/client" - controllerutils "kusionstack.io/kube-utils/controller/utils" - "kusionstack.io/kube-utils/xset/api" - "kusionstack.io/kube-utils/xset/opslifecycle" - "kusionstack.io/kube-utils/xset/subresources" -) - -func (r *RealSyncControl) cleanReplaceTargetLabels( - ctx context.Context, - needCleanLabelTargets []client.Object, - targetsNeedCleanLabels [][]string, - ownedIDs map[int]*api.ContextDetail, - currentIDs sets.Int, -) (bool, sets.Int, error) { - logger := logr.FromContext(ctx) - needUpdateContext := false - needDeleteTargetsIDs := sets.Int{} - mapOriginToNewTargetContext := r.mapReplaceOriginToNewTargetContext(ownedIDs) - mapNewToOriginTargetContext := r.mapReplaceNewToOriginTargetContext(ownedIDs) - _, err := controllerutils.SlowStartBatch(len(needCleanLabelTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) (err error) { - defer func() { - if err == nil { - logger.Info("cleanReplaceTargetLabels clean replace labels success", "kind", needCleanLabelTargets[i].GetObjectKind(), "target", needCleanLabelTargets[i].GetName(), "labels", targetsNeedCleanLabels[i]) - } - }() - target := needCleanLabelTargets[i] - needCleanLabels := targetsNeedCleanLabels[i] - var deletePatch []map[string]string - for _, labelKey := range needCleanLabels { - patchOperation := map[string]string{ - "op": "remove", - "path": fmt.Sprintf("/metadata/labels/%s", strings.ReplaceAll(labelKey, "/", "~1")), - } - deletePatch = append(deletePatch, patchOperation) - // replace finished, (1) remove ReplaceNewTargetID, ReplaceOriginTargetID key from IDs, (2) try to delete origin Target's ID - if labelKey == r.xsetLabelAnnoMgr.Value(api.XReplacePairOriginName) { - needUpdateContext = true - newTargetId, _ := GetInstanceID(r.xsetLabelAnnoMgr, target) - if originTargetContext, exist := mapOriginToNewTargetContext[newTargetId]; exist && originTargetContext != nil { - r.resourceContextControl.Remove(originTargetContext, api.EnumReplaceNewTargetIDContextDataKey) - if _, exist := currentIDs[originTargetContext.ID]; !exist { - needDeleteTargetsIDs.Insert(originTargetContext.ID) - } - } - if contextDetail, exist := ownedIDs[newTargetId]; exist { - r.resourceContextControl.Remove(contextDetail, api.EnumReplaceOriginTargetIDContextDataKey) - } - } - // replace canceled, (1) remove ReplaceNewTargetID, ReplaceOriginTargetID key from IDs, (2) try to delete new Target's ID - _, replaceIndicate := r.xsetLabelAnnoMgr.Get(target.GetLabels(), api.XReplaceIndicationLabelKey) - if !replaceIndicate && labelKey == r.xsetLabelAnnoMgr.Value(api.XReplacePairNewId) { - needUpdateContext = true - originTargetId, _ := GetInstanceID(r.xsetLabelAnnoMgr, target) - if newTargetContext, exist := mapNewToOriginTargetContext[originTargetId]; exist && newTargetContext != nil { - r.resourceContextControl.Remove(newTargetContext, api.EnumReplaceOriginTargetIDContextDataKey) - if _, exist := currentIDs[newTargetContext.ID]; !exist { - needDeleteTargetsIDs.Insert(newTargetContext.ID) - } - } - if contextDetail, exist := ownedIDs[originTargetId]; exist { - r.resourceContextControl.Remove(contextDetail, api.EnumReplaceNewTargetIDContextDataKey) - } - } - } - // patch to bytes - patchBytes, err := json.Marshal(deletePatch) - if err != nil { - return err - } - if err = r.xControl.PatchTarget(ctx, target, client.RawPatch(types.JSONPatchType, patchBytes)); err != nil { - return fmt.Errorf("failed to remove replace pair label %s/%s: %w", target.GetNamespace(), target.GetName(), err) - } - return nil - }) - - return needUpdateContext, needDeleteTargetsIDs, err -} - -func (r *RealSyncControl) replaceOriginTargets( - ctx context.Context, - instance api.XSetObject, - syncContext *SyncContext, - needReplaceOriginTargets []*TargetWrapper, - ownedIDs map[int]*api.ContextDetail, - availableContexts []*api.ContextDetail, -) (int, error) { - logger := logr.FromContext(ctx) - mapNewToOriginTargetContext := r.mapReplaceNewToOriginTargetContext(ownedIDs) - successCount, err := controllerutils.SlowStartBatch(len(needReplaceOriginTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { - originWrapper := needReplaceOriginTargets[i] - originTarget := needReplaceOriginTargets[i].Object - originTargetId, _ := GetInstanceID(r.xsetLabelAnnoMgr, originTarget) - - if ownedIDs[originTargetId] == nil { - r.Recorder.Eventf(instance, corev1.EventTypeWarning, "OriginTargetContext", "cannot found resource context id %d of origin target %s/%s", originTargetId, originTarget.GetNamespace(), originTarget.GetName()) - return fmt.Errorf("cannot found context for replace origin target %s/%s", originTarget.GetNamespace(), originTarget.GetName()) - } - - replaceRevision := r.getReplaceRevision(originTarget, syncContext) - - // add instance id and replace pair label - var newInstanceId string - var newTargetContext *api.ContextDetail - if contextDetail, exist := mapNewToOriginTargetContext[originTargetId]; exist && contextDetail != nil { - newTargetContext = contextDetail - // reuse targetContext ID if pair-relation exists - newInstanceId = fmt.Sprintf("%d", newTargetContext.ID) - logger.Info("replaceOriginTargets", "try to reuse new pod resourceContext id", newInstanceId) - } else { - if availableContexts[i] == nil { - r.Recorder.Eventf(instance, corev1.EventTypeWarning, "AvailableContext", "cannot found available context for replace origin target %s/%s", originTarget.GetNamespace(), originTarget.GetName()) - return fmt.Errorf("cannot found available context for replace new target when replacing origin target %s/%s", originTarget.GetNamespace(), originTarget.GetName()) - } - newTargetContext = availableContexts[i] - // add replace pair-relation to targetContexts for originTarget and newTarget - newInstanceId = fmt.Sprintf("%d", newTargetContext.ID) - r.resourceContextControl.Put(ownedIDs[originTargetId], api.EnumReplaceNewTargetIDContextDataKey, newInstanceId) - r.resourceContextControl.Put(ownedIDs[newTargetContext.ID], api.EnumReplaceOriginTargetIDContextDataKey, strconv.Itoa(originTargetId)) - r.resourceContextControl.Remove(ownedIDs[newTargetContext.ID], api.EnumJustCreateContextDataKey) - } - - // create target using update revision if replaced by update, otherwise using current revision - newTarget, err := NewTargetFrom(r.xsetController, r.xsetLabelAnnoMgr, instance, replaceRevision, newTargetContext.ID, - r.xsetController.GetXSetTemplatePatcher(instance), - func(object client.Object) error { - if decorationAdapter, ok := r.xsetController.(api.DecorationAdapter); ok { - // get current decoration patcher from origin target, and patch new target - if fn, err := decorationAdapter.GetDecorationPatcherByRevisions(ctx, r.Client, originTarget, originWrapper.DecorationUpdatedRevisions); err != nil { - return err - } else { - return fn(object) - } - } - return nil - }, - ) - if err != nil { - return err - } - - r.xsetLabelAnnoMgr.Set(newTarget, api.XReplacePairOriginName, originTarget.GetName()) - r.xsetLabelAnnoMgr.Set(newTarget, api.XCreatingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) - r.resourceContextControl.Put(newTargetContext, api.EnumRevisionContextDataKey, replaceRevision.GetName()) - - // create pvcs for new target - if _, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - err = r.pvcControl.CreateTargetPvcs(ctx, instance, newTarget, syncContext.ExistingPvcs) - if err != nil { - return fmt.Errorf("fail to create PVCs for target %s: %w", newTarget.GetName(), err) - } - } - - if newCreatedTarget, err := r.xControl.CreateTarget(ctx, newTarget); err == nil { - r.Recorder.Eventf(originTarget, - corev1.EventTypeNormal, - "CreatePairTarget", - "succeed to create replace pair Target %s/%s with revision %s by replace", - originTarget.GetNamespace(), - originTarget.GetName(), - replaceRevision.GetName()) - - if err := r.cacheExpectations.ExpectCreation(clientutil.ObjectKeyString(instance), r.targetGVK, newTarget.GetNamespace(), newTarget.GetName()); err != nil { - return err - } - - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:%q}}}`, r.xsetLabelAnnoMgr.Value(api.XReplacePairNewId), newInstanceId))) - if err = r.xControl.PatchTarget(ctx, originTarget, patch); err != nil { - return fmt.Errorf("fail to update origin target %s/%s pair label %s when updating by replaceUpdate: %s", originTarget.GetNamespace(), originTarget.GetName(), newCreatedTarget.GetName(), err.Error()) - } - logger.Info("replaceOriginTargets", "replacing originTarget", originTarget.GetName(), "originTargetId", originTargetId, "newTargetContextID", newInstanceId) - return nil - } else { - r.Recorder.Eventf(originTarget, - corev1.EventTypeNormal, - "ReplaceTarget", - "failed to create replace pair Target %s/%s from revision %s by replace update: %s", - originTarget.GetNamespace(), - originTarget.GetName(), - replaceRevision.GetName(), - err.Error()) - return err - } - }) - - return successCount, err -} - -func (r *RealSyncControl) dealReplaceTargets(ctx context.Context, targets []*TargetWrapper) ( - needReplaceTargets []*TargetWrapper, needCleanLabelTargets []client.Object, targetNeedCleanLabels [][]string, needDeleteTargets []client.Object, -) { - logger := logr.FromContext(ctx) - targetInstanceIdMap := make(map[string]client.Object) - targetNameMap := make(map[string]*TargetWrapper) - filteredTargets := FilterOutActiveTargetWrappers(targets) - - for _, target := range filteredTargets { - targetLabels := target.GetLabels() - - if instanceId, ok := r.xsetLabelAnnoMgr.Get(targetLabels, api.XInstanceIdLabelKey); ok { - targetInstanceIdMap[instanceId] = target - } - targetNameMap[target.GetName()] = target - } - - // deal need replace targets - for _, target := range filteredTargets { - targetLabels := target.GetLabels() - - // no replace indication label - if _, exist := r.xsetLabelAnnoMgr.Get(targetLabels, api.XReplaceIndicationLabelKey); !exist { - continue - } - - // origin target is about to scaleIn, skip replace - if opslifecycle.IsDuringOps(r.updateConfig.XsetLabelAnnoMgr, r.scaleInLifecycleAdapter, target) { - logger.Info("dealReplaceTargets", "target is during scaleIn ops lifecycle, skip replacing", target.GetName()) - continue - } - - // target is replace new created target, skip replace - if originTargetName, exist := r.xsetLabelAnnoMgr.Get(targetLabels, api.XReplacePairOriginName); exist { - if _, exist := targetNameMap[originTargetName]; exist { - continue - } - } - - // target already has a new created target for replacement - if newPairTargetId, exist := r.xsetLabelAnnoMgr.Get(targetLabels, api.XReplacePairNewId); exist { - if _, exist := targetInstanceIdMap[newPairTargetId]; exist { - continue - } - } - - needReplaceTargets = append(needReplaceTargets, target) - } - - for _, wrapper := range filteredTargets { - target := wrapper.Object - targetLabels := target.GetLabels() - _, replaceByUpdate := r.xsetLabelAnnoMgr.Get(targetLabels, api.XReplaceByReplaceUpdateLabelKey) - var needCleanLabels []string - - // target is replace new created target, skip replace - if originTargetName, exist := r.xsetLabelAnnoMgr.Get(targetLabels, api.XReplacePairOriginName); exist { - // replace pair origin target is not exist, clean label. - if originTarget, exist := targetNameMap[originTargetName]; !exist { - needCleanLabels = append(needCleanLabels, r.xsetLabelAnnoMgr.Value(api.XReplacePairOriginName)) - } else if _, exist := r.xsetLabelAnnoMgr.Get(originTarget.GetLabels(), api.XReplaceIndicationLabelKey); !exist { - // replace canceled, delete replace new target if new target is not service available - if !r.xsetController.CheckAvailable(target) { - needDeleteTargets = append(needDeleteTargets, target) - } - } else if !replaceByUpdate { - // not replace update, delete origin target when new created target is service available - if r.xsetController.CheckAvailable(target) { - needDeleteTargets = append(needDeleteTargets, originTarget.Object) - } - } - } - - if newPairTargetId, exist := r.xsetLabelAnnoMgr.Get(targetLabels, api.XReplacePairNewId); exist { - if _, exist := targetInstanceIdMap[newPairTargetId]; !exist { - needCleanLabels = append(needCleanLabels, r.xsetLabelAnnoMgr.Value(api.XReplacePairNewId)) - } - } - - if len(needCleanLabels) > 0 { - needCleanLabelTargets = append(needCleanLabelTargets, target) - targetNeedCleanLabels = append(targetNeedCleanLabels, needCleanLabels) - } - } - return needReplaceTargets, needCleanLabelTargets, targetNeedCleanLabels, needDeleteTargets -} - -func updateReplaceOriginTarget( - ctx context.Context, - c client.Client, - recorder record.EventRecorder, - xsetLabelAnnoMgr api.XSetLabelAnnotationManager, - originTargetUpdateInfo, newTargetUpdateInfo *TargetUpdateInfo, -) error { - originTarget := originTargetUpdateInfo.Object - - // 1. delete the new target if not updated - if newTargetUpdateInfo != nil { - newTarget := newTargetUpdateInfo.Object - _, deletionIndicate := xsetLabelAnnoMgr.Get(newTarget.GetLabels(), api.XDeletionIndicationLabelKey) - currentRevision, exist := newTarget.GetLabels()[appsv1.ControllerRevisionHashLabelKey] - if exist && currentRevision != originTargetUpdateInfo.UpdateRevision.GetName() && !deletionIndicate { - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, xsetLabelAnnoMgr.Value(api.XDeletionIndicationLabelKey), time.Now().UnixNano()))) - if patchErr := c.Patch(ctx, newTarget, patch); patchErr != nil { - err := fmt.Errorf("failed to delete replace pair new target %s/%s %s", - newTarget.GetNamespace(), newTarget.GetName(), patchErr.Error()) - return err - } - recorder.Eventf(originTarget, - corev1.EventTypeNormal, - "DeleteOldNewTarget", - "succeed to delete replace new Target %s/%s by label to-replace", - originTarget.GetNamespace(), - originTarget.GetName(), - ) - } - } - - // 2. replace the origin target with updated target - _, replaceIndicate := xsetLabelAnnoMgr.Get(originTarget.GetLabels(), api.XReplaceIndicationLabelKey) - replaceRevision, replaceByUpdate := xsetLabelAnnoMgr.Get(originTarget.GetLabels(), api.XReplaceByReplaceUpdateLabelKey) - if !replaceIndicate || !replaceByUpdate || replaceRevision != originTargetUpdateInfo.UpdateRevision.Name { - now := time.Now().UnixNano() - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%v", %q: "%v"}}}`, xsetLabelAnnoMgr.Value(api.XReplaceIndicationLabelKey), now, xsetLabelAnnoMgr.Value(api.XReplaceByReplaceUpdateLabelKey), originTargetUpdateInfo.UpdateRevision.Name))) - if err := c.Patch(ctx, originTarget, patch); err != nil { - return fmt.Errorf("fail to label origin target %s/%s with replace indicate label by replaceUpdate: %s", originTarget.GetNamespace(), originTarget.GetName(), err.Error()) - } - recorder.Eventf(originTarget, - corev1.EventTypeNormal, - "UpdateOriginTarget", - "succeed to update Target %s/%s by label to-replace", - originTarget.GetNamespace(), - originTarget.GetName(), - ) - } - - return nil -} - -// getReplaceRevision finds replaceNewTarget's revision from originTarget -func (r *RealSyncControl) getReplaceRevision(originTarget client.Object, syncContext *SyncContext) *appsv1.ControllerRevision { - // replace update, first find revision from label, if revision not found, just replace with updated revision - if updateRevisionName, exist := r.xsetLabelAnnoMgr.Get(originTarget.GetLabels(), api.XReplaceByReplaceUpdateLabelKey); exist { - for _, rv := range syncContext.Revisions { - if updateRevisionName == rv.Name { - return rv - } - } - return syncContext.UpdatedRevision - } - - // replace by to-replace label, just replace with current revision - targetCurrentRevisionName, exist := originTarget.GetLabels()[appsv1.ControllerRevisionHashLabelKey] - if !exist { - return syncContext.CurrentRevision - } - - for _, revision := range syncContext.Revisions { - if revision.GetName() == targetCurrentRevisionName { - return revision - } - } - - return syncContext.CurrentRevision -} - -// classify the pair relationship for Target replacement. -func classifyTargetReplacingMapping(xsetLabelAnnoMgr api.XSetLabelAnnotationManager, targetWrappers []*TargetWrapper) map[string]*TargetWrapper { - targetNameMap := make(map[string]*TargetWrapper) - targetIdMap := make(map[string]*TargetWrapper) - for _, targetWrapper := range targetWrappers { - targetNameMap[targetWrapper.GetName()] = targetWrapper - targetIdMap[strconv.Itoa(targetWrapper.ID)] = targetWrapper - } - - // old target name => new target wrapper - replaceTargetMapping := make(map[string]*TargetWrapper) - for _, targetWrapper := range targetWrappers { - if targetWrapper.Object == nil { - continue - } - name := targetWrapper.GetName() - if replacePairNewIdStr, exist := xsetLabelAnnoMgr.Get(targetWrapper.GetLabels(), api.XReplacePairNewId); exist { - if pairNewTarget, exist := targetIdMap[replacePairNewIdStr]; exist { - replaceTargetMapping[name] = pairNewTarget - // if one of pair targets is to Exclude, both targets should not scaleIn - targetWrapper.ToExclude = targetWrapper.ToExclude || pairNewTarget.ToExclude - continue - } - } else if replaceOriginStr, exist := xsetLabelAnnoMgr.Get(targetWrapper.GetLabels(), api.XReplacePairOriginName); exist { - if originTarget, exist := targetNameMap[replaceOriginStr]; exist { - id, exist := xsetLabelAnnoMgr.Get(originTarget.GetLabels(), api.XReplacePairNewId) - if exist && id == strconv.Itoa(targetWrapper.ID) { - continue - } - } - } - - // non paired target, just put it in the map - replaceTargetMapping[name] = nil - } - return replaceTargetMapping -} - -func (r *RealSyncControl) mapReplaceNewToOriginTargetContext(ownedIDs map[int]*api.ContextDetail) map[int]*api.ContextDetail { - mapNewToOriginTargetContext := make(map[int]*api.ContextDetail) - for id, contextDetail := range ownedIDs { - if val, exist := r.resourceContextControl.Get(contextDetail, api.EnumReplaceNewTargetIDContextDataKey); exist { - newTargetId, _ := strconv.ParseInt(val, 10, 32) - newTargetContextDetail, exist := ownedIDs[int(newTargetId)] - originTargetId, _ := r.resourceContextControl.Get(newTargetContextDetail, api.EnumReplaceOriginTargetIDContextDataKey) - if exist && originTargetId == strconv.Itoa(id) { - mapNewToOriginTargetContext[id] = newTargetContextDetail - } else { - mapNewToOriginTargetContext[id] = nil - } - } - } - return mapNewToOriginTargetContext -} - -func (r *RealSyncControl) mapReplaceOriginToNewTargetContext(ownedIDs map[int]*api.ContextDetail) map[int]*api.ContextDetail { - mapOriginToNewTargetContext := make(map[int]*api.ContextDetail) - for id, contextDetail := range ownedIDs { - if val, exist := r.resourceContextControl.Get(contextDetail, api.EnumReplaceOriginTargetIDContextDataKey); exist { - originTargetId, _ := strconv.ParseInt(val, 10, 32) - originTargetContextDetail, exist := ownedIDs[int(originTargetId)] - newTargetId, _ := r.resourceContextControl.Get(originTargetContextDetail, api.EnumReplaceNewTargetIDContextDataKey) - if exist && newTargetId == strconv.Itoa(id) { - mapOriginToNewTargetContext[id] = originTargetContextDetail - } else { - mapOriginToNewTargetContext[id] = nil - } - } - } - return mapOriginToNewTargetContext -} diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go deleted file mode 100644 index 61683b2..0000000 --- a/xset/synccontrols/x_scale.go +++ /dev/null @@ -1,262 +0,0 @@ -/* -Copyright 2024-2025 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package synccontrols - -import ( - "context" - "errors" - "sort" - "strconv" - - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/util/retry" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - - clientutil "kusionstack.io/kube-utils/client" - controllerutils "kusionstack.io/kube-utils/controller/utils" - "kusionstack.io/kube-utils/xset/api" - "kusionstack.io/kube-utils/xset/opslifecycle" - "kusionstack.io/kube-utils/xset/subresources" -) - -// getTargetsToDelete -// 1. finds number of diff targets from filteredTargets to do scaleIn -// 2. finds targets allowed to scale in out of diff -func (r *RealSyncControl) getTargetsToDelete(xsetObject api.XSetObject, filteredTargets []*TargetWrapper, replaceMapping map[string]*TargetWrapper, diff int) []*TargetWrapper { - var countedTargets []*TargetWrapper - for _, target := range filteredTargets { - if _, exist := replaceMapping[target.GetName()]; exist { - countedTargets = append(countedTargets, target) - } - } - - // 1. select targets to delete in first round according to diff - sort.Sort(newActiveTargetsForDeletion(countedTargets, r.xsetController.CheckReadyTime)) - if diff > len(countedTargets) { - diff = len(countedTargets) - } - - // 2. select targets to delete in second round according to replace, delete, exclude - var needDeleteTargets []*TargetWrapper - for i, target := range countedTargets { - // find targets to be scaleIn out of diff, is allowed to ops - spec := r.xsetController.GetXSetSpec(xsetObject) - _, allowed := opslifecycle.AllowOps(r.updateConfig.XsetLabelAnnoMgr, r.scaleInLifecycleAdapter, ptr.Deref(spec.ScaleStrategy.OperationDelaySeconds, 0), target) - if i >= diff && !allowed { - continue - } - - // don't scaleIn exclude target and its newTarget (if exist) - if target.ToExclude { - continue - } - - if replacePairTarget, exist := replaceMapping[target.GetName()]; exist && replacePairTarget != nil { - // don't selective scaleIn newTarget (and its originTarget) until replace finished - if replacePairTarget.ToDelete && !target.ToDelete { - continue - } - // when scaleIn origin Target, newTarget should be deleted if not service available - if !r.xsetController.CheckAvailable(target.Object) { - needDeleteTargets = append(needDeleteTargets, replacePairTarget) - } - } - needDeleteTargets = append(needDeleteTargets, target) - } - - return needDeleteTargets -} - -type ActiveTargetsForDeletion struct { - targets []*TargetWrapper - checkReadyFunc func(object client.Object) (bool, *metav1.Time) -} - -func newActiveTargetsForDeletion( - targets []*TargetWrapper, - checkReadyFunc func(object client.Object) (bool, *metav1.Time), -) *ActiveTargetsForDeletion { - return &ActiveTargetsForDeletion{ - targets: targets, - checkReadyFunc: checkReadyFunc, - } -} - -func (s *ActiveTargetsForDeletion) Len() int { return len(s.targets) } -func (s *ActiveTargetsForDeletion) Swap(i, j int) { - s.targets[i], s.targets[j] = s.targets[j], s.targets[i] -} - -// Less sort deletion order by: targetToDelete > targetToExclude > duringScaleIn > others -func (s *ActiveTargetsForDeletion) Less(i, j int) bool { - l, r := s.targets[i], s.targets[j] - - if l.ToDelete != r.ToDelete { - return l.ToDelete - } - - if l.ToExclude != r.ToExclude { - return l.ToExclude - } - - // targets which are during scaleInOps should be deleted before those not during - if l.IsDuringScaleInOps != r.IsDuringScaleInOps { - return l.IsDuringScaleInOps - } - - lReady, _ := s.checkReadyFunc(l.Object) - rReady, _ := s.checkReadyFunc(r.Object) - if lReady != rReady { - return !lReady - } - - if l.OpsPriority != nil && r.OpsPriority != nil { - if l.OpsPriority.PriorityClass != r.OpsPriority.PriorityClass { - return l.OpsPriority.PriorityClass < r.OpsPriority.PriorityClass - } - if l.OpsPriority.DeletionCost != r.OpsPriority.DeletionCost { - return l.OpsPriority.DeletionCost < r.OpsPriority.DeletionCost - } - } - - // TODO consider service available timestamps - return CompareTarget(l.Object, r.Object, s.checkReadyFunc) -} - -// doIncludeExcludeTargets do real include and exclude for targets which are allowed to in/exclude -func (r *RealSyncControl) doIncludeExcludeTargets(ctx context.Context, xset api.XSetObject, excludeTargets, includeTargets []string, availableContexts []*api.ContextDetail) error { - var excludeErrs, includeErrs []error - _, _ = controllerutils.SlowStartBatch(len(excludeTargets), controllerutils.SlowStartInitialBatchSize, false, func(idx int, _ error) (err error) { - defer func() { excludeErrs = append(excludeErrs, err) }() - return r.excludeTarget(ctx, xset, excludeTargets[idx]) - }) - _, _ = controllerutils.SlowStartBatch(len(includeTargets), controllerutils.SlowStartInitialBatchSize, false, func(idx int, _ error) (err error) { - defer func() { includeErrs = append(includeErrs, err) }() - return r.includeTarget(ctx, xset, includeTargets[idx], strconv.Itoa(availableContexts[idx].ID)) - }) - return errors.Join(append(includeErrs, excludeErrs...)...) -} - -// excludeTarget try to exclude a target from xset -func (r *RealSyncControl) excludeTarget(ctx context.Context, xsetObject api.XSetObject, targetName string) error { - target := r.xsetController.NewXObject() - if err := r.Client.Get(ctx, types.NamespacedName{Namespace: xsetObject.GetNamespace(), Name: targetName}, target); err != nil { - return err - } - - // exclude subresource - if adapter, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - volumes := adapter.GetXSpecVolumes(target) - for i := range volumes { - volume := volumes[i] - if volume.PersistentVolumeClaim == nil { - continue - } - pvc := &corev1.PersistentVolumeClaim{} - err := r.Client.Get(ctx, types.NamespacedName{Namespace: target.GetNamespace(), Name: volume.PersistentVolumeClaim.ClaimName}, pvc) - // If pvc not found, ignore it. In case of pvc is filtered out by controller-mesh - if apierrors.IsNotFound(err) { - continue - } else if err != nil { - return err - } - - r.xsetLabelAnnoMgr.Set(pvc, api.XOrphanedIndicationLabelKey, "true") - if err := r.pvcControl.OrphanPvc(ctx, xsetObject, pvc); err != nil { - return err - } - } - } - - r.xsetLabelAnnoMgr.Set(target, api.XOrphanedIndicationLabelKey, "true") - if err := r.xControl.OrphanTarget(xsetObject, target); err != nil { - return err - } - return r.cacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(xsetObject), r.targetGVK, target.GetNamespace(), target.GetName(), target.GetResourceVersion()) -} - -// includeTarget try to include a target into xset -func (r *RealSyncControl) includeTarget(ctx context.Context, xsetObject api.XSetObject, targetName, instanceId string) error { - target := r.xsetController.NewXObject() - if err := r.Client.Get(ctx, types.NamespacedName{Namespace: xsetObject.GetNamespace(), Name: targetName}, target); err != nil { - return err - } - - // exclude subresource - if adapter, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - volumes := adapter.GetXSpecVolumes(target) - for i := range volumes { - volume := volumes[i] - if volume.PersistentVolumeClaim == nil { - continue - } - pvc := &corev1.PersistentVolumeClaim{} - err := r.Client.Get(ctx, types.NamespacedName{Namespace: target.GetNamespace(), Name: volume.PersistentVolumeClaim.ClaimName}, pvc) - // If pvc not found, ignore it. In case of pvc is filtered out by controller-mesh - if apierrors.IsNotFound(err) { - continue - } else if err != nil { - return err - } - - r.xsetLabelAnnoMgr.Set(pvc, api.XInstanceIdLabelKey, instanceId) - r.xsetLabelAnnoMgr.Delete(pvc.GetLabels(), api.XOrphanedIndicationLabelKey) - if err := r.pvcControl.AdoptPvc(ctx, xsetObject, pvc); err != nil { - return err - } - } - } - - r.xsetLabelAnnoMgr.Set(target, api.XInstanceIdLabelKey, instanceId) - r.xsetLabelAnnoMgr.Delete(target.GetLabels(), api.XOrphanedIndicationLabelKey) - if err := r.xControl.AdoptTarget(xsetObject, target); err != nil { - return err - } - return r.cacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(xsetObject), r.targetGVK, target.GetNamespace(), target.GetName(), target.GetResourceVersion()) -} - -// reclaimScaleStrategy updates targetToDelete, targetToExclude, targetToInclude in scaleStrategy -func (r *RealSyncControl) reclaimScaleStrategy(ctx context.Context, deletedTargets, excludedTargets, includedTargets sets.String, xsetObject api.XSetObject) error { - xspec := r.xsetController.GetXSetSpec(xsetObject) - // reclaim TargetToDelete - toDeleteTargets := sets.NewString(xspec.ScaleStrategy.TargetToDelete...) - notDeletedTargets := toDeleteTargets.Delete(deletedTargets.List()...) - xspec.ScaleStrategy.TargetToDelete = notDeletedTargets.List() - // reclaim TargetToExclude - toExcludeTargets := sets.NewString(xspec.ScaleStrategy.TargetToExclude...) - notExcludeTargets := toExcludeTargets.Delete(excludedTargets.List()...) - xspec.ScaleStrategy.TargetToExclude = notExcludeTargets.List() - // reclaim TargetToInclude - toIncludeTargetNames := sets.NewString(xspec.ScaleStrategy.TargetToInclude...) - notIncludeTargets := toIncludeTargetNames.Delete(includedTargets.List()...) - xspec.ScaleStrategy.TargetToInclude = notIncludeTargets.List() - if err := r.xsetController.UpdateScaleStrategy(ctx, r.Client, xsetObject, &xspec.ScaleStrategy); err != nil { - return err - } - // update xsetObject.spec.scaleStrategy - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.Client.Update(ctx, xsetObject) - }); err != nil { - return err - } - return r.cacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(xsetObject), r.xsetGVK, xsetObject.GetNamespace(), xsetObject.GetName(), xsetObject.GetResourceVersion()) -} diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go deleted file mode 100644 index b864676..0000000 --- a/xset/synccontrols/x_update.go +++ /dev/null @@ -1,666 +0,0 @@ -/* -Copyright 2024-2025 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package synccontrols - -import ( - "context" - "fmt" - "sort" - "time" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" - "k8s.io/client-go/util/retry" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - - clientutil "kusionstack.io/kube-utils/client" - "kusionstack.io/kube-utils/controller/expectations" - controllerutils "kusionstack.io/kube-utils/controller/utils" - "kusionstack.io/kube-utils/xset/api" - "kusionstack.io/kube-utils/xset/opslifecycle" - "kusionstack.io/kube-utils/xset/resourcecontexts" - "kusionstack.io/kube-utils/xset/subresources" - "kusionstack.io/kube-utils/xset/xcontrol" -) - -const UnknownRevision = "__unknownRevision__" - -func (r *RealSyncControl) attachTargetUpdateInfo(_ context.Context, xsetObject api.XSetObject, syncContext *SyncContext) ([]*TargetUpdateInfo, error) { - activeTargets := FilterOutActiveTargetWrappers(syncContext.TargetWrappers) - targetUpdateInfoList := make([]*TargetUpdateInfo, len(activeTargets)) - - for i, target := range activeTargets { - updateInfo := &TargetUpdateInfo{ - TargetWrapper: syncContext.TargetWrappers[i], - } - - updateInfo.UpdateRevision = syncContext.UpdatedRevision - // decide this target current revision, or nil if not indicated - if target.GetLabels() != nil { - currentRevisionName, exist := target.GetLabels()[appsv1.ControllerRevisionHashLabelKey] - if exist { - if currentRevisionName == syncContext.UpdatedRevision.GetName() { - updateInfo.IsUpdatedRevision = true - updateInfo.CurrentRevision = syncContext.UpdatedRevision - } else { - updateInfo.IsUpdatedRevision = false - for _, rv := range syncContext.Revisions { - if currentRevisionName == rv.GetName() { - updateInfo.CurrentRevision = rv - } - } - } - } - } - - // default CurrentRevision is an empty revision - if updateInfo.CurrentRevision == nil { - updateInfo.CurrentRevision = &appsv1.ControllerRevision{ - ObjectMeta: metav1.ObjectMeta{ - Name: UnknownRevision, - }, - } - r.Recorder.Eventf(target.Object, - corev1.EventTypeWarning, - "TargetCurrentRevisionNotFound", - "target is going to be updated by recreate because: (1) controller-revision-hash label not found, or (2) not found in history revisions") - } - - var err error - spec := r.xsetController.GetXSetSpec(xsetObject) - // decide whether the TargetOpsLifecycle is during ops or not - updateInfo.RequeueForOperationDelay, updateInfo.IsAllowUpdateOps = opslifecycle.AllowOps(r.updateConfig.XsetLabelAnnoMgr, r.updateLifecycleAdapter, ptr.Deref(spec.UpdateStrategy.OperationDelaySeconds, 0), target) - // check subresource pvc template changed - if _, enabled := subresources.GetSubresourcePvcAdapter(r.xsetController); enabled { - updateInfo.PvcTmpHashChanged, err = r.pvcControl.IsTargetPvcTmpChanged(xsetObject, target.Object, syncContext.ExistingPvcs) - if err != nil { - return nil, err - } - } - targetUpdateInfoList[i] = updateInfo - } - - // attach replace info - targetUpdateInfoMap := make(map[string]*TargetUpdateInfo) - for _, targetUpdateInfo := range targetUpdateInfoList { - targetUpdateInfoMap[targetUpdateInfo.GetName()] = targetUpdateInfo - } - // originTarget's isAllowUpdateOps depends on these 2 cases: - // (1) target is during replacing but not during replaceUpdate, keep it legacy value - // (2) target is during replaceUpdate, set to "true" if newTarget is service available - for originTargetName, replacePairNewTarget := range syncContext.replacingMap { - originTargetInfo := targetUpdateInfoMap[originTargetName] - _, replaceIndicated := r.xsetLabelAnnoMgr.Get(originTargetInfo.GetLabels(), api.XReplaceIndicationLabelKey) - _, replaceByReplaceUpdate := r.xsetLabelAnnoMgr.Get(originTargetInfo.GetLabels(), api.XReplaceByReplaceUpdateLabelKey) - isReplaceUpdating := replaceIndicated && replaceByReplaceUpdate - - originTargetInfo.IsInReplace = replaceIndicated - originTargetInfo.IsInReplaceUpdate = isReplaceUpdating - - if replacePairNewTarget != nil { - // origin target is allowed to ops if new pod is serviceAvailable - newTargetSa := r.xsetController.CheckAvailable(replacePairNewTarget.Object) - originTargetInfo.IsAllowUpdateOps = originTargetInfo.IsAllowUpdateOps || newTargetSa - // attach replace new target updateInfo - ReplacePairNewTargetInfo := targetUpdateInfoMap[replacePairNewTarget.GetName()] - ReplacePairNewTargetInfo.IsInReplace = true - // in case of to-replace label is removed from origin target, new target is still in replaceUpdate - ReplacePairNewTargetInfo.IsInReplaceUpdate = replaceByReplaceUpdate - - ReplacePairNewTargetInfo.ReplacePairOriginTargetName = originTargetName - originTargetInfo.ReplacePairNewTargetInfo = ReplacePairNewTargetInfo - } - } - - // join PlaceHolder targets in updating - for _, target := range syncContext.TargetWrappers { - if !target.PlaceHolder { - continue - } - updateInfo := &TargetUpdateInfo{ - TargetWrapper: target, - UpdateRevision: syncContext.UpdatedRevision, - } - if revision, exist := r.resourceContextControl.Get(target.ContextDetail, api.EnumRevisionContextDataKey); exist && - revision == syncContext.UpdatedRevision.GetName() { - updateInfo.IsUpdatedRevision = true - } - targetUpdateInfoList = append(targetUpdateInfoList, updateInfo) - } - - return targetUpdateInfoList, nil -} - -func filterOutPlaceHolderUpdateInfos(targets []*TargetUpdateInfo) []*TargetUpdateInfo { - var filteredTargetUpdateInfos []*TargetUpdateInfo - for _, target := range targets { - if target.PlaceHolder { - continue - } - filteredTargetUpdateInfos = append(filteredTargetUpdateInfos, target) - } - return filteredTargetUpdateInfos -} - -func (r *RealSyncControl) decideTargetToUpdate(xsetController api.XSetController, xset api.XSetObject, targetInfos []*TargetUpdateInfo) []*TargetUpdateInfo { - spec := xsetController.GetXSetSpec(xset) - filteredPodInfos := r.getTargetsUpdateTargets(targetInfos) - - if spec.UpdateStrategy.RollingUpdate != nil && spec.UpdateStrategy.RollingUpdate.ByLabel != nil { - activeTargetInfos := filterOutPlaceHolderUpdateInfos(filteredPodInfos) - return r.decideTargetToUpdateByLabel(activeTargetInfos) - } - - return r.decideTargetToUpdateByPartition(xsetController, xset, filteredPodInfos) -} - -func (r *RealSyncControl) decideTargetToUpdateByLabel(targetInfos []*TargetUpdateInfo) (targetToUpdate []*TargetUpdateInfo) { - for i := range targetInfos { - if _, exist := r.xsetLabelAnnoMgr.Get(targetInfos[i].GetLabels(), api.XSetUpdateIndicationLabelKey); exist { - targetToUpdate = append(targetToUpdate, targetInfos[i]) - continue - } - - // separate decoration and xset update progress - if targetInfos[i].DecorationChanged { - if targetInfos[i].IsInReplace { - continue - } - targetInfos[i].IsUpdatedRevision = true - targetInfos[i].UpdateRevision = targetInfos[i].CurrentRevision - targetToUpdate = append(targetToUpdate, targetInfos[i]) - } - } - return targetToUpdate -} - -func (r *RealSyncControl) decideTargetToUpdateByPartition(xsetController api.XSetController, xset api.XSetObject, filteredTargetInfos []*TargetUpdateInfo) []*TargetUpdateInfo { - spec := xsetController.GetXSetSpec(xset) - replicas := ptr.Deref(spec.Replicas, 0) - currentTargetCount := int32(len(filteredTargetInfos)) - partition := int32(0) - - if spec.UpdateStrategy.RollingUpdate != nil && spec.UpdateStrategy.RollingUpdate.ByPartition != nil { - partition = ptr.Deref(spec.UpdateStrategy.RollingUpdate.ByPartition.Partition, 0) - } - - // update all or not update any replicas - if partition == 0 { - return filteredTargetInfos - } - if partition >= replicas { - return nil - } - - // partial update replicas - ordered := newOrderedTargetUpdateInfos(filteredTargetInfos, xsetController.CheckReadyTime) - sort.Sort(ordered) - targetToUpdate := ordered.targets[:replicas-partition] - // separate decoration and xset update progress - for i := replicas - partition; i < int32Min(replicas, currentTargetCount); i++ { - if ordered.targets[i].DecorationChanged { - ordered.targets[i].IsUpdatedRevision = true - ordered.targets[i].UpdateRevision = ordered.targets[i].CurrentRevision - targetToUpdate = append(targetToUpdate, ordered.targets[i]) - } - } - return targetToUpdate -} - -// when sort targets to choose update, only sort (1) replace origin targets, (2) non-exclude targets -func (r *RealSyncControl) getTargetsUpdateTargets(targetInfos []*TargetUpdateInfo) (filteredTargetInfos []*TargetUpdateInfo) { - for _, targetInfo := range targetInfos { - if targetInfo.ReplacePairOriginTargetName != "" { - continue - } - if targetInfo.PlaceHolder { - if _, isReplaceNewTarget := r.resourceContextControl.Get(targetInfo.ContextDetail, api.EnumReplaceOriginTargetIDContextDataKey); isReplaceNewTarget { - continue - } - } - filteredTargetInfos = append(filteredTargetInfos, targetInfo) - } - return filteredTargetInfos -} - -func newOrderedTargetUpdateInfos( - targetInfos []*TargetUpdateInfo, - checkReadyFunc func(object client.Object) (bool, *metav1.Time), -) *orderByDefault { - return &orderByDefault{ - targets: targetInfos, - checkReadyFunc: checkReadyFunc, - } -} - -type orderByDefault struct { - targets []*TargetUpdateInfo - checkReadyFunc func(object client.Object) (bool, *metav1.Time) -} - -func (o *orderByDefault) Len() int { - return len(o.targets) -} - -func (o *orderByDefault) Swap(i, j int) { o.targets[i], o.targets[j] = o.targets[j], o.targets[i] } - -func (o *orderByDefault) Less(i, j int) bool { - l, r := o.targets[i], o.targets[j] - if l.IsUpdatedRevision != r.IsUpdatedRevision { - return l.IsUpdatedRevision - } - - if l.IsDuringUpdateOps != r.IsDuringUpdateOps { - return l.IsDuringUpdateOps - } - - if l.IsInReplaceUpdate != r.IsInReplaceUpdate { - return l.IsInReplaceUpdate - } - - if l.PlaceHolder != r.PlaceHolder { - return r.PlaceHolder - } - - if l.PlaceHolder && r.PlaceHolder { - return true - } - - lReady, _ := o.checkReadyFunc(l.Object) - rReady, _ := o.checkReadyFunc(r.Object) - if lReady != rReady { - return !lReady - } - - if l.DecorationChanged != r.DecorationChanged { - return l.DecorationChanged - } - - if l.OpsPriority != nil && r.OpsPriority != nil { - if l.OpsPriority.PriorityClass != r.OpsPriority.PriorityClass { - return l.OpsPriority.PriorityClass < r.OpsPriority.PriorityClass - } - if l.OpsPriority.DeletionCost != r.OpsPriority.DeletionCost { - return l.OpsPriority.DeletionCost < r.OpsPriority.DeletionCost - } - } - - return CompareTarget(l.Object, r.Object, o.checkReadyFunc) -} - -type UpdateConfig struct { - XsetController api.XSetController - XsetLabelAnnoMgr api.XSetLabelAnnotationManager - Client client.Client - TargetControl xcontrol.TargetControl - ResourceContextControl resourcecontexts.ResourceContextControl - Recorder record.EventRecorder - - scaleInLifecycleAdapter api.LifecycleAdapter - updateLifecycleAdapter api.LifecycleAdapter - - CacheExpectations expectations.CacheExpectationsInterface - TargetGVK schema.GroupVersionKind -} - -type TargetUpdater interface { - Setup(config *UpdateConfig, xset api.XSetObject) - FulfillTargetUpdatedInfo(ctx context.Context, revision *appsv1.ControllerRevision, targetUpdateInfo *TargetUpdateInfo) error - BeginUpdateTarget(ctx context.Context, syncContext *SyncContext, targetCh chan *TargetUpdateInfo) (bool, error) - FilterAllowOpsTargets(ctx context.Context, targetToUpdate []*TargetUpdateInfo, ownedIDs map[int]*api.ContextDetail, syncContext *SyncContext, targetCh chan *TargetUpdateInfo) (*time.Duration, error) - UpgradeTarget(ctx context.Context, targetInfo *TargetUpdateInfo) error - GetTargetUpdateFinishStatus(ctx context.Context, targetUpdateInfo *TargetUpdateInfo) (bool, string, error) - FinishUpdateTarget(ctx context.Context, targetInfo *TargetUpdateInfo, finishByCancelUpdate bool) error -} - -type GenericTargetUpdater struct { - OwnerObject api.XSetObject - - *UpdateConfig -} - -func (u *GenericTargetUpdater) Setup(config *UpdateConfig, xset api.XSetObject) { - u.UpdateConfig = config - u.OwnerObject = xset -} - -func (u *GenericTargetUpdater) BeginUpdateTarget(_ context.Context, syncContext *SyncContext, targetCh chan *TargetUpdateInfo) (bool, error) { - succCount, err := controllerutils.SlowStartBatch(len(targetCh), controllerutils.SlowStartInitialBatchSize, false, func(int, error) error { - targetInfo := <-targetCh - u.Recorder.Eventf(targetInfo.Object, corev1.EventTypeNormal, "TargetUpdateLifecycle", "try to begin TargetOpsLifecycle for updating Target of XSet") - - if updated, err := opslifecycle.BeginWithCleaningOld(u.XsetLabelAnnoMgr, u.Client, u.updateLifecycleAdapter, targetInfo.Object, func(obj client.Object) (bool, error) { - if !targetInfo.OnlyMetadataChanged && !targetInfo.InPlaceUpdateSupport { - return opslifecycle.WhenBeginDelete(u.XsetLabelAnnoMgr, obj) - } - return false, nil - }); err != nil { - return fmt.Errorf("fail to begin TargetOpsLifecycle for updating Target %s/%s: %s", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) - } else if updated { - // add an expectation for this target update, before next reconciling - if err := u.CacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(u.OwnerObject), u.TargetGVK, targetInfo.GetNamespace(), targetInfo.GetName(), targetInfo.GetResourceVersion()); err != nil { - return err - } - } - return nil - }) - - updating := succCount > 0 - if err != nil { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetUpdate, err, "UpdateFailed", err.Error()) - return updating, err - } else { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetUpdate, nil, "Updated", "") - } - return updating, nil -} - -func (u *GenericTargetUpdater) FilterAllowOpsTargets(ctx context.Context, candidates []*TargetUpdateInfo, ownedIDs map[int]*api.ContextDetail, _ *SyncContext, targetCh chan *TargetUpdateInfo) (*time.Duration, error) { - var recordedRequeueAfter *time.Duration - needUpdateContext := false - for i := range candidates { - targetInfo := candidates[i] - - if !targetInfo.PlaceHolder { - if !targetInfo.IsAllowUpdateOps { - continue - } - if targetInfo.RequeueForOperationDelay != nil { - u.Recorder.Eventf(targetInfo, corev1.EventTypeNormal, "TargetUpdateLifecycle", "delay Target update for %f seconds", targetInfo.RequeueForOperationDelay.Seconds()) - if recordedRequeueAfter == nil || *targetInfo.RequeueForOperationDelay < *recordedRequeueAfter { - recordedRequeueAfter = targetInfo.RequeueForOperationDelay - } - continue - } - } - - targetInfo.IsAllowUpdateOps = true - - if targetInfo.IsUpdatedRevision && !targetInfo.PvcTmpHashChanged && !targetInfo.DecorationChanged { - continue - } - - if _, exist := ownedIDs[targetInfo.ID]; !exist { - u.Recorder.Eventf(u.OwnerObject, corev1.EventTypeWarning, "TargetBeforeUpdate", "target %s/%s is not allowed to update because cannot find context id %s in resourceContext", targetInfo.GetNamespace(), targetInfo.GetName(), targetInfo.GetLabels()[u.XsetLabelAnnoMgr.Value(api.XInstanceIdLabelKey)]) - continue - } - - if !u.ResourceContextControl.Contains(ownedIDs[targetInfo.ID], api.EnumRevisionContextDataKey, targetInfo.UpdateRevision.GetName()) { - needUpdateContext = true - u.ResourceContextControl.Put(ownedIDs[targetInfo.ID], api.EnumRevisionContextDataKey, targetInfo.UpdateRevision.GetName()) - } - - spec := u.XsetController.GetXSetSpec(u.OwnerObject) - - // mark targetContext "TargetRecreateUpgrade" if upgrade by recreate - isRecreateUpdatePolicy := spec.UpdateStrategy.UpdatePolicy == api.XSetRecreateTargetUpdateStrategyType - if (!targetInfo.OnlyMetadataChanged && !targetInfo.InPlaceUpdateSupport) || isRecreateUpdatePolicy { - u.ResourceContextControl.Put(ownedIDs[targetInfo.ID], api.EnumRecreateUpdateContextDataKey, "true") - } - - // add decoration revision to target context - if targetInfo.DecorationChanged { - if val, ok := u.ResourceContextControl.Get(ownedIDs[targetInfo.ID], api.EnumTargetDecorationRevisionKey); !ok || val != targetInfo.DecorationUpdatedRevisions { - needUpdateContext = true - u.ResourceContextControl.Put(ownedIDs[targetInfo.ID], api.EnumTargetDecorationRevisionKey, targetInfo.DecorationUpdatedRevisions) - } - } - - if targetInfo.PlaceHolder { - continue - } - - // if Target has not been updated, update it. - targetCh <- candidates[i] - } - // mark Target to use updated revision before updating it. - if needUpdateContext { - u.Recorder.Eventf(u.OwnerObject, corev1.EventTypeNormal, "UpdateToTargetContext", "try to update ResourceContext for XSet") - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return u.ResourceContextControl.UpdateToTargetContext(ctx, u.OwnerObject, ownedIDs) - }) - return recordedRequeueAfter, err - } - return recordedRequeueAfter, nil -} - -func (u *GenericTargetUpdater) FinishUpdateTarget(_ context.Context, targetInfo *TargetUpdateInfo, finishByCancelUpdate bool) error { - if finishByCancelUpdate { - // cancel update lifecycle - return opslifecycle.CancelOpsLifecycle(u.XsetLabelAnnoMgr, u.Client, u.updateLifecycleAdapter, targetInfo.Object) - } - - // target is ops finished, finish the lifecycle gracefully - if updated, err := opslifecycle.Finish(u.XsetLabelAnnoMgr, u.Client, u.updateLifecycleAdapter, targetInfo.Object); err != nil { - return fmt.Errorf("failed to finish TargetOpsLifecycle for updating Target %s/%s: %s", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) - } else if updated { - // add an expectation for this target update, before next reconciling - if err := u.CacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(u.OwnerObject), u.TargetGVK, targetInfo.GetNamespace(), targetInfo.GetName(), targetInfo.GetResourceVersion()); err != nil { - return err - } - u.Recorder.Eventf(targetInfo.Object, - corev1.EventTypeNormal, - "UpdateReady", "target %s/%s update finished", targetInfo.GetNamespace(), targetInfo.GetName()) - } - return nil -} - -// Support users to define inPlaceOnlyTargetUpdater and register through RegisterInPlaceOnlyUpdater -var inPlaceOnlyTargetUpdater TargetUpdater - -func RegisterInPlaceOnlyUpdater(targetUpdater TargetUpdater) { - inPlaceOnlyTargetUpdater = targetUpdater -} - -// Support users to define inPlaceIfPossibleUpdater and register through RegistryInPlaceIfPossibleUpdater -var inPlaceIfPossibleUpdater TargetUpdater - -func RegisterInPlaceIfPossibleUpdater(targetUpdater TargetUpdater) { - inPlaceIfPossibleUpdater = targetUpdater -} - -func (r *RealSyncControl) newTargetUpdater(xset api.XSetObject) TargetUpdater { - spec := r.xsetController.GetXSetSpec(xset) - var targetUpdater TargetUpdater - switch spec.UpdateStrategy.UpdatePolicy { - case api.XSetRecreateTargetUpdateStrategyType: - targetUpdater = &recreateTargetUpdater{} - case api.XSetInPlaceOnlyTargetUpdateStrategyType: - if inPlaceOnlyTargetUpdater != nil { - targetUpdater = inPlaceOnlyTargetUpdater - } else if inPlaceIfPossibleUpdater != nil { - // In case of using native K8s, Target is only allowed to update with container image, so InPlaceOnly policy is - // implemented with InPlaceIfPossible policy as default for compatibility. - targetUpdater = inPlaceIfPossibleUpdater - } else { - // if none of InplaceOnly and InplaceIfPossible updater is registered, use default Recreate updater - targetUpdater = &recreateTargetUpdater{} - } - case api.XSetReplaceTargetUpdateStrategyType: - targetUpdater = &replaceUpdateTargetUpdater{} - default: - if inPlaceIfPossibleUpdater != nil { - targetUpdater = inPlaceIfPossibleUpdater - } else { - targetUpdater = &recreateTargetUpdater{} - } - } - targetUpdater.Setup(r.updateConfig, xset) - return targetUpdater -} - -func (u *GenericTargetUpdater) RecreateTarget(ctx context.Context, targetInfo *TargetUpdateInfo) error { - if err := u.TargetControl.DeleteTarget(ctx, targetInfo.Object); err != nil { - return fmt.Errorf("fail to delete Target %s/%s when updating by recreate: %v", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) - } - - u.Recorder.Eventf(targetInfo.Object, - corev1.EventTypeNormal, - "UpdateTarget", - "succeed to update Target %s/%s to from revision %s to revision %s by recreate", - targetInfo.GetNamespace(), - targetInfo.GetName(), - targetInfo.CurrentRevision.GetName(), - targetInfo.UpdateRevision.GetName()) - - return nil -} - -type recreateTargetUpdater struct { - GenericTargetUpdater -} - -func (u *recreateTargetUpdater) FulfillTargetUpdatedInfo(_ context.Context, _ *appsv1.ControllerRevision, _ *TargetUpdateInfo) error { - return nil -} - -func (u *recreateTargetUpdater) UpgradeTarget(ctx context.Context, targetInfo *TargetUpdateInfo) error { - return u.GenericTargetUpdater.RecreateTarget(ctx, targetInfo) -} - -func (u *recreateTargetUpdater) GetTargetUpdateFinishStatus(_ context.Context, targetInfo *TargetUpdateInfo) (finished bool, msg string, err error) { - // Recreate policy always treat Target as update not finished - return targetInfo.IsUpdatedRevision && !targetInfo.DecorationChanged, "", nil -} - -type replaceUpdateTargetUpdater struct { - GenericTargetUpdater -} - -func (u *replaceUpdateTargetUpdater) Setup(config *UpdateConfig, xset api.XSetObject) { - u.GenericTargetUpdater.Setup(config, xset) -} - -func (u *replaceUpdateTargetUpdater) BeginUpdateTarget(ctx context.Context, syncContext *SyncContext, targetCh chan *TargetUpdateInfo) (bool, error) { - succCount, err := controllerutils.SlowStartBatch(len(targetCh), controllerutils.SlowStartInitialBatchSize, false, func(int, error) error { - targetInfo := <-targetCh - if targetInfo.ReplacePairNewTargetInfo != nil { - replacePairNewTarget := targetInfo.ReplacePairNewTargetInfo.Object - newTargetRevision, exist := replacePairNewTarget.GetLabels()[appsv1.ControllerRevisionHashLabelKey] - if exist && newTargetRevision == targetInfo.UpdateRevision.GetName() { - return nil - } - if _, exist := u.XsetLabelAnnoMgr.Get(replacePairNewTarget.GetLabels(), api.XDeletionIndicationLabelKey); exist { - return nil - } - - u.Recorder.Eventf(targetInfo.Object, - corev1.EventTypeNormal, - "ReplaceUpdateTarget", - "label to-delete on new pair target %s/%s because it is not updated revision, current revision: %s, updated revision: %s", - replacePairNewTarget.GetNamespace(), - replacePairNewTarget.GetName(), - newTargetRevision, - syncContext.UpdatedRevision.GetName()) - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, u.XsetLabelAnnoMgr.Value(api.XDeletionIndicationLabelKey), time.Now().UnixNano()))) - if patchErr := u.Client.Patch(ctx, targetInfo.ReplacePairNewTargetInfo.Object, patch); patchErr != nil { - err := fmt.Errorf("failed to delete replace pair new target %s/%s %s", - targetInfo.ReplacePairNewTargetInfo.GetNamespace(), targetInfo.ReplacePairNewTargetInfo.GetName(), patchErr.Error()) - return err - } - } - return nil - }) - - return succCount > 0, err -} - -func (u *replaceUpdateTargetUpdater) FilterAllowOpsTargets(_ context.Context, candidates []*TargetUpdateInfo, _ map[int]*api.ContextDetail, _ *SyncContext, targetCh chan *TargetUpdateInfo) (requeueAfter *time.Duration, err error) { - activeTargetToUpdate := filterOutPlaceHolderUpdateInfos(candidates) - for i, targetInfo := range activeTargetToUpdate { - if targetInfo.IsUpdatedRevision && !targetInfo.PvcTmpHashChanged && !targetInfo.DecorationChanged { - continue - } - - targetCh <- activeTargetToUpdate[i] - } - return nil, err -} - -func (u *replaceUpdateTargetUpdater) FulfillTargetUpdatedInfo(_ context.Context, _ *appsv1.ControllerRevision, _ *TargetUpdateInfo) (err error) { - return -} - -func (u *replaceUpdateTargetUpdater) UpgradeTarget(ctx context.Context, targetInfo *TargetUpdateInfo) error { - return updateReplaceOriginTarget(ctx, u.Client, u.Recorder, u.XsetLabelAnnoMgr, targetInfo, targetInfo.ReplacePairNewTargetInfo) -} - -func (u *replaceUpdateTargetUpdater) GetTargetUpdateFinishStatus(_ context.Context, targetUpdateInfo *TargetUpdateInfo) (finished bool, msg string, err error) { - replaceNewTargetInfo := targetUpdateInfo.ReplacePairNewTargetInfo - if replaceNewTargetInfo == nil { - return - } - - return u.isTargetUpdatedServiceAvailable(replaceNewTargetInfo) -} - -func (u *replaceUpdateTargetUpdater) FinishUpdateTarget(ctx context.Context, targetInfo *TargetUpdateInfo, finishByCancelUpdate bool) error { - if finishByCancelUpdate { - // cancel replace update by removing to-replace and replace-by-update label from origin target - if targetInfo.IsInReplace { - patch := client.RawPatch(types.MergePatchType, fmt.Appendf(nil, `{"metadata":{"labels":{"%s":null, "%s":null}}}`, u.XsetLabelAnnoMgr.Value(api.XReplaceIndicationLabelKey), u.XsetLabelAnnoMgr.Value(api.XReplaceByReplaceUpdateLabelKey))) - if err := u.TargetControl.PatchTarget(ctx, targetInfo.Object, patch); err != nil { - return fmt.Errorf("failed to patch replace pair target %s/%s %w when cancel replace update", targetInfo.GetNamespace(), targetInfo.GetName(), err) - } - } - return nil - } - - ReplacePairNewTargetInfo := targetInfo.ReplacePairNewTargetInfo - if ReplacePairNewTargetInfo != nil { - if _, exist := u.XsetLabelAnnoMgr.Get(targetInfo.GetLabels(), api.XDeletionIndicationLabelKey); !exist { - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, u.XsetLabelAnnoMgr.Value(api.XDeletionIndicationLabelKey), time.Now().UnixNano()))) - if err := u.TargetControl.PatchTarget(ctx, targetInfo.Object, patch); err != nil { - return fmt.Errorf("failed to delete replace pair origin target %s/%s %s", targetInfo.GetNamespace(), targetInfo.ReplacePairNewTargetInfo.GetName(), err.Error()) - } - } - } - return nil -} - -func (u *GenericTargetUpdater) isTargetUpdatedServiceAvailable(targetInfo *TargetUpdateInfo) (finished bool, msg string, err error) { - // check decoration changed - if targetInfo.DecorationChanged { - return false, "decoration changed", nil - } - - if targetInfo.GetLabels() == nil { - return false, "no labels on target", nil - } - if targetInfo.IsInReplace && targetInfo.ReplacePairNewTargetInfo != nil { - return false, "replace origin target", nil - } - - if u.XsetController.CheckAvailable(targetInfo.Object) { - return true, "", nil - } - - return false, "target not service available", nil -} - -func int32Min(l, r int32) int32 { - if l < r { - return l - } - - return r -} diff --git a/xset/synccontrols/x_utils.go b/xset/synccontrols/x_utils.go deleted file mode 100644 index 3fc7271..0000000 --- a/xset/synccontrols/x_utils.go +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package synccontrols - -import ( - "context" - "fmt" - "strconv" - "time" - - appsv1 "k8s.io/api/apps/v1" - apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - clientutils "kusionstack.io/kube-utils/client" - "kusionstack.io/kube-utils/condition" - controllerutils "kusionstack.io/kube-utils/controller/utils" - "kusionstack.io/kube-utils/xset/api" -) - -func GetInstanceID(xsetLabelAnnoMgr api.XSetLabelAnnotationManager, target client.Object) (int, error) { - if target.GetLabels() == nil { - return -1, fmt.Errorf("no labels found for instance ID") - } - - instanceIdLabelKey := xsetLabelAnnoMgr.Value(api.XInstanceIdLabelKey) - val, exist := target.GetLabels()[instanceIdLabelKey] - if !exist { - return -1, fmt.Errorf("failed to find instance ID label %s", instanceIdLabelKey) - } - - id, err := strconv.ParseInt(val, 10, 32) - if err != nil { - // ignore invalid target instance ID - return -1, fmt.Errorf("failed to parse instance ID with value %s: %w", val, err) - } - - return int(id), nil -} - -func NewTargetFrom(setController api.XSetController, xsetLabelAnnoMgr api.XSetLabelAnnotationManager, owner api.XSetObject, revision *appsv1.ControllerRevision, id int, updateFuncs ...func(client.Object) error) (client.Object, error) { - targetObj, err := setController.GetXObjectFromRevision(revision) - if err != nil { - return nil, err - } - - meta := setController.XSetMeta() - ownerRef := metav1.NewControllerRef(owner, meta.GroupVersionKind()) - targetObj.SetOwnerReferences(append(targetObj.GetOwnerReferences(), *ownerRef)) - targetObj.SetNamespace(owner.GetNamespace()) - targetObj.SetGenerateName(GetTargetsPrefix(owner.GetName())) - - if IsTargetNamingSuffixPolicyPersistentSequence(setController.GetXSetSpec(owner)) { - targetObj.SetName(fmt.Sprintf("%s%d", targetObj.GetGenerateName(), id)) - } - - xsetLabelAnnoMgr.Set(targetObj, api.XInstanceIdLabelKey, fmt.Sprintf("%d", id)) - targetObj.GetLabels()[appsv1.ControllerRevisionHashLabelKey] = revision.GetName() - controlByXSet(xsetLabelAnnoMgr, targetObj) - - for _, fn := range updateFuncs { - if err := fn(targetObj); err != nil { - return targetObj, err - } - } - - return targetObj, nil -} - -const ConditionUpdatePeriodBackOff = 30 * time.Second - -func AddOrUpdateCondition(status *api.XSetStatus, conditionType api.XSetConditionType, err error, reason, message string) { - condStatus := metav1.ConditionTrue - if err != nil { - condStatus = metav1.ConditionFalse - } - - existCond := condition.GetCondition(status.Conditions, string(conditionType)) - if existCond != nil && existCond.Reason == reason && existCond.Status == condStatus { - now := metav1.Now() - if now.Sub(existCond.LastTransitionTime.Time) < ConditionUpdatePeriodBackOff { - return - } - } - - cond := condition.NewCondition(string(conditionType), condStatus, reason, message) - status.Conditions = condition.SetCondition(status.Conditions, *cond) - - // update condition last transition time - for i := range status.Conditions { - c := status.Conditions[i] - if c.Type == string(conditionType) { - status.Conditions[i].LastTransitionTime = metav1.Now() - return - } - } -} - -func GetTargetsPrefix(controllerName string) string { - // use the dash (if the name isn't too long) to make the target name a bit prettier - prefix := fmt.Sprintf("%s-", controllerName) - if len(apimachineryvalidation.NameIsDNSSubdomain(prefix, true)) != 0 { - prefix = controllerName - } - return prefix -} - -func IsTargetUpdatedRevision(target client.Object, revision string) bool { - if target.GetLabels() == nil { - return false - } - - return target.GetLabels()[appsv1.ControllerRevisionHashLabelKey] == revision -} - -func ObjectKeyString(obj client.Object) string { - if obj.GetNamespace() == "" { - return obj.GetName() - } - return obj.GetNamespace() + "/" + obj.GetName() -} - -func controlByXSet(xsetLabelAnnoMgr api.XSetLabelAnnotationManager, obj client.Object) { - if obj.GetLabels() == nil { - obj.SetLabels(map[string]string{}) - } - if v, ok := xsetLabelAnnoMgr.Get(obj.GetLabels(), api.ControlledByXSetLabel); !ok || v != "true" { - xsetLabelAnnoMgr.Set(obj, api.ControlledByXSetLabel, "true") - } -} - -func IsControlledByXSet(xsetLabelManager api.XSetLabelAnnotationManager, obj client.Object) bool { - if obj.GetLabels() == nil { - return false - } - - v, ok := xsetLabelManager.Get(obj.GetLabels(), api.ControlledByXSetLabel) - return ok && v == "true" -} - -func ApplyTemplatePatcher(ctx context.Context, xsetController api.XSetController, c client.Client, xset api.XSetObject, targets []*TargetWrapper) error { - _, patchErr := controllerutils.SlowStartBatch(len(targets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { - if targets[i].Object == nil || targets[i].PlaceHolder { - return nil - } - _, err := clientutils.UpdateOnConflict(ctx, c, c, targets[i].Object, xsetController.GetXSetTemplatePatcher(xset)) - return err - }) - return patchErr -} - -func CompareTarget(l, r client.Object, checkReadyFunc func(object client.Object) (bool, *metav1.Time)) bool { - // If both targets are ready, the latest ready one is smaller - lReady, lReadyTime := checkReadyFunc(l) - rReady, rReadyTime := checkReadyFunc(r) - if lReady && rReady && !lReadyTime.Equal(rReadyTime) { - return afterOrZero(lReadyTime, rReadyTime) - } - // Empty creation time targets < newer targets < older targets - lCreationTime, rCreationTime := l.GetCreationTimestamp(), r.GetCreationTimestamp() - if !(&lCreationTime).Equal(&rCreationTime) { - return afterOrZero(&lCreationTime, &rCreationTime) - } - return false -} - -func afterOrZero(t1, t2 *metav1.Time) bool { - if t1.Time.IsZero() || t2.Time.IsZero() { - return t1.Time.IsZero() - } - return t1.After(t2.Time) -} - -func IsTargetNamingSuffixPolicyPersistentSequence(xsetSpec *api.XSetSpec) bool { - if xsetSpec == nil || xsetSpec.NamingStrategy == nil { - return false - } - return xsetSpec.NamingStrategy.TargetNamingSuffixPolicy == api.TargetNamingSuffixPolicyPersistentSequence -} diff --git a/xset/xcontrol/target_control.go b/xset/xcontrol/target_control.go deleted file mode 100644 index 1d5378e..0000000 --- a/xset/xcontrol/target_control.go +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package xcontrol - -import ( - "context" - "errors" - "fmt" - "reflect" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" - - "kusionstack.io/kube-utils/controller/mixin" - refmanagerutil "kusionstack.io/kube-utils/controller/refmanager" - "kusionstack.io/kube-utils/xset/api" -) - -const ( - FieldIndexOwnerRefUID = "ownerRefUID" -) - -type TargetControl interface { - GetFilteredTargets(ctx context.Context, selector *metav1.LabelSelector, owner api.XSetObject) ([]client.Object, []client.Object, error) - CreateTarget(ctx context.Context, target client.Object) (client.Object, error) - DeleteTarget(ctx context.Context, target client.Object) error - UpdateTarget(ctx context.Context, target client.Object) error - PatchTarget(ctx context.Context, target client.Object, patch client.Patch) error - OrphanTarget(xset api.XSetObject, target client.Object) error - AdoptTarget(xset api.XSetObject, target client.Object) error -} - -type targetControl struct { - client client.Client - schema *runtime.Scheme - - xsetController api.XSetController - xGVK schema.GroupVersionKind -} - -func NewTargetControl(mixin *mixin.ReconcilerMixin, xsetController api.XSetController) (TargetControl, error) { - if err := setUpCache(mixin.Cache, xsetController); err != nil { - return nil, err - } - - xMeta := xsetController.XMeta() - gvk := xMeta.GroupVersionKind() - return &targetControl{ - client: mixin.Client, - schema: mixin.Scheme, - xsetController: xsetController, - xGVK: gvk, - }, nil -} - -func (r *targetControl) GetFilteredTargets(ctx context.Context, selector *metav1.LabelSelector, owner api.XSetObject) ([]client.Object, []client.Object, error) { - targetList := r.xsetController.NewXObjectList() - if err := r.client.List(ctx, targetList, &client.ListOptions{ - Namespace: owner.GetNamespace(), - FieldSelector: fields.OneTermEqualSelector(FieldIndexOwnerRefUID, string(owner.GetUID())), - }); err != nil { - return nil, nil, err - } - - targetListVal := reflect.Indirect(reflect.ValueOf(targetList)) - itemsVal := targetListVal.FieldByName("Items") - if !itemsVal.IsValid() { - return nil, nil, fmt.Errorf("target list items is invalid") - } - - var items []client.Object - if itemsVal.Kind() == reflect.Slice { - items = make([]client.Object, itemsVal.Len()) - for i := 0; i < itemsVal.Len(); i++ { - itemVal := itemsVal.Index(i).Addr().Interface() - items[i] = itemVal.(client.Object) - } - } else { - return nil, nil, fmt.Errorf("target list items is invalid") - } - - allTargets, err := r.getTargets(items, selector, owner) - if err != nil { - return nil, nil, err - } - - items = filterOutInactiveTargets(r.xsetController, items) - filteredTargets, err := r.getTargets(items, selector, owner) - - return filteredTargets, allTargets, err -} - -func (r *targetControl) CreateTarget(ctx context.Context, target client.Object) (client.Object, error) { - if err := r.client.Create(ctx, target); err != nil { - return nil, fmt.Errorf("failed to create target: %s", err.Error()) - } - return target, nil -} - -func (r *targetControl) DeleteTarget(ctx context.Context, target client.Object) error { - return r.client.Delete(ctx, target) -} - -func (r *targetControl) UpdateTarget(ctx context.Context, target client.Object) error { - return r.client.Update(ctx, target) -} - -func (r *targetControl) PatchTarget(ctx context.Context, target client.Object, patch client.Patch) error { - return r.client.Patch(ctx, target, patch) -} - -func (r *targetControl) OrphanTarget(xset api.XSetObject, target client.Object) error { - spec := r.xsetController.GetXSetSpec(xset) - if spec.Selector.MatchLabels == nil { - return nil - } - - if target.GetLabels() == nil { - target.SetLabels(make(map[string]string)) - } - if target.GetAnnotations() == nil { - target.SetAnnotations(make(map[string]string)) - } - - refWriter := refmanagerutil.NewOwnerRefWriter(r.client) - if err := refWriter.Release(context.TODO(), xset, target); err != nil { - return fmt.Errorf("failed to orphan target: %s", err.Error()) - } - - return nil -} - -func (r *targetControl) AdoptTarget(xset api.XSetObject, target client.Object) error { - spec := r.xsetController.GetXSetSpec(xset) - if spec.Selector.MatchLabels == nil { - return nil - } - - refWriter := refmanagerutil.NewOwnerRefWriter(r.client) - matcher, err := refmanagerutil.LabelSelectorAsMatch(spec.Selector) - if err != nil { - return fmt.Errorf("fail to create labelSelector matcher: %s", err.Error()) - } - refManager := refmanagerutil.NewObjectControllerRefManager(refWriter, xset, xset.GetObjectKind().GroupVersionKind(), matcher) - - if _, err = refManager.Claim(context.TODO(), target); err != nil { - return fmt.Errorf("failed to adopt target: %s", err.Error()) - } - - return nil -} - -func (r *targetControl) getTargets(candidates []client.Object, selector *metav1.LabelSelector, xset api.XSetObject) ([]client.Object, error) { - // Use RefManager to adopt/orphan as needed. - writer := refmanagerutil.NewOwnerRefWriter(r.client) - matcher, err := refmanagerutil.LabelSelectorAsMatch(selector) - if err != nil { - return nil, fmt.Errorf("fail to create labelSelector matcher: %s", err.Error()) - } - refManager := refmanagerutil.NewObjectControllerRefManager(writer, xset, xset.GetObjectKind().GroupVersionKind(), matcher) - - var claimObjs []client.Object - var errList []error - for _, obj := range candidates { - ok, err := refManager.Claim(context.TODO(), obj) - if err != nil { - errList = append(errList, err) - continue - } - if ok { - claimObjs = append(claimObjs, obj) - } - } - - return claimObjs, errors.Join(errList...) -} - -func setUpCache(cache cache.Cache, controller api.XSetController) error { - if err := cache.IndexField(context.TODO(), controller.NewXObject(), FieldIndexOwnerRefUID, func(object client.Object) []string { - ownerRef := metav1.GetControllerOf(object) - if ownerRef == nil || ownerRef.Kind != controller.XSetMeta().Kind { - return nil - } - return []string{string(ownerRef.UID)} - }); err != nil { - return fmt.Errorf("failed to index by field for x->xset %s: %s", FieldIndexOwnerRefUID, err.Error()) - } - return nil -} - -func filterOutInactiveTargets(xsetController api.XSetController, targets []client.Object) []client.Object { - var filteredTarget []client.Object - for i := range targets { - target := targets[i] - if xsetController.CheckInactive(target) { - continue - } - filteredTarget = append(filteredTarget, target) - } - return filteredTarget -} diff --git a/xset/xset_controller.go b/xset/xset_controller.go deleted file mode 100644 index 20f72ec..0000000 --- a/xset/xset_controller.go +++ /dev/null @@ -1,368 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package xset - -import ( - "context" - "errors" - "fmt" - "time" - - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/clock" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - clientutil "kusionstack.io/kube-utils/client" - "kusionstack.io/kube-utils/controller/expectations" - "kusionstack.io/kube-utils/controller/history" - "kusionstack.io/kube-utils/controller/mixin" - "kusionstack.io/kube-utils/xset/api" - "kusionstack.io/kube-utils/xset/api/validation" - "kusionstack.io/kube-utils/xset/resourcecontexts" - "kusionstack.io/kube-utils/xset/revisionowner" - "kusionstack.io/kube-utils/xset/subresources" - "kusionstack.io/kube-utils/xset/synccontrols" - "kusionstack.io/kube-utils/xset/xcontrol" -) - -type xSetCommonReconciler struct { - mixin.ReconcilerMixin - - XSetController api.XSetController - meta metav1.TypeMeta - finalizerName string - xsetGVK schema.GroupVersionKind - - // reconcile logic helpers - cacheExpectations *expectations.CacheExpectations - targetControl xcontrol.TargetControl - pvcControl subresources.PvcControl - syncControl synccontrols.SyncControl - revisionManager history.HistoryManager - resourceContextControl resourcecontexts.ResourceContextControl -} - -func SetUpWithManager(mgr ctrl.Manager, xsetController api.XSetController) error { - if err := validation.ValidateXSetController(xsetController); err != nil { - return err - } - resourceContextAdapter := resourcecontexts.GetResourceContextAdapter(xsetController) - if err := validation.ValidateResourceContextAdapter(resourceContextAdapter); err != nil { - return err - } - - reconcilerMixin := mixin.NewReconcilerMixin(xsetController.ControllerName(), mgr) - xsetLabelManager := api.GetXSetLabelAnnotationManager(xsetController) - xsetMeta := xsetController.XSetMeta() - xsetGVK := xsetMeta.GroupVersionKind() - resourceContextMeta := resourceContextAdapter.ResourceContextMeta() - resourceContextGVK := resourceContextMeta.GroupVersionKind() - targetMeta := xsetController.XMeta() - - targetControl, err := xcontrol.NewTargetControl(reconcilerMixin, xsetController) - if err != nil { - return err - } - cacheExpectations := expectations.NewxCacheExpectations(reconcilerMixin.Client, reconcilerMixin.Scheme, clock.RealClock{}) - resourceContextControl := resourcecontexts.NewRealResourceContextControl(reconcilerMixin.Client, xsetController, resourceContextAdapter, resourceContextGVK, cacheExpectations) - pvcControl, err := subresources.NewRealPvcControl(reconcilerMixin, cacheExpectations, xsetLabelManager, xsetController) - if err != nil { - return errors.New("failed to create pvc control") - } - syncControl := synccontrols.NewRealSyncControl(reconcilerMixin, xsetController, targetControl, pvcControl, xsetLabelManager, resourceContextControl, cacheExpectations) - revisionControl := history.NewRevisionControl(reconcilerMixin.Client, reconcilerMixin.Client) - revisionOwner := revisionowner.NewRevisionOwner(xsetController, targetControl) - revisionManager := history.NewHistoryManager(revisionControl, revisionOwner) - - reconciler := &xSetCommonReconciler{ - targetControl: targetControl, - ReconcilerMixin: *reconcilerMixin, - XSetController: xsetController, - meta: xsetController.XSetMeta(), - finalizerName: xsetController.FinalizerName(), - pvcControl: pvcControl, - syncControl: syncControl, - revisionManager: revisionManager, - resourceContextControl: resourceContextControl, - cacheExpectations: cacheExpectations, - xsetGVK: xsetGVK, - } - - c, err := controller.New(xsetController.ControllerName(), mgr, controller.Options{ - MaxConcurrentReconciles: 5, - Reconciler: reconciler, - }) - if err != nil { - return fmt.Errorf("failed to create controller: %s", err.Error()) - } - - if err := c.Watch(&source.Kind{Type: xsetController.NewXSetObject()}, &handler.EnqueueRequestForObject{}); err != nil { - return fmt.Errorf("failed to watch %s: %s", xsetController.XSetMeta().Kind, err.Error()) - } - - if err := c.Watch(&source.Kind{Type: xsetController.NewXObject()}, &handler.EnqueueRequestForOwner{ - IsController: true, - OwnerType: xsetController.NewXSetObject(), - }, predicate.Funcs{ - CreateFunc: func(event event.CreateEvent) bool { - return synccontrols.IsControlledByXSet(xsetLabelManager, event.Object) - }, - UpdateFunc: func(updateEvent event.UpdateEvent) bool { - return synccontrols.IsControlledByXSet(xsetLabelManager, updateEvent.ObjectNew) || - synccontrols.IsControlledByXSet(xsetLabelManager, updateEvent.ObjectOld) - }, - DeleteFunc: func(deleteEvent event.DeleteEvent) bool { - return synccontrols.IsControlledByXSet(xsetLabelManager, deleteEvent.Object) - }, - GenericFunc: func(genericEvent event.GenericEvent) bool { - return synccontrols.IsControlledByXSet(xsetLabelManager, genericEvent.Object) - }, - }); err != nil { - return fmt.Errorf("failed to watch %s: %s", targetMeta.Kind, err.Error()) - } - - // watch for decoration changed - if adapter, ok := xsetController.(api.DecorationAdapter); ok { - err = adapter.WatchDecoration(c) - if err != nil { - return err - } - } - - return nil -} - -func (r *xSetCommonReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - kind := r.meta.Kind - key := req.String() - ctx = logr.NewContext(ctx, r.Logger.WithValues(kind, key)) - logger := logr.FromContext(ctx) - instance := r.XSetController.NewXSetObject() - if err := r.Client.Get(ctx, req.NamespacedName, instance); err != nil { - if !apierrors.IsNotFound(err) { - logger.Error(err, "failed to find object") - return reconcile.Result{}, err - } - - logger.Info("object deleted") - r.cacheExpectations.DeleteExpectations(req.String()) - return ctrl.Result{}, nil - } - - // if cacheExpectation not fulfilled, shortcut this reconciling till informer cache is updated. - if satisfied := r.cacheExpectations.SatisfiedExpectations(req.String()); !satisfied { - logger.Info("not satisfied to reconcile") - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil - } - - if instance.GetDeletionTimestamp() != nil { - if controllerutil.ContainsFinalizer(instance, r.finalizerName) { - // reclaim target sub resources before remove finalizers - if err := r.ensureReclaimTargetSubResources(ctx, instance); err != nil { - return ctrl.Result{}, err - } - // reclaim decoration ownerReferences before remove finalizers - if err := r.ensureReclaimOwnerReferences(ctx, instance); err != nil { - return ctrl.Result{}, err - } - if cleaned, err := r.ensureReclaimTargetsDeletion(ctx, instance); !cleaned || err != nil { - // reclaim targets deletion before remove finalizers - return ctrl.Result{}, err - } - // reclaim owner IDs in ResourceContextControl - if err := r.resourceContextControl.UpdateToTargetContext(ctx, instance, nil); err != nil { - return ctrl.Result{}, err - } - } - return ctrl.Result{}, clientutil.RemoveFinalizerAndUpdate(ctx, r.Client, instance, r.finalizerName) - } - - if !controllerutil.ContainsFinalizer(instance, r.finalizerName) { - return ctrl.Result{}, clientutil.AddFinalizerAndUpdate(ctx, r.Client, instance, r.finalizerName) - } - - currentRevision, updatedRevision, revisions, collisionCount, _, err := r.revisionManager.ConstructRevisions(ctx, instance) - if err != nil { - return ctrl.Result{}, fmt.Errorf("fail to construct revision for %s %s: %s", kind, key, err.Error()) - } - - xsetStatus := r.XSetController.GetXSetStatus(instance) - newStatus := xsetStatus.DeepCopy() - newStatus.UpdatedRevision = updatedRevision.Name - newStatus.CurrentRevision = currentRevision.Name - newStatus.CollisionCount = &collisionCount - syncContext := &synccontrols.SyncContext{ - Revisions: revisions, - CurrentRevision: currentRevision, - UpdatedRevision: updatedRevision, - NewStatus: newStatus, - } - - requeueAfter, syncErr := r.doSync(ctx, instance, syncContext) - if syncErr != nil { - logger.Error(syncErr, "failed to sync") - } - - newStatus = r.syncControl.CalculateStatus(ctx, instance, syncContext) - // update status anyway - if err := r.updateStatus(ctx, instance, newStatus); err != nil { - return requeueResult(requeueAfter), fmt.Errorf("fail to update status of %s %s: %s", kind, req, err.Error()) - } - return requeueResult(requeueAfter), syncErr -} - -func (r *xSetCommonReconciler) doSync(ctx context.Context, instance api.XSetObject, syncContext *synccontrols.SyncContext) (*time.Duration, error) { - synced, err := r.syncControl.SyncTargets(ctx, instance, syncContext) - if err != nil || synced { - return nil, err - } - - err = r.syncControl.Replace(ctx, instance, syncContext) - if err != nil { - return nil, err - } - - _, scaleRequeueAfter, scaleErr := r.syncControl.Scale(ctx, instance, syncContext) - _, updateRequeueAfter, updateErr := r.syncControl.Update(ctx, instance, syncContext) - patcherErr := synccontrols.ApplyTemplatePatcher(ctx, r.XSetController, r.Client, instance, syncContext.TargetWrappers) - - err = errors.Join(scaleErr, updateErr, patcherErr) - if updateRequeueAfter != nil && (scaleRequeueAfter == nil || *updateRequeueAfter < *scaleRequeueAfter) { - return updateRequeueAfter, err - } - return scaleRequeueAfter, err -} - -func (r *xSetCommonReconciler) ensureReclaimTargetSubResources(ctx context.Context, xset api.XSetObject) error { - if _, enabled := subresources.GetSubresourcePvcAdapter(r.XSetController); enabled { - err := r.ensureReclaimPvcs(ctx, xset) - if err != nil { - return err - } - } - return nil -} - -// ensureReclaimPvcs removes xset ownerReference from pvcs if RetainPvcWhenXSetDeleted. -// This allows pvcs to be retained for other xsets with same pvc template. -func (r *xSetCommonReconciler) ensureReclaimPvcs(ctx context.Context, xset api.XSetObject) error { - if !r.pvcControl.RetainPvcWhenXSetDeleted(xset) { - return nil - } - var needReclaimPvcs []*corev1.PersistentVolumeClaim - pvcs, err := r.pvcControl.GetFilteredPvcs(ctx, xset) - if err != nil { - return err - } - // reclaim pvcs if RetainPvcWhenXSetDeleted - for i := range pvcs { - owned := pvcs[i].OwnerReferences != nil && len(pvcs[i].OwnerReferences) > 0 - if owned { - needReclaimPvcs = append(needReclaimPvcs, pvcs[i]) - } - } - for i := range needReclaimPvcs { - if err := r.pvcControl.OrphanPvc(ctx, xset, needReclaimPvcs[i]); err != nil { - return err - } - } - return nil -} - -func (r *xSetCommonReconciler) ensureReclaimTargetsDeletion(ctx context.Context, instance api.XSetObject) (bool, error) { - xSetSpec := r.XSetController.GetXSetSpec(instance) - _, targets, err := r.targetControl.GetFilteredTargets(ctx, xSetSpec.Selector, instance) - if err != nil { - return false, fmt.Errorf("fail to get filtered Targets: %s", err.Error()) - } - // if targets are deleted, return true - if len(targets) == 0 { - return true, nil - } - // wait for all targets are terminating - for i := range targets { - target := targets[i] - if target.GetDeletionTimestamp() == nil { - r.Recorder.Eventf(instance, corev1.EventTypeNormal, "TargetsDeleted", "waiting for models to be deleted gracefully before xset deleted %s/%s", instance.GetNamespace(), instance.GetName()) - return false, r.syncControl.BatchDeleteTargetsByLabel(ctx, r.targetControl, targets) - } - } - return true, nil -} - -// ensureReclaimOwnerReferences removes decoration ownerReference from filteredPods if xset is deleting. -func (r *xSetCommonReconciler) ensureReclaimOwnerReferences(ctx context.Context, instance api.XSetObject) error { - decorationAdapter, ok := r.XSetController.(api.DecorationAdapter) - if !ok { - return nil - } - xSetSpec := r.XSetController.GetXSetSpec(instance) - _, filteredTargets, err := r.targetControl.GetFilteredTargets(ctx, xSetSpec.Selector, instance) - if err != nil { - return fmt.Errorf("fail to get filtered Targets: %s", err.Error()) - } - // reclaim decoration ownerReferences on filteredPods - gvk := decorationAdapter.GetDecorationGroupVersionKind() - for i := range filteredTargets { - if len(filteredTargets[i].GetOwnerReferences()) == 0 { - continue - } - var newOwnerRefs []metav1.OwnerReference - for j := range filteredTargets[i].GetOwnerReferences() { - if filteredTargets[i].GetOwnerReferences()[j].Kind == gvk.Kind { - continue - } - newOwnerRefs = append(newOwnerRefs, filteredTargets[i].GetOwnerReferences()[j]) - } - if len(newOwnerRefs) != len(filteredTargets[i].GetOwnerReferences()) { - filteredTargets[i].SetOwnerReferences(newOwnerRefs) - if err := r.targetControl.UpdateTarget(ctx, filteredTargets[i]); err != nil { - return err - } - } - } - return nil -} - -func (r *xSetCommonReconciler) updateStatus(ctx context.Context, instance api.XSetObject, status *api.XSetStatus) error { - r.XSetController.SetXSetStatus(instance, status) - if err := r.Client.Status().Update(ctx, instance); err != nil { - return fmt.Errorf("fail to update status of %s: %s", instance.GetName(), err.Error()) - } - return r.cacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(instance), r.xsetGVK, instance.GetNamespace(), instance.GetName(), instance.GetResourceVersion()) -} - -func requeueResult(requeueTime *time.Duration) reconcile.Result { - if requeueTime != nil { - if *requeueTime == 0 { - return reconcile.Result{Requeue: true} - } - return reconcile.Result{RequeueAfter: *requeueTime} - } - return reconcile.Result{} -}