diff --git a/build/components/versions.yml b/build/components/versions.yml index 438feffc2b..c35bd7f2e6 100644 --- a/build/components/versions.yml +++ b/build/components/versions.yml @@ -3,7 +3,7 @@ firmware: libvirt: v10.9.0 edk2: stable202411 core: - 3p-kubevirt: v1.6.2-v12n.20 + 3p-kubevirt: dvp/set-memory-limits-while-hotplugging 3p-containerized-data-importer: v1.60.3-v12n.17 distribution: 2.8.3 package: diff --git a/images/virt-artifact/werf.inc.yaml b/images/virt-artifact/werf.inc.yaml index f30560fba6..767eff694a 100644 --- a/images/virt-artifact/werf.inc.yaml +++ b/images/virt-artifact/werf.inc.yaml @@ -15,6 +15,7 @@ secrets: shell: install: - | + echo rebuild 33 echo "Git clone {{ $gitRepoName }} repository..." git clone --depth=1 $(cat /run/secrets/SOURCE_REPO)/{{ $gitRepoUrl }} --branch {{ $tag }} /src/kubevirt diff --git a/images/virtualization-artifact/pkg/common/annotations/annotations.go b/images/virtualization-artifact/pkg/common/annotations/annotations.go index c70c6542cb..00e93d6805 100644 --- a/images/virtualization-artifact/pkg/common/annotations/annotations.go +++ b/images/virtualization-artifact/pkg/common/annotations/annotations.go @@ -90,9 +90,10 @@ const ( AnnVMRestartRequested = AnnAPIGroupV + "/vm-restart-requested" // AnnVMOPWorkloadUpdate is an annotation on vmop that represents a vmop created by workload-updater controller. - AnnVMOPWorkloadUpdate = AnnAPIGroupV + "/workload-update" - AnnVMOPWorkloadUpdateImage = AnnAPIGroupV + "/workload-update-image" - AnnVMOPWorkloadUpdateNodePlacementSum = AnnAPIGroupV + "/workload-update-node-placement-sum" + AnnVMOPWorkloadUpdate = AnnAPIGroupV + "/workload-update" + AnnVMOPWorkloadUpdateImage = AnnAPIGroupV + "/workload-update-image" + AnnVMOPWorkloadUpdateNodePlacementSum = AnnAPIGroupV + "/workload-update-node-placement-sum" + AnnVMOPWorkloadUpdateHotplugResourcesSum = AnnAPIGroupV + "/workload-update-hotplug-resources-sum" // AnnVMRestore is an annotation on a resource that indicates it was created by the vmrestore controller; the value is the UID of the `VirtualMachineRestore` resource. AnnVMRestore = AnnAPIGroupV + "/vmrestore" // AnnVMOPEvacuation is an annotation on vmop that represents a vmop created by evacuation controller diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm.go b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm.go index 33ce9d7e02..2321386fc1 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm.go @@ -33,6 +33,7 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/common/array" "github.com/deckhouse/virtualization-controller/pkg/common/resource_builder" "github.com/deckhouse/virtualization-controller/pkg/common/vm" + "github.com/deckhouse/virtualization-controller/pkg/featuregates" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) @@ -46,6 +47,9 @@ const ( // GenericCPUModel specifies the base CPU model for Features and Discovery CPU model types. GenericCPUModel = "qemu64" + + MaxMemorySizeForHotplug = 256 * 1024 * 1024 * 1024 // 256 Gi (safely limit to not overlap somewhat conservative 38 bit physical address space) + EnableMemoryHotplugThreshold = 1 * 1024 * 1024 * 1024 // 1 Gi (no hotplug for VMs with less than 1Gi) ) type KVVMOptions struct { @@ -269,7 +273,25 @@ func (b *KVVM) SetCPU(cores int, coreFraction string) error { return nil } +// SetMemory sets memory in kvvm. +// There are 2 possibilities to set memory: +// 1. Use domain.memory.guest field: it enabled memory hotplugging, but not set resources.limits. +// 2. Explicitly set limits and requests in domain.resources. No hotplugging in this scenario. +// +// (1) is a new approach, and (2) should be respected for Running VMs started by previous version of the controller. func (b *KVVM) SetMemory(memorySize resource.Quantity) { + // Support for VMs started with memory size in requests-limits. + // TODO delete this in the future (around 3-4 more versions after enabling memory hotplug by default). + if b.ResourceExists && isVMRunningWithMemoryResources(b.Resource) { + b.setMemoryNonHotpluggable(memorySize) + return + } + b.setMemoryHotpluggable(memorySize) +} + +// setMemoryNonHotpluggable translates memory size to requests and limits in KVVM. +// Note: this is a first implementation, memory hotplug is not compatible with this strategy. +func (b *KVVM) setMemoryNonHotpluggable(memorySize resource.Quantity) { res := &b.Resource.Spec.Template.Spec.Domain.Resources if res.Requests == nil { res.Requests = make(map[corev1.ResourceName]resource.Quantity) @@ -281,6 +303,57 @@ func (b *KVVM) SetMemory(memorySize resource.Quantity) { res.Limits[corev1.ResourceMemory] = memorySize } +// setMemoryHotpluggable translates memory size to settings in domain.memory field. +// This field is compatible with memory hotplug. +// Also, remove requests-limits for memory if any. +func (b *KVVM) setMemoryHotpluggable(memorySize resource.Quantity) { + domain := &b.Resource.Spec.Template.Spec.Domain + + currentMaxGuest := int64(-1) + if domain.Memory != nil && domain.Memory.MaxGuest != nil { + currentMaxGuest = domain.Memory.MaxGuest.Value() + } + + domain.Memory = &virtv1.Memory{ + Guest: &memorySize, + } + + // Set maxMemory to enable hotplug for mem size >= 1Gi. + hotplugThreshold := resource.NewQuantity(EnableMemoryHotplugThreshold, resource.BinarySI) + if featuregates.Default().Enabled(featuregates.HotplugMemoryWithLiveMigration) { + if memorySize.Cmp(*hotplugThreshold) >= 0 { + maxMemory := resource.NewQuantity(MaxMemorySizeForHotplug, resource.BinarySI) + domain.Memory.MaxGuest = maxMemory + } + } + // Set maxGuest to 0 if hotplug is disabled now (mem size < 1Gi) and maxGuest was previously set. + // Zero value is just a flag to patch memory and remove maxGuest before updating kvvm. + if memorySize.Cmp(*hotplugThreshold) == -1 && currentMaxGuest > 0 { + domain.Memory.MaxGuest = resource.NewQuantity(0, resource.BinarySI) + } + + // Remove memory limits and requests if set by previous implementation. + res := &b.Resource.Spec.Template.Spec.Domain.Resources + delete(res.Requests, corev1.ResourceMemory) + delete(res.Limits, corev1.ResourceMemory) +} + +func isVMRunningWithMemoryResources(kvvm *virtv1.VirtualMachine) bool { + if kvvm == nil { + return false + } + + if kvvm.Status.PrintableStatus != virtv1.VirtualMachineStatusRunning { + return false + } + + res := kvvm.Spec.Template.Spec.Domain.Resources + _, hasMemoryRequests := res.Requests[corev1.ResourceMemory] + _, hasMemoryLimits := res.Limits[corev1.ResourceMemory] + + return hasMemoryRequests && hasMemoryLimits +} + func GetCPURequest(cores int, coreFraction string) (*resource.Quantity, error) { if coreFraction == "" { return GetCPULimit(cores), nil @@ -473,7 +546,7 @@ func (b *KVVM) SetProvisioning(p *v1alpha2.Provisioning) error { } } -func (b *KVVM) SetOsType(osType v1alpha2.OsType) error { +func (b *KVVM) SetOSType(osType v1alpha2.OsType) error { switch osType { case v1alpha2.Windows: // Need for `029-use-OFVM_CODE-for-linux.patch` diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_test.go b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_test.go index 8a14050251..237bb4b5b4 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_test.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_test.go @@ -119,25 +119,25 @@ func TestSetAffinity(t *testing.T) { } } -func TestSetOsType(t *testing.T) { +func TestSetOSType(t *testing.T) { name := "test-name" namespace := "test-namespace" t.Run("Change from Windows to Generic should remove TPM", func(t *testing.T) { builder := NewEmptyKVVM(types.NamespacedName{Name: name, Namespace: namespace}, KVVMOptions{}) - err := builder.SetOsType(v1alpha2.Windows) + err := builder.SetOSType(v1alpha2.Windows) if err != nil { - t.Fatalf("SetOsType(Windows) failed: %v", err) + t.Fatalf("SetOSType(Windows) failed: %v", err) } if builder.Resource.Spec.Template.Spec.Domain.Devices.TPM == nil { t.Error("TPM should be present after setting Windows OS") } - err = builder.SetOsType(v1alpha2.GenericOs) + err = builder.SetOSType(v1alpha2.GenericOs) if err != nil { - t.Fatalf("SetOsType(GenericOs) failed: %v", err) + t.Fatalf("SetOSType(GenericOs) failed: %v", err) } if builder.Resource.Spec.Template.Spec.Domain.Devices.TPM != nil { diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go index 8e09166b0d..d0cd26e305 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go @@ -99,7 +99,7 @@ func ApplyVirtualMachineSpec( if err := kvvm.SetRunPolicy(vm.Spec.RunPolicy); err != nil { return err } - if err := kvvm.SetOsType(vm.Spec.OsType); err != nil { + if err := kvvm.SetOSType(vm.Spec.OsType); err != nil { return err } if err := kvvm.SetBootloader(vm.Spec.Bootloader); err != nil { diff --git a/images/virtualization-artifact/pkg/controller/reconciler/reconciler.go b/images/virtualization-artifact/pkg/controller/reconciler/reconciler.go index d839d491b0..7de44a3339 100644 --- a/images/virtualization-artifact/pkg/controller/reconciler/reconciler.go +++ b/images/virtualization-artifact/pkg/controller/reconciler/reconciler.go @@ -19,6 +19,7 @@ package reconciler import ( "context" "errors" + "fmt" "reflect" "strings" "time" @@ -102,7 +103,8 @@ handlersLoop: switch { case err == nil: // OK. case errors.Is(err, ErrStopHandlerChain): - log.Debug("Handler chain execution stopped") + msg := fmt.Sprintf("Handler %s stopped chain execution", name) + log.Debug(msg) result = MergeResults(result, res) break handlersLoop case k8serrors.IsConflict(err): diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go index 530551a938..f80cf5cae7 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go @@ -20,12 +20,15 @@ import ( "context" "errors" "fmt" + "reflect" "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/component-base/featuregate" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -33,6 +36,7 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization-controller/pkg/common/network" "github.com/deckhouse/virtualization-controller/pkg/common/object" + "github.com/deckhouse/virtualization-controller/pkg/common/patch" vmutil "github.com/deckhouse/virtualization-controller/pkg/common/vm" "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" @@ -52,11 +56,18 @@ type syncVolumesService interface { SyncVolumes(ctx context.Context, s state.VirtualMachineState, restartRequired bool) (reconcile.Result, error) } -func NewSyncKvvmHandler(dvcrSettings *dvcr.Settings, client client.Client, recorder eventrecord.EventRecorderLogger, syncVolumesService syncVolumesService) *SyncKvvmHandler { +func NewSyncKvvmHandler( + dvcrSettings *dvcr.Settings, + client client.Client, + recorder eventrecord.EventRecorderLogger, + featureGate featuregate.FeatureGate, + syncVolumesService syncVolumesService, +) *SyncKvvmHandler { return &SyncKvvmHandler{ dvcrSettings: dvcrSettings, client: client, recorder: recorder, + featureGate: featureGate, syncVolumesService: syncVolumesService, } } @@ -65,6 +76,7 @@ type SyncKvvmHandler struct { client client.Client recorder eventrecord.EventRecorderLogger dvcrSettings *dvcr.Settings + featureGate featuregate.FeatureGate syncVolumesService syncVolumesService } @@ -298,24 +310,14 @@ func (h *SyncKvvmHandler) syncKVVM(ctx context.Context, s state.VirtualMachineSt } return true, nil case h.isVMStopped(s.VirtualMachine().Current(), kvvm, pod): - // KVVM must be updated when the VM is stopped because all its components, - // like VirtualDisk and other resources, - // can be changed during the restoration process. + // KVVM should be updated when VM become stopped. + // It is safe to update KVVM at this point in general and also all related resources + // can be changed during the restoration process: e.g. VirtualDisks, VMIPs, etc. // For example, the PVC of the VirtualDisk will be changed, // and the volume with this PVC must be updated in the KVVM specification. - hasVMChanges, err := h.detectVMSpecChanges(ctx, s) - if err != nil { - return false, fmt.Errorf("detect changes on the stopped internal virtual machine: %w", err) - } - hasVMClassChanges, err := h.detectVMClassSpecChanges(ctx, s) + err := h.updateKVVM(ctx, s) if err != nil { - return false, fmt.Errorf("detect changes on the stopped internal virtual machine: %w", err) - } - if hasVMChanges || hasVMClassChanges { - err := h.updateKVVM(ctx, s) - if err != nil { - return false, fmt.Errorf("update stopped internal virtual machine: %w", err) - } + return false, fmt.Errorf("update internal virtual machine in 'Stopped' state: %w", err) } return true, nil case h.hasNoneDisruptiveChanges(s.VirtualMachine().Current(), kvvm, kvvmi, allChanges): @@ -370,17 +372,48 @@ func (h *SyncKvvmHandler) updateKVVM(ctx context.Context, s state.VirtualMachine return fmt.Errorf("the virtual machine is empty, please report a bug") } - kvvm, err := MakeKVVMFromVMSpec(ctx, s) + newKVVM, err := MakeKVVMFromVMSpec(ctx, s) if err != nil { - return fmt.Errorf("failed to prepare the internal virtual machine: %w", err) + return fmt.Errorf("update internal virtual machine: make kvvm from the virtual machine spec: %w", err) } - if err = h.client.Update(ctx, kvvm); err != nil { - return fmt.Errorf("failed to create the internal virtual machine: %w", err) - } + // Check for changes to skip unneeded updated. + isChanged, err := IsKVVMChanged(ctx, s, newKVVM) + if err != nil { + return fmt.Errorf("update internal virtual machine: detect changes: %w", err) + } + + if isChanged { + memory := newKVVM.Spec.Template.Spec.Domain.Memory + if memory != nil && memory.MaxGuest != nil && memory.MaxGuest.IsZero() { + // Zero maxGuest is a special value to patch KVVM to unset maxGuest. + // Set it to nil for next update call. + memory.MaxGuest = nil + + // 2 operations: remove memory.maxGuest; set memory.guest. + // Remove is not enough, remove and set are needed both to pass the kubevirt vm-validator webhook. + patchBytes, err := patch.NewJSONPatch( + patch.WithRemove("/spec/template/spec/domain/memory/maxGuest"), + patch.WithReplace("/spec/template/spec/domain/memory/guest", memory.Guest.String()), + ).Bytes() + if err != nil { + return fmt.Errorf("prepare json patch to unset memory.maxGuest: %w", err) + } + + if err = h.client.Patch(ctx, newKVVM, client.RawPatch(types.JSONPatchType, patchBytes)); err != nil { + return fmt.Errorf("patch internal virtual machine to unset memory.maxGuest: %w", err) + } + } - log.Info("Update KubeVirt VM done", "name", kvvm.Name) - log.Debug("Update KubeVirt VM done", "name", kvvm.Name, "kvvm", kvvm) + if err = h.client.Update(ctx, newKVVM); err != nil { + return fmt.Errorf("update internal virtual machine: %w", err) + } + + log.Info("Update internal virtual machine done", "name", newKVVM.Name) + log.Debug("Update internal virtual machine done", "name", newKVVM.Name, "kvvm", newKVVM) + } else { + log.Debug("Update internal virtual machine is not needed", "name", newKVVM.Name, "kvvm", newKVVM) + } return nil } @@ -407,7 +440,7 @@ func MakeKVVMFromVMSpec(ctx context.Context, s state.VirtualMachineState) (*virt bdState := NewBlockDeviceState(s) err = bdState.Reload(ctx) if err != nil { - return nil, fmt.Errorf("failed to relaod blockdevice state for the virtual machine: %w", err) + return nil, fmt.Errorf("failed to reload blockdevice state for the virtual machine: %w", err) } class, err := s.Class(ctx) if err != nil { @@ -454,6 +487,25 @@ func MakeKVVMFromVMSpec(ctx context.Context, s state.VirtualMachineState) (*virt return newKVVM, nil } +// IsKVVMChanged returns whether kvvm spec or special annotations are changed. +func IsKVVMChanged(ctx context.Context, s state.VirtualMachineState, kvvm *virtv1.VirtualMachine) (bool, error) { + currentKVVM, err := s.KVVM(ctx) + if err != nil { + return false, fmt.Errorf("get current kvvm: %w", err) + } + + isChanged := currentKVVM.Annotations[annotations.AnnVMLastAppliedSpec] != kvvm.Annotations[annotations.AnnVMLastAppliedSpec] + + if !isChanged { + isChanged = currentKVVM.Annotations[annotations.AnnVMClassLastAppliedSpec] != kvvm.Annotations[annotations.AnnVMClassLastAppliedSpec] + } + + if !isChanged { + isChanged = !reflect.DeepEqual(kvvm.Spec, currentKVVM.Spec) + } + return isChanged, nil +} + func (h *SyncKvvmHandler) loadLastAppliedSpec(vm *v1alpha2.VirtualMachine, kvvm *virtv1.VirtualMachine) *v1alpha2.VirtualMachineSpec { if kvvm == nil || vm == nil { return nil @@ -526,7 +578,7 @@ func (h *SyncKvvmHandler) detectSpecChanges( // Compare VM spec applied to the underlying KVVM // with the current VM spec (maybe edited by the user). - specChanges := vmchange.CompareVMSpecs(lastSpec, currentSpec) + specChanges := vmchange.NewVMSpecComparator(h.featureGate).Compare(lastSpec, currentSpec) log.Info(fmt.Sprintf("detected VM changes: empty %v, disruptive %v, actionType %v", specChanges.IsEmpty(), specChanges.IsDisruptive(), specChanges.ActionType())) log.Info(fmt.Sprintf("detected VM changes JSON: %s", specChanges.ToJSON())) @@ -564,36 +616,6 @@ func (h *SyncKvvmHandler) isVMStopped( return isVMStopped(kvvm) && (!isKVVMICreated(kvvm) || podStopped) } -// detectVMSpecChanges returns true and no error if specification has changes. -func (h *SyncKvvmHandler) detectVMSpecChanges(ctx context.Context, s state.VirtualMachineState) (bool, error) { - currentKvvm, err := s.KVVM(ctx) - if err != nil { - return false, err - } - - newKvvm, err := MakeKVVMFromVMSpec(ctx, s) - if err != nil { - return false, err - } - - return currentKvvm.Annotations[annotations.AnnVMLastAppliedSpec] != newKvvm.Annotations[annotations.AnnVMLastAppliedSpec], nil -} - -// detectVMClassSpecChanges returns true and no error if specification has changes. -func (h *SyncKvvmHandler) detectVMClassSpecChanges(ctx context.Context, s state.VirtualMachineState) (bool, error) { - currentKvvm, err := s.KVVM(ctx) - if err != nil { - return false, err - } - - newKvvm, err := MakeKVVMFromVMSpec(ctx, s) - if err != nil { - return false, err - } - - return currentKvvm.Annotations[annotations.AnnVMClassLastAppliedSpec] != newKvvm.Annotations[annotations.AnnVMClassLastAppliedSpec], nil -} - // canApplyChanges returns true if changes can be applied right now. // // Wait if changes are disruptive, and approval mode is manual, and VM is still running. diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go index 1ed545dfdc..f94934c295 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go @@ -35,6 +35,7 @@ import ( vmservice "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/service" "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" "github.com/deckhouse/virtualization-controller/pkg/eventrecord" + "github.com/deckhouse/virtualization-controller/pkg/featuregates" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" ) @@ -136,7 +137,7 @@ var _ = Describe("SyncKvvmHandler", func() { } reconcile := func() { - h := NewSyncKvvmHandler(nil, fakeClient, recorder, vmservice.NewMigrationVolumesService(fakeClient, MakeKVVMFromVMSpec, 10*time.Second)) + h := NewSyncKvvmHandler(nil, fakeClient, recorder, featuregates.Default(), vmservice.NewMigrationVolumesService(fakeClient, MakeKVVMFromVMSpec, 10*time.Second)) _, err := h.Handle(ctx, vmState) Expect(err).NotTo(HaveOccurred()) err = resource.Update(context.Background()) diff --git a/images/virtualization-artifact/pkg/controller/vm/vm_controller.go b/images/virtualization-artifact/pkg/controller/vm/vm_controller.go index 6a4e231a88..21b3ba6100 100644 --- a/images/virtualization-artifact/pkg/controller/vm/vm_controller.go +++ b/images/virtualization-artifact/pkg/controller/vm/vm_controller.go @@ -76,7 +76,7 @@ func SetupController( internal.NewPodHandler(client), internal.NewSizePolicyHandler(), internal.NewNetworkInterfaceHandler(featuregates.Default()), - internal.NewSyncKvvmHandler(dvcrSettings, client, recorder, migrateVolumesService), + internal.NewSyncKvvmHandler(dvcrSettings, client, recorder, featuregates.Default(), migrateVolumesService), internal.NewSyncPowerStateHandler(client, recorder), internal.NewSyncMetadataHandler(client), internal.NewLifeCycleHandler(client, recorder), diff --git a/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go b/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go new file mode 100644 index 0000000000..c863cf1b46 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go @@ -0,0 +1,65 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmchange + +import ( + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/component-base/featuregate" + + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" + "github.com/deckhouse/virtualization-controller/pkg/featuregates" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +type comparatorMemory struct { + featureGate featuregate.FeatureGate +} + +func NewComparatorMemory(featureGate featuregate.FeatureGate) VMSpecFieldComparator { + return &comparatorMemory{ + featureGate: featureGate, + } +} + +// Compare detects changes in memory size. +// It is aware of hotplug mechanism. If hotplug is disabled it requires +// restart if memory.size is changed. If hotplug is enabled, it allows +// changing "on the fly". Also, it requires restart if hotplug boundary +// is crossed. +// Note: memory hotplug is enabled if VM has more than 1Gi of RAM. +func (c *comparatorMemory) Compare(current, desired *v1alpha2.VirtualMachineSpec) []FieldChange { + hotplugThreshold := resource.NewQuantity(kvbuilder.EnableMemoryHotplugThreshold, resource.BinarySI) + isHotpluggable := current.Memory.Size.Cmp(*hotplugThreshold) > 0 + isHotpluggableDesired := desired.Memory.Size.Cmp(*hotplugThreshold) > 0 + + actionType := ActionRestart + if isHotpluggable && isHotpluggableDesired { + actionType = ActionApplyImmediate + } + + // Restart required to decrease memory size. (current > desired) + if current.Memory.Size.Cmp(desired.Memory.Size) == 1 { + actionType = ActionRestart + } + + // Require reboot if memory hotplug is not enabled. + if !c.featureGate.Enabled(featuregates.HotplugMemoryWithLiveMigration) { + actionType = ActionRestart + } + + return compareQuantity("memory.size", current.Memory.Size, desired.Memory.Size, resource.Quantity{}, actionType) +} diff --git a/images/virtualization-artifact/pkg/controller/vmchange/comparators.go b/images/virtualization-artifact/pkg/controller/vmchange/comparators.go index 8c44383fe9..529d6bed7e 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/comparators.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/comparators.go @@ -17,8 +17,6 @@ limitations under the License. package vmchange import ( - "k8s.io/apimachinery/pkg/api/resource" - "github.com/deckhouse/virtualization/api/core/v1alpha2" ) @@ -153,11 +151,6 @@ func compareCPU(current, desired *v1alpha2.VirtualMachineSpec) []FieldChange { return nil } -// compareMemory returns changes in the memory section. -func compareMemory(current, desired *v1alpha2.VirtualMachineSpec) []FieldChange { - return compareQuantity("memory.size", current.Memory.Size, desired.Memory.Size, resource.Quantity{}, ActionRestart) -} - func compareProvisioning(current, desired *v1alpha2.VirtualMachineSpec) []FieldChange { changes := compareEmpty( "provisioning", diff --git a/images/virtualization-artifact/pkg/controller/vmchange/compare.go b/images/virtualization-artifact/pkg/controller/vmchange/compare.go index 54350e7d96..aa1f44259b 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/compare.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/compare.go @@ -17,31 +17,30 @@ limitations under the License. package vmchange import ( + "k8s.io/component-base/featuregate" + "github.com/deckhouse/virtualization/api/core/v1alpha2" ) type SpecFieldsComparator func(prev, next *v1alpha2.VirtualMachineSpec) []FieldChange -var specComparators = []SpecFieldsComparator{ - compareVirtualMachineClass, - compareRunPolicy, - compareVirtualMachineIPAddress, - compareTopologySpreadConstraints, - compareAffinity, - compareNodeSelector, - comparePriorityClassName, - compareTolerations, - compareDisruptions, - compareTerminationGracePeriodSeconds, - compareEnableParavirtualization, - compareOSType, - compareBootloader, - compareCPU, - compareMemory, - compareBlockDevices, - compareProvisioning, - compareNetworks, - compareUSBDevices, +type VMSpecFieldComparator interface { + Compare(prev, next *v1alpha2.VirtualMachineSpec) []FieldChange +} + +type vmSpecFieldsComparatorWithFn struct { + fn func(prev, next *v1alpha2.VirtualMachineSpec) []FieldChange +} + +func (v *vmSpecFieldsComparatorWithFn) Compare(prev, next *v1alpha2.VirtualMachineSpec) []FieldChange { + if v.fn == nil { + return nil + } + return v.fn(prev, next) +} + +func vmSpecFieldComparator(fn SpecFieldsComparator) VMSpecFieldComparator { + return &vmSpecFieldsComparatorWithFn{fn: fn} } type VMClassSpecFieldsComparator func(prev, next *v1alpha2.VirtualMachineClassSpec) []FieldChange @@ -51,18 +50,45 @@ var vmclassSpecComparators = []VMClassSpecFieldsComparator{ compareVMClassTolerations, } -func CompareSpecs(prev, next *v1alpha2.VirtualMachineSpec, prevClass, nextClass *v1alpha2.VirtualMachineClassSpec) SpecChanges { - specChanges := CompareVMSpecs(prev, next) - specClassChanges := CompareClassSpecs(prevClass, nextClass) - specChanges.Add(specClassChanges.GetAll()...) - return specChanges +type VMSpecComparator struct { + featureGate featuregate.FeatureGate +} + +func NewVMSpecComparator(featureGate featuregate.FeatureGate) *VMSpecComparator { + return &VMSpecComparator{ + featureGate: featureGate, + } +} + +func (v *VMSpecComparator) comparators() []VMSpecFieldComparator { + return []VMSpecFieldComparator{ + vmSpecFieldComparator(compareVirtualMachineClass), + vmSpecFieldComparator(compareRunPolicy), + vmSpecFieldComparator(compareVirtualMachineIPAddress), + vmSpecFieldComparator(compareTopologySpreadConstraints), + vmSpecFieldComparator(compareAffinity), + vmSpecFieldComparator(compareNodeSelector), + vmSpecFieldComparator(comparePriorityClassName), + vmSpecFieldComparator(compareTolerations), + vmSpecFieldComparator(compareDisruptions), + vmSpecFieldComparator(compareTerminationGracePeriodSeconds), + vmSpecFieldComparator(compareEnableParavirtualization), + vmSpecFieldComparator(compareOSType), + vmSpecFieldComparator(compareBootloader), + vmSpecFieldComparator(compareCPU), + NewComparatorMemory(v.featureGate), + vmSpecFieldComparator(compareBlockDevices), + vmSpecFieldComparator(compareProvisioning), + vmSpecFieldComparator(compareNetworks), + vmSpecFieldComparator(compareUSBDevices), + } } -func CompareVMSpecs(prev, next *v1alpha2.VirtualMachineSpec) SpecChanges { +func (v *VMSpecComparator) Compare(prev, next *v1alpha2.VirtualMachineSpec) SpecChanges { specChanges := SpecChanges{} - for _, comparator := range specComparators { - changes := comparator(prev, next) + for _, comparator := range v.comparators() { + changes := comparator.Compare(prev, next) if HasChanges(changes) { specChanges.Add(changes...) } diff --git a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go index 2ab26e66a8..e428b4fa0c 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go @@ -21,8 +21,10 @@ import ( "testing" "github.com/stretchr/testify/require" + "k8s.io/component-base/featuregate" "sigs.k8s.io/yaml" + "github.com/deckhouse/virtualization-controller/pkg/featuregates" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) @@ -31,6 +33,7 @@ func TestActionRequiredOnCompare(t *testing.T) { title string currentSpec string desiredSpec string + features []featuregate.Feature assertFn func(t *testing.T, changes SpecChanges) }{ { @@ -43,6 +46,7 @@ cpu: cpu: cores: 3 `, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("cpu.cores", ChangeReplace), @@ -60,6 +64,7 @@ cpu: cores: 2 coreFraction: 40% `, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("cpu.coreFraction", ChangeReplace), @@ -77,6 +82,7 @@ cpu: cores: 6 coreFraction: 40% `, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("cpu", ChangeReplace), @@ -93,6 +99,7 @@ cpu: cores: 2 coreFraction: 100% `, + nil, assertChanges( actionRequired(ActionNone), requirePathOperation("cpu.coreFraction", ChangeAdd), @@ -109,21 +116,119 @@ cpu: cpu: cores: 2 `, + nil, assertChanges( actionRequired(ActionNone), requirePathOperation("cpu.coreFraction", ChangeRemove), ), }, { - "restart on memory.size change", + "restart on memory.size change: no hotplug", + ` +memory: + size: 256Mi +`, + ` +memory: + size: 512Mi +`, + nil, + assertChanges( + actionRequired(ActionRestart), + requirePathOperation("memory.size", ChangeReplace), + ), + }, + { + "restart on memory.size change: enable hotplug", + ` +memory: + size: 384Mi +`, ` memory: size: 2Gi +`, + nil, + assertChanges( + actionRequired(ActionRestart), + requirePathOperation("memory.size", ChangeReplace), + ), + }, + { + "restart on memory.size change: disable hotplug", + ` +memory: + size: 3Gi `, ` memory: - size: 1Gi + size: 128Mi `, + nil, + assertChanges( + actionRequired(ActionRestart), + requirePathOperation("memory.size", ChangeReplace), + ), + }, + { + "immediate apply on memory.size increase when hotplug is enabled", + ` +memory: + size: 2Gi +`, + ` +memory: + size: 4Gi +`, + []featuregate.Feature{featuregates.HotplugMemoryWithLiveMigration}, + assertChanges( + actionRequired(ActionApplyImmediate), + requirePathOperation("memory.size", ChangeReplace), + ), + }, + { + "restart on memory.size increase when hotplug is disabled", + ` +memory: + size: 2Gi +`, + ` +memory: + size: 4Gi +`, + nil, + assertChanges( + actionRequired(ActionRestart), + requirePathOperation("memory.size", ChangeReplace), + ), + }, + { + "restart on memory.size reduce when hotplug is enabled", + ` +memory: + size: 4Gi +`, + ` +memory: + size: 2Gi +`, + []featuregate.Feature{featuregates.HotplugMemoryWithLiveMigration}, + assertChanges( + actionRequired(ActionRestart), + requirePathOperation("memory.size", ChangeReplace), + ), + }, + { + "restart on memory.size reduce when hotplug is disabled", + ` +memory: + size: 4Gi +`, + ` +memory: + size: 2Gi +`, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("memory.size", ChangeReplace), @@ -137,6 +242,7 @@ blockDeviceRefs: - kind: VirtualImage name: linux `, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("blockDeviceRefs", ChangeAdd), @@ -150,6 +256,7 @@ blockDeviceRefs: name: linux `, ``, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("blockDeviceRefs", ChangeRemove), @@ -169,6 +276,7 @@ blockDeviceRefs: - kind: VirtualImage name: linux `, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("blockDeviceRefs.0", ChangeAdd), @@ -188,6 +296,7 @@ blockDeviceRefs: - kind: VirtualImage name: linux `, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("blockDeviceRefs.0", ChangeRemove), @@ -209,6 +318,7 @@ blockDeviceRefs: - kind: VirtualImage name: linux `, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("blockDeviceRefs.0", ChangeReplace), @@ -244,6 +354,7 @@ blockDeviceRefs: - kind: VirtualImage name: linux `, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("blockDeviceRefs.0", ChangeReplace), @@ -261,6 +372,7 @@ provisioning: userData: | #cloudinit `, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("provisioning", ChangeAdd), @@ -276,6 +388,7 @@ provisioning: name: cloud-init-secret `, "", + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("provisioning", ChangeRemove), @@ -296,6 +409,7 @@ provisioning: userData: | #cloudinit `, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("provisioning", ChangeReplace), @@ -317,6 +431,7 @@ provisioning: kind: Secret name: provisioning-secret `, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("provisioning.userDataRef.name", ChangeReplace), @@ -330,6 +445,7 @@ enableParavirtualization: true ` enableParavirtualization: false `, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("enableParavirtualization", ChangeReplace), @@ -343,6 +459,7 @@ enableParavirtualization: false ` enableParavirtualization: true `, + nil, assertChanges( actionRequired(ActionRestart), requirePathOperation("enableParavirtualization", ChangeReplace), @@ -364,6 +481,7 @@ networks: name: net1 id: 2 `, + nil, assertChanges( actionRequired(ActionNone), requirePathOperation("networks", ChangeReplace), @@ -377,7 +495,12 @@ networks: currentSpec := loadVMSpec(t, tt.currentSpec) desiredSpec := loadVMSpec(t, tt.desiredSpec) - changes = CompareVMSpecs(currentSpec, desiredSpec) + gate := featuregates.Default() + if len(tt.features) > 0 { + gate = newFeatureGate(t, tt.features...) + } + + changes = NewVMSpecComparator(gate).Compare(currentSpec, desiredSpec) defer func() { if t.Failed() { @@ -448,3 +571,23 @@ func changesToYAML(changes SpecChanges) string { res, _ := yaml.Marshal(status) return string(res) } + +func newFeatureGate(t *testing.T, enabled ...featuregate.Feature) featuregate.FeatureGate { + t.Helper() + + gate, setFromMap, err := featuregates.NewUnlocked() + if err != nil { + t.Fatalf("failed to create feature gate: %v", err) + } + + featureMap := map[string]bool{} + for _, feature := range enabled { + featureMap[string(feature)] = true + } + + if err = setFromMap(featureMap); err != nil { + t.Fatalf("failed to set USB feature gate: %v", err) + } + + return gate +} diff --git a/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go b/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go new file mode 100644 index 0000000000..c2bbdeb6f2 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go @@ -0,0 +1,80 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + virtv1 "kubevirt.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "github.com/deckhouse/virtualization-controller/pkg/common/object" + "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" + "github.com/deckhouse/virtualization-controller/pkg/logger" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +const hotplugHandler = "HotplugHandler" + +func NewHotplugHandler(client client.Client, migration OneShotMigration) *HotplugHandler { + return &HotplugHandler{ + client: client, + oneShotMigration: migration, + } +} + +type HotplugHandler struct { + client client.Client + oneShotMigration OneShotMigration +} + +func (h *HotplugHandler) Handle(ctx context.Context, vm *v1alpha2.VirtualMachine) (reconcile.Result, error) { + if vm == nil || !vm.GetDeletionTimestamp().IsZero() { + return reconcile.Result{}, nil + } + + kvvmi := &virtv1.VirtualMachineInstance{} + if err := h.client.Get(ctx, object.NamespacedName(vm), kvvmi); err != nil { + return reconcile.Result{}, client.IgnoreNotFound(err) + } + + cond, _ := conditions.GetKVVMICondition(virtv1.VirtualMachineInstanceMemoryChange, kvvmi.Status.Conditions) + if cond.Status != corev1.ConditionTrue { + return reconcile.Result{}, nil + } + + log := logger.FromContext(ctx).With(logger.SlogHandler(hotplugHandler)) + ctx = logger.ToContext(ctx, log) + + migrate, err := h.oneShotMigration.OnceMigrate(ctx, vm, annotations.AnnVMOPWorkloadUpdateHotplugResourcesSum, getHotplugResourcesSum(vm)) + if migrate { + log.Info("The virtual machine was triggered to migrate by the hotplug resources handler.") + } + + return reconcile.Result{}, err +} + +func (h *HotplugHandler) Name() string { + return hotplugHandler +} + +func getHotplugResourcesSum(vm *v1alpha2.VirtualMachine) string { + return vm.Spec.Memory.Size.String() +} diff --git a/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug_test.go b/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug_test.go new file mode 100644 index 0000000000..71606517c8 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug_test.go @@ -0,0 +1,119 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "context" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + virtv1 "kubevirt.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" + "github.com/deckhouse/virtualization-controller/pkg/common/testutil" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +var _ = Describe("TestHotplugResourcesHandler", func() { + const ( + name = "vm-hotplug-resources" + namespace = "default" + ) + + var ( + serviceCompleteErr = errors.New("service is complete") + ctx = testutil.ContextBackgroundWithNoOpLogger() + fakeClient client.Client + ) + + AfterEach(func() { + fakeClient = nil + }) + + newVMAndKVVMI := func(hasHotMemoryChange bool) (*v1alpha2.VirtualMachine, *virtv1.VirtualMachineInstance) { + vm := vmbuilder.NewEmpty(name, namespace) + kvvmi := newEmptyKVVMI(name, namespace) + + if hasHotMemoryChange { + kvvmi.Status.Conditions = append(kvvmi.Status.Conditions, virtv1.VirtualMachineInstanceCondition{ + Type: virtv1.VirtualMachineInstanceMemoryChange, + Status: corev1.ConditionTrue, + }) + } + return vm, kvvmi + } + + newOnceMigrationMock := func(shouldMigrate bool) *OneShotMigrationMock { + return &OneShotMigrationMock{ + OnceMigrateFunc: func(ctx context.Context, vm *v1alpha2.VirtualMachine, annotationKey, annotationExpectedValue string) (bool, error) { + if shouldMigrate { + return true, serviceCompleteErr + } + return false, nil + }, + } + } + + type testResourcesSettings struct { + hasHotMemoryChangeCondition bool + shouldMigrate bool + } + + DescribeTable("HotplugResourcesHandler should return serviceCompleteErr if migration executed", + func(settings testResourcesSettings) { + vm, kvvmi := newVMAndKVVMI(settings.hasHotMemoryChangeCondition) + fakeClient = setupEnvironment(vm, kvvmi) + + mockMigration := newOnceMigrationMock(settings.shouldMigrate) + + h := NewHotplugHandler(fakeClient, mockMigration) + _, err := h.Handle(ctx, vm) + + if settings.hasHotMemoryChangeCondition && !settings.shouldMigrate { + Expect(err).ToNot(HaveOccurred()) + } else if settings.hasHotMemoryChangeCondition { + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(serviceCompleteErr)) + } else { + Expect(err).NotTo(HaveOccurred()) + } + }, + Entry( + "Migration should be executed on hotMemoryChange condition", + testResourcesSettings{ + hasHotMemoryChangeCondition: true, + shouldMigrate: true, + }, + ), + Entry( + "Migration should not be executed the second time", + testResourcesSettings{ + hasHotMemoryChangeCondition: true, + shouldMigrate: false, + }, + ), + Entry( + "Migration should not be executed without hotMemoryChange condition", + testResourcesSettings{ + hasHotMemoryChangeCondition: false, + }, + ), + ) +}) diff --git a/images/virtualization-artifact/pkg/controller/workload-updater/internal/watcher/kvvmi.go b/images/virtualization-artifact/pkg/controller/workload-updater/internal/watcher/kvvmi.go index a473c88654..59eec3b3cd 100644 --- a/images/virtualization-artifact/pkg/controller/workload-updater/internal/watcher/kvvmi.go +++ b/images/virtualization-artifact/pkg/controller/workload-updater/internal/watcher/kvvmi.go @@ -54,8 +54,9 @@ func (w *KVVMIWatcher) Watch(mgr manager.Manager, ctr controller.Controller) err CreateFunc: func(e event.TypedCreateEvent[*virtv1.VirtualMachineInstance]) bool { return false }, DeleteFunc: func(e event.TypedDeleteEvent[*virtv1.VirtualMachineInstance]) bool { return false }, UpdateFunc: func(e event.TypedUpdateEvent[*virtv1.VirtualMachineInstance]) bool { - cond, _ := conditions.GetKVVMICondition(conditions.VirtualMachineInstanceNodePlacementNotMatched, e.ObjectNew.Status.Conditions) - return cond.Status == corev1.ConditionTrue + nodePlacementCondition, _ := conditions.GetKVVMICondition(conditions.VirtualMachineInstanceNodePlacementNotMatched, e.ObjectNew.Status.Conditions) + hotMemoryChangeCondition, _ := conditions.GetKVVMICondition(virtv1.VirtualMachineInstanceMemoryChange, e.ObjectNew.Status.Conditions) + return nodePlacementCondition.Status == corev1.ConditionTrue || hotMemoryChangeCondition.Status == corev1.ConditionTrue }, }, ), diff --git a/images/virtualization-artifact/pkg/controller/workload-updater/workload_updater_controller.go b/images/virtualization-artifact/pkg/controller/workload-updater/workload_updater_controller.go index fc8b474683..8240277b39 100644 --- a/images/virtualization-artifact/pkg/controller/workload-updater/workload_updater_controller.go +++ b/images/virtualization-artifact/pkg/controller/workload-updater/workload_updater_controller.go @@ -27,6 +27,7 @@ import ( "github.com/deckhouse/deckhouse/pkg/log" "github.com/deckhouse/virtualization-controller/pkg/controller/workload-updater/internal/handler" "github.com/deckhouse/virtualization-controller/pkg/controller/workload-updater/internal/service" + "github.com/deckhouse/virtualization-controller/pkg/featuregates" "github.com/deckhouse/virtualization-controller/pkg/logger" ) @@ -48,6 +49,10 @@ func SetupController( handler.NewFirmwareHandler(client, service.NewOneShotMigrationService(client, "firmware-update-"), firmwareImage, namespace, virtControllerName), handler.NewNodePlacementHandler(client, service.NewOneShotMigrationService(client, "nodeplacement-update-")), } + if featuregates.Default().Enabled(featuregates.HotplugMemoryWithLiveMigration) { + hotplugHandler := handler.NewHotplugHandler(client, service.NewOneShotMigrationService(client, "hotplug-resources-")) + handlers = append(handlers, hotplugHandler) + } r := NewReconciler(client, handlers) c, err := controller.New(ControllerName, mgr, controller.Options{ diff --git a/images/virtualization-artifact/pkg/featuregates/featuregate.go b/images/virtualization-artifact/pkg/featuregates/featuregate.go index 38e06075fe..8d567ae151 100644 --- a/images/virtualization-artifact/pkg/featuregates/featuregate.go +++ b/images/virtualization-artifact/pkg/featuregates/featuregate.go @@ -30,6 +30,7 @@ const ( VolumeMigration featuregate.Feature = "VolumeMigration" TargetMigration featuregate.Feature = "TargetMigration" USB featuregate.Feature = "USB" + HotplugMemoryWithLiveMigration featuregate.Feature = "HotplugMemoryWithLiveMigration" ) var featureSpecs = map[featuregate.Feature]featuregate.FeatureSpec{ @@ -57,6 +58,11 @@ var featureSpecs = map[featuregate.Feature]featuregate.FeatureSpec{ LockToDefault: true, PreRelease: featuregate.Alpha, }, + HotplugMemoryWithLiveMigration: { + Default: false, + LockToDefault: version.GetEdition() == version.EditionCE, + PreRelease: featuregate.Alpha, + }, } var ( diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index b34e7ec2c2..e46d2362fc 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -277,3 +277,15 @@ properties: enum: - "text" - "json" + featureGates: + type: array + description: | + Enable experimental or early access features. + + - `HotplugCPUWithLiveMigration` — enable live changing of cpu cores number. (Not available in CE); + - `HotplugMemoryWithLiveMigration` — enable live changing of memory size. (Not available in CE); + items: + type: string + enum: + - "HotplugCPUWithLiveMigration" + - "HotplugMemoryWithLiveMigration" diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index 442193611c..5aaf65aae6 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -174,3 +174,12 @@ properties: Работает для следующих компонентов: - `virtualization-controller` + featureGates: + type: array + description: | + Включение экспериментальных или недостаточно обкатанных возможностей. + + - `HotplugCPUWithLiveMigration` — включить изменение количества ядер процессора без перезагрузки. (Не доступно в CE); + - `HotplugMemoryWithLiveMigration` — включить изменение размера памяти без перезагрузки. (Не доступно в CE); + items: + type: string diff --git a/templates/kubevirt/kubevirt.yaml b/templates/kubevirt/kubevirt.yaml index 47c8ed8b67..55ad3fcd6c 100644 --- a/templates/kubevirt/kubevirt.yaml +++ b/templates/kubevirt/kubevirt.yaml @@ -356,11 +356,28 @@ env: patch: '{"spec":{"template":{"metadata":{"labels":{"security.deckhouse.io/security-policy-exception": "virt-handler-ds"}}}}}' type: strategic - # Change host path for directory with capabilities xml files. We have custom qemu with different - # machine types thus it conflicts with the original kubevirt. +{{ define "virt-handler-rewrite-host-path-volumes"}} +volumes: +# Directory with capabilities xml files. We have custom qemu with different +# machine types thus it conflicts with the original kubevirt. +- name: node-labeller + hostPath: + path: /var/run/d8-virtualization/node-labeller +# Other directories to communicate with virt-launcher. +# Also rewrite to prevent errors. +- name: libvirt-runtimes + hostPath: + path: /var/run/d8-virtualization/libvirt-runtimes +- name: virt-share-dir + hostPath: + path: /var/run/d8-virtualization/kubevirt +- name: virt-private-dir + hostPath: + path: /var/run/d8-virtualization/kubevirt-private +{{- end }} - resourceName: virt-handler resourceType: DaemonSet - patch: '{"spec":{"template":{"spec":{"volumes":[{"name":"node-labeller","hostPath":{"path":"/var/run/d8-virtualization/node-labeller"}}]}}}}' + patch: '{"spec":{"template":{"spec": {{include "virt-handler-rewrite-host-path-volumes" . | fromYaml | toJson }} }}}' type: strategic imagePullPolicy: IfNotPresent diff --git a/templates/virtualization-controller/_helpers.tpl b/templates/virtualization-controller/_helpers.tpl index 8db602dd1e..3d799bf0fd 100644 --- a/templates/virtualization-controller/_helpers.tpl +++ b/templates/virtualization-controller/_helpers.tpl @@ -117,3 +117,16 @@ true - name: KUBE_APISERVER_FEATURE_GATES value: {{ .Values.virtualization.internal.kubeAPIServerFeatureGates | toJson | quote }} {{- end }} + +{{- define "virtualization-controller.feature-gates-flag-args-item" }} +{{- $gates := list }} +{{- if (.Values.global.enabledModules | has "sdn") }} +{{- $gates = append $gates "SDN=true" }} +{{- end }} +{{- range $feat := .Values.virtualization.internal.moduleConfig.featureGates }} +{{- $gates = append $gates (printf "%s=true" $feat)}} +{{- end }} +{{- if gt (len $gates) 0 }} +- --feature-gates={{$gates | join ","}} +{{- end }} +{{- end }} diff --git a/templates/virtualization-controller/deployment.yaml b/templates/virtualization-controller/deployment.yaml index 1ee253d2b0..8634a067c9 100644 --- a/templates/virtualization-controller/deployment.yaml +++ b/templates/virtualization-controller/deployment.yaml @@ -81,10 +81,8 @@ spec: {{- include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" . | nindent 10 }} image: {{ include "helm_lib_module_image" (list . "virtualizationController") }} imagePullPolicy: IfNotPresent - {{- if (.Values.global.enabledModules | has "sdn") }} args: - - --feature-gates=SDN=true - {{- end }} + {{ include "virtualization-controller.feature-gates-flag-args-item" . | nindent 12 }} volumeMounts: - mountPath: /tmp/k8s-webhook-server/serving-certs name: admission-webhook-secret