diff --git a/aks-node-controller/pkg/gen/aksnodeconfig/v1/kubelet_config.pb.go b/aks-node-controller/pkg/gen/aksnodeconfig/v1/kubelet_config.pb.go index dc61bfbc049..255c6cb98e7 100644 --- a/aks-node-controller/pkg/gen/aksnodeconfig/v1/kubelet_config.pb.go +++ b/aks-node-controller/pkg/gen/aksnodeconfig/v1/kubelet_config.pb.go @@ -501,6 +501,36 @@ type KubeletConfigFileConfig struct { // Default: ["pods"] // +optional. EnforceNodeAllocatable []string `protobuf:"bytes,37,rep,name=enforce_node_allocatable,json=enforceNodeAllocatable,proto3" json:"enforce_node_allocatable,omitempty"` + // evictionSoft is a map of signal names to quantities that defines soft eviction thresholds. + // Each signal listed here must also have a corresponding entry in evictionSoftGracePeriod. + // Soft eviction terminates pods gracefully (respecting terminationGracePeriodSeconds, capped by + // evictionMaxPodGracePeriod) once the threshold is breached for the configured grace period. + // Used by AKS Node Memory Hardening (F2/F5). + // +optional. + EvictionSoft map[string]string `protobuf:"bytes,41,rep,name=eviction_soft,json=evictionSoft,proto3" json:"eviction_soft,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // evictionSoftGracePeriod is a map of signal names to durations defining how long the soft + // eviction threshold must be breached before triggering eviction. Each entry must correspond + // to a signal listed in evictionSoft. Used by AKS Node Memory Hardening (F2/F5). + // +optional. + EvictionSoftGracePeriod map[string]string `protobuf:"bytes,42,rep,name=eviction_soft_grace_period,json=evictionSoftGracePeriod,proto3" json:"eviction_soft_grace_period,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // evictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when + // terminating pods in response to a soft eviction threshold being met. Setting this value + // caps the pod's terminationGracePeriodSeconds during soft eviction. Used by AKS Node + // Memory Hardening (F2/F5). + // +optional. + EvictionMaxPodGracePeriod int32 `protobuf:"varint,43,opt,name=eviction_max_pod_grace_period,json=evictionMaxPodGracePeriod,proto3" json:"eviction_max_pod_grace_period,omitempty"` + // kubeReservedCgroup is the absolute name of the cgroup the kubelet should manage for the + // kube-reserved compute resources. When enforce-node-allocatable contains "kube-reserved", + // this cgroup must exist before kubelet starts. Example: "/kubelet.slice". + // Used by AKS Node Memory Hardening (F2/F5). + // +optional. + KubeReservedCgroup string `protobuf:"bytes,44,opt,name=kube_reserved_cgroup,json=kubeReservedCgroup,proto3" json:"kube_reserved_cgroup,omitempty"` + // systemReservedCgroup is the absolute name of the cgroup the kubelet should manage for the + // system-reserved compute resources. When enforce-node-allocatable contains "system-reserved", + // this cgroup must exist before kubelet starts. Example: "/system.slice". + // Used by AKS Node Memory Hardening (F2/F5). + // +optional. + SystemReservedCgroup string `protobuf:"bytes,45,opt,name=system_reserved_cgroup,json=systemReservedCgroup,proto3" json:"system_reserved_cgroup,omitempty"` // A comma separated whitelist of unsafe sysctls or sysctl patterns (ending in *). // Unsafe sysctl groups are kernel.shm*, kernel.msg*, kernel.sem, fs.mqueue.*, and net.*. // These sysctls are namespaced but not allowed by default. @@ -810,6 +840,41 @@ func (x *KubeletConfigFileConfig) GetEnforceNodeAllocatable() []string { return nil } +func (x *KubeletConfigFileConfig) GetEvictionSoft() map[string]string { + if x != nil { + return x.EvictionSoft + } + return nil +} + +func (x *KubeletConfigFileConfig) GetEvictionSoftGracePeriod() map[string]string { + if x != nil { + return x.EvictionSoftGracePeriod + } + return nil +} + +func (x *KubeletConfigFileConfig) GetEvictionMaxPodGracePeriod() int32 { + if x != nil { + return x.EvictionMaxPodGracePeriod + } + return 0 +} + +func (x *KubeletConfigFileConfig) GetKubeReservedCgroup() string { + if x != nil { + return x.KubeReservedCgroup + } + return "" +} + +func (x *KubeletConfigFileConfig) GetSystemReservedCgroup() string { + if x != nil { + return x.SystemReservedCgroup + } + return "" +} + func (x *KubeletConfigFileConfig) GetAllowedUnsafeSysctls() []string { if x != nil { return x.AllowedUnsafeSysctls @@ -1221,7 +1286,7 @@ var file_aksnodeconfig_v1_kubelet_config_proto_rawDesc = []byte{ 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x89, 0x15, + 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa8, 0x19, 0x0a, 0x17, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x1f, 0x0a, @@ -1350,104 +1415,138 @@ var file_aksnodeconfig_v1_kubelet_config_proto_rawDesc = []byte{ 0x72, 0x63, 0x65, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x25, 0x20, 0x03, 0x28, 0x09, 0x52, 0x16, 0x65, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x12, 0x34, 0x0a, 0x16, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x75, 0x6e, - 0x73, 0x61, 0x66, 0x65, 0x5f, 0x73, 0x79, 0x73, 0x63, 0x74, 0x6c, 0x73, 0x18, 0x26, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x14, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x55, 0x6e, 0x73, 0x61, 0x66, - 0x65, 0x53, 0x79, 0x73, 0x63, 0x74, 0x6c, 0x73, 0x12, 0x37, 0x0a, 0x15, 0x73, 0x65, 0x72, 0x69, - 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x75, 0x6c, 0x6c, - 0x73, 0x18, 0x27, 0x20, 0x01, 0x28, 0x08, 0x48, 0x09, 0x52, 0x13, 0x73, 0x65, 0x72, 0x69, 0x61, - 0x6c, 0x69, 0x7a, 0x65, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x50, 0x75, 0x6c, 0x6c, 0x73, 0x88, 0x01, - 0x01, 0x12, 0x27, 0x0a, 0x0f, 0x73, 0x65, 0x63, 0x63, 0x6f, 0x6d, 0x70, 0x5f, 0x64, 0x65, 0x66, - 0x61, 0x75, 0x6c, 0x74, 0x18, 0x28, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x73, 0x65, 0x63, 0x63, - 0x6f, 0x6d, 0x70, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x1a, 0x3f, 0x0a, 0x11, 0x45, 0x76, - 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x61, 0x72, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3f, 0x0a, 0x11, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x47, 0x61, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x41, 0x0a, 0x13, - 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x45, 0x6e, + 0x6c, 0x65, 0x12, 0x60, 0x0a, 0x0d, 0x65, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, + 0x6f, 0x66, 0x74, 0x18, 0x29, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x61, 0x6b, 0x73, 0x6e, + 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x75, 0x62, + 0x65, 0x6c, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x45, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x6f, 0x66, + 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x65, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x53, 0x6f, 0x66, 0x74, 0x12, 0x83, 0x01, 0x0a, 0x1a, 0x65, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x73, 0x6f, 0x66, 0x74, 0x5f, 0x67, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x65, 0x72, + 0x69, 0x6f, 0x64, 0x18, 0x2a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x61, 0x6b, 0x73, 0x6e, + 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x75, 0x62, + 0x65, 0x6c, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x45, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x6f, 0x66, + 0x74, 0x47, 0x72, 0x61, 0x63, 0x65, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x17, 0x65, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x6f, 0x66, 0x74, 0x47, + 0x72, 0x61, 0x63, 0x65, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x12, 0x40, 0x0a, 0x1d, 0x65, 0x76, + 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x67, + 0x72, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x18, 0x2b, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x19, 0x65, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x61, 0x78, 0x50, 0x6f, + 0x64, 0x47, 0x72, 0x61, 0x63, 0x65, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x12, 0x30, 0x0a, 0x14, + 0x6b, 0x75, 0x62, 0x65, 0x5f, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x5f, 0x63, 0x67, + 0x72, 0x6f, 0x75, 0x70, 0x18, 0x2c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6b, 0x75, 0x62, 0x65, + 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x43, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x34, + 0x0a, 0x16, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x5f, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x64, 0x5f, 0x63, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x2d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, + 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x43, 0x67, + 0x72, 0x6f, 0x75, 0x70, 0x12, 0x34, 0x0a, 0x16, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x5f, + 0x75, 0x6e, 0x73, 0x61, 0x66, 0x65, 0x5f, 0x73, 0x79, 0x73, 0x63, 0x74, 0x6c, 0x73, 0x18, 0x26, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x14, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x55, 0x6e, 0x73, + 0x61, 0x66, 0x65, 0x53, 0x79, 0x73, 0x63, 0x74, 0x6c, 0x73, 0x12, 0x37, 0x0a, 0x15, 0x73, 0x65, + 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x75, + 0x6c, 0x6c, 0x73, 0x18, 0x27, 0x20, 0x01, 0x28, 0x08, 0x48, 0x09, 0x52, 0x13, 0x73, 0x65, 0x72, + 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x50, 0x75, 0x6c, 0x6c, 0x73, + 0x88, 0x01, 0x01, 0x12, 0x27, 0x0a, 0x0f, 0x73, 0x65, 0x63, 0x63, 0x6f, 0x6d, 0x70, 0x5f, 0x64, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x18, 0x28, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x73, 0x65, + 0x63, 0x63, 0x6f, 0x6d, 0x70, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x1a, 0x3f, 0x0a, 0x11, + 0x45, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x61, 0x72, 0x64, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3f, 0x0a, + 0x11, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x47, 0x61, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x41, + 0x0a, 0x13, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x1a, 0x3f, 0x0a, 0x11, 0x4b, 0x75, 0x62, 0x65, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x1a, 0x3f, 0x0a, 0x11, 0x45, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x6f, + 0x66, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x1a, 0x4a, 0x0a, 0x1c, 0x45, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, + 0x6f, 0x66, 0x74, 0x47, 0x72, 0x61, 0x63, 0x65, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, - 0x3f, 0x0a, 0x11, 0x4b, 0x75, 0x62, 0x65, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x72, - 0x64, 0x5f, 0x71, 0x70, 0x73, 0x42, 0x22, 0x0a, 0x20, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, - 0x67, 0x63, 0x5f, 0x68, 0x69, 0x67, 0x68, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, - 0x64, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x42, 0x21, 0x0a, 0x1f, 0x5f, 0x69, 0x6d, - 0x61, 0x67, 0x65, 0x5f, 0x67, 0x63, 0x5f, 0x6c, 0x6f, 0x77, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, - 0x68, 0x6f, 0x6c, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x42, 0x12, 0x0a, 0x10, - 0x5f, 0x63, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x71, 0x6f, 0x73, - 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x70, 0x6f, 0x64, 0x73, 0x42, 0x11, 0x0a, - 0x0f, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x70, 0x69, 0x64, 0x73, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, - 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x63, 0x66, 0x73, 0x5f, 0x71, 0x75, 0x6f, - 0x74, 0x61, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x5f, 0x73, 0x77, 0x61, 0x70, - 0x5f, 0x6f, 0x6e, 0x42, 0x1a, 0x0a, 0x18, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x5f, 0x6c, 0x6f, 0x67, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x42, - 0x18, 0x0a, 0x16, 0x5f, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x5f, 0x69, 0x6d, - 0x61, 0x67, 0x65, 0x5f, 0x70, 0x75, 0x6c, 0x6c, 0x73, 0x22, 0xf2, 0x01, 0x0a, 0x15, 0x4b, 0x75, - 0x62, 0x65, 0x6c, 0x65, 0x74, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x3f, 0x0a, 0x04, 0x78, 0x35, 0x30, 0x39, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x2b, 0x2e, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x58, 0x35, 0x30, 0x39, - 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x04, - 0x78, 0x35, 0x30, 0x39, 0x12, 0x48, 0x0a, 0x07, 0x77, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, - 0x57, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x77, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x12, 0x4e, - 0x0a, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x6f, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x30, 0x2e, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x41, 0x6e, 0x6f, 0x6e, - 0x79, 0x6d, 0x6f, 0x75, 0x73, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x6f, 0x75, 0x73, 0x22, 0x41, - 0x0a, 0x19, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x58, 0x35, 0x30, 0x39, 0x41, 0x75, 0x74, - 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0e, 0x63, - 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x41, 0x46, 0x69, 0x6c, - 0x65, 0x22, 0x55, 0x0a, 0x1c, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x57, 0x65, 0x62, 0x68, - 0x6f, 0x6f, 0x6b, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x63, - 0x61, 0x63, 0x68, 0x65, 0x5f, 0x74, 0x74, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x63, 0x61, 0x63, 0x68, 0x65, 0x54, 0x74, 0x6c, 0x22, 0x3a, 0x0a, 0x1e, 0x4b, 0x75, 0x62, 0x65, - 0x6c, 0x65, 0x74, 0x41, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x6f, 0x75, 0x73, 0x41, 0x75, 0x74, 0x68, - 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, - 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, - 0x62, 0x6c, 0x65, 0x64, 0x22, 0x73, 0x0a, 0x14, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x41, - 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, - 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, - 0x12, 0x47, 0x0a, 0x07, 0x77, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x2d, 0x2e, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x57, 0x65, 0x62, 0x68, - 0x6f, 0x6f, 0x6b, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x07, 0x77, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x22, 0x85, 0x01, 0x0a, 0x1b, 0x4b, 0x75, - 0x62, 0x65, 0x6c, 0x65, 0x74, 0x57, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x41, 0x75, 0x74, 0x68, - 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x14, 0x63, 0x61, 0x63, - 0x68, 0x65, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x74, 0x74, - 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x63, 0x61, 0x63, 0x68, 0x65, 0x41, 0x75, - 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x54, 0x74, 0x6c, 0x12, 0x34, 0x0a, 0x16, 0x63, - 0x61, 0x63, 0x68, 0x65, 0x5f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, - 0x64, 0x5f, 0x74, 0x74, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x63, 0x61, 0x63, - 0x68, 0x65, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x54, 0x74, - 0x6c, 0x2a, 0x61, 0x0a, 0x0b, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x44, 0x69, 0x73, 0x6b, - 0x12, 0x1c, 0x0a, 0x18, 0x4b, 0x55, 0x42, 0x45, 0x4c, 0x45, 0x54, 0x5f, 0x44, 0x49, 0x53, 0x4b, - 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x18, - 0x0a, 0x14, 0x4b, 0x55, 0x42, 0x45, 0x4c, 0x45, 0x54, 0x5f, 0x44, 0x49, 0x53, 0x4b, 0x5f, 0x4f, - 0x53, 0x5f, 0x44, 0x49, 0x53, 0x4b, 0x10, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x4b, 0x55, 0x42, 0x45, - 0x4c, 0x45, 0x54, 0x5f, 0x44, 0x49, 0x53, 0x4b, 0x5f, 0x54, 0x45, 0x4d, 0x50, 0x5f, 0x44, 0x49, - 0x53, 0x4b, 0x10, 0x02, 0x42, 0x5a, 0x5a, 0x58, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x61, - 0x6b, 0x65, 0x72, 0x2f, 0x61, 0x6b, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2d, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, 0x65, 0x6e, 0x2f, - 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x76, 0x31, - 0x3b, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x76, 0x31, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, + 0x13, 0x0a, 0x11, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x5f, 0x71, 0x70, 0x73, 0x42, 0x22, 0x0a, 0x20, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x67, + 0x63, 0x5f, 0x68, 0x69, 0x67, 0x68, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, + 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x42, 0x21, 0x0a, 0x1f, 0x5f, 0x69, 0x6d, 0x61, + 0x67, 0x65, 0x5f, 0x67, 0x63, 0x5f, 0x6c, 0x6f, 0x77, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, + 0x6f, 0x6c, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x42, 0x12, 0x0a, 0x10, 0x5f, + 0x63, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x71, 0x6f, 0x73, 0x42, + 0x0b, 0x0a, 0x09, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x70, 0x6f, 0x64, 0x73, 0x42, 0x11, 0x0a, 0x0f, + 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x70, 0x69, 0x64, 0x73, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x42, + 0x10, 0x0a, 0x0e, 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x63, 0x66, 0x73, 0x5f, 0x71, 0x75, 0x6f, 0x74, + 0x61, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x5f, 0x73, 0x77, 0x61, 0x70, 0x5f, + 0x6f, 0x6e, 0x42, 0x1a, 0x0a, 0x18, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x5f, 0x6c, 0x6f, 0x67, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x42, 0x18, + 0x0a, 0x16, 0x5f, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x5f, 0x69, 0x6d, 0x61, + 0x67, 0x65, 0x5f, 0x70, 0x75, 0x6c, 0x6c, 0x73, 0x22, 0xf2, 0x01, 0x0a, 0x15, 0x4b, 0x75, 0x62, + 0x65, 0x6c, 0x65, 0x74, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x3f, 0x0a, 0x04, 0x78, 0x35, 0x30, 0x39, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2b, 0x2e, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x58, 0x35, 0x30, 0x39, 0x41, + 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x04, 0x78, + 0x35, 0x30, 0x39, 0x12, 0x48, 0x0a, 0x07, 0x77, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x57, + 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x77, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x12, 0x4e, 0x0a, + 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x6f, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x30, 0x2e, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x41, 0x6e, 0x6f, 0x6e, 0x79, + 0x6d, 0x6f, 0x75, 0x73, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x6f, 0x75, 0x73, 0x22, 0x41, 0x0a, + 0x19, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x58, 0x35, 0x30, 0x39, 0x41, 0x75, 0x74, 0x68, + 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x41, 0x46, 0x69, 0x6c, 0x65, + 0x22, 0x55, 0x0a, 0x1c, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x57, 0x65, 0x62, 0x68, 0x6f, + 0x6f, 0x6b, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x61, + 0x63, 0x68, 0x65, 0x5f, 0x74, 0x74, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, + 0x61, 0x63, 0x68, 0x65, 0x54, 0x74, 0x6c, 0x22, 0x3a, 0x0a, 0x1e, 0x4b, 0x75, 0x62, 0x65, 0x6c, + 0x65, 0x74, 0x41, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x6f, 0x75, 0x73, 0x41, 0x75, 0x74, 0x68, 0x65, + 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, + 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, + 0x6c, 0x65, 0x64, 0x22, 0x73, 0x0a, 0x14, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x41, 0x75, + 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6d, + 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, + 0x47, 0x0a, 0x07, 0x77, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2d, 0x2e, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x57, 0x65, 0x62, 0x68, 0x6f, + 0x6f, 0x6b, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x07, 0x77, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x22, 0x85, 0x01, 0x0a, 0x1b, 0x4b, 0x75, 0x62, + 0x65, 0x6c, 0x65, 0x74, 0x57, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x41, 0x75, 0x74, 0x68, 0x6f, + 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x14, 0x63, 0x61, 0x63, 0x68, + 0x65, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x74, 0x74, 0x6c, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x63, 0x61, 0x63, 0x68, 0x65, 0x41, 0x75, 0x74, + 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x54, 0x74, 0x6c, 0x12, 0x34, 0x0a, 0x16, 0x63, 0x61, + 0x63, 0x68, 0x65, 0x5f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, + 0x5f, 0x74, 0x74, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x63, 0x61, 0x63, 0x68, + 0x65, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x54, 0x74, 0x6c, + 0x2a, 0x61, 0x0a, 0x0b, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x44, 0x69, 0x73, 0x6b, 0x12, + 0x1c, 0x0a, 0x18, 0x4b, 0x55, 0x42, 0x45, 0x4c, 0x45, 0x54, 0x5f, 0x44, 0x49, 0x53, 0x4b, 0x5f, + 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x18, 0x0a, + 0x14, 0x4b, 0x55, 0x42, 0x45, 0x4c, 0x45, 0x54, 0x5f, 0x44, 0x49, 0x53, 0x4b, 0x5f, 0x4f, 0x53, + 0x5f, 0x44, 0x49, 0x53, 0x4b, 0x10, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x4b, 0x55, 0x42, 0x45, 0x4c, + 0x45, 0x54, 0x5f, 0x44, 0x49, 0x53, 0x4b, 0x5f, 0x54, 0x45, 0x4d, 0x50, 0x5f, 0x44, 0x49, 0x53, + 0x4b, 0x10, 0x02, 0x42, 0x5a, 0x5a, 0x58, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x61, 0x6b, + 0x65, 0x72, 0x2f, 0x61, 0x6b, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2d, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, + 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x76, 0x31, 0x3b, + 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x76, 0x31, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1463,7 +1562,7 @@ func file_aksnodeconfig_v1_kubelet_config_proto_rawDescGZIP() []byte { } var file_aksnodeconfig_v1_kubelet_config_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_aksnodeconfig_v1_kubelet_config_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_aksnodeconfig_v1_kubelet_config_proto_msgTypes = make([]protoimpl.MessageInfo, 16) var file_aksnodeconfig_v1_kubelet_config_proto_goTypes = []any{ (KubeletDisk)(0), // 0: aksnodeconfig.v1.KubeletDisk (*KubeletConfig)(nil), // 1: aksnodeconfig.v1.KubeletConfig @@ -1480,6 +1579,8 @@ var file_aksnodeconfig_v1_kubelet_config_proto_goTypes = []any{ nil, // 12: aksnodeconfig.v1.KubeletConfigFileConfig.FeatureGatesEntry nil, // 13: aksnodeconfig.v1.KubeletConfigFileConfig.SystemReservedEntry nil, // 14: aksnodeconfig.v1.KubeletConfigFileConfig.KubeReservedEntry + nil, // 15: aksnodeconfig.v1.KubeletConfigFileConfig.EvictionSoftEntry + nil, // 16: aksnodeconfig.v1.KubeletConfigFileConfig.EvictionSoftGracePeriodEntry } var file_aksnodeconfig_v1_kubelet_config_proto_depIdxs = []int32{ 9, // 0: aksnodeconfig.v1.KubeletConfig.kubelet_flags:type_name -> aksnodeconfig.v1.KubeletConfig.KubeletFlagsEntry @@ -1492,15 +1593,17 @@ var file_aksnodeconfig_v1_kubelet_config_proto_depIdxs = []int32{ 12, // 7: aksnodeconfig.v1.KubeletConfigFileConfig.feature_gates:type_name -> aksnodeconfig.v1.KubeletConfigFileConfig.FeatureGatesEntry 13, // 8: aksnodeconfig.v1.KubeletConfigFileConfig.system_reserved:type_name -> aksnodeconfig.v1.KubeletConfigFileConfig.SystemReservedEntry 14, // 9: aksnodeconfig.v1.KubeletConfigFileConfig.kube_reserved:type_name -> aksnodeconfig.v1.KubeletConfigFileConfig.KubeReservedEntry - 4, // 10: aksnodeconfig.v1.KubeletAuthentication.x509:type_name -> aksnodeconfig.v1.KubeletX509Authentication - 5, // 11: aksnodeconfig.v1.KubeletAuthentication.webhook:type_name -> aksnodeconfig.v1.KubeletWebhookAuthentication - 6, // 12: aksnodeconfig.v1.KubeletAuthentication.anonymous:type_name -> aksnodeconfig.v1.KubeletAnonymousAuthentication - 8, // 13: aksnodeconfig.v1.KubeletAuthorization.webhook:type_name -> aksnodeconfig.v1.KubeletWebhookAuthorization - 14, // [14:14] is the sub-list for method output_type - 14, // [14:14] is the sub-list for method input_type - 14, // [14:14] is the sub-list for extension type_name - 14, // [14:14] is the sub-list for extension extendee - 0, // [0:14] is the sub-list for field type_name + 15, // 10: aksnodeconfig.v1.KubeletConfigFileConfig.eviction_soft:type_name -> aksnodeconfig.v1.KubeletConfigFileConfig.EvictionSoftEntry + 16, // 11: aksnodeconfig.v1.KubeletConfigFileConfig.eviction_soft_grace_period:type_name -> aksnodeconfig.v1.KubeletConfigFileConfig.EvictionSoftGracePeriodEntry + 4, // 12: aksnodeconfig.v1.KubeletAuthentication.x509:type_name -> aksnodeconfig.v1.KubeletX509Authentication + 5, // 13: aksnodeconfig.v1.KubeletAuthentication.webhook:type_name -> aksnodeconfig.v1.KubeletWebhookAuthentication + 6, // 14: aksnodeconfig.v1.KubeletAuthentication.anonymous:type_name -> aksnodeconfig.v1.KubeletAnonymousAuthentication + 8, // 15: aksnodeconfig.v1.KubeletAuthorization.webhook:type_name -> aksnodeconfig.v1.KubeletWebhookAuthorization + 16, // [16:16] is the sub-list for method output_type + 16, // [16:16] is the sub-list for method input_type + 16, // [16:16] is the sub-list for extension type_name + 16, // [16:16] is the sub-list for extension extendee + 0, // [0:16] is the sub-list for field type_name } func init() { file_aksnodeconfig_v1_kubelet_config_proto_init() } @@ -1515,7 +1618,7 @@ func file_aksnodeconfig_v1_kubelet_config_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_aksnodeconfig_v1_kubelet_config_proto_rawDesc, NumEnums: 1, - NumMessages: 14, + NumMessages: 16, NumExtensions: 0, NumServices: 0, }, diff --git a/aks-node-controller/proto/aksnodeconfig/v1/kubelet_config.proto b/aks-node-controller/proto/aksnodeconfig/v1/kubelet_config.proto index e4ea65dffd6..aafdc17b94b 100644 --- a/aks-node-controller/proto/aksnodeconfig/v1/kubelet_config.proto +++ b/aks-node-controller/proto/aksnodeconfig/v1/kubelet_config.proto @@ -393,6 +393,41 @@ message KubeletConfigFileConfig { +optional. */ repeated string enforce_node_allocatable = 37; + /* evictionSoft is a map of signal names to quantities that defines soft eviction thresholds. + Each signal listed here must also have a corresponding entry in evictionSoftGracePeriod. + Soft eviction terminates pods gracefully (respecting terminationGracePeriodSeconds, capped by + evictionMaxPodGracePeriod) once the threshold is breached for the configured grace period. + Used by AKS Node Memory Hardening (F2/F5). + +optional. */ + map eviction_soft = 41; + + /* evictionSoftGracePeriod is a map of signal names to durations defining how long the soft + eviction threshold must be breached before triggering eviction. Each entry must correspond + to a signal listed in evictionSoft. Used by AKS Node Memory Hardening (F2/F5). + +optional. */ + map eviction_soft_grace_period = 42; + + /* evictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when + terminating pods in response to a soft eviction threshold being met. Setting this value + caps the pod's terminationGracePeriodSeconds during soft eviction. Used by AKS Node + Memory Hardening (F2/F5). + +optional. */ + int32 eviction_max_pod_grace_period = 43; + + /* kubeReservedCgroup is the absolute name of the cgroup the kubelet should manage for the + kube-reserved compute resources. When enforce-node-allocatable contains "kube-reserved", + this cgroup must exist before kubelet starts. Example: "/kubelet.slice". + Used by AKS Node Memory Hardening (F2/F5). + +optional. */ + string kube_reserved_cgroup = 44; + + /* systemReservedCgroup is the absolute name of the cgroup the kubelet should manage for the + system-reserved compute resources. When enforce-node-allocatable contains "system-reserved", + this cgroup must exist before kubelet starts. Example: "/system.slice". + Used by AKS Node Memory Hardening (F2/F5). + +optional. */ + string system_reserved_cgroup = 45; + /* A comma separated whitelist of unsafe sysctls or sysctl patterns (ending in *). Unsafe sysctl groups are kernel.shm*, kernel.msg*, kernel.sem, fs.mqueue.*, and net.*. These sysctls are namespaced but not allowed by default. diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 1bccaf924c7..259e13753c6 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -809,6 +809,17 @@ EOF local tls_bootstrapping_start_time_filepath="/opt/azure/containers/tls-bootstrap-start-time" date +"%F %T.%3N" > "${tls_bootstrapping_start_time_filepath}" + # Node Memory Hardening (F2/F5): if the RP rendered --kube-reserved-cgroup or + # --system-reserved-cgroup, ensure the corresponding systemd slices exist before + # kubelet starts so its NodeAllocatable enforcement loop can find them. The + # helper is a no-op when neither value is present (back-compat with non-hardened pools). + resolveKubeletReservedCgroups + if [ -n "${KUBE_RESERVED_CGROUP}" ] || [ -n "${SYSTEM_RESERVED_CGROUP}" ]; then + if ! logs_to_events "AKS.CSE.ensureKubelet.ensureKubeletCgroupHierarchy" ensureKubeletCgroupHierarchy; then + exit $ERR_KUBELET_START_FAIL + fi + fi + # start kubelet.service without waiting for the main process to start, though check whether it has entered a failed state after enablement if ! systemctlEnableAndStartNoBlock kubelet 240; then # append kubelet status to CSE output to ensure we can see it diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 00a940a4e31..2d35e19b275 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -1452,4 +1452,132 @@ function get_sandbox_image_from_containerd_config() { echo "$sandbox_image" } + +# ensureKubeletCgroupHierarchy creates the systemd slices used by kubelet for the +# kube-reserved and system-reserved enforcement tiers (Node Memory Hardening F2/F5). +# It MUST be called before kubelet starts so that /kubelet.slice and /system.slice +# exist and are managed by systemd before the first kubelet enforcement pass. +# +# The function: +# - Asserts cgroupv2 unified hierarchy (cgroupv1 is not supported by this feature +# because mixed/legacy hierarchies cannot reliably enforce per-slice MemoryMax). +# - Drops a /etc/systemd/system/kubelet.slice unit (system.slice ships with systemd). +# - Triggers `systemctl daemon-reload` and `systemctl start kubelet.slice` so the +# cgroup is materialised at /sys/fs/cgroup/kubelet.slice prior to kubelet boot. +# +# Inputs (env, all optional — the function is a no-op if the RP did not opt in): +# KUBE_RESERVED_CGROUP — absolute cgroup name, e.g. "/kubelet.slice" +# SYSTEM_RESERVED_CGROUP — absolute cgroup name, e.g. "/system.slice" + +# resolveKubeletReservedCgroups exports KUBE_RESERVED_CGROUP and SYSTEM_RESERVED_CGROUP +# from either the kubelet config-file JSON (when KUBELET_CONFIG_FILE_ENABLED=true) or +# from KUBELET_FLAGS as a fallback. Both vars are unset (empty string) when the RP did +# not opt the pool into Node Memory Hardening, which keeps ensureKubeletCgroupHierarchy +# a no-op for non-hardened pools. +# +# When kubelet config-file mode is enabled, --kube-reserved-cgroup / +# --system-reserved-cgroup are filtered out of KUBELET_FLAGS by the RP +# (TranslatedKubeletConfigFlags) and rendered into kubeletconfig.json instead, so +# we must source the cgroup names from the JSON in that mode. +resolveKubeletReservedCgroups() { + KUBE_RESERVED_CGROUP="" + SYSTEM_RESERVED_CGROUP="" + if [ "${KUBELET_CONFIG_FILE_ENABLED:-}" = "true" ] && [ -n "${KUBELET_CONFIG_FILE_CONTENT:-}" ]; then + KUBE_RESERVED_CGROUP=$(echo "${KUBELET_CONFIG_FILE_CONTENT}" | base64 -d | jq -r '.kubeReservedCgroup // ""') + SYSTEM_RESERVED_CGROUP=$(echo "${KUBELET_CONFIG_FILE_CONTENT}" | base64 -d | jq -r '.systemReservedCgroup // ""') + else + KUBE_RESERVED_CGROUP=$(extract_value_from_kubelet_flags "${KUBELET_FLAGS:-}" "kube-reserved-cgroup") + SYSTEM_RESERVED_CGROUP=$(extract_value_from_kubelet_flags "${KUBELET_FLAGS:-}" "system-reserved-cgroup") + fi + export KUBE_RESERVED_CGROUP SYSTEM_RESERVED_CGROUP +} + +ensureKubeletCgroupHierarchy() { + if [ -z "${KUBE_RESERVED_CGROUP:-}" ] && [ -z "${SYSTEM_RESERVED_CGROUP:-}" ]; then + return 0 + fi + + # Path overrides exist for ShellSpec coverage; production callers leave them at + # their defaults. + local cgroupv2_marker="${CGROUPV2_MARKER_PATH:-/sys/fs/cgroup/cgroup.controllers}" + local kubelet_slice_unit="${KUBELET_SLICE_UNIT_PATH:-/etc/systemd/system/kubelet.slice}" + local kubelet_dropin_dir="${KUBELET_SERVICE_DROPIN_DIR:-/etc/systemd/system/kubelet.service.d}" + + # Assert cgroupv2 unified hierarchy. The canonical marker is the presence of + # /sys/fs/cgroup/cgroup.controllers, which only exists under cgroupv2. + if [ ! -f "${cgroupv2_marker}" ]; then + echo "ensureKubeletCgroupHierarchy: cgroupv2 unified hierarchy not detected; node memory hardening cgroup enforcement requires cgroupv2" + return 1 + fi + + # Validate supported values: only /kubelet.slice (or bare kubelet.slice) is + # supported for KUBE_RESERVED_CGROUP, and only /system.slice for + # SYSTEM_RESERVED_CGROUP (a built-in systemd slice). Reject any other value + # explicitly so kubelet doesn't fail later with an opaque enforcement error. + case "${KUBE_RESERVED_CGROUP:-}" in + ""|"/kubelet.slice"|"kubelet.slice") ;; + *) + echo "ensureKubeletCgroupHierarchy: unsupported KUBE_RESERVED_CGROUP=${KUBE_RESERVED_CGROUP}; only /kubelet.slice is supported" + return 1 + ;; + esac + case "${SYSTEM_RESERVED_CGROUP:-}" in + ""|"/system.slice"|"system.slice") ;; + *) + echo "ensureKubeletCgroupHierarchy: unsupported SYSTEM_RESERVED_CGROUP=${SYSTEM_RESERVED_CGROUP}; only /system.slice is supported" + return 1 + ;; + esac + + # /system.slice is a built-in systemd slice; we only need to create kubelet.slice. + if [ "${KUBE_RESERVED_CGROUP:-}" = "/kubelet.slice" ] || [ "${KUBE_RESERVED_CGROUP:-}" = "kubelet.slice" ]; then + if [ ! -f "${kubelet_slice_unit}" ]; then + mkdir -p "$(dirname "${kubelet_slice_unit}")" + # [Install] WantedBy=slices.target ensures the slice is pulled in by + # systemd on every boot (including post-reboot), not only the current + # provisioning boot. Combined with the Before=kubelet.service drop-in + # below this guarantees /sys/fs/cgroup/kubelet.slice is materialised + # before kubelet starts, so NodeAllocatable enforcement does not race. + tee "${kubelet_slice_unit}" > /dev/null <<'EOF' +[Unit] +Description=Slice for kubelet kube-reserved enforcement (AKS Node Memory Hardening) +Before=slices.target +DefaultDependencies=no + +[Slice] + +[Install] +WantedBy=slices.target +EOF + chmod 0644 "${kubelet_slice_unit}" + + # Drop-in on kubelet.service so systemd starts kubelet.slice first + # on every boot. This survives reboots without depending on the + # one-shot `systemctl start` below. + mkdir -p "${kubelet_dropin_dir}" + tee "${kubelet_dropin_dir}/10-kubelet-slice.conf" > /dev/null <<'EOF' +[Unit] +Wants=kubelet.slice +After=kubelet.slice +EOF + chmod 0644 "${kubelet_dropin_dir}/10-kubelet-slice.conf" + + systemctl daemon-reload + + # Enable the slice so it is started on subsequent boots. + if ! systemctl enable kubelet.slice; then + echo "ensureKubeletCgroupHierarchy: failed to enable kubelet.slice" + return 1 + fi + fi + + # Materialise the cgroup tree at /sys/fs/cgroup/kubelet.slice before kubelet starts on this boot. + if ! systemctl start kubelet.slice; then + echo "ensureKubeletCgroupHierarchy: failed to start kubelet.slice" + return 1 + fi + fi + + return 0 +} #HELPERSEOF diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index 61ef460608b..7baac10552a 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -2229,6 +2229,24 @@ type AKSKubeletConfiguration struct { imagefs.available: "15%" +optional. */ EvictionHard map[string]string `json:"evictionHard,omitempty"` + /* evictionSoft is a map of signal names to quantities that defines soft eviction thresholds. + For example: {"memory.available": "300Mi"}. + Each signal listed here must also have a corresponding entry in evictionSoftGracePeriod. + Soft eviction terminates pods gracefully (respecting terminationGracePeriodSeconds, capped by + evictionMaxPodGracePeriod) once the threshold is breached for the configured grace period. + +optional. */ + EvictionSoft map[string]string `json:"evictionSoft,omitempty"` + /* evictionSoftGracePeriod is a map of signal names to durations defining how long the soft + eviction threshold must be breached before triggering eviction. Example: + {"memory.available": "30s", "nodefs.available": "2m"}. + Each entry must correspond to a signal listed in evictionSoft. + +optional. */ + EvictionSoftGracePeriod map[string]string `json:"evictionSoftGracePeriod,omitempty"` + /* evictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when + terminating pods in response to a soft eviction threshold being met. Setting this value + caps the pod's terminationGracePeriodSeconds during soft eviction. + +optional. */ + EvictionMaxPodGracePeriod int32 `json:"evictionMaxPodGracePeriod,omitempty"` /* protectKernelDefaults, if true, causes the Kubelet to error if kernel flags are not as it expects. Otherwise the Kubelet will attempt to modify kernel flags to match its expectation. @@ -2308,6 +2326,16 @@ type AKSKubeletConfiguration struct { Default: ["pods"] +optional. */ EnforceNodeAllocatable []string `json:"enforceNodeAllocatable,omitempty"` + /* kubeReservedCgroup is the absolute name of the cgroup the kubelet should manage + for the kube-reserved compute resources. When enforce-node-allocatable contains + "kube-reserved", this cgroup must exist before kubelet starts. Example: "/kubelet.slice". + +optional. */ + KubeReservedCgroup string `json:"kubeReservedCgroup,omitempty"` + /* systemReservedCgroup is the absolute name of the cgroup the kubelet should manage + for the system-reserved compute resources. When enforce-node-allocatable contains + "system-reserved", this cgroup must exist before kubelet starts. Example: "/system.slice". + +optional. */ + SystemReservedCgroup string `json:"systemReservedCgroup,omitempty"` /* A comma separated whitelist of unsafe sysctls or sysctl patterns (ending in *). Unsafe sysctl groups are kernel.shm*, kernel.msg*, kernel.sem, fs.mqueue.*, and net.*. These sysctls are namespaced but not allowed by default. diff --git a/pkg/agent/utils.go b/pkg/agent/utils.go index a1bae0c8e5e..357ce667390 100644 --- a/pkg/agent/utils.go +++ b/pkg/agent/utils.go @@ -44,6 +44,9 @@ var TranslatedKubeletConfigFlags = map[string]bool{ "--cluster-domain": true, "--max-pods": true, "--eviction-hard": true, + "--eviction-soft": true, + "--eviction-soft-grace-period": true, + "--eviction-max-pod-grace-period": true, "--node-status-update-frequency": true, "--node-status-report-frequency": true, "--image-gc-high-threshold": true, @@ -51,6 +54,8 @@ var TranslatedKubeletConfigFlags = map[string]bool{ "--event-qps": true, "--pod-max-pids": true, "--enforce-node-allocatable": true, + "--kube-reserved-cgroup": true, + "--system-reserved-cgroup": true, "--streaming-connection-idle-timeout": true, "--rotate-certificates": true, "--rotate-server-certificates": true, @@ -500,6 +505,8 @@ func getAKSKubeletConfiguration(kc map[string]string) *datamodel.AKSKubeletConfi EventRecordQPS: strToInt32Ptr(kc["--event-qps"]), PodPidsLimit: strToInt64Ptr(kc["--pod-max-pids"]), EnforceNodeAllocatable: strings.Split(kc["--enforce-node-allocatable"], ","), + KubeReservedCgroup: kc["--kube-reserved-cgroup"], + SystemReservedCgroup: kc["--system-reserved-cgroup"], StreamingConnectionIdleTimeout: datamodel.Duration(kc["--streaming-connection-idle-timeout"]), RotateCertificates: strToBool(kc["--rotate-certificates"]), ServerTLSBootstrap: strToBool(kc["--rotate-server-certificates"]), @@ -592,7 +599,22 @@ func GetKubeletConfigFileContent(kc map[string]string, customKc *datamodel.Custo // EvictionHard. // default: "memory.available<750Mi,nodefs.available<10%,nodefs.inodesFree<5%". if eh, ok := kc["--eviction-hard"]; ok && eh != "" { - kubeletConfig.EvictionHard = strKeyValToMap(eh, ",", "<") + kubeletConfig.EvictionHard = strKeyValToMap(eh, "<") + } + + // EvictionSoft (e.g. "memory.available<500Mi,nodefs.available<15%,imagefs.available<20%"). + if es, ok := kc["--eviction-soft"]; ok && es != "" { + kubeletConfig.EvictionSoft = strKeyValToMap(es, "<") + } + + // EvictionSoftGracePeriod (e.g. "memory.available=30s,nodefs.available=2m,imagefs.available=2m"). + if esg, ok := kc["--eviction-soft-grace-period"]; ok && esg != "" { + kubeletConfig.EvictionSoftGracePeriod = strKeyValToMap(esg, "=") + } + + // EvictionMaxPodGracePeriod (integer seconds, e.g. "60"). + if v, ok := kc["--eviction-max-pod-grace-period"]; ok && v != "" { + kubeletConfig.EvictionMaxPodGracePeriod = strToInt32(v) } // feature gates. @@ -601,8 +623,8 @@ func GetKubeletConfigFileContent(kc map[string]string, customKc *datamodel.Custo // system reserve and kube reserve. // looks like "cpu=100m,memory=1638Mi". - kubeletConfig.SystemReserved = strKeyValToMap(kc["--system-reserved"], ",", "=") - kubeletConfig.KubeReserved = strKeyValToMap(kc["--kube-reserved"], ",", "=") + kubeletConfig.SystemReserved = strKeyValToMap(kc["--system-reserved"], "=") + kubeletConfig.KubeReserved = strKeyValToMap(kc["--kube-reserved"], "=") // Settings from customKubeletConfig, only take if it's set. setCustomKubeletConfig(customKc, kubeletConfig) @@ -653,9 +675,9 @@ func strToInt64Ptr(str string) *int64 { return &i } -func strKeyValToMap(str string, strDelim string, pairDelim string) map[string]string { +func strKeyValToMap(str string, pairDelim string) map[string]string { m := make(map[string]string) - pairs := strings.Split(str, strDelim) + pairs := strings.Split(str, ",") for _, pairRaw := range pairs { pair := strings.Split(pairRaw, pairDelim) if len(pair) == numInPair { diff --git a/pkg/agent/utils_test.go b/pkg/agent/utils_test.go index b8b42429ff2..defeb7601d2 100644 --- a/pkg/agent/utils_test.go +++ b/pkg/agent/utils_test.go @@ -514,6 +514,89 @@ func TestGetKubeletConfigFileCustomKCShouldOverrideValuesPassedInKc(t *testing.T } } +func TestGetKubeletConfigFileNodeMemoryHardeningFields(t *testing.T) { + // Verifies AgentBaker renders the new Node Memory Hardening kubelet args + // (soft eviction + cgroup tiering) into the generated kubelet config file. + // Uses JSON unmarshaling rather than a brittle text snapshot so that future + // non-related additions to AKSKubeletConfiguration do not break this test. + kc := getExampleKcWithNodeStatusReportFrequency() + kc["--eviction-soft"] = "memory.available<500Mi,nodefs.available<15%,imagefs.available<20%" + kc["--eviction-soft-grace-period"] = "memory.available=30s,nodefs.available=2m,imagefs.available=2m" + kc["--eviction-max-pod-grace-period"] = "60" + kc["--enforce-node-allocatable"] = "pods,kube-reserved,system-reserved" + kc["--kube-reserved-cgroup"] = "/kubelet.slice" + kc["--system-reserved-cgroup"] = "/system.slice" + + configFileStr := GetKubeletConfigFileContent(kc, nil) + + var got struct { + EvictionSoft map[string]string `json:"evictionSoft"` + EvictionSoftGracePeriod map[string]string `json:"evictionSoftGracePeriod"` + EvictionMaxPodGracePeriod int32 `json:"evictionMaxPodGracePeriod"` + EnforceNodeAllocatable []string `json:"enforceNodeAllocatable"` + KubeReservedCgroup string `json:"kubeReservedCgroup"` + SystemReservedCgroup string `json:"systemReservedCgroup"` + } + if err := json.Unmarshal([]byte(configFileStr), &got); err != nil { + t.Fatalf("failed to unmarshal generated kubelet config: %v\nconfig: %s", err, configFileStr) + } + + wantSoft := map[string]string{ + "memory.available": "500Mi", + "nodefs.available": "15%", + "imagefs.available": "20%", + } + if diff := cmp.Diff(wantSoft, got.EvictionSoft); diff != "" { + t.Errorf("evictionSoft mismatch (-want +got):\n%s", diff) + } + + wantSoftGrace := map[string]string{ + "memory.available": "30s", + "nodefs.available": "2m", + "imagefs.available": "2m", + } + if diff := cmp.Diff(wantSoftGrace, got.EvictionSoftGracePeriod); diff != "" { + t.Errorf("evictionSoftGracePeriod mismatch (-want +got):\n%s", diff) + } + + if got.EvictionMaxPodGracePeriod != 60 { + t.Errorf("evictionMaxPodGracePeriod=%d, want 60", got.EvictionMaxPodGracePeriod) + } + + wantEnforce := []string{"pods", "kube-reserved", "system-reserved"} + if diff := cmp.Diff(wantEnforce, got.EnforceNodeAllocatable); diff != "" { + t.Errorf("enforceNodeAllocatable mismatch (-want +got):\n%s", diff) + } + + if got.KubeReservedCgroup != "/kubelet.slice" { + t.Errorf("kubeReservedCgroup=%q, want %q", got.KubeReservedCgroup, "/kubelet.slice") + } + if got.SystemReservedCgroup != "/system.slice" { + t.Errorf("systemReservedCgroup=%q, want %q", got.SystemReservedCgroup, "/system.slice") + } +} + +func TestGetKubeletConfigFileNodeMemoryHardeningFieldsOmittedByDefault(t *testing.T) { + // Backward-compat: when the RP does not pass the new flags, the generated + // kubelet config must NOT contain the new fields. This guards the 6-month + // VHD support window — non-hardened pools must see no change to these fields. + kc := getExampleKcWithNodeStatusReportFrequency() + + configFileStr := GetKubeletConfigFileContent(kc, nil) + + for _, field := range []string{ + `"evictionSoft"`, + `"evictionSoftGracePeriod"`, + `"evictionMaxPodGracePeriod"`, + `"kubeReservedCgroup"`, + `"systemReservedCgroup"`, + } { + if strings.Contains(configFileStr, field) { + t.Errorf("expected %s to be omitted from kubelet config when not set, got:\n%s", field, configFileStr) + } + } +} + func TestIsTLSBootstrappingEnabledWithHardCodedToken(t *testing.T) { cases := []struct { tlsBootstrapToken *string @@ -698,7 +781,7 @@ var _ = Describe("Assert datamodel.CSEStatus can be used to parse output JSON", It("When cse output format is correct and contains call known fields", func() { testMessage := `{"ExitCode": "51", "Output": "test", "Error": "", - "ExecDuration": "39", "KernelStartTime": "kernel start time", + "ExecDuration": "39", "KernelStartTime": "kernel start time", "SystemdSummary": "systemd summary", "CSEStartTime": "cse start time", "GuestAgentStartTime": "guest agent start time", "BootDatapoints": {"dp1": "1"}}` var cseStatus datamodel.CSEStatus @@ -741,7 +824,7 @@ var _ = Describe("Assert datamodel.CSEStatus can be used to parse output JSON", }) It("When Error is missing", func() { - testMessage := `{ "ExitCode": "51", "Output": "test", + testMessage := `{ "ExitCode": "51", "Output": "test", "Error": "", "ExecDuration": "39", "Error": }` var cseStatus datamodel.CSEStatus err := json.Unmarshal([]byte(testMessage), &cseStatus) @@ -757,7 +840,7 @@ var _ = Describe("Assert datamodel.CSEStatus can be used to parse output JSON", }) It("When SystemdSummary is missing", func() { - testMessage := `{ "ExitCode": "51", "Output": "test", + testMessage := `{ "ExitCode": "51", "Output": "test", "Error": "", "ExecDuration": "39", "SystemdSummary": }` var cseStatus datamodel.CSEStatus err := json.Unmarshal([]byte(testMessage), &cseStatus) @@ -773,7 +856,7 @@ var _ = Describe("Assert datamodel.CSEStatus can be used to parse output JSON", }) It("When GuestAgentStartTime is missing", func() { - testMessage := `{ "ExitCode": "51", "Output": "test", + testMessage := `{ "ExitCode": "51", "Output": "test", "Error": "", "ExecDuration": "39", "GuestAgentStartTime": }` var cseStatus datamodel.CSEStatus err := json.Unmarshal([]byte(testMessage), &cseStatus) @@ -781,7 +864,7 @@ var _ = Describe("Assert datamodel.CSEStatus can be used to parse output JSON", }) It("When BootDatapoints is missing", func() { - testMessage := `{ "ExitCode": "51", "Output": "test", + testMessage := `{ "ExitCode": "51", "Output": "test", "Error": "", "ExecDuration": "39", "BootDatapoints": }` var cseStatus datamodel.CSEStatus err := json.Unmarshal([]byte(testMessage), &cseStatus) @@ -797,7 +880,7 @@ var _ = Describe("Assert datamodel.CSEStatus can be used to parse output JSON", }) It("when ExecDuration is an integer", func() { - testMessage := `{ "ExitCode": "51", "Output": "test", + testMessage := `{ "ExitCode": "51", "Output": "test", "Error": "", "ExecDuration": 39}` var cseStatus datamodel.CSEStatus err := json.Unmarshal([]byte(testMessage), &cseStatus) diff --git a/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh index 79ab03650eb..09a25f81734 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh @@ -706,4 +706,151 @@ EOF The output should equal "unknown" End End + + Describe 'resolveKubeletReservedCgroups' + It 'is a no-op when neither config-file mode nor flags carry the cgroup names' + KUBELET_CONFIG_FILE_ENABLED="false" + KUBELET_CONFIG_FILE_CONTENT="" + KUBELET_FLAGS="--node-ip=10.0.0.1 --rotate-certificates=true" + When call resolveKubeletReservedCgroups + The variable KUBE_RESERVED_CGROUP should equal "" + The variable SYSTEM_RESERVED_CGROUP should equal "" + The status should be success + End + + It 'extracts cgroup names from KUBELET_FLAGS in flag mode' + KUBELET_CONFIG_FILE_ENABLED="false" + KUBELET_CONFIG_FILE_CONTENT="" + KUBELET_FLAGS="--kube-reserved-cgroup=/kubelet.slice --system-reserved-cgroup=/system.slice --node-ip=10.0.0.1" + When call resolveKubeletReservedCgroups + The variable KUBE_RESERVED_CGROUP should equal "/kubelet.slice" + The variable SYSTEM_RESERVED_CGROUP should equal "/system.slice" + The status should be success + End + + It 'extracts cgroup names from KUBELET_CONFIG_FILE_CONTENT in config-file mode' + KUBELET_CONFIG_FILE_ENABLED="true" + KUBELET_CONFIG_FILE_CONTENT=$(cat spec/parts/linux/cloud-init/artifacts/kubelet_mocks/config_file/node_hardening_enabled.json | base64 -w 0) + # Even though flags carry the values, config-file mode must win. + KUBELET_FLAGS="--kube-reserved-cgroup=/wrong.slice --system-reserved-cgroup=/wrong.slice" + When call resolveKubeletReservedCgroups + The variable KUBE_RESERVED_CGROUP should equal "/kubelet.slice" + The variable SYSTEM_RESERVED_CGROUP should equal "/system.slice" + The status should be success + End + + It 'returns empty strings in config-file mode when the JSON does not opt into hardening' + KUBELET_CONFIG_FILE_ENABLED="true" + KUBELET_CONFIG_FILE_CONTENT=$(cat spec/parts/linux/cloud-init/artifacts/kubelet_mocks/config_file/server_tls_bootstrap_enabled.json | base64 -w 0) + KUBELET_FLAGS="" + When call resolveKubeletReservedCgroups + The variable KUBE_RESERVED_CGROUP should equal "" + The variable SYSTEM_RESERVED_CGROUP should equal "" + The status should be success + End + End + + Describe 'ensureKubeletCgroupHierarchy' + # Use a per-test temp directory and point the function's path overrides at + # it so the slice unit file and drop-in are created in a sandbox rather + # than under /etc/systemd. The cgroupv2 marker is similarly redirected to + # a temp file so we can toggle its presence without touching the host. + setup_paths() { + ENSURE_CGROUP_TMPDIR=$(mktemp -d) + export CGROUPV2_MARKER_PATH="${ENSURE_CGROUP_TMPDIR}/cgroup.controllers" + export KUBELET_SLICE_UNIT_PATH="${ENSURE_CGROUP_TMPDIR}/kubelet.slice" + export KUBELET_SERVICE_DROPIN_DIR="${ENSURE_CGROUP_TMPDIR}/kubelet.service.d" + : > "${CGROUPV2_MARKER_PATH}" # cgroupv2 present by default + } + cleanup_paths() { + [ -n "${ENSURE_CGROUP_TMPDIR:-}" ] && rm -rf "${ENSURE_CGROUP_TMPDIR}" + } + + It 'is a no-op when neither cgroup is configured' + KUBE_RESERVED_CGROUP="" + SYSTEM_RESERVED_CGROUP="" + When call ensureKubeletCgroupHierarchy + The status should be success + The output should equal "" + End + + It 'rejects unsupported KUBE_RESERVED_CGROUP values' + setup_paths + KUBE_RESERVED_CGROUP="/custom.slice" + SYSTEM_RESERVED_CGROUP="" + When call ensureKubeletCgroupHierarchy + cleanup_paths + The status should be failure + The output should include "unsupported KUBE_RESERVED_CGROUP=/custom.slice" + End + + It 'rejects unsupported SYSTEM_RESERVED_CGROUP values' + setup_paths + KUBE_RESERVED_CGROUP="" + SYSTEM_RESERVED_CGROUP="/custom.slice" + When call ensureKubeletCgroupHierarchy + cleanup_paths + The status should be failure + The output should include "unsupported SYSTEM_RESERVED_CGROUP=/custom.slice" + End + + It 'fails when the cgroupv2 unified hierarchy is not detected' + setup_paths + rm -f "${CGROUPV2_MARKER_PATH}" + KUBE_RESERVED_CGROUP="/kubelet.slice" + SYSTEM_RESERVED_CGROUP="" + When call ensureKubeletCgroupHierarchy + cleanup_paths + The status should be failure + The output should include "cgroupv2 unified hierarchy not detected" + End + + It 'creates kubelet.slice and the kubelet.service drop-in for /kubelet.slice' + setup_paths + systemctl() { return 0; } + KUBE_RESERVED_CGROUP="/kubelet.slice" + SYSTEM_RESERVED_CGROUP="/system.slice" + When call ensureKubeletCgroupHierarchy + slice_contents=$(cat "${KUBELET_SLICE_UNIT_PATH}" 2>/dev/null) + dropin_contents=$(cat "${KUBELET_SERVICE_DROPIN_DIR}/10-kubelet-slice.conf" 2>/dev/null) + cleanup_paths + The status should be success + The variable slice_contents should include "Description=Slice for kubelet kube-reserved enforcement" + The variable slice_contents should include "WantedBy=slices.target" + The variable dropin_contents should include "Wants=kubelet.slice" + The variable dropin_contents should include "After=kubelet.slice" + End + + It 'returns failure when systemctl enable kubelet.slice fails' + setup_paths + systemctl() { + if [ "$1" = "enable" ]; then + return 1 + fi + return 0 + } + KUBE_RESERVED_CGROUP="/kubelet.slice" + SYSTEM_RESERVED_CGROUP="" + When call ensureKubeletCgroupHierarchy + cleanup_paths + The status should be failure + The output should include "failed to enable kubelet.slice" + End + + It 'returns failure when systemctl start kubelet.slice fails' + setup_paths + systemctl() { + if [ "$1" = "start" ]; then + return 1 + fi + return 0 + } + KUBE_RESERVED_CGROUP="/kubelet.slice" + SYSTEM_RESERVED_CGROUP="" + When call ensureKubeletCgroupHierarchy + cleanup_paths + The status should be failure + The output should include "failed to start kubelet.slice" + End + End End diff --git a/spec/parts/linux/cloud-init/artifacts/kubelet_mocks/config_file/node_hardening_enabled.json b/spec/parts/linux/cloud-init/artifacts/kubelet_mocks/config_file/node_hardening_enabled.json new file mode 100644 index 00000000000..167d355d10b --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/kubelet_mocks/config_file/node_hardening_enabled.json @@ -0,0 +1,6 @@ +{ + "kind": "KubeletConfiguration", + "apiVersion": "kubelet.config.k8s.io/v1beta1", + "kubeReservedCgroup": "/kubelet.slice", + "systemReservedCgroup": "/system.slice" +}