diff --git a/PROJECT b/PROJECT index b7b0e02..2c2e1ed 100644 --- a/PROJECT +++ b/PROJECT @@ -22,4 +22,11 @@ resources: kind: Node path: k8s.io/api/core/v1 version: v1 +- api: + crdVersion: v1 + controller: true + domain: readiness.node.x-k8s.io + kind: NodeReadinessRuleReport + path: sigs.k8s.io/node-readiness-controller/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/api/v1alpha1/nodereadinessrulereport_types.go b/api/v1alpha1/nodereadinessrulereport_types.go new file mode 100644 index 0000000..6184121 --- /dev/null +++ b/api/v1alpha1/nodereadinessrulereport_types.go @@ -0,0 +1,194 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// RuleStatus defines the result of evaluating a NodeReadinessRule's criteria against a Node. +// +kubebuilder:validation:Enum=Matched;Unmatched;Error +type RuleStatus string + +const ( + // RuleStatusMatched indicates that the Node successfully met all criteria + // (both NodeSelector and Conditions) defined in the NodeReadinessRule. + // When in this state, the controller should ensure the corresponding Taint is applied. + RuleStatusMatched RuleStatus = "Matched" + + // RuleStatusUnmatched indicates that the Node did not meet the criteria + // defined in the NodeReadinessRule (e.g., label mismatch or condition not satisfied). + // When in this state, the controller should ensure the corresponding Taint is absent. + RuleStatusUnmatched RuleStatus = "Unmatched" + + // RuleStatusError indicates that a programmatic or configuration error occurred + // during the evaluation process (e.g., an invalid or unparseable NodeSelector). + // The controller cannot safely determine if the taint should be present or absent. + RuleStatusError RuleStatus = "Error" +) + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster,shortName=nrrp +// +kubebuilder:printcolumn:name="Node",type=string,JSONPath=`.spec.nodeName`,description="The Node this report belongs to." +// +kubebuilder:printcolumn:name="Matched Rules",type=integer,JSONPath=`.status.summary.matchedRules`,description="Number of rules matching this node." +// +kubebuilder:printcolumn:name="UnMatched Rules",type=integer,JSONPath=`.status.summary.unMatchedRules`,description="Number of rules not matching this node." +// +kubebuilder:printcolumn:name="Applied Taints",type=integer,JSONPath=`.status.summary.appliedTaints`,description="Number of taints currently applied." +// +kubebuilder:printcolumn:name="Errors",type=integer,JSONPath=`.status.summary.errors`,description="Number of evaluation errors." +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// NodeReadinessRuleReport is the Schema for the nodereadinessrulereports API. +type NodeReadinessRuleReport struct { + metav1.TypeMeta `json:",inline"` + + // metadata is a standard object metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitzero"` + + // spec defines the desired state of NodeReadinessRuleReport + // +required + Spec NodeReadinessRuleReportSpec `json:"spec,omitzero"` + + // status defines the observed state of NodeReadinessRuleReport + // +optional + Status NodeReadinessRuleReportStatus `json:"status,omitzero"` +} + +// NodeReadinessRuleReportSpec defines the desired state of NodeReadinessRuleReport. +type NodeReadinessRuleReportSpec struct { + // nodeName specifies the exact name of the target Kubernetes Node. + // This object establishes a strict 1:1 relationship with the specified node, + // acting as the single source of truth for all rules and statuses applied to it. + // Because it binds this resource to a specific physical or virtual machine, it cannot be changed once set. + // + // The validation constraints enforce standard Kubernetes resource naming + // (RFC 1123 DNS Subdomain format), as defined in upstream apimachinery: + // https://github.com/kubernetes/apimachinery/blob/master/pkg/util/validation/validation.go#L198 + // + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="nodeName is immutable and cannot be changed once set" + NodeName string `json:"nodeName,omitempty"` +} + +// NodeReadinessRuleReportStatus defines the observed state of NodeReadinessRuleReport. +// +kubebuilder:validation:MinProperties=1 +type NodeReadinessRuleReportStatus struct { + // readinessReports provides detailed insight into the rule's assessment for individual Nodes. + // This is primarily used for auditing and debugging why specific Nodes were or + // were not targeted by the rule. + // + // +optional + // +listType=map + // +listMapKey=ruleName + // +kubebuilder:validation:MaxItems=100 + ReadinessReports []ReadinessReport `json:"readinessReports,omitempty"` + + // summary provides a quick overview of the rules applied to this node. + // + // +optional + Summary ReportSummary `json:"summary,omitempty,omitzero"` +} + +// ReadinessReport defines the outcome of evaluating a single NodeReadinessRule against a specific Node. +type ReadinessReport struct { + // ruleName is the name of the NodeReadinessRule being evaluated. + // It acts as a direct reference to the NodeReadinessRule that generated this report entry. + // + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + RuleName string `json:"ruleName,omitempty"` + + // reason contains a concise, machine-readable string detailing the primary outcome + // of the evaluation (e.g., "SelectorMismatch", "CriteriaMet", "ConditionNotFound"). + // + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=256 + Reason string `json:"reason,omitempty"` + + // message is a comprehensive, human-readable explanation providing further + // context about the evaluation result or any specific errors encountered. + // + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=10240 + Message string `json:"message,omitempty"` + + // ruleStatus indicates the overall outcome of the rule's criteria against the Node. + // This reflects whether the Node successfully matched the rule's NodeSelector and Conditions. + // + // +required + RuleStatus RuleStatus `json:"ruleStatus,omitempty"` + + // taintStatus reflects the observed state of the rule's specified taint on the Node. + // It indicates whether the taint is currently Present or Absent. + // + // +required + TaintStatus TaintStatus `json:"taintStatus,omitempty"` + + // lastEvaluationTime records the exact moment the controller most recently + // assessed this rule against the Node. This helps identify stale reports. + // + // +required + LastEvaluationTime metav1.Time `json:"lastEvaluationTime,omitempty,omitzero"` +} + +// ReportSummary aggregates the results to provide a high-level overview. +// +kubebuilder:validation:MinProperties=1 +type ReportSummary struct { + // matchedRules is the total number of rules currently matching this node. + // + // +optional + // +kubebuilder:validation:Minimum=0 + MatchedRules *int32 `json:"matchedRules,omitempty"` + + // unMatchedRules is the total number of rules currently not matching this node. + // + // +optional + // +kubebuilder:validation:Minimum=0 + UnMatchedRules *int32 `json:"unMatchedRules,omitempty"` + + // appliedTaints is the total number of taints successfully applied by the controller. + // + // +optional + // +kubebuilder:validation:Minimum=0 + AppliedTaints *int32 `json:"appliedTaints,omitempty"` + + // errors is the total number of rules that failed to evaluate properly. + // + // +optional + // +kubebuilder:validation:Minimum=0 + Errors *int32 `json:"errors,omitempty"` +} + +// +kubebuilder:object:root=true + +// NodeReadinessRuleReportList contains a list of NodeReadinessRuleReport. +type NodeReadinessRuleReportList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitzero"` + Items []NodeReadinessRuleReport `json:"items"` +} + +func init() { + objectTypes = append(objectTypes, &NodeReadinessRuleReport{}, &NodeReadinessRuleReportList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index e8c4e61..bf7c0c6 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -185,6 +185,103 @@ func (in *NodeReadinessRuleList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeReadinessRuleReport) DeepCopyInto(out *NodeReadinessRuleReport) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeReadinessRuleReport. +func (in *NodeReadinessRuleReport) DeepCopy() *NodeReadinessRuleReport { + if in == nil { + return nil + } + out := new(NodeReadinessRuleReport) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NodeReadinessRuleReport) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeReadinessRuleReportList) DeepCopyInto(out *NodeReadinessRuleReportList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NodeReadinessRuleReport, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeReadinessRuleReportList. +func (in *NodeReadinessRuleReportList) DeepCopy() *NodeReadinessRuleReportList { + if in == nil { + return nil + } + out := new(NodeReadinessRuleReportList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NodeReadinessRuleReportList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeReadinessRuleReportSpec) DeepCopyInto(out *NodeReadinessRuleReportSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeReadinessRuleReportSpec. +func (in *NodeReadinessRuleReportSpec) DeepCopy() *NodeReadinessRuleReportSpec { + if in == nil { + return nil + } + out := new(NodeReadinessRuleReportSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeReadinessRuleReportStatus) DeepCopyInto(out *NodeReadinessRuleReportStatus) { + *out = *in + if in.ReadinessReports != nil { + in, out := &in.ReadinessReports, &out.ReadinessReports + *out = make([]ReadinessReport, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Summary.DeepCopyInto(&out.Summary) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeReadinessRuleReportStatus. +func (in *NodeReadinessRuleReportStatus) DeepCopy() *NodeReadinessRuleReportStatus { + if in == nil { + return nil + } + out := new(NodeReadinessRuleReportStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeReadinessRuleSpec) DeepCopyInto(out *NodeReadinessRuleSpec) { *out = *in @@ -241,3 +338,54 @@ func (in *NodeReadinessRuleStatus) DeepCopy() *NodeReadinessRuleStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReadinessReport) DeepCopyInto(out *ReadinessReport) { + *out = *in + in.LastEvaluationTime.DeepCopyInto(&out.LastEvaluationTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReadinessReport. +func (in *ReadinessReport) DeepCopy() *ReadinessReport { + if in == nil { + return nil + } + out := new(ReadinessReport) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReportSummary) DeepCopyInto(out *ReportSummary) { + *out = *in + if in.MatchedRules != nil { + in, out := &in.MatchedRules, &out.MatchedRules + *out = new(int32) + **out = **in + } + if in.UnMatchedRules != nil { + in, out := &in.UnMatchedRules, &out.UnMatchedRules + *out = new(int32) + **out = **in + } + if in.AppliedTaints != nil { + in, out := &in.AppliedTaints, &out.AppliedTaints + *out = new(int32) + **out = **in + } + if in.Errors != nil { + in, out := &in.Errors, &out.Errors + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReportSummary. +func (in *ReportSummary) DeepCopy() *ReportSummary { + if in == nil { + return nil + } + out := new(ReportSummary) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/main.go b/cmd/main.go index 34652e9..9686b28 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -22,16 +22,13 @@ import ( "net/http" "os" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. "go.uber.org/zap/zapcore" - _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/client-go/rest" - "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -52,7 +49,6 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(nodereadinessiov1alpha1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -136,17 +132,31 @@ func main() { Controller: readinessController, } - // Setup controllers with manager + ruleReportReconciler := &controller.NodeReadinessRuleReportReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Controller: readinessController, + } + + // Setup Rule Reconciler. ctx := ctrl.SetupSignalHandler() if err := ruleReconciler.SetupWithManager(ctx, mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NodeReadinessRule") os.Exit(1) } + + // Setup Node Reconciler if err := nodeReconciler.SetupWithManager(ctx, mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Node") os.Exit(1) } + // Setup Report Reconciler + if err := ruleReportReconciler.SetupWithManager(ctx, mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NodeReadinessRuleReport") + os.Exit(1) + } + // Setup webhook (conditional based on flag) if enableWebhook { nodeReadinessWebhook := webhook.NewNodeReadinessRuleWebhook(mgr.GetClient()) @@ -158,6 +168,7 @@ func main() { } else { setupLog.Info("webhook disabled") } + // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/config/crd/bases/readiness.node.x-k8s.io_nodereadinessrulereports.yaml b/config/crd/bases/readiness.node.x-k8s.io_nodereadinessrulereports.yaml new file mode 100644 index 0000000..09d42e5 --- /dev/null +++ b/config/crd/bases/readiness.node.x-k8s.io_nodereadinessrulereports.yaml @@ -0,0 +1,195 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: nodereadinessrulereports.readiness.node.x-k8s.io +spec: + group: readiness.node.x-k8s.io + names: + kind: NodeReadinessRuleReport + listKind: NodeReadinessRuleReportList + plural: nodereadinessrulereports + shortNames: + - nrrp + singular: nodereadinessrulereport + scope: Cluster + versions: + - additionalPrinterColumns: + - description: The Node this report belongs to. + jsonPath: .spec.nodeName + name: Node + type: string + - description: Number of rules matching this node. + jsonPath: .status.summary.matchedRules + name: Matched Rules + type: integer + - description: Number of rules not matching this node. + jsonPath: .status.summary.unMatchedRules + name: UnMatched Rules + type: integer + - description: Number of taints currently applied. + jsonPath: .status.summary.appliedTaints + name: Applied Taints + type: integer + - description: Number of evaluation errors. + jsonPath: .status.summary.errors + name: Errors + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeReadinessRuleReport is the Schema for the nodereadinessrulereports + API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of NodeReadinessRuleReport + properties: + nodeName: + description: |- + nodeName specifies the exact name of the target Kubernetes Node. + This object establishes a strict 1:1 relationship with the specified node, + acting as the single source of truth for all rules and statuses applied to it. + Because it binds this resource to a specific physical or virtual machine, it cannot be changed once set. + + The validation constraints enforce standard Kubernetes resource naming + (RFC 1123 DNS Subdomain format), as defined in upstream apimachinery: + https://github.com/kubernetes/apimachinery/blob/master/pkg/util/validation/validation.go#L198 + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + x-kubernetes-validations: + - message: nodeName is immutable and cannot be changed once set + rule: self == oldSelf + required: + - nodeName + type: object + status: + description: status defines the observed state of NodeReadinessRuleReport + minProperties: 1 + properties: + readinessReports: + description: |- + readinessReports provides detailed insight into the rule's assessment for individual Nodes. + This is primarily used for auditing and debugging why specific Nodes were or + were not targeted by the rule. + items: + description: ReadinessReport defines the outcome of evaluating a + single NodeReadinessRule against a specific Node. + properties: + lastEvaluationTime: + description: |- + lastEvaluationTime records the exact moment the controller most recently + assessed this rule against the Node. This helps identify stale reports. + format: date-time + type: string + message: + description: |- + message is a comprehensive, human-readable explanation providing further + context about the evaluation result or any specific errors encountered. + maxLength: 10240 + minLength: 1 + type: string + reason: + description: |- + reason contains a concise, machine-readable string detailing the primary outcome + of the evaluation (e.g., "SelectorMismatch", "CriteriaMet", "ConditionNotFound"). + maxLength: 256 + minLength: 1 + type: string + ruleName: + description: |- + ruleName is the name of the NodeReadinessRule being evaluated. + It acts as a direct reference to the NodeReadinessRule that generated this report entry. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + ruleStatus: + description: |- + ruleStatus indicates the overall outcome of the rule's criteria against the Node. + This reflects whether the Node successfully matched the rule's NodeSelector and Conditions. + enum: + - Matched + - Unmatched + - Error + type: string + taintStatus: + description: |- + taintStatus reflects the observed state of the rule's specified taint on the Node. + It indicates whether the taint is currently Present or Absent. + enum: + - Present + - Absent + type: string + required: + - lastEvaluationTime + - ruleName + - ruleStatus + - taintStatus + type: object + maxItems: 100 + type: array + x-kubernetes-list-map-keys: + - ruleName + x-kubernetes-list-type: map + summary: + description: summary provides a quick overview of the rules applied + to this node. + minProperties: 1 + properties: + appliedTaints: + description: appliedTaints is the total number of taints successfully + applied by the controller. + format: int32 + minimum: 0 + type: integer + errors: + description: errors is the total number of rules that failed to + evaluate properly. + format: int32 + minimum: 0 + type: integer + matchedRules: + description: matchedRules is the total number of rules currently + matching this node. + format: int32 + minimum: 0 + type: integer + unMatchedRules: + description: unMatchedRules is the total number of rules currently + not matching this node. + format: int32 + minimum: 0 + type: integer + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 79ab98e..707a4df 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,4 +3,5 @@ # It should be run by config/default resources: - bases/readiness.node.x-k8s.io_nodereadinessrules.yaml +- bases/readiness.node.x-k8s.io_nodereadinessrulereports.yaml # +kubebuilder:scaffold:crdkustomizeresource diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 6fbbd3a..da80120 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -22,6 +22,11 @@ resources: # default, aiding admins in cluster management. Those roles are # not used by the nrrcontroller itself. You can comment the following lines # if you do not want those helpers be installed with your Project. +- nodereadinessrulereport_admin_role.yaml +- nodereadinessrulereport_editor_role.yaml +- nodereadinessrulereport_viewer_role.yaml - nodereadinessrule_admin_role.yaml - nodereadinessrule_editor_role.yaml - nodereadinessrule_viewer_role.yaml + + diff --git a/config/rbac/nodereadinessrulereport_admin_role.yaml b/config/rbac/nodereadinessrulereport_admin_role.yaml new file mode 100644 index 0000000..1be71dd --- /dev/null +++ b/config/rbac/nodereadinessrulereport_admin_role.yaml @@ -0,0 +1,27 @@ +# This rule is not used by the project nrrcontroller itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants full permissions ('*') over readiness.node.x-k8s.io. +# This role is intended for users authorized to modify roles and bindings within the cluster, +# enabling them to delegate specific permissions to other users or groups as needed. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nrrcontroller + app.kubernetes.io/managed-by: kustomize + name: nodereadinessrulereport-admin-role +rules: +- apiGroups: + - readiness.node.x-k8s.io + resources: + - nodereadinessrulereports + verbs: + - '*' +- apiGroups: + - readiness.node.x-k8s.io + resources: + - nodereadinessrulereports/status + verbs: + - get diff --git a/config/rbac/nodereadinessrulereport_editor_role.yaml b/config/rbac/nodereadinessrulereport_editor_role.yaml new file mode 100644 index 0000000..370ed84 --- /dev/null +++ b/config/rbac/nodereadinessrulereport_editor_role.yaml @@ -0,0 +1,33 @@ +# This rule is not used by the project nrrcontroller itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants permissions to create, update, and delete resources within the readiness.node.x-k8s.io. +# This role is intended for users who need to manage these resources +# but should not control RBAC or manage permissions for others. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nrrcontroller + app.kubernetes.io/managed-by: kustomize + name: nodereadinessrulereport-editor-role +rules: +- apiGroups: + - readiness.node.x-k8s.io + resources: + - nodereadinessrulereports + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - readiness.node.x-k8s.io + resources: + - nodereadinessrulereports/status + verbs: + - get diff --git a/config/rbac/nodereadinessrulereport_viewer_role.yaml b/config/rbac/nodereadinessrulereport_viewer_role.yaml new file mode 100644 index 0000000..96b6248 --- /dev/null +++ b/config/rbac/nodereadinessrulereport_viewer_role.yaml @@ -0,0 +1,29 @@ +# This rule is not used by the project nrrcontroller itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants read-only access to readiness.node.x-k8s.io resources. +# This role is intended for users who need visibility into these resources +# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nrrcontroller + app.kubernetes.io/managed-by: kustomize + name: nodereadinessrulereport-viewer-role +rules: +- apiGroups: + - readiness.node.x-k8s.io + resources: + - nodereadinessrulereports + verbs: + - get + - list + - watch +- apiGroups: + - readiness.node.x-k8s.io + resources: + - nodereadinessrulereports/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 0d36c19..ebd1cef 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,13 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - "" resources: @@ -33,6 +40,7 @@ rules: - apiGroups: - readiness.node.x-k8s.io resources: + - nodereadinessrulereports - nodereadinessrules verbs: - create @@ -45,12 +53,14 @@ rules: - apiGroups: - readiness.node.x-k8s.io resources: + - nodereadinessrulereports/finalizers - nodereadinessrules/finalizers verbs: - update - apiGroups: - readiness.node.x-k8s.io resources: + - nodereadinessrulereports/status - nodereadinessrules/status verbs: - get diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 24d8f2d..5430f77 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -1,4 +1,5 @@ ## Append samples of your project ## resources: - v1alpha1_nodereadinessrule.yaml +- v1alpha1_nodereadinessrulereport.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/v1alpha1_nodereadinessrulereport.yaml b/config/samples/v1alpha1_nodereadinessrulereport.yaml new file mode 100644 index 0000000..e7b67d6 --- /dev/null +++ b/config/samples/v1alpha1_nodereadinessrulereport.yaml @@ -0,0 +1,9 @@ +apiVersion: readiness.node.x-k8s.io/v1alpha1 +kind: NodeReadinessRuleReport +metadata: + labels: + app.kubernetes.io/name: nrrcontroller + app.kubernetes.io/managed-by: kustomize + name: nodereadinessrulereport-sample +spec: + # TODO(user): Add fields here diff --git a/go.mod b/go.mod index e628a39..eb25381 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( k8s.io/api v0.34.0 k8s.io/apimachinery v0.34.0 k8s.io/client-go v0.34.0 + k8s.io/klog/v2 v2.130.1 sigs.k8s.io/controller-runtime v0.22.1 ) @@ -91,7 +92,6 @@ require ( k8s.io/apiextensions-apiserver v0.34.0 // indirect k8s.io/apiserver v0.34.0 // indirect k8s.io/component-base v0.34.0 // indirect - k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect diff --git a/internal/controller/nodereadinessrulereport_controller.go b/internal/controller/nodereadinessrulereport_controller.go new file mode 100644 index 0000000..8e230fb --- /dev/null +++ b/internal/controller/nodereadinessrulereport_controller.go @@ -0,0 +1,272 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "sort" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + readinessv1alpha1 "sigs.k8s.io/node-readiness-controller/api/v1alpha1" +) + +// NodeReadinessRuleReportReconciler reconciles a NodeReadinessRuleReport object. +type NodeReadinessRuleReportReconciler struct { + client.Client + Scheme *runtime.Scheme + + Controller *RuleReadinessController +} + +func (r *NodeReadinessRuleReportReconciler) SetupWithManager(_ context.Context, mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + Named("nodereadinessrulereport"). + For(&corev1.Node{}). + Owns(&readinessv1alpha1.NodeReadinessRuleReport{}). + Watches( + &readinessv1alpha1.NodeReadinessRule{}, + handler.EnqueueRequestsFromMapFunc(r.mapRuleToNodes), + ). + Complete(r) +} + +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch +// +kubebuilder:rbac:groups=readiness.node.x-k8s.io,resources=nodereadinessrules,verbs=get;list;watch +// +kubebuilder:rbac:groups=readiness.node.x-k8s.io,resources=nodereadinessrulereports,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=readiness.node.x-k8s.io,resources=nodereadinessrulereports/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=readiness.node.x-k8s.io,resources=nodereadinessrulereports/finalizers,verbs=update + +func (r *NodeReadinessRuleReportReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + + // 1. Fetch the Target Node + var node corev1.Node + if err := r.Get(ctx, req.NamespacedName, &node); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + log = log.WithValues("node", node.Name) + ctx = ctrl.LoggerInto(ctx, log) + log.Info("Generating NodeReadinessRules report") + + // 2. Fetch all NodeReadinessRules + var ruleList readinessv1alpha1.NodeReadinessRuleList + if err := r.List(ctx, &ruleList); err != nil { + log.Error(err, "Failed to list NodeReadinessRules") + return ctrl.Result{}, err + } + + // 3. Evaluate all rules against this Node + readinessReports := make([]readinessv1alpha1.ReadinessReport, 0, len(ruleList.Items)) + for _, rule := range ruleList.Items { + log.Info("Evaluating rule", "ruleName", rule.Name) + report := r.evaluateNode(ctx, node, rule) + log.Info("Rule report", "ruleName", rule.Name, "report", report) + readinessReports = append(readinessReports, report) + } + + sort.SliceStable(readinessReports, func(i, j int) bool { + return readinessReports[i].RuleName < readinessReports[j].RuleName + }) + + // 4. Calculate the Summary metrics + var matchedRules, unMatchedRules, appliedTaints, evalErrors int32 + + for _, result := range readinessReports { + if result.RuleStatus == readinessv1alpha1.RuleStatusMatched { + matchedRules++ + } + + if result.RuleStatus == readinessv1alpha1.RuleStatusUnmatched { + unMatchedRules++ + } + + if result.TaintStatus == readinessv1alpha1.TaintStatusPresent { + appliedTaints++ + } + + if result.RuleStatus == readinessv1alpha1.RuleStatusError { + evalErrors++ + } + } + + // Create the summary object + reportSummary := readinessv1alpha1.ReportSummary{ + MatchedRules: &matchedRules, + UnMatchedRules: &unMatchedRules, + AppliedTaints: &appliedTaints, + Errors: &evalErrors, + } + log.V(5).Info("Rules summary report", "summary", reportSummary) + // 5. Initialize the NodeReadinessRuleReport Object. + report := &readinessv1alpha1.NodeReadinessRuleReport{ + ObjectMeta: metav1.ObjectMeta{ + Name: getNodeReadinessRuleReportName(node.Name), + }, + } + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, report, func() error { + if err := controllerutil.SetControllerReference(&node, report, r.Scheme); err != nil { + return err + } + report.Spec.NodeName = node.Name + return nil + }) + + if err != nil { + log.Error(err, "Failed to create or update NodeReadinessReport Spec") + return ctrl.Result{}, err + } + + oldStatus := report.Status.DeepCopy() + + report.Status.ReadinessReports = readinessReports + report.Status.Summary = reportSummary + + if !equality.Semantic.DeepEqual(oldStatus, &report.Status) { + if err := r.Status().Update(ctx, report); err != nil { + log.Error(err, "Failed to update NodeReadinessReport Status") + return ctrl.Result{}, err + } + log.Info("Successfully updated NodeReadinessReport Status") + } + + if op != controllerutil.OperationResultNone { + log.Info("Successfully reconciled NodeReadinessReport Spec", "operation", op) + } + + return ctrl.Result{}, nil +} + +// evaluateNode checks a single Node against a single NodeReadinessRule +// and returns a populated NodeReadinessRuleReport for the NodeReadinessRule. +func (r *NodeReadinessRuleReportReconciler) evaluateNode(ctx context.Context, node corev1.Node, rule readinessv1alpha1.NodeReadinessRule) readinessv1alpha1.ReadinessReport { + log := ctrl.LoggerFrom(ctx) + + result := readinessv1alpha1.ReadinessReport{ + RuleName: rule.Name, + RuleStatus: readinessv1alpha1.RuleStatusUnmatched, + TaintStatus: readinessv1alpha1.TaintStatusAbsent, + LastEvaluationTime: metav1.Now(), + } + + // 1. Evaluate the NodeSelector + selector, err := metav1.LabelSelectorAsSelector(&rule.Spec.NodeSelector) + if err != nil { + log.Error(err, "Failed to convert node selector") + result.Reason = "InvalidSelector" + result.Message = fmt.Sprintf("Failed to parse NodeSelector: %v", err) + result.RuleStatus = readinessv1alpha1.RuleStatusError + return result + } + + if !selector.Matches(labels.Set(node.Labels)) { + result.Reason = "SelectorMismatch" + result.Message = "Node labels do not match the rule's NodeSelector." + return result + } + + // 2. Check if the Taint is currently applied on the Node + if hasTaint(&node, &rule.Spec.Taint) { + result.TaintStatus = readinessv1alpha1.TaintStatusPresent + } + + // 3. Evaluate the Conditions + for _, req := range rule.Spec.Conditions { + conditionFound := false + + for _, nodeCond := range node.Status.Conditions { + if string(nodeCond.Type) == req.Type { + conditionFound = true + if nodeCond.Status != req.RequiredStatus { + result.Reason = "ConditionStatusMismatch" + result.Message = fmt.Sprintf("Condition '%s' is '%s', required '%s'.", req.Type, nodeCond.Status, req.RequiredStatus) + return result + } + break // Found the condition and it matched, move to the next requirement + } + } + + if !conditionFound { + log.Info("Condition not found", "type", req.Type) + result.Reason = "ConditionNotFound" + result.Message = fmt.Sprintf("Required condition '%s' was not found on the Node.", req.Type) + return result + } + } + + // If we reach this point, the Node matched BOTH the selector and all conditions. + result.Reason = "CriteriaMet" + result.Message = "Node successfully matches all rule criteria." + result.RuleStatus = readinessv1alpha1.RuleStatusMatched + + return result +} + +// mapRuleToNodes is triggered whenever a NodeReadinessRule is Created, Updated, or Deleted. +// It queues a Reconcile request for every Node in the cluster. +func (r *NodeReadinessRuleReportReconciler) mapRuleToNodes(ctx context.Context, obj client.Object) []reconcile.Request { + log := ctrl.LoggerFrom(ctx) + + var nodeList corev1.NodeList + if err := r.List(ctx, &nodeList); err != nil { + log.Error(err, "Failed to list nodes in mapRuleToNodes") + return nil + } + + requests := make([]reconcile.Request, 0, len(nodeList.Items)) + for _, node := range nodeList.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: node.Name, + }, + }) + } + + return requests +} + +// hasTaint is a small helper to check if a specific taint exists on a Node. +func hasTaint(node *corev1.Node, targetTaint *corev1.Taint) bool { + for _, t := range node.Spec.Taints { + if t.Key == targetTaint.Key && t.Value == targetTaint.Value && t.Effect == targetTaint.Effect { + return true + } + } + return false +} + +func getNodeReadinessRuleReportName(nodeName string) string { + return fmt.Sprintf("nrr-report-%s", nodeName) +} diff --git a/internal/controller/nodereadinessrulereport_controller_test.go b/internal/controller/nodereadinessrulereport_controller_test.go new file mode 100644 index 0000000..2962dba --- /dev/null +++ b/internal/controller/nodereadinessrulereport_controller_test.go @@ -0,0 +1,496 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + readinessv1alpha1 "sigs.k8s.io/node-readiness-controller/api/v1alpha1" +) + +var _ = Describe("NodeReadinessRuleReport Controller", func() { + var ( + ctx context.Context + reconciler *NodeReadinessRuleReportReconciler + ) + + BeforeEach(func() { + ctx = context.Background() + + // Clean up Nodes + var nodeList corev1.NodeList + Expect(k8sClient.List(ctx, &nodeList)).To(Succeed()) + for _, obj := range nodeList.Items { + if len(obj.Finalizers) > 0 { + obj.Finalizers = nil + _ = k8sClient.Update(ctx, &obj) + } + _ = k8sClient.Delete(ctx, &obj) + } + + // Clean up Rules + var ruleList readinessv1alpha1.NodeReadinessRuleList + Expect(k8sClient.List(ctx, &ruleList)).To(Succeed()) + for _, obj := range ruleList.Items { + if len(obj.Finalizers) > 0 { + obj.Finalizers = nil + _ = k8sClient.Update(ctx, &obj) + } + _ = k8sClient.Delete(ctx, &obj) + } + + // Clean up Reports + var reportList readinessv1alpha1.NodeReadinessRuleReportList + Expect(k8sClient.List(ctx, &reportList)).To(Succeed()) + for _, obj := range reportList.Items { + _ = k8sClient.Delete(ctx, &obj) + } + + // Wait for the API server to actually finish purging the objects + Eventually(func() bool { + var currentRules readinessv1alpha1.NodeReadinessRuleList + var currentNodes corev1.NodeList + var currentReports readinessv1alpha1.NodeReadinessRuleReportList + + _ = k8sClient.List(ctx, ¤tRules) + _ = k8sClient.List(ctx, ¤tNodes) + _ = k8sClient.List(ctx, ¤tReports) + + return len(currentRules.Items) == 0 && + len(currentNodes.Items) == 0 && + len(currentReports.Items) == 0 + }, time.Second*5, time.Millisecond*100).Should(BeTrue(), "Failed to clean up prior test resources") + + // Initialize the reconciler for the fresh, clean test + reconciler = &NodeReadinessRuleReportReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + } + }) + + AfterEach(func() { + // 1. Clean up all NodeReadinessRuleReports + var reportList readinessv1alpha1.NodeReadinessRuleReportList + Expect(k8sClient.List(ctx, &reportList)).To(Succeed()) + for _, report := range reportList.Items { + _ = k8sClient.Delete(ctx, &report) + } + + // 2. Clean up all NodeReadinessRules + var ruleList readinessv1alpha1.NodeReadinessRuleList + Expect(k8sClient.List(ctx, &ruleList)).To(Succeed()) + for _, rule := range ruleList.Items { + if len(rule.Finalizers) > 0 { + rule.Finalizers = nil + _ = k8sClient.Update(ctx, &rule) + } + _ = k8sClient.Delete(ctx, &rule) + } + + // 3. Clean up all Nodes + var nodeList corev1.NodeList + Expect(k8sClient.List(ctx, &nodeList)).To(Succeed()) + for _, node := range nodeList.Items { + if len(node.Finalizers) > 0 { + node.Finalizers = nil + _ = k8sClient.Update(ctx, &node) + } + _ = k8sClient.Delete(ctx, &node) + } + }) + + Context("When reconciling a Node", func() { + It("Should create a Report with no matched rules if no rules exist", func() { + nodeName := "test-node-empty" + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + } + Expect(k8sClient.Create(ctx, node)).Should(Succeed()) + + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: nodeName}} + _, err := reconciler.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + + reportLookupKey := types.NamespacedName{Name: getNodeReadinessRuleReportName(nodeName)} + createdReport := &readinessv1alpha1.NodeReadinessRuleReport{} + + Eventually(func() error { + return k8sClient.Get(ctx, reportLookupKey, createdReport) + }, time.Second*2, time.Millisecond*50).Should(Succeed()) + + Expect(createdReport.Spec.NodeName).Should(Equal(nodeName)) + Expect(createdReport.Status.ReadinessReports).Should(BeEmpty()) + Expect(*createdReport.Status.Summary.MatchedRules).Should(Equal(int32(0))) + }) + + It("Should accurately evaluate a Node against a matching Rule", func() { + ruleName := "test-rule-match" + nodeName := "test-node-match" + + rule := &readinessv1alpha1.NodeReadinessRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: ruleName, + }, + Spec: readinessv1alpha1.NodeReadinessRuleSpec{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"env": "production"}, + }, + Conditions: []readinessv1alpha1.ConditionRequirement{ + {Type: "Ready", RequiredStatus: corev1.ConditionTrue}, + }, + Taint: corev1.Taint{ + Key: "dedicated", + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + }, + EnforcementMode: readinessv1alpha1.EnforcementModeContinuous, + }, + } + Expect(k8sClient.Create(ctx, rule)).Should(Succeed()) + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Labels: map[string]string{"env": "production"}, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + {Key: "dedicated", Value: "true", Effect: corev1.TaintEffectNoSchedule}, + }, + }, + } + Expect(k8sClient.Create(ctx, node)).Should(Succeed()) + + node.Status = corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: "Ready", Status: corev1.ConditionTrue}, + }, + } + Expect(k8sClient.Status().Update(ctx, node)).Should(Succeed()) + + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: nodeName}} + Eventually(func() error { + _, err := reconciler.Reconcile(ctx, req) + if err != nil { + return err + } + + reportLookupKey := types.NamespacedName{Name: getNodeReadinessRuleReportName(nodeName)} + createdReport := &readinessv1alpha1.NodeReadinessRuleReport{} + if err := k8sClient.Get(ctx, reportLookupKey, createdReport); err != nil { + return err + } + if len(createdReport.Status.ReadinessReports) == 0 { + return fmt.Errorf("cache not synced yet") + } + return nil + }, time.Second*5, time.Millisecond*100).Should(Succeed()) + + // Immediate Assertions + reportLookupKey := types.NamespacedName{Name: getNodeReadinessRuleReportName(nodeName)} + createdReport := &readinessv1alpha1.NodeReadinessRuleReport{} + Expect(k8sClient.Get(ctx, reportLookupKey, createdReport)).Should(Succeed()) + + Expect(*createdReport.Status.Summary.MatchedRules).Should(Equal(int32(1))) + Expect(*createdReport.Status.Summary.AppliedTaints).Should(Equal(int32(1))) + + report := createdReport.Status.ReadinessReports[0] + Expect(report.RuleName).Should(Equal(ruleName)) + Expect(report.RuleStatus).Should(Equal(readinessv1alpha1.RuleStatusMatched)) + Expect(report.TaintStatus).Should(Equal(readinessv1alpha1.TaintStatusPresent)) + Expect(report.Reason).Should(Equal("CriteriaMet")) + }) + + It("Should mark a rule as Unmatched if NodeSelector fails", func() { + ruleName := "test-rule-mismatch-label" + nodeName := "test-node-mismatch-label" + + rule := &readinessv1alpha1.NodeReadinessRule{ + ObjectMeta: metav1.ObjectMeta{Name: ruleName}, + Spec: readinessv1alpha1.NodeReadinessRuleSpec{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"env": "production"}, + }, + Conditions: []readinessv1alpha1.ConditionRequirement{ + {Type: "Ready", RequiredStatus: corev1.ConditionTrue}, + }, + Taint: corev1.Taint{ + Key: "dedicated", + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + }, + EnforcementMode: readinessv1alpha1.EnforcementModeContinuous, + }, + } + Expect(k8sClient.Create(ctx, rule)).Should(Succeed()) + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Labels: map[string]string{"env": "staging"}, + }, + } + Expect(k8sClient.Create(ctx, node)).Should(Succeed()) + + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: nodeName}} + Eventually(func() error { + _, err := reconciler.Reconcile(ctx, req) + return err + }, time.Second*2, time.Millisecond*100).Should(Succeed()) + + reportLookupKey := types.NamespacedName{Name: getNodeReadinessRuleReportName(nodeName)} + createdReport := &readinessv1alpha1.NodeReadinessRuleReport{} + + Eventually(func() string { + _ = k8sClient.Get(ctx, reportLookupKey, createdReport) + if len(createdReport.Status.ReadinessReports) > 0 { + return createdReport.Status.ReadinessReports[0].Reason + } + return "" + }, time.Second*2, time.Millisecond*100).Should(Equal("SelectorMismatch")) + }) + + It("Should mark a rule as Unmatched if a required condition status does not match", func() { + ruleName := "rule-condition-mismatch" + nodeName := "node-condition-mismatch" + + rule := &readinessv1alpha1.NodeReadinessRule{ + ObjectMeta: metav1.ObjectMeta{Name: ruleName}, + Spec: readinessv1alpha1.NodeReadinessRuleSpec{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"env": "staging"}, + }, + Conditions: []readinessv1alpha1.ConditionRequirement{ + {Type: "Ready", RequiredStatus: corev1.ConditionTrue}, + }, + Taint: corev1.Taint{ + Key: "dedicated", + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + }, + EnforcementMode: readinessv1alpha1.EnforcementModeContinuous, + }, + } + Expect(k8sClient.Create(ctx, rule)).Should(Succeed()) + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Labels: map[string]string{"env": "staging"}, + }, + } + Expect(k8sClient.Create(ctx, node)).Should(Succeed()) + + node.Status = corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: "Ready", Status: corev1.ConditionFalse}, + }, + } + Expect(k8sClient.Status().Update(ctx, node)).Should(Succeed()) + + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: nodeName}} + Eventually(func() error { + _, err := reconciler.Reconcile(ctx, req) + return err + }, time.Second*2, time.Millisecond*100).Should(Succeed()) + + reportLookupKey := types.NamespacedName{Name: getNodeReadinessRuleReportName(nodeName)} + createdReport := &readinessv1alpha1.NodeReadinessRuleReport{} + + Eventually(func() string { + _ = k8sClient.Get(ctx, reportLookupKey, createdReport) + if len(createdReport.Status.ReadinessReports) > 0 { + return createdReport.Status.ReadinessReports[0].Reason + } + return "" + }, time.Second*2, time.Millisecond*100).Should(Equal("ConditionStatusMismatch")) + }) + + It("Should mark a rule as Unmatched if a required condition is completely missing", func() { + ruleName := "rule-condition-missing" + nodeName := "node-condition-missing" + + rule := &readinessv1alpha1.NodeReadinessRule{ + ObjectMeta: metav1.ObjectMeta{Name: ruleName}, + Spec: readinessv1alpha1.NodeReadinessRuleSpec{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"env": "staging"}, + }, + Conditions: []readinessv1alpha1.ConditionRequirement{ + {Type: "CustomReady", RequiredStatus: corev1.ConditionTrue}, + }, + Taint: corev1.Taint{ + Key: "dedicated", + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + }, + EnforcementMode: readinessv1alpha1.EnforcementModeContinuous, + }, + } + Expect(k8sClient.Create(ctx, rule)).Should(Succeed()) + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Labels: map[string]string{"env": "staging"}, + }, + } + Expect(k8sClient.Create(ctx, node)).Should(Succeed()) + + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: nodeName}} + Eventually(func() error { + _, err := reconciler.Reconcile(ctx, req) + return err + }, time.Second*2, time.Millisecond*100).Should(Succeed()) + + reportLookupKey := types.NamespacedName{Name: getNodeReadinessRuleReportName(nodeName)} + createdReport := &readinessv1alpha1.NodeReadinessRuleReport{} + + Eventually(func() string { + _ = k8sClient.Get(ctx, reportLookupKey, createdReport) + if len(createdReport.Status.ReadinessReports) > 0 { + return createdReport.Status.ReadinessReports[0].Reason + } + return "" + }, time.Second*2, time.Millisecond*100).Should(Equal("ConditionNotFound")) + }) + + It("Should identify when a Node matches the rule but is missing the expected Taint", func() { + ruleName := "rule-taint-absent" + nodeName := "node-taint-absent" + + rule := &readinessv1alpha1.NodeReadinessRule{ + ObjectMeta: metav1.ObjectMeta{Name: ruleName}, + Spec: readinessv1alpha1.NodeReadinessRuleSpec{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"env": "production"}, + }, + Taint: corev1.Taint{ + Key: "special-hardware", Value: "true", Effect: corev1.TaintEffectNoSchedule, + }, + Conditions: []readinessv1alpha1.ConditionRequirement{ + {Type: "Ready", RequiredStatus: corev1.ConditionTrue}, + }, + EnforcementMode: readinessv1alpha1.EnforcementModeContinuous, + }, + } + Expect(k8sClient.Create(ctx, rule)).Should(Succeed()) + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Labels: map[string]string{"env": "staging"}, + }, + } + Expect(k8sClient.Create(ctx, node)).Should(Succeed()) + + node.Status = corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: "Ready", Status: corev1.ConditionTrue}, + }, + } + Expect(k8sClient.Status().Update(ctx, node)).Should(Succeed()) + + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: nodeName}} + Eventually(func() error { + _, err := reconciler.Reconcile(ctx, req) + return err + }, time.Second*2, time.Millisecond*100).Should(Succeed()) + + reportLookupKey := types.NamespacedName{Name: getNodeReadinessRuleReportName(nodeName)} + createdReport := &readinessv1alpha1.NodeReadinessRuleReport{} + + Eventually(func() readinessv1alpha1.TaintStatus { + _ = k8sClient.Get(ctx, reportLookupKey, createdReport) + if len(createdReport.Status.ReadinessReports) > 0 { + return createdReport.Status.ReadinessReports[0].TaintStatus + } + return "" + }, time.Second*2, time.Millisecond*100).Should(Equal(readinessv1alpha1.TaintStatusAbsent)) + }) + + It("Should generate an error report if the Rule has an invalid NodeSelector", func() { + ruleName := "rule-invalid-selector" + nodeName := "node-invalid-selector" + + rule := &readinessv1alpha1.NodeReadinessRule{ + ObjectMeta: metav1.ObjectMeta{Name: ruleName}, + Spec: readinessv1alpha1.NodeReadinessRuleSpec{ + NodeSelector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "env", + Operator: "InvalidOperator", // This will cause parsing to fail + Values: []string{"prod"}, + }, + }, + }, + Conditions: []readinessv1alpha1.ConditionRequirement{ + {Type: "Ready", RequiredStatus: corev1.ConditionTrue}, + }, + Taint: corev1.Taint{ + Key: "dedicated", + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + }, + EnforcementMode: readinessv1alpha1.EnforcementModeContinuous, + }, + } + Expect(k8sClient.Create(ctx, rule)).Should(Succeed()) + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Labels: map[string]string{"env": "staging"}, + }, + } + Expect(k8sClient.Create(ctx, node)).Should(Succeed()) + + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: nodeName}} + Eventually(func() error { + _, err := reconciler.Reconcile(ctx, req) + return err + }, time.Second*2, time.Millisecond*100).Should(Succeed()) + + reportLookupKey := types.NamespacedName{Name: getNodeReadinessRuleReportName(nodeName)} + createdReport := &readinessv1alpha1.NodeReadinessRuleReport{} + + Eventually(func() readinessv1alpha1.RuleStatus { + _ = k8sClient.Get(ctx, reportLookupKey, createdReport) + if len(createdReport.Status.ReadinessReports) > 0 { + return createdReport.Status.ReadinessReports[0].RuleStatus + } + return "" + }, time.Second*2, time.Millisecond*100).Should(Equal(readinessv1alpha1.RuleStatusError)) + + // Verify the error was tallied in the summary + Expect(*createdReport.Status.Summary.Errors).Should(Equal(int32(1))) + }) + }) +})