feat/static-pod-initialization (#125)

examples/staticpods-readiness/README.md

@@ -0,0 +1,39 @@
+# Node Readiness Examples (Static Pods)
+
+This example demonstrates how NRC can be used alongside [NPD (Node Problem Detector)](https://github.com/kubernetes/node-problem-detector) to make sure that all static pods are mirrored in the API server to avoid over-committing issues. See [#115325](https://github.com/kubernetes/kubernetes/issues/115325), [#47264](https://github.com/kubernetes/website/issues/47264), and [#126870](https://github.com/kubernetes/kubernetes/pull/126870) for reference.
+
+## Deployment Steps
+
+1. Deploy a testing cluster. For this example we use Kind with a mounted static pod manifest for testing:
+
+   ```bash
+   kind create cluster --config examples/staticpods-readiness/kind-cluster-config.yaml
+   ```
+
+2. Install NRC:
+
+   ```bash
+   VERSION=v0.2.0
+   kubectl apply -f https://github.com/kubernetes-sigs/node-readiness-controller/releases/download/${VERSION}/crds.yaml
+   kubectl apply -f https://github.com/kubernetes-sigs/node-readiness-controller/releases/download/${VERSION}/install.yaml
+   ```
+3. Taint The node:
+   ```bash
+   kubectl taint node kind-worker readiness.k8s.io/StaticPodsMissing=pending:NoSchedule
+   ```
+4. Deploy NPD as a DaemonSet with the static pods readiness configuration:
+
+   ```bash
+   ./examples/staticpods-readiness/setup-staticpods-readiness.sh
+   ```
+
+   This deploys NPD with:
+   - Init container that downloads `kubectl`, `yq`, and `jq`
+   - Custom plugin that checks if static pods are mirrored
+   - RBAC permissions to read pods and nodes
+
+5. Apply the node readiness rule:
+   ```bash
+      kubectl apply -f examples/staticpods-readiness/staticpods-readiness-rule.yaml
+   ```
+   If the static pods was successfully created, NRC would read the condition added by NPD and the taint should be removed.

examples/staticpods-readiness/kind-cluster-config.yaml

@@ -0,0 +1,10 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+  extraMounts:
+  - hostPath: examples/staticpods-readiness/test-staticpod.yaml
+    containerPath: /etc/kubernetes/manifests/test-staticpod.yaml
+    readOnly: true
+

examples/staticpods-readiness/npd/npd-ds.yaml

@@ -0,0 +1,113 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: node-problem-detector-staticpods
+  namespace: kube-system
+  labels:
+    app: node-problem-detector
+    component: staticpods-monitor
+spec:
+  selector:
+    matchLabels:
+      app: node-problem-detector
+      component: staticpods-monitor
+  template:
+    metadata:
+      labels:
+        app: node-problem-detector
+        component: staticpods-monitor
+    spec:
+      serviceAccountName: node-problem-detector-staticpods
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      - operator: Exists
+        effect: NoExecute
+      initContainers:
+      # Download tools (kubectl, yq, jq) for the check script
+      - name: install-tools
+        image: busybox:1.36
+        command:
+        - sh
+        - -c
+        - |
+          set -e
+          ARCH=$(uname -m)
+          case $ARCH in
+            x86_64) ARCH="amd64" ;;
+            aarch64) ARCH="arm64" ;;
+          esac
+
+          # Install kubectl
+          KUBECTL_VERSION="v1.35.0"
+          wget -O /tools-bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl"
+          chmod +x /tools-bin/kubectl
+
+          # Install yq
+          YQ_VERSION="v4.44.1"
+          wget -O /tools-bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH}"
+          chmod +x /tools-bin/yq
+
+          # Install jq
+          JQ_VERSION="1.7.1"
+          wget -O /tools-bin/jq "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-${ARCH}"
+          chmod +x /tools-bin/jq
+        volumeMounts:
+        - name: tools-bin
+          mountPath: /tools-bin
+      containers:
+      - name: node-problem-detector
+        image: registry.k8s.io/node-problem-detector/node-problem-detector:v0.8.19
+        command:
+        - /node-problem-detector
+        - --logtostderr
+        - --config.custom-plugin-monitor=/config/staticpods-plugin.json
+        securityContext:
+          privileged: true
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: PATH
+          value: /tools-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+        volumeMounts:
+        - name: config
+          mountPath: /config
+          readOnly: true
+        - name: plugin
+          mountPath: /config/plugin
+          readOnly: true
+        - name: static-pods
+          mountPath: /etc/kubernetes/manifests
+          readOnly: true
+        - name: tools-bin
+          mountPath: /tools-bin
+          readOnly: true
+        resources:
+          limits:
+            cpu: 100m
+            memory: 128Mi
+          requests:
+            cpu: 20m
+            memory: 64Mi
+      volumes:
+      - name: config
+        configMap:
+          name: npd-staticpods-config
+          items:
+          - key: staticpods-plugin.json
+            path: staticpods-plugin.json
+      - name: plugin
+        configMap:
+          name: npd-staticpods-config
+          defaultMode: 0755
+          items:
+          - key: check-staticpods.sh
+            path: check-staticpods.sh
+      - name: static-pods
+        hostPath:
+          path: /etc/kubernetes/manifests
+          type: DirectoryOrCreate
+      - name: tools-bin
+        emptyDir: {}

examples/staticpods-readiness/npd/npd-rbac.yaml

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: node-problem-detector-staticpods
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: node-problem-detector-staticpods
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: node-problem-detector-staticpods
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: node-problem-detector-staticpods
+subjects:
+- kind: ServiceAccount
+  name: node-problem-detector-staticpods
+  namespace: kube-system

examples/staticpods-readiness/npd/npd-staticpods-config.yaml

@@ -0,0 +1,118 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: npd-staticpods-config
+  namespace: kube-system
+data:
+  staticpods-plugin.json: |
+    {
+      "plugin": "custom",
+      "pluginConfig": {
+        "invoke_interval": "30s",
+        "timeout": "10s",
+        "max_output_length": 80,
+        "concurrency": 1
+      },
+      "source": "staticpods-monitor",
+      "conditions": [
+        {
+          "type": "StaticPodsMissing",
+          "reason": "AllStaticPodsMirrored",
+          "message": "All static pods have mirror pods in API server"
+        }
+      ],
+      "rules": [
+        {
+          "type": "permanent",
+          "condition": "StaticPodsMissing",
+          "reason": "StaticPodsNotMirrored",
+          "path": "/config/plugin/check-staticpods.sh"
+        }
+      ]
+    }
+
+  check-staticpods.sh: |
+    #!/bin/bash
+
+    set -euo pipefail
+
+    # Exit code constants
+    readonly OK=0
+    readonly NONOK=1
+    readonly UNKNOWN=2
+
+    STATIC_PODS_DIR="/etc/kubernetes/manifests"
+    NODE_NAME="${NODE_NAME:-$(hostname)}"
+
+
+    # Check required tools
+    for tool in kubectl yq jq; do
+      if ! command -v $tool &> /dev/null; then
+        echo "$tool not found in PATH"
+        exit $UNKNOWN
+      fi
+    done
+
+    parse_json_manifest() {
+
+        # The yaml file might be a multi-document yaml , but given that kubelet
+        # only reads the first document we will skip the rest.
+        NAME=$(jq -r '.metadata.name' $1) 
+        NAMESPACE=$(jq -r '.metadata.namespace // "default"' $1) 
+
+        # if NAME is empty , we'll suppose that the file is 
+        # syntaxically incorrect and exit.
+        if [ "$NAME" = "" ];then 
+          echo "manifest $1 is invalid"
+          exit $UNKNOWN
+        fi
+
+        FNAME="$NAME-$NODE_NAME"
+
+        RESULT=$(kubectl get pod "$FNAME" -n "$NAMESPACE" -o=jsonpath='{.metadata.name}' 2>/dev/null)
+        if [ "$RESULT" = "" ];then
+            echo "Static pod $FNAME is missing a mirror pod"
+            exit "$NONOK"
+        fi
+    }
+
+
+    parse_yaml_manifest() {
+
+        NAME=$(yq -r '.metadata.name' $1 | head -1) 
+        NAMESPACE=$(yq -r '.metadata.namespace // "default"' $1 | head -1)
+
+        # if NAME is empty , we'll suppose that the file is 
+        # syntaxically incorrect and exit.
+        if [ "$NAME" = "" ];then 
+            echo "manifest $1 is invalid"
+            exit $UNKNOWN
+        fi
+
+        FNAME="$NAME-$NODE_NAME"
+
+        RESULT=$(kubectl get pod "$FNAME" -n "$NAMESPACE" -o=jsonpath='{.metadata.name}' 2>/dev/null)
+        if [ "$RESULT" = "" ];then
+            echo "Static pod $FNAME is missing a mirror pod"
+            exit "$NONOK"
+        fi
+    }
+
+    for file in "$STATIC_PODS_DIR"/*
+    do
+        # we read the first non blank line , if it starts with '{'
+        # after trimming then it's json , otherwise we will consider it
+        # as yaml. (It's not 100% accurate , but it's a good guess work).
+
+        # we read the first line and strip the leading whitespaces
+        fline=$(grep -m 1 . $file | awk '{$1=$1};1')
+
+        fchar="${fline:0:1}"
+        if [ $fchar = "{" ]; then
+            parse_json_manifest "$file"    
+        else
+            parse_yaml_manifest "$file"
+        fi
+
+    done
+    exit $OK

examples/staticpods-readiness/setup-staticpods-readiness.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Copyright The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+kubectl apply -f "$SCRIPT_DIR/npd/npd-rbac.yaml"
+kubectl apply -f "$SCRIPT_DIR/npd/npd-staticpods-config.yaml"
+kubectl apply -f "$SCRIPT_DIR/npd/npd-ds.yaml"

examples/staticpods-readiness/staticpods-readiness-rule.yaml

@@ -0,0 +1,15 @@
+apiVersion: readiness.node.x-k8s.io/v1alpha1
+kind: NodeReadinessRule
+metadata:
+  name: static-pods-readiness-rule
+spec:
+  nodeSelector:
+    matchLabels: {}
+  conditions:
+    - type: "StaticPodsMissing"
+      requiredStatus: "False"
+  taint:
+    key: "readiness.k8s.io/StaticPodsMissing"
+    effect: "NoSchedule"
+
+  enforcementMode: "bootstrap-only"

examples/staticpods-readiness/test-staticpod.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-static-pod
+  labels:
+    app: test-static-pod
+spec:
+  containers:
+  - name: pause
+    image: registry.k8s.io/pause:3.9
+    resources:
+      requests:
+        cpu: 10m
+        memory: 16Mi
+      limits:
+        cpu: 10m
+        memory: 16Mi