diff --git a/examples/staticpods-readiness/README.md b/examples/staticpods-readiness/README.md new file mode 100644 index 0000000..b96a382 --- /dev/null +++ b/examples/staticpods-readiness/README.md @@ -0,0 +1,39 @@ +# Node Readiness Examples (Static Pods) + +This example demonstrates how NRC can be used alongside [NPD (Node Problem Detector)](https://github.com/kubernetes/node-problem-detector) to make sure that all static pods are mirrored in the API server to avoid over-committing issues. See [#115325](https://github.com/kubernetes/kubernetes/issues/115325), [#47264](https://github.com/kubernetes/website/issues/47264), and [#126870](https://github.com/kubernetes/kubernetes/pull/126870) for reference. + +## Deployment Steps + +1. Deploy a testing cluster. For this example we use Kind with a mounted static pod manifest for testing: + + ```bash + kind create cluster --config examples/staticpods-readiness/kind-cluster-config.yaml + ``` + +2. Install NRC: + + ```bash + VERSION=v0.2.0 + kubectl apply -f https://github.com/kubernetes-sigs/node-readiness-controller/releases/download/${VERSION}/crds.yaml + kubectl apply -f https://github.com/kubernetes-sigs/node-readiness-controller/releases/download/${VERSION}/install.yaml + ``` +3. Taint The node: + ```bash + kubectl taint node kind-worker readiness.k8s.io/StaticPodsMissing=pending:NoSchedule + ``` +4. Deploy NPD as a DaemonSet with the static pods readiness configuration: + + ```bash + ./examples/staticpods-readiness/setup-staticpods-readiness.sh + ``` + + This deploys NPD with: + - Init container that downloads `kubectl`, `yq`, and `jq` + - Custom plugin that checks if static pods are mirrored + - RBAC permissions to read pods and nodes + +5. Apply the node readiness rule: + ```bash + kubectl apply -f examples/staticpods-readiness/staticpods-readiness-rule.yaml + ``` + If the static pods was successfully created, NRC would read the condition added by NPD and the taint should be removed. \ No newline at end of file diff --git a/examples/staticpods-readiness/kind-cluster-config.yaml b/examples/staticpods-readiness/kind-cluster-config.yaml new file mode 100644 index 0000000..2f9a12f --- /dev/null +++ b/examples/staticpods-readiness/kind-cluster-config.yaml @@ -0,0 +1,10 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker + extraMounts: + - hostPath: examples/staticpods-readiness/test-staticpod.yaml + containerPath: /etc/kubernetes/manifests/test-staticpod.yaml + readOnly: true + diff --git a/examples/staticpods-readiness/npd/npd-ds.yaml b/examples/staticpods-readiness/npd/npd-ds.yaml new file mode 100644 index 0000000..9813de4 --- /dev/null +++ b/examples/staticpods-readiness/npd/npd-ds.yaml @@ -0,0 +1,113 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-problem-detector-staticpods + namespace: kube-system + labels: + app: node-problem-detector + component: staticpods-monitor +spec: + selector: + matchLabels: + app: node-problem-detector + component: staticpods-monitor + template: + metadata: + labels: + app: node-problem-detector + component: staticpods-monitor + spec: + serviceAccountName: node-problem-detector-staticpods + tolerations: + - operator: Exists + effect: NoSchedule + - operator: Exists + effect: NoExecute + initContainers: + # Download tools (kubectl, yq, jq) for the check script + - name: install-tools + image: busybox:1.36 + command: + - sh + - -c + - | + set -e + ARCH=$(uname -m) + case $ARCH in + x86_64) ARCH="amd64" ;; + aarch64) ARCH="arm64" ;; + esac + + # Install kubectl + KUBECTL_VERSION="v1.35.0" + wget -O /tools-bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl" + chmod +x /tools-bin/kubectl + + # Install yq + YQ_VERSION="v4.44.1" + wget -O /tools-bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH}" + chmod +x /tools-bin/yq + + # Install jq + JQ_VERSION="1.7.1" + wget -O /tools-bin/jq "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-${ARCH}" + chmod +x /tools-bin/jq + volumeMounts: + - name: tools-bin + mountPath: /tools-bin + containers: + - name: node-problem-detector + image: registry.k8s.io/node-problem-detector/node-problem-detector:v0.8.19 + command: + - /node-problem-detector + - --logtostderr + - --config.custom-plugin-monitor=/config/staticpods-plugin.json + securityContext: + privileged: true + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: PATH + value: /tools-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + volumeMounts: + - name: config + mountPath: /config + readOnly: true + - name: plugin + mountPath: /config/plugin + readOnly: true + - name: static-pods + mountPath: /etc/kubernetes/manifests + readOnly: true + - name: tools-bin + mountPath: /tools-bin + readOnly: true + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 20m + memory: 64Mi + volumes: + - name: config + configMap: + name: npd-staticpods-config + items: + - key: staticpods-plugin.json + path: staticpods-plugin.json + - name: plugin + configMap: + name: npd-staticpods-config + defaultMode: 0755 + items: + - key: check-staticpods.sh + path: check-staticpods.sh + - name: static-pods + hostPath: + path: /etc/kubernetes/manifests + type: DirectoryOrCreate + - name: tools-bin + emptyDir: {} \ No newline at end of file diff --git a/examples/staticpods-readiness/npd/npd-rbac.yaml b/examples/staticpods-readiness/npd/npd-rbac.yaml new file mode 100644 index 0000000..0d222b0 --- /dev/null +++ b/examples/staticpods-readiness/npd/npd-rbac.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-problem-detector-staticpods + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: node-problem-detector-staticpods +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - pods + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: node-problem-detector-staticpods +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: node-problem-detector-staticpods +subjects: +- kind: ServiceAccount + name: node-problem-detector-staticpods + namespace: kube-system \ No newline at end of file diff --git a/examples/staticpods-readiness/npd/npd-staticpods-config.yaml b/examples/staticpods-readiness/npd/npd-staticpods-config.yaml new file mode 100644 index 0000000..2068b01 --- /dev/null +++ b/examples/staticpods-readiness/npd/npd-staticpods-config.yaml @@ -0,0 +1,118 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: npd-staticpods-config + namespace: kube-system +data: + staticpods-plugin.json: | + { + "plugin": "custom", + "pluginConfig": { + "invoke_interval": "30s", + "timeout": "10s", + "max_output_length": 80, + "concurrency": 1 + }, + "source": "staticpods-monitor", + "conditions": [ + { + "type": "StaticPodsMissing", + "reason": "AllStaticPodsMirrored", + "message": "All static pods have mirror pods in API server" + } + ], + "rules": [ + { + "type": "permanent", + "condition": "StaticPodsMissing", + "reason": "StaticPodsNotMirrored", + "path": "/config/plugin/check-staticpods.sh" + } + ] + } + + check-staticpods.sh: | + #!/bin/bash + + set -euo pipefail + + # Exit code constants + readonly OK=0 + readonly NONOK=1 + readonly UNKNOWN=2 + + STATIC_PODS_DIR="/etc/kubernetes/manifests" + NODE_NAME="${NODE_NAME:-$(hostname)}" + + + # Check required tools + for tool in kubectl yq jq; do + if ! command -v $tool &> /dev/null; then + echo "$tool not found in PATH" + exit $UNKNOWN + fi + done + + parse_json_manifest() { + + # The yaml file might be a multi-document yaml , but given that kubelet + # only reads the first document we will skip the rest. + NAME=$(jq -r '.metadata.name' $1) + NAMESPACE=$(jq -r '.metadata.namespace // "default"' $1) + + # if NAME is empty , we'll suppose that the file is + # syntaxically incorrect and exit. + if [ "$NAME" = "" ];then + echo "manifest $1 is invalid" + exit $UNKNOWN + fi + + FNAME="$NAME-$NODE_NAME" + + RESULT=$(kubectl get pod "$FNAME" -n "$NAMESPACE" -o=jsonpath='{.metadata.name}' 2>/dev/null) + if [ "$RESULT" = "" ];then + echo "Static pod $FNAME is missing a mirror pod" + exit "$NONOK" + fi + } + + + parse_yaml_manifest() { + + NAME=$(yq -r '.metadata.name' $1 | head -1) + NAMESPACE=$(yq -r '.metadata.namespace // "default"' $1 | head -1) + + # if NAME is empty , we'll suppose that the file is + # syntaxically incorrect and exit. + if [ "$NAME" = "" ];then + echo "manifest $1 is invalid" + exit $UNKNOWN + fi + + FNAME="$NAME-$NODE_NAME" + + RESULT=$(kubectl get pod "$FNAME" -n "$NAMESPACE" -o=jsonpath='{.metadata.name}' 2>/dev/null) + if [ "$RESULT" = "" ];then + echo "Static pod $FNAME is missing a mirror pod" + exit "$NONOK" + fi + } + + for file in "$STATIC_PODS_DIR"/* + do + # we read the first non blank line , if it starts with '{' + # after trimming then it's json , otherwise we will consider it + # as yaml. (It's not 100% accurate , but it's a good guess work). + + # we read the first line and strip the leading whitespaces + fline=$(grep -m 1 . $file | awk '{$1=$1};1') + + fchar="${fline:0:1}" + if [ $fchar = "{" ]; then + parse_json_manifest "$file" + else + parse_yaml_manifest "$file" + fi + + done + exit $OK \ No newline at end of file diff --git a/examples/staticpods-readiness/setup-staticpods-readiness.sh b/examples/staticpods-readiness/setup-staticpods-readiness.sh new file mode 100755 index 0000000..8850792 --- /dev/null +++ b/examples/staticpods-readiness/setup-staticpods-readiness.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +kubectl apply -f "$SCRIPT_DIR/npd/npd-rbac.yaml" +kubectl apply -f "$SCRIPT_DIR/npd/npd-staticpods-config.yaml" +kubectl apply -f "$SCRIPT_DIR/npd/npd-ds.yaml" diff --git a/examples/staticpods-readiness/staticpods-readiness-rule.yaml b/examples/staticpods-readiness/staticpods-readiness-rule.yaml new file mode 100644 index 0000000..ef72044 --- /dev/null +++ b/examples/staticpods-readiness/staticpods-readiness-rule.yaml @@ -0,0 +1,15 @@ +apiVersion: readiness.node.x-k8s.io/v1alpha1 +kind: NodeReadinessRule +metadata: + name: static-pods-readiness-rule +spec: + nodeSelector: + matchLabels: {} + conditions: + - type: "StaticPodsMissing" + requiredStatus: "False" + taint: + key: "readiness.k8s.io/StaticPodsMissing" + effect: "NoSchedule" + + enforcementMode: "bootstrap-only" diff --git a/examples/staticpods-readiness/test-staticpod.yaml b/examples/staticpods-readiness/test-staticpod.yaml new file mode 100644 index 0000000..e547ca7 --- /dev/null +++ b/examples/staticpods-readiness/test-staticpod.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-static-pod + labels: + app: test-static-pod +spec: + containers: + - name: pause + image: registry.k8s.io/pause:3.9 + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 10m + memory: 16Mi \ No newline at end of file