From e927ca456fce02628012795084487dddf667fdb0 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Sun, 19 Jul 2026 10:39:12 -0500 Subject: [PATCH] fix(observability): make a cold install-observability succeed from zero MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Installing the observability stack on a fresh cluster fails partway through and needs manual recovery. Independent causes, all fixed here: - OpenTelemetryCollector CR install was racy. It sat in the kustomize bundle, so the first `kustomize build | kubectl apply` created it before its CRD and webhook existed. The task already applies it in a dedicated step, but that step only waited for the operator HelmRelease Ready + CRD Established — neither of which guarantees the operator's admission webhook is serving with an injected caBundle (both are async after Helm install), so it still failed with x509 / "no endpoints" on a cold cluster. Fix: drop the CR from the bundle, and in the dedicated step wait for the operator Deployment and retry the apply until the webhook is ready. - cert-manager cainjector OOMKilled (exit 137) on its 64Mi limit once the stack's CRDs/webhooks landed, so it never injected the OTel operator's webhook caBundle. Raise the limit to 256Mi (its footprint scales with cluster object count). - Loki's default memcached chunk/results caches request ~8-9Gi each and stay Pending on a single-node cluster, leaving the Loki HelmRelease permanently not-Ready. Disable them — unneeded for a SingleBinary + filesystem setup. After this, `task install-observability` completes clean from a fresh cluster with no manual intervention. --- Taskfile.yml | 18 ++++++++++++++++-- components/cert-manager/cert-manager-hr.yaml | 11 ++++++++--- components/observability/kustomization.yaml | 6 +++++- components/observability/loki-hr.yaml | 9 +++++++++ 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 48b45eb..08f011f 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -444,8 +444,22 @@ tasks: - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmsingle -n telemetry-system --timeout=120s || true - echo "⏳ Waiting for OpenTelemetryCollector CRD to be Established …" - kubectl wait --for=condition=Established crd/opentelemetrycollectors.opentelemetry.io --timeout={{.WAIT_TIMEOUT}} - - echo "➡️ Applying OpenTelemetryCollector CR …" - - kubectl apply -f {{.REPO_DIR}}/components/observability/otel-collector/opentelemetry-collector.yaml + - echo "➡️ Applying OpenTelemetryCollector CR (waiting for the operator webhook) …" + - | + set -euo pipefail + # HR-Ready + CRD-Established do NOT guarantee the operator's admission + # webhook is serving with an injected caBundle — both happen + # asynchronously after the Helm install. Wait for the operator + # Deployment, then retry the apply so a cold-start caBundle-injection + # lag can't fail the install with an x509 / "no endpoints" webhook error. + kubectl -n telemetry-system wait --for=condition=Available \ + deploy -l app.kubernetes.io/name=opentelemetry-operator --timeout={{.WAIT_TIMEOUT}} + for i in $(seq 1 30); do + kubectl apply -f {{.REPO_DIR}}/components/observability/otel-collector/opentelemetry-collector.yaml && break + [ "$i" = 30 ] && { echo "OpenTelemetryCollector apply failed after retries" >&2; exit 1; } + echo " operator webhook not ready yet, retrying ($i)…" >&2 + sleep 5 + done - echo "⏳ Waiting for OTel Collector DaemonSet …" - kubectl -n telemetry-system rollout status daemonset/otel-collector-collector --timeout={{.WAIT_TIMEOUT}} - echo "➡️ Applying Grafana Instance (after Operator CRDs are ready) …" diff --git a/components/cert-manager/cert-manager-hr.yaml b/components/cert-manager/cert-manager-hr.yaml index a28dcfe..0723eda 100644 --- a/components/cert-manager/cert-manager-hr.yaml +++ b/components/cert-manager/cert-manager-hr.yaml @@ -47,11 +47,16 @@ spec: memory: 64Mi # CA Injector configuration + # cainjector caches every CRD, APIService, and webhook config in the + # cluster to inject CA bundles, so its memory scales with cluster object + # count. 64Mi is enough for a bare cluster but OOMKills (exit 137) once the + # optional observability stack adds its CRDs/webhooks — which then blocks + # webhook caBundle injection (e.g. the OTel operator's), so give it room. cainjector: resources: requests: cpu: 5m - memory: 16Mi + memory: 32Mi limits: - cpu: 50m - memory: 64Mi \ No newline at end of file + cpu: 100m + memory: 256Mi diff --git a/components/observability/kustomization.yaml b/components/observability/kustomization.yaml index 945898a..bb61b33 100644 --- a/components/observability/kustomization.yaml +++ b/components/observability/kustomization.yaml @@ -9,7 +9,11 @@ resources: - tempo-hr.yaml - otel-collector/helm-repository.yaml - otel-collector/helm-release-operator.yaml - - otel-collector/opentelemetry-collector.yaml + # NOTE: otel-collector/opentelemetry-collector.yaml is intentionally NOT + # listed here. It is an OpenTelemetryCollector CR whose CRD and validating + # webhook are installed by the operator HelmRelease above. Applying it in + # this bundle races the operator, so the install-observability task applies + # it in a dedicated step after the CRD is Established (and the webhook ready). # Prometheus Operator CRDs - https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.81.0/stripped-down-crds.yaml diff --git a/components/observability/loki-hr.yaml b/components/observability/loki-hr.yaml index 6c78137..7a4b97e 100644 --- a/components/observability/loki-hr.yaml +++ b/components/observability/loki-hr.yaml @@ -59,6 +59,15 @@ spec: cpu: 200m memory: 512Mi + # The chart enables memcached chunk/results caches by default, each + # requesting ~8-9Gi of memory — they stay Pending (Insufficient memory) on + # a single-node kind cluster, leaving the Loki HelmRelease permanently + # not-Ready. They add nothing for a SingleBinary + filesystem dev setup. + chunksCache: + enabled: false + resultsCache: + enabled: false + # Disable all other components backend: replicas: 0