fix(observability): make a cold install-observability succeed from zero

Taskfile.yml

@@ -444,8 +444,22 @@ tasks:
       - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmsingle -n telemetry-system --timeout=120s || true
       - echo "⏳ Waiting for OpenTelemetryCollector CRD to be Established …"
       - kubectl wait --for=condition=Established crd/opentelemetrycollectors.opentelemetry.io --timeout={{.WAIT_TIMEOUT}}
-      - echo "➡️  Applying OpenTelemetryCollector CR …"
-      - kubectl apply -f {{.REPO_DIR}}/components/observability/otel-collector/opentelemetry-collector.yaml
+      - echo "➡️  Applying OpenTelemetryCollector CR (waiting for the operator webhook) …"
+      - |
+        set -euo pipefail
+        # HR-Ready + CRD-Established do NOT guarantee the operator's admission
+        # webhook is serving with an injected caBundle — both happen
+        # asynchronously after the Helm install. Wait for the operator
+        # Deployment, then retry the apply so a cold-start caBundle-injection
+        # lag can't fail the install with an x509 / "no endpoints" webhook error.
+        kubectl -n telemetry-system wait --for=condition=Available \
+          deploy -l app.kubernetes.io/name=opentelemetry-operator --timeout={{.WAIT_TIMEOUT}}
+        for i in $(seq 1 30); do
+          kubectl apply -f {{.REPO_DIR}}/components/observability/otel-collector/opentelemetry-collector.yaml && break
+          [ "$i" = 30 ] && { echo "OpenTelemetryCollector apply failed after retries" >&2; exit 1; }
+          echo "  operator webhook not ready yet, retrying ($i)…" >&2
+          sleep 5
+        done
       - echo "⏳ Waiting for OTel Collector DaemonSet …"
       - kubectl -n telemetry-system rollout status daemonset/otel-collector-collector --timeout={{.WAIT_TIMEOUT}}
       - echo "➡️  Applying Grafana Instance (after Operator CRDs are ready) …"

components/cert-manager/cert-manager-hr.yaml

@@ -47,11 +47,16 @@ spec:
           memory: 64Mi
 
     # CA Injector configuration
+    # cainjector caches every CRD, APIService, and webhook config in the
+    # cluster to inject CA bundles, so its memory scales with cluster object
+    # count. 64Mi is enough for a bare cluster but OOMKills (exit 137) once the
+    # optional observability stack adds its CRDs/webhooks — which then blocks
+    # webhook caBundle injection (e.g. the OTel operator's), so give it room.
     cainjector:
       resources:
         requests:
           cpu: 5m
-          memory: 16Mi
+          memory: 32Mi
         limits:
-          cpu: 50m
-          memory: 64Mi
+          cpu: 100m
+          memory: 256Mi

components/observability/kustomization.yaml

@@ -9,7 +9,11 @@ resources:
   - tempo-hr.yaml
   - otel-collector/helm-repository.yaml
   - otel-collector/helm-release-operator.yaml
-  - otel-collector/opentelemetry-collector.yaml
+  # NOTE: otel-collector/opentelemetry-collector.yaml is intentionally NOT
+  # listed here. It is an OpenTelemetryCollector CR whose CRD and validating
+  # webhook are installed by the operator HelmRelease above. Applying it in
+  # this bundle races the operator, so the install-observability task applies
+  # it in a dedicated step after the CRD is Established (and the webhook ready).
   # Prometheus Operator CRDs
   - https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.81.0/stripped-down-crds.yaml
 

components/observability/loki-hr.yaml

@@ -59,6 +59,15 @@ spec:
           cpu: 200m
           memory: 512Mi
 
+    # The chart enables memcached chunk/results caches by default, each
+    # requesting ~8-9Gi of memory — they stay Pending (Insufficient memory) on
+    # a single-node kind cluster, leaving the Loki HelmRelease permanently
+    # not-Ready. They add nothing for a SingleBinary + filesystem dev setup.
+    chunksCache:
+      enabled: false
+    resultsCache:
+      enabled: false
+
     # Disable all other components
     backend:
       replicas: 0