diff --git a/Taskfile.yml b/Taskfile.yml index 48b45eb..08f011f 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -444,8 +444,22 @@ tasks: - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmsingle -n telemetry-system --timeout=120s || true - echo "⏳ Waiting for OpenTelemetryCollector CRD to be Established …" - kubectl wait --for=condition=Established crd/opentelemetrycollectors.opentelemetry.io --timeout={{.WAIT_TIMEOUT}} - - echo "➡️ Applying OpenTelemetryCollector CR …" - - kubectl apply -f {{.REPO_DIR}}/components/observability/otel-collector/opentelemetry-collector.yaml + - echo "➡️ Applying OpenTelemetryCollector CR (waiting for the operator webhook) …" + - | + set -euo pipefail + # HR-Ready + CRD-Established do NOT guarantee the operator's admission + # webhook is serving with an injected caBundle — both happen + # asynchronously after the Helm install. Wait for the operator + # Deployment, then retry the apply so a cold-start caBundle-injection + # lag can't fail the install with an x509 / "no endpoints" webhook error. + kubectl -n telemetry-system wait --for=condition=Available \ + deploy -l app.kubernetes.io/name=opentelemetry-operator --timeout={{.WAIT_TIMEOUT}} + for i in $(seq 1 30); do + kubectl apply -f {{.REPO_DIR}}/components/observability/otel-collector/opentelemetry-collector.yaml && break + [ "$i" = 30 ] && { echo "OpenTelemetryCollector apply failed after retries" >&2; exit 1; } + echo " operator webhook not ready yet, retrying ($i)…" >&2 + sleep 5 + done - echo "⏳ Waiting for OTel Collector DaemonSet …" - kubectl -n telemetry-system rollout status daemonset/otel-collector-collector --timeout={{.WAIT_TIMEOUT}} - echo "➡️ Applying Grafana Instance (after Operator CRDs are ready) …" diff --git a/components/cert-manager/cert-manager-hr.yaml b/components/cert-manager/cert-manager-hr.yaml index a28dcfe..0723eda 100644 --- a/components/cert-manager/cert-manager-hr.yaml +++ b/components/cert-manager/cert-manager-hr.yaml @@ -47,11 +47,16 @@ spec: memory: 64Mi # CA Injector configuration + # cainjector caches every CRD, APIService, and webhook config in the + # cluster to inject CA bundles, so its memory scales with cluster object + # count. 64Mi is enough for a bare cluster but OOMKills (exit 137) once the + # optional observability stack adds its CRDs/webhooks — which then blocks + # webhook caBundle injection (e.g. the OTel operator's), so give it room. cainjector: resources: requests: cpu: 5m - memory: 16Mi + memory: 32Mi limits: - cpu: 50m - memory: 64Mi \ No newline at end of file + cpu: 100m + memory: 256Mi diff --git a/components/observability/kustomization.yaml b/components/observability/kustomization.yaml index 945898a..bb61b33 100644 --- a/components/observability/kustomization.yaml +++ b/components/observability/kustomization.yaml @@ -9,7 +9,11 @@ resources: - tempo-hr.yaml - otel-collector/helm-repository.yaml - otel-collector/helm-release-operator.yaml - - otel-collector/opentelemetry-collector.yaml + # NOTE: otel-collector/opentelemetry-collector.yaml is intentionally NOT + # listed here. It is an OpenTelemetryCollector CR whose CRD and validating + # webhook are installed by the operator HelmRelease above. Applying it in + # this bundle races the operator, so the install-observability task applies + # it in a dedicated step after the CRD is Established (and the webhook ready). # Prometheus Operator CRDs - https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.81.0/stripped-down-crds.yaml diff --git a/components/observability/loki-hr.yaml b/components/observability/loki-hr.yaml index 6c78137..7a4b97e 100644 --- a/components/observability/loki-hr.yaml +++ b/components/observability/loki-hr.yaml @@ -59,6 +59,15 @@ spec: cpu: 200m memory: 512Mi + # The chart enables memcached chunk/results caches by default, each + # requesting ~8-9Gi of memory — they stay Pending (Insufficient memory) on + # a single-node kind cluster, leaving the Loki HelmRelease permanently + # not-Ready. They add nothing for a SingleBinary + filesystem dev setup. + chunksCache: + enabled: false + resultsCache: + enabled: false + # Disable all other components backend: replicas: 0