feat(observability): split into composable per-component tasks

Taskfile.yml

@@ -107,7 +107,15 @@ tasks:
       - echo "  install-cert-manager    Deploy cert-manager + CSI driver"
       - echo "  install-kyverno         Deploy Kyverno policy engine"
       - echo "  install-envoy-gateway-operator  Deploy Envoy Gateway"
-      - echo "  install-observability   [OPTIONAL] Deploy telemetry stack"
+      - echo ""
+      - echo "📊 Observability (composable):"
+      - echo "  install-observability       [OPTIONAL] Deploy full telemetry stack"
+      - echo "  install-prometheus-crds     Prometheus Operator CRDs"
+      - echo "  install-victoria-metrics    Victoria Metrics (vmagent + vmsingle)"
+      - echo "  install-otel-collector      OpenTelemetry Operator + Collector"
+      - echo "  install-grafana             Grafana Operator + instance + datasources"
+      - echo "  install-loki                Loki for log aggregation"
+      - echo "  install-tempo               Tempo for distributed tracing"
       - echo ""
       - echo "🔨 CI/Development:"
       - echo "  kind-load-image         Load images into KIND"
@@ -407,26 +415,16 @@ tasks:
       - echo "   To find the actual NodePort assignments, run:"
       - echo "   kubectl get svc -n envoy-gateway-system -l app.kubernetes.io/name=envoy"
 
-  install-observability:
-    desc: "[OPTIONAL] Deploy complete observability stack (Victoria Metrics, Loki, Tempo, Grafana, OTel)"
+  install-prometheus-crds:
+    desc: "Deploy Prometheus Operator CRDs"
     silent: true
     cmds:
       - task: ensure-repo
-      - echo "➡️  Reconciling Observability Stack …"
-      - kustomize build {{.REPO_DIR}}/components/observability | kubectl apply -f -
-      - echo "⏳ Waiting for Grafana Operator HelmRelease …"
-      - kubectl -n flux-system wait helmrelease/grafana-operator --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
-      - echo "⏳ Waiting for Victoria Metrics Stack HelmRelease …"
-      - kubectl -n flux-system wait helmrelease/vm --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
-      - echo "⏳ Waiting for Loki HelmRelease …"
-      - kubectl -n flux-system wait helmrelease/loki --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
-      - echo "⏳ Waiting for Tempo HelmRelease …"
-      - kubectl -n flux-system wait helmrelease/tempo --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
-      - echo "⏳ Waiting for OpenTelemetry Operator HelmRelease …"
-      - kubectl -n flux-system wait helmrelease/opentelemetry-operator --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
+      - echo "➡️  Reconciling Prometheus Operator CRDs …"
+      - kustomize build {{.REPO_DIR}}/components/observability/prometheus-crds | kubectl apply --server-side --force-conflicts -f -
       - |
         set -euo pipefail
-        echo "⏳ Waiting for Prometheus CRDs to be Established…"
+        echo "⏳ Waiting for Prometheus CRDs to be Established …"
         for crd in \
           servicemonitors.monitoring.coreos.com \
           podmonitors.monitoring.coreos.com \
@@ -436,22 +434,106 @@ tasks:
           alertmanagers.monitoring.coreos.com \
           thanosrulers.monitoring.coreos.com
         do
-          echo "$crd"
+          echo "  $crd"
           kubectl wait --for=condition=Established "crd/${crd}" --timeout=120s || true
         done
-      - echo "⏳ Waiting for vmagent & vmsingle to be Ready (best-effort)…"
-      - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmagent -n telemetry-system --timeout=120s || true
-      - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmsingle -n telemetry-system --timeout=120s || true
+      - echo "✅ Prometheus Operator CRDs are ready"
+
+  install-victoria-metrics:
+    desc: "Deploy Victoria Metrics (vmagent + vmsingle)"
+    silent: true
+    deps: [install-prometheus-crds]
+    cmds:
+      - task: ensure-repo
+      - echo "➡️  Reconciling Victoria Metrics …"
+      - kustomize build {{.REPO_DIR}}/components/observability/victoria-metrics | kubectl apply -f -
+      - echo "⏳ Waiting for Victoria Metrics Stack HelmRelease …"
+      - kubectl -n flux-system wait helmrelease/vm --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
+      - echo "⏳ Waiting for vmagent & vmsingle to be Ready …"
+      - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmagent -n victoria-metrics-system --timeout={{.WAIT_TIMEOUT}} || true
+      - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmsingle -n victoria-metrics-system --timeout={{.WAIT_TIMEOUT}} || true
+      - echo "✅ Victoria Metrics is ready"
+
+  install-otel-collector:
+    desc: "Deploy OpenTelemetry Operator + Collector"
+    silent: true
+    cmds:
+      - task: ensure-repo
+      - echo "➡️  Reconciling OpenTelemetry Operator …"
+      - kustomize build {{.REPO_DIR}}/components/observability/otel-collector | kubectl apply -f -
+      - echo "⏳ Waiting for OpenTelemetry Operator HelmRelease …"
+      - kubectl -n flux-system wait helmrelease/opentelemetry-operator --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
       - echo "⏳ Waiting for OpenTelemetryCollector CRD to be Established …"
       - kubectl wait --for=condition=Established crd/opentelemetrycollectors.opentelemetry.io --timeout={{.WAIT_TIMEOUT}}
-      - echo "➡️  Applying OpenTelemetryCollector CR …"
-      - kubectl apply -f {{.REPO_DIR}}/components/observability/otel-collector/opentelemetry-collector.yaml
+      - |
+        echo "⏳ Waiting for OTel webhook to be ready …"
+        sleep 10
+      - echo "➡️  Applying OpenTelemetryCollector CR (with retry for webhook readiness) …"
+      - |
+        set -euo pipefail
+        for i in 1 2 3 4 5; do
+          if kubectl apply -f {{.REPO_DIR}}/components/observability/otel-collector/opentelemetry-collector.yaml 2>/dev/null; then
+            echo "  CR applied successfully"
+            break
+          fi
+          echo "  Attempt $i failed, retrying in 10s …"
+          sleep 10
+        done
       - echo "⏳ Waiting for OTel Collector DaemonSet …"
-      - kubectl -n telemetry-system rollout status daemonset/otel-collector-collector --timeout={{.WAIT_TIMEOUT}}
+      - kubectl -n otel-collector-system rollout status daemonset/otel-collector-collector --timeout={{.WAIT_TIMEOUT}}
+      - echo "✅ OpenTelemetry Collector is ready"
+
+  install-grafana:
+    desc: "Deploy Grafana Operator + instance + datasources"
+    silent: true
+    deps: [install-victoria-metrics]
+    cmds:
+      - task: ensure-repo
+      - echo "➡️  Reconciling Grafana Operator …"
+      - kustomize build {{.REPO_DIR}}/components/observability/grafana | kubectl apply -f -
+      - echo "⏳ Waiting for Grafana Operator HelmRelease …"
+      - kubectl -n flux-system wait helmrelease/grafana-operator --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
       - echo "➡️  Applying Grafana Instance (after Operator CRDs are ready) …"
-      - kubectl apply -f {{.REPO_DIR}}/components/observability/grafana-instance.yaml
+      - kubectl apply -f {{.REPO_DIR}}/components/observability/grafana/grafana-instance.yaml
       - echo "➡️  Applying Grafana Datasources …"
-      - kustomize build {{.REPO_DIR}}/components/observability/datasources | kubectl apply -f -
+      - kustomize build {{.REPO_DIR}}/components/observability/grafana/datasources | kubectl apply -f -
+      - echo "✅ Grafana is ready"
+      - echo ""
+      - 'echo "📊 Access Grafana at: http://localhost:30000"'
+      - 'echo "   Username: admin"'
+      - 'echo "   Password: datum123"'
+
+  install-loki:
+    desc: "Deploy Loki for log aggregation"
+    silent: true
+    cmds:
+      - task: ensure-repo
+      - echo "➡️  Reconciling Loki …"
+      - kustomize build {{.REPO_DIR}}/components/observability/loki | kubectl apply -f -
+      - echo "⏳ Waiting for Loki HelmRelease …"
+      - kubectl -n flux-system wait helmrelease/loki --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
+      - echo "✅ Loki is ready"
+
+  install-tempo:
+    desc: "Deploy Tempo for distributed tracing"
+    silent: true
+    cmds:
+      - task: ensure-repo
+      - echo "➡️  Reconciling Tempo …"
+      - kustomize build {{.REPO_DIR}}/components/observability/tempo | kubectl apply -f -
+      - echo "⏳ Waiting for Tempo HelmRelease …"
+      - kubectl -n flux-system wait helmrelease/tempo --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
+      - echo "✅ Tempo is ready"
+
+  install-observability:
+    desc: "[OPTIONAL] Deploy complete observability stack (Victoria Metrics, Loki, Tempo, Grafana, OTel)"
+    silent: true
+    cmds:
+      - task: install-victoria-metrics
+      - task: install-otel-collector
+      - task: install-loki
+      - task: install-tempo
+      - task: install-grafana
       - echo "✅ Observability stack is ready"
       - echo ""
       - 'echo "📊 Access Grafana at: http://localhost:30000"'

components/observability/README.md

@@ -4,39 +4,61 @@ Optional observability stack for test infrastructure, providing metrics, logs, a
 
 ## Overview
 
-This component is an **optional add-on** deployed after core infrastructure. It provides comprehensive telemetry capabilities using Grafana, Victoria Metrics, Loki, and Tempo.
+This component is an **optional add-on** deployed after core infrastructure. It provides comprehensive telemetry capabilities using Grafana, Victoria Metrics, Loki, Tempo, and the OpenTelemetry Collector.
 
-## Components
+The stack is split into composable subcomponents — each deploys into its own namespace so it can be installed, upgraded, or removed independently (`kubectl delete ns <component>-system` cleanly uninstalls).
 
-- **Grafana**: Visualization and dashboards
-- **Victoria Metrics**: Metrics collection and storage
-- **Loki**: Log aggregation and storage
-- **Tempo**: Distributed tracing storage
-- **Promtail**: Log collection agent
+## Subcomponents
+
+| Component         | Namespace                 | Purpose                               |
+| ----------------- | ------------------------- | ------------------------------------- |
+| `prometheus-crds` | (cluster-scoped)          | Prometheus Operator CRDs              |
+| `victoria-metrics`| `victoria-metrics-system` | Metrics collection and storage        |
+| `otel-collector`  | `otel-collector-system`   | OpenTelemetry DaemonSet collector     |
+| `loki`            | `loki-system`             | Log aggregation and storage           |
+| `tempo`           | `tempo-system`            | Distributed tracing storage           |
+| `grafana`         | `grafana-system`          | Visualization, dashboards, datasources|
 
 ## Access
 
 - **Grafana UI**: Available at NodePort 30000 (admin/datum123)
-- **Default Datasources**: Victoria Metrics (metrics), Loki (logs), Tempo (traces)
+- **Default Datasources**: Victoria Metrics (metrics), Loki (logs), Tempo (traces), Alertmanager
 
 ## Prerequisites
 
 Core test infrastructure must be running before deploying observability components.
 
 ## Deployment
 
-Deploy using the `install-observability` task target:
+Deploy the whole stack:
 
 ```bash
 task install-observability
 ```
 
-This deploys all components and configures datasources automatically.
+Or install components individually:
+
+```bash
+task install-prometheus-crds
+task install-victoria-metrics
+task install-otel-collector
+task install-loki
+task install-tempo
+task install-grafana
+```
 
 ## Removal
 
-To remove the observability stack:
+To remove an individual component:
+
+```bash
+kubectl delete namespace <component>-system
+```
+
+To remove the whole stack:
 
 ```bash
-kubectl delete namespace observability
+for ns in grafana-system tempo-system loki-system otel-collector-system victoria-metrics-system; do
+  kubectl delete namespace "$ns"
+done
 ```

components/observability/datasources/alertmanager-datasource.yaml → components/observability/grafana/datasources/alertmanager-datasource.yaml

@@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1
 kind: GrafanaDatasource
 metadata:
   name: alertmanager
-  namespace: telemetry-system
+  namespace: grafana-system
 spec:
   instanceSelector:
     matchLabels:
@@ -11,7 +11,7 @@ spec:
     name: Alertmanager
     type: alertmanager
     access: proxy
-    url: http://vmalertmanager-telemetry-system-vm.telemetry-system.svc.cluster.local:9093
+    url: http://vmalertmanager-victoria-metrics-system-vm.victoria-metrics-system.svc.cluster.local:9093
     isDefault: true
     editable: true
     jsonData:

components/observability/datasources/loki-datasource.yaml → components/observability/grafana/datasources/loki-datasource.yaml

@@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1
 kind: GrafanaDatasource
 metadata:
   name: loki-datasource
-  namespace: telemetry-system
+  namespace: grafana-system
   labels:
     dashboards: "grafana"
 spec:
@@ -13,7 +13,7 @@ spec:
     name: Loki
     type: loki
     access: proxy
-    url: http://telemetry-system-loki.telemetry-system.svc.cluster.local:3100
+    url: http://loki-system-loki.loki-system.svc.cluster.local:3100
     isDefault: false
     editable: true
     jsonData:

components/observability/datasources/tempo-datasource.yaml → components/observability/grafana/datasources/tempo-datasource.yaml

@@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1
 kind: GrafanaDatasource
 metadata:
   name: tempo-datasource
-  namespace: telemetry-system
+  namespace: grafana-system
   labels:
     dashboards: "grafana"
 spec:
@@ -13,7 +13,7 @@ spec:
     name: Tempo
     type: tempo
     access: proxy
-    url: http://telemetry-system-tempo.telemetry-system.svc.cluster.local:3100
+    url: http://tempo-system-tempo.tempo-system.svc.cluster.local:3100
     uid: tempo-datasource
     isDefault: false
     editable: true

components/observability/datasources/vm-datasource.yaml → components/observability/grafana/datasources/vm-datasource.yaml

@@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1
 kind: GrafanaDatasource
 metadata:
   name: victoria-metrics-datasource
-  namespace: telemetry-system
+  namespace: grafana-system
   labels:
     dashboards: "grafana"
 spec:
@@ -13,7 +13,7 @@ spec:
     name: VictoriaMetrics
     type: prometheus
     access: proxy
-    url: http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428
+    url: http://vmsingle-victoria-metrics-system-vm.victoria-metrics-system.svc.cluster.local:8428
     uid: victoria-metrics-datasource
     isDefault: true
     editable: true

components/observability/grafana-instance.yaml → components/observability/grafana/grafana-instance.yaml

@@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1
 kind: Grafana
 metadata:
   name: grafana
-  namespace: telemetry-system
+  namespace: grafana-system
   labels:
     dashboards: "grafana"
 spec:

components/observability/grafana-operator-hr.yaml → components/observability/grafana/grafana-operator-hr.yaml

@@ -6,7 +6,7 @@ metadata:
 spec:
   interval: 15m
   timeout: 5m
-  targetNamespace: telemetry-system
+  targetNamespace: grafana-system
   chart:
     spec:
       chart: grafana-operator

components/observability/grafana/helm-repositories.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana
+  namespace: flux-system
+spec:
+  interval: 12h
+  url: https://grafana.github.io/helm-charts/

components/observability/grafana/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - helm-repositories.yaml
+  - grafana-operator-hr.yaml
+  - grafana-instance.yaml
+  - datasources

components/observability/grafana/namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: grafana-system
+  labels:
+    name: grafana-system
+    dashboards: "grafana"

components/observability/kustomization.yaml

@@ -1,15 +1,24 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - namespace.yaml
-  - helm-repositories.yaml
-  - grafana-operator-hr.yaml
-  - victoria-metrics-hr.yaml
-  - loki-hr.yaml
-  - tempo-hr.yaml
-  - otel-collector/helm-repository.yaml
-  - otel-collector/helm-release-operator.yaml
-  - otel-collector/opentelemetry-collector.yaml
-  # Prometheus Operator CRDs
-  - https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.81.0/stripped-down-crds.yaml
-
+  - prometheus-crds
+  - victoria-metrics
+  - otel-collector
+  - loki
+  - tempo
+  - grafana
+# When the full observability stack is installed together, grafana-operator is
+# present, so flip the VM chart's grafana-dashboard generation back on. The
+# per-component (composable) VM install leaves this at false so it does not
+# depend on the Grafana CRDs.
+patches:
+  - target:
+      group: helm.toolkit.fluxcd.io
+      version: v2
+      kind: HelmRelease
+      name: vm
+      namespace: flux-system
+    patch: |
+      - op: replace
+        path: /spec/values/defaultDashboards/grafanaOperator/enabled
+        value: true