From a3f944a5716c86ab8bdf171c7e608d1879e7c584 Mon Sep 17 00:00:00 2001
From: Scot Wells <wells.scot@gmail.com>
Date: Wed, 15 Apr 2026 20:54:22 -0500
Subject: [PATCH 1/3] feat(observability): split into composable per-component
 tasks

The monolithic install-observability task deploys the entire telemetry
stack as a single blob, which doesn't fit on resource-constrained CI
runners. Downstream repos like resource-metrics only need Victoria
Metrics + OTel Collector for their e2e tests.

Split the observability stack into per-component kustomization
directories and Taskfile tasks so consumers can install only what they
need. The existing install-observability task is preserved as a thin
composite that calls all sub-tasks, maintaining full backward
compatibility.

New tasks:
- install-prometheus-crds
- install-victoria-metrics (depends on prometheus-crds)
- install-otel-collector (with webhook retry logic)
- install-grafana (depends on victoria-metrics)
- install-loki
- install-tempo

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 Taskfile.yml                                  | 130 ++++++++++++++----
 .../datasources/alertmanager-datasource.yaml  |  19 +++
 .../grafana/datasources/kustomization.yaml    |   7 +
 .../grafana/datasources/loki-datasource.yaml  |  26 ++++
 .../grafana/datasources/tempo-datasource.yaml |  54 ++++++++
 .../grafana/datasources/vm-datasource.yaml    |  21 +++
 .../{ => grafana}/grafana-instance.yaml       |   0
 .../{ => grafana}/grafana-operator-hr.yaml    |   0
 .../grafana/helm-repositories.yaml            |  18 +++
 .../observability/grafana/kustomization.yaml  |   8 ++
 .../observability/grafana/namespace.yaml      |   7 +
 components/observability/kustomization.yaml   |  20 ++-
 .../observability/loki/helm-repositories.yaml |  18 +++
 .../observability/loki/kustomization.yaml     |   6 +
 .../observability/{ => loki}/loki-hr.yaml     |   0
 components/observability/loki/namespace.yaml  |   7 +
 .../otel-collector/kustomization.yaml         |   1 +
 .../otel-collector/namespace.yaml             |   7 +
 .../prometheus-crds/kustomization.yaml        |   6 +
 .../prometheus-crds/namespace.yaml            |   7 +
 .../tempo/helm-repositories.yaml              |  18 +++
 .../observability/tempo/kustomization.yaml    |   6 +
 components/observability/tempo/namespace.yaml |   7 +
 .../observability/{ => tempo}/tempo-hr.yaml   |   0
 .../victoria-metrics/helm-repositories.yaml   |  18 +++
 .../victoria-metrics/kustomization.yaml       |   6 +
 .../victoria-metrics/namespace.yaml           |   7 +
 .../victoria-metrics-hr.yaml                  |   0
 28 files changed, 394 insertions(+), 30 deletions(-)
 create mode 100644 components/observability/grafana/datasources/alertmanager-datasource.yaml
 create mode 100644 components/observability/grafana/datasources/kustomization.yaml
 create mode 100644 components/observability/grafana/datasources/loki-datasource.yaml
 create mode 100644 components/observability/grafana/datasources/tempo-datasource.yaml
 create mode 100644 components/observability/grafana/datasources/vm-datasource.yaml
 rename components/observability/{ => grafana}/grafana-instance.yaml (100%)
 rename components/observability/{ => grafana}/grafana-operator-hr.yaml (100%)
 create mode 100644 components/observability/grafana/helm-repositories.yaml
 create mode 100644 components/observability/grafana/kustomization.yaml
 create mode 100644 components/observability/grafana/namespace.yaml
 create mode 100644 components/observability/loki/helm-repositories.yaml
 create mode 100644 components/observability/loki/kustomization.yaml
 rename components/observability/{ => loki}/loki-hr.yaml (100%)
 create mode 100644 components/observability/loki/namespace.yaml
 create mode 100644 components/observability/otel-collector/namespace.yaml
 create mode 100644 components/observability/prometheus-crds/kustomization.yaml
 create mode 100644 components/observability/prometheus-crds/namespace.yaml
 create mode 100644 components/observability/tempo/helm-repositories.yaml
 create mode 100644 components/observability/tempo/kustomization.yaml
 create mode 100644 components/observability/tempo/namespace.yaml
 rename components/observability/{ => tempo}/tempo-hr.yaml (100%)
 create mode 100644 components/observability/victoria-metrics/helm-repositories.yaml
 create mode 100644 components/observability/victoria-metrics/kustomization.yaml
 create mode 100644 components/observability/victoria-metrics/namespace.yaml
 rename components/observability/{ => victoria-metrics}/victoria-metrics-hr.yaml (100%)

diff --git a/Taskfile.yml b/Taskfile.yml
index 48b45eb..6abeab4 100644
--- a/Taskfile.yml
+++ b/Taskfile.yml
@@ -107,7 +107,15 @@ tasks:
       - echo "  install-cert-manager    Deploy cert-manager + CSI driver"
       - echo "  install-kyverno         Deploy Kyverno policy engine"
       - echo "  install-envoy-gateway-operator  Deploy Envoy Gateway"
-      - echo "  install-observability   [OPTIONAL] Deploy telemetry stack"
+      - echo ""
+      - echo "📊 Observability (composable):"
+      - echo "  install-observability       [OPTIONAL] Deploy full telemetry stack"
+      - echo "  install-prometheus-crds     Prometheus Operator CRDs"
+      - echo "  install-victoria-metrics    Victoria Metrics (vmagent + vmsingle)"
+      - echo "  install-otel-collector      OpenTelemetry Operator + Collector"
+      - echo "  install-grafana             Grafana Operator + instance + datasources"
+      - echo "  install-loki                Loki for log aggregation"
+      - echo "  install-tempo               Tempo for distributed tracing"
       - echo ""
       - echo "🔨 CI/Development:"
       - echo "  kind-load-image         Load images into KIND"
@@ -407,26 +415,16 @@ tasks:
       - echo "   To find the actual NodePort assignments, run:"
       - echo "   kubectl get svc -n envoy-gateway-system -l app.kubernetes.io/name=envoy"
 
-  install-observability:
-    desc: "[OPTIONAL] Deploy complete observability stack (Victoria Metrics, Loki, Tempo, Grafana, OTel)"
+  install-prometheus-crds:
+    desc: "Deploy Prometheus Operator CRDs"
     silent: true
     cmds:
       - task: ensure-repo
-      - echo "➡️  Reconciling Observability Stack …"
-      - kustomize build {{.REPO_DIR}}/components/observability | kubectl apply -f -
-      - echo "⏳ Waiting for Grafana Operator HelmRelease …"
-      - kubectl -n flux-system wait helmrelease/grafana-operator --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
-      - echo "⏳ Waiting for Victoria Metrics Stack HelmRelease …"
-      - kubectl -n flux-system wait helmrelease/vm --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
-      - echo "⏳ Waiting for Loki HelmRelease …"
-      - kubectl -n flux-system wait helmrelease/loki --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
-      - echo "⏳ Waiting for Tempo HelmRelease …"
-      - kubectl -n flux-system wait helmrelease/tempo --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
-      - echo "⏳ Waiting for OpenTelemetry Operator HelmRelease …"
-      - kubectl -n flux-system wait helmrelease/opentelemetry-operator --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
+      - echo "➡️  Reconciling Prometheus Operator CRDs …"
+      - kustomize build {{.REPO_DIR}}/components/observability/prometheus-crds | kubectl apply --server-side --force-conflicts -f -
       - |
         set -euo pipefail
-        echo "⏳ Waiting for Prometheus CRDs to be Established…"
+        echo "⏳ Waiting for Prometheus CRDs to be Established …"
         for crd in \
           servicemonitors.monitoring.coreos.com \
           podmonitors.monitoring.coreos.com \
@@ -436,22 +434,106 @@ tasks:
           alertmanagers.monitoring.coreos.com \
           thanosrulers.monitoring.coreos.com
         do
-          echo "$crd"
+          echo "  $crd"
           kubectl wait --for=condition=Established "crd/${crd}" --timeout=120s || true
         done
-      - echo "⏳ Waiting for vmagent & vmsingle to be Ready (best-effort)…"
-      - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmagent -n telemetry-system --timeout=120s || true
-      - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmsingle -n telemetry-system --timeout=120s || true
+      - echo "✅ Prometheus Operator CRDs are ready"
+
+  install-victoria-metrics:
+    desc: "Deploy Victoria Metrics (vmagent + vmsingle)"
+    silent: true
+    deps: [install-prometheus-crds]
+    cmds:
+      - task: ensure-repo
+      - echo "➡️  Reconciling Victoria Metrics …"
+      - kustomize build {{.REPO_DIR}}/components/observability/victoria-metrics | kubectl apply -f -
+      - echo "⏳ Waiting for Victoria Metrics Stack HelmRelease …"
+      - kubectl -n flux-system wait helmrelease/vm --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
+      - echo "⏳ Waiting for vmagent & vmsingle to be Ready …"
+      - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmagent -n telemetry-system --timeout={{.WAIT_TIMEOUT}} || true
+      - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmsingle -n telemetry-system --timeout={{.WAIT_TIMEOUT}} || true
+      - echo "✅ Victoria Metrics is ready"
+
+  install-otel-collector:
+    desc: "Deploy OpenTelemetry Operator + Collector"
+    silent: true
+    cmds:
+      - task: ensure-repo
+      - echo "➡️  Reconciling OpenTelemetry Operator …"
+      - kustomize build {{.REPO_DIR}}/components/observability/otel-collector | kubectl apply -f -
+      - echo "⏳ Waiting for OpenTelemetry Operator HelmRelease …"
+      - kubectl -n flux-system wait helmrelease/opentelemetry-operator --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
       - echo "⏳ Waiting for OpenTelemetryCollector CRD to be Established …"
       - kubectl wait --for=condition=Established crd/opentelemetrycollectors.opentelemetry.io --timeout={{.WAIT_TIMEOUT}}
-      - echo "➡️  Applying OpenTelemetryCollector CR …"
-      - kubectl apply -f {{.REPO_DIR}}/components/observability/otel-collector/opentelemetry-collector.yaml
+      - |
+        echo "⏳ Waiting for OTel webhook to be ready …"
+        sleep 10
+      - echo "➡️  Applying OpenTelemetryCollector CR (with retry for webhook readiness) …"
+      - |
+        set -euo pipefail
+        for i in 1 2 3 4 5; do
+          if kubectl apply -f {{.REPO_DIR}}/components/observability/otel-collector/opentelemetry-collector.yaml 2>/dev/null; then
+            echo "  CR applied successfully"
+            break
+          fi
+          echo "  Attempt $i failed, retrying in 10s …"
+          sleep 10
+        done
       - echo "⏳ Waiting for OTel Collector DaemonSet …"
       - kubectl -n telemetry-system rollout status daemonset/otel-collector-collector --timeout={{.WAIT_TIMEOUT}}
+      - echo "✅ OpenTelemetry Collector is ready"
+
+  install-grafana:
+    desc: "Deploy Grafana Operator + instance + datasources"
+    silent: true
+    deps: [install-victoria-metrics]
+    cmds:
+      - task: ensure-repo
+      - echo "➡️  Reconciling Grafana Operator …"
+      - kustomize build {{.REPO_DIR}}/components/observability/grafana | kubectl apply -f -
+      - echo "⏳ Waiting for Grafana Operator HelmRelease …"
+      - kubectl -n flux-system wait helmrelease/grafana-operator --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
       - echo "➡️  Applying Grafana Instance (after Operator CRDs are ready) …"
-      - kubectl apply -f {{.REPO_DIR}}/components/observability/grafana-instance.yaml
+      - kubectl apply -f {{.REPO_DIR}}/components/observability/grafana/grafana-instance.yaml
       - echo "➡️  Applying Grafana Datasources …"
-      - kustomize build {{.REPO_DIR}}/components/observability/datasources | kubectl apply -f -
+      - kustomize build {{.REPO_DIR}}/components/observability/grafana/datasources | kubectl apply -f -
+      - echo "✅ Grafana is ready"
+      - echo ""
+      - 'echo "📊 Access Grafana at: http://localhost:30000"'
+      - 'echo "   Username: admin"'
+      - 'echo "   Password: datum123"'
+
+  install-loki:
+    desc: "Deploy Loki for log aggregation"
+    silent: true
+    cmds:
+      - task: ensure-repo
+      - echo "➡️  Reconciling Loki …"
+      - kustomize build {{.REPO_DIR}}/components/observability/loki | kubectl apply -f -
+      - echo "⏳ Waiting for Loki HelmRelease …"
+      - kubectl -n flux-system wait helmrelease/loki --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
+      - echo "✅ Loki is ready"
+
+  install-tempo:
+    desc: "Deploy Tempo for distributed tracing"
+    silent: true
+    cmds:
+      - task: ensure-repo
+      - echo "➡️  Reconciling Tempo …"
+      - kustomize build {{.REPO_DIR}}/components/observability/tempo | kubectl apply -f -
+      - echo "⏳ Waiting for Tempo HelmRelease …"
+      - kubectl -n flux-system wait helmrelease/tempo --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
+      - echo "✅ Tempo is ready"
+
+  install-observability:
+    desc: "[OPTIONAL] Deploy complete observability stack (Victoria Metrics, Loki, Tempo, Grafana, OTel)"
+    silent: true
+    cmds:
+      - task: install-victoria-metrics
+      - task: install-otel-collector
+      - task: install-loki
+      - task: install-tempo
+      - task: install-grafana
       - echo "✅ Observability stack is ready"
       - echo ""
       - 'echo "📊 Access Grafana at: http://localhost:30000"'
diff --git a/components/observability/grafana/datasources/alertmanager-datasource.yaml b/components/observability/grafana/datasources/alertmanager-datasource.yaml
new file mode 100644
index 0000000..fae8cf3
--- /dev/null
+++ b/components/observability/grafana/datasources/alertmanager-datasource.yaml
@@ -0,0 +1,19 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDatasource
+metadata:
+  name: alertmanager
+  namespace: telemetry-system
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  datasource:
+    name: Alertmanager
+    type: alertmanager
+    access: proxy
+    url: http://vmalertmanager-telemetry-system-vm.telemetry-system.svc.cluster.local:9093
+    isDefault: true
+    editable: true
+    jsonData:
+      implementation: prometheus
+      handleGrafanaManagedAlerts: true
diff --git a/components/observability/grafana/datasources/kustomization.yaml b/components/observability/grafana/datasources/kustomization.yaml
new file mode 100644
index 0000000..76fe8ff
--- /dev/null
+++ b/components/observability/grafana/datasources/kustomization.yaml
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - vm-datasource.yaml
+  - loki-datasource.yaml
+  - tempo-datasource.yaml
+  - alertmanager-datasource.yaml
diff --git a/components/observability/grafana/datasources/loki-datasource.yaml b/components/observability/grafana/datasources/loki-datasource.yaml
new file mode 100644
index 0000000..e1e5f21
--- /dev/null
+++ b/components/observability/grafana/datasources/loki-datasource.yaml
@@ -0,0 +1,26 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDatasource
+metadata:
+  name: loki-datasource
+  namespace: telemetry-system
+  labels:
+    dashboards: "grafana"
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  datasource:
+    name: Loki
+    type: loki
+    access: proxy
+    url: http://telemetry-system-loki.telemetry-system.svc.cluster.local:3100
+    isDefault: false
+    editable: true
+    jsonData:
+      maxLines: 1000
+      derivedFields:
+        - datasourceUid: tempo-datasource
+          matcherRegex: "traceID=(\\w+)"
+          name: "TraceID"
+          url: "$${__value.raw}"
+          urlDisplayLabel: "View Trace"
diff --git a/components/observability/grafana/datasources/tempo-datasource.yaml b/components/observability/grafana/datasources/tempo-datasource.yaml
new file mode 100644
index 0000000..db66274
--- /dev/null
+++ b/components/observability/grafana/datasources/tempo-datasource.yaml
@@ -0,0 +1,54 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDatasource
+metadata:
+  name: tempo-datasource
+  namespace: telemetry-system
+  labels:
+    dashboards: "grafana"
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  datasource:
+    name: Tempo
+    type: tempo
+    access: proxy
+    url: http://telemetry-system-tempo.telemetry-system.svc.cluster.local:3100
+    uid: tempo-datasource
+    isDefault: false
+    editable: true
+    jsonData:
+      httpMethod: GET
+      tracesToLogs:
+        datasourceUid: loki-datasource
+        tags:
+          - job
+          - instance
+          - pod
+          - namespace
+        mappedTags:
+          - key: service.name
+            value: service
+        mapTagNamesEnabled: true
+        spanStartTimeShift: "-1h"
+        spanEndTimeShift: "1h"
+        filterByTraceID: true
+        filterBySpanID: false
+      tracesToMetrics:
+        datasourceUid: victoria-metrics-datasource
+        tags:
+          - job
+          - instance
+          - pod
+          - namespace
+        queries:
+          - name: "Request Rate"
+            query: 'rate(tempo_request_duration_seconds_count[$__rate_interval])'
+      serviceMap:
+        datasourceUid: victoria-metrics-datasource
+      nodeGraph:
+        enabled: true
+      search:
+        hide: false
+      lokiSearch:
+        datasourceUid: loki-datasource
diff --git a/components/observability/grafana/datasources/vm-datasource.yaml b/components/observability/grafana/datasources/vm-datasource.yaml
new file mode 100644
index 0000000..7dbb59d
--- /dev/null
+++ b/components/observability/grafana/datasources/vm-datasource.yaml
@@ -0,0 +1,21 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDatasource
+metadata:
+  name: victoria-metrics-datasource
+  namespace: telemetry-system
+  labels:
+    dashboards: "grafana"
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  datasource:
+    name: VictoriaMetrics
+    type: prometheus
+    access: proxy
+    url: http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428
+    uid: victoria-metrics-datasource
+    isDefault: true
+    editable: true
+    jsonData:
+      timeInterval: "30s"
diff --git a/components/observability/grafana-instance.yaml b/components/observability/grafana/grafana-instance.yaml
similarity index 100%
rename from components/observability/grafana-instance.yaml
rename to components/observability/grafana/grafana-instance.yaml
diff --git a/components/observability/grafana-operator-hr.yaml b/components/observability/grafana/grafana-operator-hr.yaml
similarity index 100%
rename from components/observability/grafana-operator-hr.yaml
rename to components/observability/grafana/grafana-operator-hr.yaml
diff --git a/components/observability/grafana/helm-repositories.yaml b/components/observability/grafana/helm-repositories.yaml
new file mode 100644
index 0000000..2678f3a
--- /dev/null
+++ b/components/observability/grafana/helm-repositories.yaml
@@ -0,0 +1,18 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: vm
+  namespace: flux-system
+spec:
+  interval: 12h
+  url: https://victoriametrics.github.io/helm-charts/
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana
+  namespace: flux-system
+spec:
+  interval: 12h
+  url: https://grafana.github.io/helm-charts/
diff --git a/components/observability/grafana/kustomization.yaml b/components/observability/grafana/kustomization.yaml
new file mode 100644
index 0000000..57d0c20
--- /dev/null
+++ b/components/observability/grafana/kustomization.yaml
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - helm-repositories.yaml
+  - grafana-operator-hr.yaml
+  - grafana-instance.yaml
+  - datasources
diff --git a/components/observability/grafana/namespace.yaml b/components/observability/grafana/namespace.yaml
new file mode 100644
index 0000000..17def55
--- /dev/null
+++ b/components/observability/grafana/namespace.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: telemetry-system
+  labels:
+    name: telemetry-system
+    dashboards: "grafana"
\ No newline at end of file
diff --git a/components/observability/kustomization.yaml b/components/observability/kustomization.yaml
index 945898a..429a936 100644
--- a/components/observability/kustomization.yaml
+++ b/components/observability/kustomization.yaml
@@ -1,15 +1,23 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  # Shared prerequisites
   - namespace.yaml
   - helm-repositories.yaml
-  - grafana-operator-hr.yaml
-  - victoria-metrics-hr.yaml
-  - loki-hr.yaml
-  - tempo-hr.yaml
+  # Prometheus Operator CRDs
+  - https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.81.0/stripped-down-crds.yaml
+  # Victoria Metrics
+  - victoria-metrics/victoria-metrics-hr.yaml
+  # Grafana
+  - grafana/grafana-operator-hr.yaml
+  - grafana/grafana-instance.yaml
+  - datasources
+  # Loki
+  - loki/loki-hr.yaml
+  # Tempo
+  - tempo/tempo-hr.yaml
+  # OpenTelemetry Collector
   - otel-collector/helm-repository.yaml
   - otel-collector/helm-release-operator.yaml
   - otel-collector/opentelemetry-collector.yaml
-  # Prometheus Operator CRDs
-  - https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.81.0/stripped-down-crds.yaml
 
diff --git a/components/observability/loki/helm-repositories.yaml b/components/observability/loki/helm-repositories.yaml
new file mode 100644
index 0000000..2678f3a
--- /dev/null
+++ b/components/observability/loki/helm-repositories.yaml
@@ -0,0 +1,18 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: vm
+  namespace: flux-system
+spec:
+  interval: 12h
+  url: https://victoriametrics.github.io/helm-charts/
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana
+  namespace: flux-system
+spec:
+  interval: 12h
+  url: https://grafana.github.io/helm-charts/
diff --git a/components/observability/loki/kustomization.yaml b/components/observability/loki/kustomization.yaml
new file mode 100644
index 0000000..c0854db
--- /dev/null
+++ b/components/observability/loki/kustomization.yaml
@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - helm-repositories.yaml
+  - loki-hr.yaml
diff --git a/components/observability/loki-hr.yaml b/components/observability/loki/loki-hr.yaml
similarity index 100%
rename from components/observability/loki-hr.yaml
rename to components/observability/loki/loki-hr.yaml
diff --git a/components/observability/loki/namespace.yaml b/components/observability/loki/namespace.yaml
new file mode 100644
index 0000000..17def55
--- /dev/null
+++ b/components/observability/loki/namespace.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: telemetry-system
+  labels:
+    name: telemetry-system
+    dashboards: "grafana"
\ No newline at end of file
diff --git a/components/observability/otel-collector/kustomization.yaml b/components/observability/otel-collector/kustomization.yaml
index 133b3ad..0ef3709 100644
--- a/components/observability/otel-collector/kustomization.yaml
+++ b/components/observability/otel-collector/kustomization.yaml
@@ -1,6 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - namespace.yaml
   - helm-repository.yaml
   - helm-release-operator.yaml
   - opentelemetry-collector.yaml
diff --git a/components/observability/otel-collector/namespace.yaml b/components/observability/otel-collector/namespace.yaml
new file mode 100644
index 0000000..17def55
--- /dev/null
+++ b/components/observability/otel-collector/namespace.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: telemetry-system
+  labels:
+    name: telemetry-system
+    dashboards: "grafana"
\ No newline at end of file
diff --git a/components/observability/prometheus-crds/kustomization.yaml b/components/observability/prometheus-crds/kustomization.yaml
new file mode 100644
index 0000000..e95bdfd
--- /dev/null
+++ b/components/observability/prometheus-crds/kustomization.yaml
@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  # Prometheus Operator CRDs
+  - https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.81.0/stripped-down-crds.yaml
diff --git a/components/observability/prometheus-crds/namespace.yaml b/components/observability/prometheus-crds/namespace.yaml
new file mode 100644
index 0000000..17def55
--- /dev/null
+++ b/components/observability/prometheus-crds/namespace.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: telemetry-system
+  labels:
+    name: telemetry-system
+    dashboards: "grafana"
\ No newline at end of file
diff --git a/components/observability/tempo/helm-repositories.yaml b/components/observability/tempo/helm-repositories.yaml
new file mode 100644
index 0000000..2678f3a
--- /dev/null
+++ b/components/observability/tempo/helm-repositories.yaml
@@ -0,0 +1,18 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: vm
+  namespace: flux-system
+spec:
+  interval: 12h
+  url: https://victoriametrics.github.io/helm-charts/
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana
+  namespace: flux-system
+spec:
+  interval: 12h
+  url: https://grafana.github.io/helm-charts/
diff --git a/components/observability/tempo/kustomization.yaml b/components/observability/tempo/kustomization.yaml
new file mode 100644
index 0000000..15865a2
--- /dev/null
+++ b/components/observability/tempo/kustomization.yaml
@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - helm-repositories.yaml
+  - tempo-hr.yaml
diff --git a/components/observability/tempo/namespace.yaml b/components/observability/tempo/namespace.yaml
new file mode 100644
index 0000000..17def55
--- /dev/null
+++ b/components/observability/tempo/namespace.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: telemetry-system
+  labels:
+    name: telemetry-system
+    dashboards: "grafana"
\ No newline at end of file
diff --git a/components/observability/tempo-hr.yaml b/components/observability/tempo/tempo-hr.yaml
similarity index 100%
rename from components/observability/tempo-hr.yaml
rename to components/observability/tempo/tempo-hr.yaml
diff --git a/components/observability/victoria-metrics/helm-repositories.yaml b/components/observability/victoria-metrics/helm-repositories.yaml
new file mode 100644
index 0000000..2678f3a
--- /dev/null
+++ b/components/observability/victoria-metrics/helm-repositories.yaml
@@ -0,0 +1,18 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: vm
+  namespace: flux-system
+spec:
+  interval: 12h
+  url: https://victoriametrics.github.io/helm-charts/
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana
+  namespace: flux-system
+spec:
+  interval: 12h
+  url: https://grafana.github.io/helm-charts/
diff --git a/components/observability/victoria-metrics/kustomization.yaml b/components/observability/victoria-metrics/kustomization.yaml
new file mode 100644
index 0000000..5159172
--- /dev/null
+++ b/components/observability/victoria-metrics/kustomization.yaml
@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - helm-repositories.yaml
+  - victoria-metrics-hr.yaml
diff --git a/components/observability/victoria-metrics/namespace.yaml b/components/observability/victoria-metrics/namespace.yaml
new file mode 100644
index 0000000..17def55
--- /dev/null
+++ b/components/observability/victoria-metrics/namespace.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: telemetry-system
+  labels:
+    name: telemetry-system
+    dashboards: "grafana"
\ No newline at end of file
diff --git a/components/observability/victoria-metrics-hr.yaml b/components/observability/victoria-metrics/victoria-metrics-hr.yaml
similarity index 100%
rename from components/observability/victoria-metrics-hr.yaml
rename to components/observability/victoria-metrics/victoria-metrics-hr.yaml

From b9515a4e12b6f4685088718b0fadc3f4c9503566 Mon Sep 17 00:00:00 2001
From: Scot Wells <wells.scot@gmail.com>
Date: Thu, 16 Apr 2026 09:58:18 -0500
Subject: [PATCH 2/3] refactor(observability): remove duplicated shared files;
 give each component its own namespace
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Problem 1: the composable split had copy-pasted namespace.yaml,
helm-repositories.yaml, and datasources/ into each component
subdirectory while leaving the originals at the root. That duplicated
~200 lines and caused the root kustomize build to fail on conflicting
resources.

- Deleted the root-level namespace.yaml, helm-repositories.yaml, and
  datasources/ directory.
- Kept a single copy of the datasources under grafana/datasources/ (where
  the Grafana instance lives).
- Pared each component's helm-repositories.yaml to just the repo that
  component actually consumes. Gave loki/tempo distinct HelmRepository
  names (loki-charts, tempo-charts) so the composed root kustomize build
  does not fail on duplicate source.toolkit.fluxcd.io resources.
- Root components/observability/kustomization.yaml now references the six
  component subdirectories only.
- Dropped prometheus-crds/namespace.yaml entirely — the kustomization
  only installs cluster-scoped CRDs so no namespace is needed.

Problem 2: everything still deployed to telemetry-system, which defeats
the point of per-component composition. Each component now has its own
namespace so kubectl delete ns <x> cleanly uninstalls it:

- victoria-metrics-system (was telemetry-system)
- grafana-system
- loki-system
- tempo-system
- otel-collector-system

Cross-component references are now fully qualified service DNS names:

- Grafana datasources point at vmsingle/vmalertmanager in
  victoria-metrics-system, loki-system-loki in loki-system, and
  tempo-system-tempo in tempo-system.
- The OTel Collector's otlp, loki, and prometheusremotewrite exporters
  point at the new FQDNs.
- VMAlert's datasource, notifier, and remoteWrite URLs use the new
  victoria-metrics-system service names.
- VM defaultDashboards.grafanaOperator.allowCrossNamespaceImport is now
  true so dashboards created in victoria-metrics-system can target the
  Grafana CR in grafana-system.

Taskfile's per-component waits updated to reference the new namespaces
(vmagent/vmsingle in victoria-metrics-system, otel-collector-collector
DaemonSet in otel-collector-system).

README refreshed to document the subcomponent layout, namespaces, and
removal procedure.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 Taskfile.yml                                  |  6 +--
 components/observability/README.md            | 46 +++++++++++-----
 .../datasources/alertmanager-datasource.yaml  | 19 -------
 .../datasources/kustomization.yaml            |  7 ---
 .../datasources/loki-datasource.yaml          | 26 ---------
 .../datasources/tempo-datasource.yaml         | 54 -------------------
 .../datasources/vm-datasource.yaml            | 21 --------
 .../datasources/alertmanager-datasource.yaml  |  4 +-
 .../grafana/datasources/loki-datasource.yaml  |  4 +-
 .../grafana/datasources/tempo-datasource.yaml |  4 +-
 .../grafana/datasources/vm-datasource.yaml    |  4 +-
 .../grafana/grafana-instance.yaml             |  2 +-
 .../grafana/grafana-operator-hr.yaml          |  2 +-
 .../grafana/helm-repositories.yaml            |  9 ----
 .../observability/grafana/namespace.yaml      |  6 +--
 .../observability/helm-repositories.yaml      | 18 -------
 components/observability/kustomization.yaml   | 26 +++------
 .../observability/loki/helm-repositories.yaml | 11 +---
 components/observability/loki/loki-hr.yaml    |  4 +-
 components/observability/loki/namespace.yaml  |  5 +-
 components/observability/namespace.yaml       |  7 ---
 .../otel-collector/helm-release-operator.yaml |  2 +-
 .../otel-collector/namespace.yaml             |  5 +-
 .../opentelemetry-collector.yaml              |  8 +--
 .../prometheus-crds/kustomization.yaml        |  3 +-
 .../prometheus-crds/namespace.yaml            |  7 ---
 .../tempo/helm-repositories.yaml              | 11 +---
 components/observability/tempo/namespace.yaml |  5 +-
 components/observability/tempo/tempo-hr.yaml  |  4 +-
 .../victoria-metrics/helm-repositories.yaml   |  9 ----
 .../victoria-metrics/namespace.yaml           |  5 +-
 .../victoria-metrics/victoria-metrics-hr.yaml | 10 ++--
 32 files changed, 81 insertions(+), 273 deletions(-)
 delete mode 100644 components/observability/datasources/alertmanager-datasource.yaml
 delete mode 100644 components/observability/datasources/kustomization.yaml
 delete mode 100644 components/observability/datasources/loki-datasource.yaml
 delete mode 100644 components/observability/datasources/tempo-datasource.yaml
 delete mode 100644 components/observability/datasources/vm-datasource.yaml
 delete mode 100644 components/observability/helm-repositories.yaml
 delete mode 100644 components/observability/namespace.yaml
 delete mode 100644 components/observability/prometheus-crds/namespace.yaml

diff --git a/Taskfile.yml b/Taskfile.yml
index 6abeab4..6387fdb 100644
--- a/Taskfile.yml
+++ b/Taskfile.yml
@@ -450,8 +450,8 @@ tasks:
       - echo "⏳ Waiting for Victoria Metrics Stack HelmRelease …"
       - kubectl -n flux-system wait helmrelease/vm --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
       - echo "⏳ Waiting for vmagent & vmsingle to be Ready …"
-      - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmagent -n telemetry-system --timeout={{.WAIT_TIMEOUT}} || true
-      - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmsingle -n telemetry-system --timeout={{.WAIT_TIMEOUT}} || true
+      - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmagent -n victoria-metrics-system --timeout={{.WAIT_TIMEOUT}} || true
+      - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmsingle -n victoria-metrics-system --timeout={{.WAIT_TIMEOUT}} || true
       - echo "✅ Victoria Metrics is ready"
 
   install-otel-collector:
@@ -480,7 +480,7 @@ tasks:
           sleep 10
         done
       - echo "⏳ Waiting for OTel Collector DaemonSet …"
-      - kubectl -n telemetry-system rollout status daemonset/otel-collector-collector --timeout={{.WAIT_TIMEOUT}}
+      - kubectl -n otel-collector-system rollout status daemonset/otel-collector-collector --timeout={{.WAIT_TIMEOUT}}
       - echo "✅ OpenTelemetry Collector is ready"
 
   install-grafana:
diff --git a/components/observability/README.md b/components/observability/README.md
index a683d3d..5a285c4 100644
--- a/components/observability/README.md
+++ b/components/observability/README.md
@@ -4,20 +4,25 @@ Optional observability stack for test infrastructure, providing metrics, logs, a
 
 ## Overview
 
-This component is an **optional add-on** deployed after core infrastructure. It provides comprehensive telemetry capabilities using Grafana, Victoria Metrics, Loki, and Tempo.
+This component is an **optional add-on** deployed after core infrastructure. It provides comprehensive telemetry capabilities using Grafana, Victoria Metrics, Loki, Tempo, and the OpenTelemetry Collector.
 
-## Components
+The stack is split into composable subcomponents — each deploys into its own namespace so it can be installed, upgraded, or removed independently (`kubectl delete ns <component>-system` cleanly uninstalls).
 
-- **Grafana**: Visualization and dashboards
-- **Victoria Metrics**: Metrics collection and storage
-- **Loki**: Log aggregation and storage
-- **Tempo**: Distributed tracing storage
-- **Promtail**: Log collection agent
+## Subcomponents
+
+| Component         | Namespace                 | Purpose                               |
+| ----------------- | ------------------------- | ------------------------------------- |
+| `prometheus-crds` | (cluster-scoped)          | Prometheus Operator CRDs              |
+| `victoria-metrics`| `victoria-metrics-system` | Metrics collection and storage        |
+| `otel-collector`  | `otel-collector-system`   | OpenTelemetry DaemonSet collector     |
+| `loki`            | `loki-system`             | Log aggregation and storage           |
+| `tempo`           | `tempo-system`            | Distributed tracing storage           |
+| `grafana`         | `grafana-system`          | Visualization, dashboards, datasources|
 
 ## Access
 
 - **Grafana UI**: Available at NodePort 30000 (admin/datum123)
-- **Default Datasources**: Victoria Metrics (metrics), Loki (logs), Tempo (traces)
+- **Default Datasources**: Victoria Metrics (metrics), Loki (logs), Tempo (traces), Alertmanager
 
 ## Prerequisites
 
@@ -25,18 +30,35 @@ Core test infrastructure must be running before deploying observability componen
 
 ## Deployment
 
-Deploy using the `install-observability` task target:
+Deploy the whole stack:
 
 ```bash
 task install-observability
 ```
 
-This deploys all components and configures datasources automatically.
+Or install components individually:
+
+```bash
+task install-prometheus-crds
+task install-victoria-metrics
+task install-otel-collector
+task install-loki
+task install-tempo
+task install-grafana
+```
 
 ## Removal
 
-To remove the observability stack:
+To remove an individual component:
+
+```bash
+kubectl delete namespace <component>-system
+```
+
+To remove the whole stack:
 
 ```bash
-kubectl delete namespace observability
+for ns in grafana-system tempo-system loki-system otel-collector-system victoria-metrics-system; do
+  kubectl delete namespace "$ns"
+done
 ```
diff --git a/components/observability/datasources/alertmanager-datasource.yaml b/components/observability/datasources/alertmanager-datasource.yaml
deleted file mode 100644
index fae8cf3..0000000
--- a/components/observability/datasources/alertmanager-datasource.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-apiVersion: grafana.integreatly.org/v1beta1
-kind: GrafanaDatasource
-metadata:
-  name: alertmanager
-  namespace: telemetry-system
-spec:
-  instanceSelector:
-    matchLabels:
-      dashboards: "grafana"
-  datasource:
-    name: Alertmanager
-    type: alertmanager
-    access: proxy
-    url: http://vmalertmanager-telemetry-system-vm.telemetry-system.svc.cluster.local:9093
-    isDefault: true
-    editable: true
-    jsonData:
-      implementation: prometheus
-      handleGrafanaManagedAlerts: true
diff --git a/components/observability/datasources/kustomization.yaml b/components/observability/datasources/kustomization.yaml
deleted file mode 100644
index 76fe8ff..0000000
--- a/components/observability/datasources/kustomization.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - vm-datasource.yaml
-  - loki-datasource.yaml
-  - tempo-datasource.yaml
-  - alertmanager-datasource.yaml
diff --git a/components/observability/datasources/loki-datasource.yaml b/components/observability/datasources/loki-datasource.yaml
deleted file mode 100644
index e1e5f21..0000000
--- a/components/observability/datasources/loki-datasource.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-apiVersion: grafana.integreatly.org/v1beta1
-kind: GrafanaDatasource
-metadata:
-  name: loki-datasource
-  namespace: telemetry-system
-  labels:
-    dashboards: "grafana"
-spec:
-  instanceSelector:
-    matchLabels:
-      dashboards: "grafana"
-  datasource:
-    name: Loki
-    type: loki
-    access: proxy
-    url: http://telemetry-system-loki.telemetry-system.svc.cluster.local:3100
-    isDefault: false
-    editable: true
-    jsonData:
-      maxLines: 1000
-      derivedFields:
-        - datasourceUid: tempo-datasource
-          matcherRegex: "traceID=(\\w+)"
-          name: "TraceID"
-          url: "$${__value.raw}"
-          urlDisplayLabel: "View Trace"
diff --git a/components/observability/datasources/tempo-datasource.yaml b/components/observability/datasources/tempo-datasource.yaml
deleted file mode 100644
index db66274..0000000
--- a/components/observability/datasources/tempo-datasource.yaml
+++ /dev/null
@@ -1,54 +0,0 @@
-apiVersion: grafana.integreatly.org/v1beta1
-kind: GrafanaDatasource
-metadata:
-  name: tempo-datasource
-  namespace: telemetry-system
-  labels:
-    dashboards: "grafana"
-spec:
-  instanceSelector:
-    matchLabels:
-      dashboards: "grafana"
-  datasource:
-    name: Tempo
-    type: tempo
-    access: proxy
-    url: http://telemetry-system-tempo.telemetry-system.svc.cluster.local:3100
-    uid: tempo-datasource
-    isDefault: false
-    editable: true
-    jsonData:
-      httpMethod: GET
-      tracesToLogs:
-        datasourceUid: loki-datasource
-        tags:
-          - job
-          - instance
-          - pod
-          - namespace
-        mappedTags:
-          - key: service.name
-            value: service
-        mapTagNamesEnabled: true
-        spanStartTimeShift: "-1h"
-        spanEndTimeShift: "1h"
-        filterByTraceID: true
-        filterBySpanID: false
-      tracesToMetrics:
-        datasourceUid: victoria-metrics-datasource
-        tags:
-          - job
-          - instance
-          - pod
-          - namespace
-        queries:
-          - name: "Request Rate"
-            query: 'rate(tempo_request_duration_seconds_count[$__rate_interval])'
-      serviceMap:
-        datasourceUid: victoria-metrics-datasource
-      nodeGraph:
-        enabled: true
-      search:
-        hide: false
-      lokiSearch:
-        datasourceUid: loki-datasource
diff --git a/components/observability/datasources/vm-datasource.yaml b/components/observability/datasources/vm-datasource.yaml
deleted file mode 100644
index 7dbb59d..0000000
--- a/components/observability/datasources/vm-datasource.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-apiVersion: grafana.integreatly.org/v1beta1
-kind: GrafanaDatasource
-metadata:
-  name: victoria-metrics-datasource
-  namespace: telemetry-system
-  labels:
-    dashboards: "grafana"
-spec:
-  instanceSelector:
-    matchLabels:
-      dashboards: "grafana"
-  datasource:
-    name: VictoriaMetrics
-    type: prometheus
-    access: proxy
-    url: http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428
-    uid: victoria-metrics-datasource
-    isDefault: true
-    editable: true
-    jsonData:
-      timeInterval: "30s"
diff --git a/components/observability/grafana/datasources/alertmanager-datasource.yaml b/components/observability/grafana/datasources/alertmanager-datasource.yaml
index fae8cf3..6c2ac3e 100644
--- a/components/observability/grafana/datasources/alertmanager-datasource.yaml
+++ b/components/observability/grafana/datasources/alertmanager-datasource.yaml
@@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1
 kind: GrafanaDatasource
 metadata:
   name: alertmanager
-  namespace: telemetry-system
+  namespace: grafana-system
 spec:
   instanceSelector:
     matchLabels:
@@ -11,7 +11,7 @@ spec:
     name: Alertmanager
     type: alertmanager
     access: proxy
-    url: http://vmalertmanager-telemetry-system-vm.telemetry-system.svc.cluster.local:9093
+    url: http://vmalertmanager-victoria-metrics-system-vm.victoria-metrics-system.svc.cluster.local:9093
     isDefault: true
     editable: true
     jsonData:
diff --git a/components/observability/grafana/datasources/loki-datasource.yaml b/components/observability/grafana/datasources/loki-datasource.yaml
index e1e5f21..803dd2e 100644
--- a/components/observability/grafana/datasources/loki-datasource.yaml
+++ b/components/observability/grafana/datasources/loki-datasource.yaml
@@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1
 kind: GrafanaDatasource
 metadata:
   name: loki-datasource
-  namespace: telemetry-system
+  namespace: grafana-system
   labels:
     dashboards: "grafana"
 spec:
@@ -13,7 +13,7 @@ spec:
     name: Loki
     type: loki
     access: proxy
-    url: http://telemetry-system-loki.telemetry-system.svc.cluster.local:3100
+    url: http://loki-system-loki.loki-system.svc.cluster.local:3100
     isDefault: false
     editable: true
     jsonData:
diff --git a/components/observability/grafana/datasources/tempo-datasource.yaml b/components/observability/grafana/datasources/tempo-datasource.yaml
index db66274..bc7ecb3 100644
--- a/components/observability/grafana/datasources/tempo-datasource.yaml
+++ b/components/observability/grafana/datasources/tempo-datasource.yaml
@@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1
 kind: GrafanaDatasource
 metadata:
   name: tempo-datasource
-  namespace: telemetry-system
+  namespace: grafana-system
   labels:
     dashboards: "grafana"
 spec:
@@ -13,7 +13,7 @@ spec:
     name: Tempo
     type: tempo
     access: proxy
-    url: http://telemetry-system-tempo.telemetry-system.svc.cluster.local:3100
+    url: http://tempo-system-tempo.tempo-system.svc.cluster.local:3100
     uid: tempo-datasource
     isDefault: false
     editable: true
diff --git a/components/observability/grafana/datasources/vm-datasource.yaml b/components/observability/grafana/datasources/vm-datasource.yaml
index 7dbb59d..61777c0 100644
--- a/components/observability/grafana/datasources/vm-datasource.yaml
+++ b/components/observability/grafana/datasources/vm-datasource.yaml
@@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1
 kind: GrafanaDatasource
 metadata:
   name: victoria-metrics-datasource
-  namespace: telemetry-system
+  namespace: grafana-system
   labels:
     dashboards: "grafana"
 spec:
@@ -13,7 +13,7 @@ spec:
     name: VictoriaMetrics
     type: prometheus
     access: proxy
-    url: http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428
+    url: http://vmsingle-victoria-metrics-system-vm.victoria-metrics-system.svc.cluster.local:8428
     uid: victoria-metrics-datasource
     isDefault: true
     editable: true
diff --git a/components/observability/grafana/grafana-instance.yaml b/components/observability/grafana/grafana-instance.yaml
index 852559f..32e1ec5 100644
--- a/components/observability/grafana/grafana-instance.yaml
+++ b/components/observability/grafana/grafana-instance.yaml
@@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1
 kind: Grafana
 metadata:
   name: grafana
-  namespace: telemetry-system
+  namespace: grafana-system
   labels:
     dashboards: "grafana"
 spec:
diff --git a/components/observability/grafana/grafana-operator-hr.yaml b/components/observability/grafana/grafana-operator-hr.yaml
index 40bfd16..135a3f1 100644
--- a/components/observability/grafana/grafana-operator-hr.yaml
+++ b/components/observability/grafana/grafana-operator-hr.yaml
@@ -6,7 +6,7 @@ metadata:
 spec:
   interval: 15m
   timeout: 5m
-  targetNamespace: telemetry-system
+  targetNamespace: grafana-system
   chart:
     spec:
       chart: grafana-operator
diff --git a/components/observability/grafana/helm-repositories.yaml b/components/observability/grafana/helm-repositories.yaml
index 2678f3a..83fa999 100644
--- a/components/observability/grafana/helm-repositories.yaml
+++ b/components/observability/grafana/helm-repositories.yaml
@@ -1,15 +1,6 @@
 ---
 apiVersion: source.toolkit.fluxcd.io/v1
 kind: HelmRepository
-metadata:
-  name: vm
-  namespace: flux-system
-spec:
-  interval: 12h
-  url: https://victoriametrics.github.io/helm-charts/
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
 metadata:
   name: grafana
   namespace: flux-system
diff --git a/components/observability/grafana/namespace.yaml b/components/observability/grafana/namespace.yaml
index 17def55..a2b112f 100644
--- a/components/observability/grafana/namespace.yaml
+++ b/components/observability/grafana/namespace.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: telemetry-system
+  name: grafana-system
   labels:
-    name: telemetry-system
-    dashboards: "grafana"
\ No newline at end of file
+    name: grafana-system
+    dashboards: "grafana"
diff --git a/components/observability/helm-repositories.yaml b/components/observability/helm-repositories.yaml
deleted file mode 100644
index 2678f3a..0000000
--- a/components/observability/helm-repositories.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: vm
-  namespace: flux-system
-spec:
-  interval: 12h
-  url: https://victoriametrics.github.io/helm-charts/
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: grafana
-  namespace: flux-system
-spec:
-  interval: 12h
-  url: https://grafana.github.io/helm-charts/
diff --git a/components/observability/kustomization.yaml b/components/observability/kustomization.yaml
index 429a936..36d7401 100644
--- a/components/observability/kustomization.yaml
+++ b/components/observability/kustomization.yaml
@@ -1,23 +1,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  # Shared prerequisites
-  - namespace.yaml
-  - helm-repositories.yaml
-  # Prometheus Operator CRDs
-  - https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.81.0/stripped-down-crds.yaml
-  # Victoria Metrics
-  - victoria-metrics/victoria-metrics-hr.yaml
-  # Grafana
-  - grafana/grafana-operator-hr.yaml
-  - grafana/grafana-instance.yaml
-  - datasources
-  # Loki
-  - loki/loki-hr.yaml
-  # Tempo
-  - tempo/tempo-hr.yaml
-  # OpenTelemetry Collector
-  - otel-collector/helm-repository.yaml
-  - otel-collector/helm-release-operator.yaml
-  - otel-collector/opentelemetry-collector.yaml
-
+  - prometheus-crds
+  - victoria-metrics
+  - otel-collector
+  - loki
+  - tempo
+  - grafana
diff --git a/components/observability/loki/helm-repositories.yaml b/components/observability/loki/helm-repositories.yaml
index 2678f3a..40716b5 100644
--- a/components/observability/loki/helm-repositories.yaml
+++ b/components/observability/loki/helm-repositories.yaml
@@ -2,16 +2,7 @@
 apiVersion: source.toolkit.fluxcd.io/v1
 kind: HelmRepository
 metadata:
-  name: vm
-  namespace: flux-system
-spec:
-  interval: 12h
-  url: https://victoriametrics.github.io/helm-charts/
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: grafana
+  name: loki-charts
   namespace: flux-system
 spec:
   interval: 12h
diff --git a/components/observability/loki/loki-hr.yaml b/components/observability/loki/loki-hr.yaml
index 6c78137..f31800b 100644
--- a/components/observability/loki/loki-hr.yaml
+++ b/components/observability/loki/loki-hr.yaml
@@ -6,14 +6,14 @@ metadata:
 spec:
   interval: 15m
   timeout: 10m
-  targetNamespace: telemetry-system
+  targetNamespace: loki-system
   chart:
     spec:
       chart: loki
       version: "6.16.0"
       sourceRef:
         kind: HelmRepository
-        name: grafana
+        name: loki-charts
         namespace: flux-system
   install:
     createNamespace: false
diff --git a/components/observability/loki/namespace.yaml b/components/observability/loki/namespace.yaml
index 17def55..de84677 100644
--- a/components/observability/loki/namespace.yaml
+++ b/components/observability/loki/namespace.yaml
@@ -1,7 +1,6 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: telemetry-system
+  name: loki-system
   labels:
-    name: telemetry-system
-    dashboards: "grafana"
\ No newline at end of file
+    name: loki-system
diff --git a/components/observability/namespace.yaml b/components/observability/namespace.yaml
deleted file mode 100644
index 17def55..0000000
--- a/components/observability/namespace.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: telemetry-system
-  labels:
-    name: telemetry-system
-    dashboards: "grafana"
\ No newline at end of file
diff --git a/components/observability/otel-collector/helm-release-operator.yaml b/components/observability/otel-collector/helm-release-operator.yaml
index 428169b..e134346 100644
--- a/components/observability/otel-collector/helm-release-operator.yaml
+++ b/components/observability/otel-collector/helm-release-operator.yaml
@@ -5,7 +5,7 @@ metadata:
   namespace: flux-system
 spec:
   interval: 5m
-  targetNamespace: telemetry-system
+  targetNamespace: otel-collector-system
   chart:
     spec:
       chart: opentelemetry-operator
diff --git a/components/observability/otel-collector/namespace.yaml b/components/observability/otel-collector/namespace.yaml
index 17def55..7ded824 100644
--- a/components/observability/otel-collector/namespace.yaml
+++ b/components/observability/otel-collector/namespace.yaml
@@ -1,7 +1,6 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: telemetry-system
+  name: otel-collector-system
   labels:
-    name: telemetry-system
-    dashboards: "grafana"
\ No newline at end of file
+    name: otel-collector-system
diff --git a/components/observability/otel-collector/opentelemetry-collector.yaml b/components/observability/otel-collector/opentelemetry-collector.yaml
index cc24973..00044c5 100644
--- a/components/observability/otel-collector/opentelemetry-collector.yaml
+++ b/components/observability/otel-collector/opentelemetry-collector.yaml
@@ -2,7 +2,7 @@ apiVersion: opentelemetry.io/v1beta1
 kind: OpenTelemetryCollector
 metadata:
   name: otel-collector
-  namespace: telemetry-system
+  namespace: otel-collector-system
 spec:
   managementState: managed
   mode: daemonset
@@ -117,15 +117,15 @@ spec:
 
     exporters:
       otlphttp/tempo:
-        endpoint: http://telemetry-system-tempo.telemetry-system.svc.cluster.local:4318
+        endpoint: http://tempo-system-tempo.tempo-system.svc.cluster.local:4318
         tls:
           insecure: true
       prometheusremotewrite:
-        endpoint: http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428/api/v1/write
+        endpoint: http://vmsingle-victoria-metrics-system-vm.victoria-metrics-system.svc.cluster.local:8428/api/v1/write
         tls:
           insecure: true
       otlphttp/loki:
-        endpoint: http://telemetry-system-loki.telemetry-system.svc.cluster.local:3100/otlp
+        endpoint: http://loki-system-loki.loki-system.svc.cluster.local:3100/otlp
         tls:
           insecure: true
 
diff --git a/components/observability/prometheus-crds/kustomization.yaml b/components/observability/prometheus-crds/kustomization.yaml
index e95bdfd..d644887 100644
--- a/components/observability/prometheus-crds/kustomization.yaml
+++ b/components/observability/prometheus-crds/kustomization.yaml
@@ -1,6 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - namespace.yaml
-  # Prometheus Operator CRDs
+  # Prometheus Operator CRDs (cluster-scoped, no namespace required)
   - https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.81.0/stripped-down-crds.yaml
diff --git a/components/observability/prometheus-crds/namespace.yaml b/components/observability/prometheus-crds/namespace.yaml
deleted file mode 100644
index 17def55..0000000
--- a/components/observability/prometheus-crds/namespace.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: telemetry-system
-  labels:
-    name: telemetry-system
-    dashboards: "grafana"
\ No newline at end of file
diff --git a/components/observability/tempo/helm-repositories.yaml b/components/observability/tempo/helm-repositories.yaml
index 2678f3a..9180816 100644
--- a/components/observability/tempo/helm-repositories.yaml
+++ b/components/observability/tempo/helm-repositories.yaml
@@ -2,16 +2,7 @@
 apiVersion: source.toolkit.fluxcd.io/v1
 kind: HelmRepository
 metadata:
-  name: vm
-  namespace: flux-system
-spec:
-  interval: 12h
-  url: https://victoriametrics.github.io/helm-charts/
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: grafana
+  name: tempo-charts
   namespace: flux-system
 spec:
   interval: 12h
diff --git a/components/observability/tempo/namespace.yaml b/components/observability/tempo/namespace.yaml
index 17def55..43194c4 100644
--- a/components/observability/tempo/namespace.yaml
+++ b/components/observability/tempo/namespace.yaml
@@ -1,7 +1,6 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: telemetry-system
+  name: tempo-system
   labels:
-    name: telemetry-system
-    dashboards: "grafana"
\ No newline at end of file
+    name: tempo-system
diff --git a/components/observability/tempo/tempo-hr.yaml b/components/observability/tempo/tempo-hr.yaml
index ae04e8b..d86ae2d 100644
--- a/components/observability/tempo/tempo-hr.yaml
+++ b/components/observability/tempo/tempo-hr.yaml
@@ -6,14 +6,14 @@ metadata:
 spec:
   interval: 15m
   timeout: 10m
-  targetNamespace: telemetry-system
+  targetNamespace: tempo-system
   chart:
     spec:
       chart: tempo
       version: "1.10.3"
       sourceRef:
         kind: HelmRepository
-        name: grafana
+        name: tempo-charts
         namespace: flux-system
   install:
     createNamespace: false
diff --git a/components/observability/victoria-metrics/helm-repositories.yaml b/components/observability/victoria-metrics/helm-repositories.yaml
index 2678f3a..2fe274a 100644
--- a/components/observability/victoria-metrics/helm-repositories.yaml
+++ b/components/observability/victoria-metrics/helm-repositories.yaml
@@ -7,12 +7,3 @@ metadata:
 spec:
   interval: 12h
   url: https://victoriametrics.github.io/helm-charts/
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: grafana
-  namespace: flux-system
-spec:
-  interval: 12h
-  url: https://grafana.github.io/helm-charts/
diff --git a/components/observability/victoria-metrics/namespace.yaml b/components/observability/victoria-metrics/namespace.yaml
index 17def55..84aee69 100644
--- a/components/observability/victoria-metrics/namespace.yaml
+++ b/components/observability/victoria-metrics/namespace.yaml
@@ -1,7 +1,6 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: telemetry-system
+  name: victoria-metrics-system
   labels:
-    name: telemetry-system
-    dashboards: "grafana"
\ No newline at end of file
+    name: victoria-metrics-system
diff --git a/components/observability/victoria-metrics/victoria-metrics-hr.yaml b/components/observability/victoria-metrics/victoria-metrics-hr.yaml
index fa8cd66..05ef53d 100644
--- a/components/observability/victoria-metrics/victoria-metrics-hr.yaml
+++ b/components/observability/victoria-metrics/victoria-metrics-hr.yaml
@@ -6,7 +6,7 @@ metadata:
 spec:
   interval: 15m
   timeout: 10m
-  targetNamespace: telemetry-system
+  targetNamespace: victoria-metrics-system
   chart:
     spec:
       chart: victoria-metrics-k8s-stack
@@ -32,7 +32,7 @@ spec:
           instanceSelector:
             matchLabels:
               dashboards: "grafana"
-          allowCrossNamespaceImport: false
+          allowCrossNamespaceImport: true
 
     # Victoria Metrics Single configuration
     vmsingle:
@@ -84,13 +84,13 @@ spec:
         evaluationInterval: "15s"
         # Configure datasource for querying metrics
         datasource:
-          url: "http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428"
+          url: "http://vmsingle-victoria-metrics-system-vm.victoria-metrics-system.svc.cluster.local:8428"
         # Configure notifier to use Alertmanager
         notifier:
-          url: "http://vmalertmanager-telemetry-system-vm.telemetry-system.svc.cluster.local:9093"
+          url: "http://vmalertmanager-victoria-metrics-system-vm.victoria-metrics-system.svc.cluster.local:9093"
         # Enable remote write for storing alert state
         remoteWrite:
-          url: "http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428/api/v1/write"
+          url: "http://vmsingle-victoria-metrics-system-vm.victoria-metrics-system.svc.cluster.local:8428/api/v1/write"
 
     # Alertmanager - enabled for test environment
     alertmanager:

From b3453bd18cbf5f531d464b5ac98db8d0c5ca438d Mon Sep 17 00:00:00 2001
From: Scot Wells <wells.scot@gmail.com>
Date: Thu, 16 Apr 2026 10:18:05 -0500
Subject: [PATCH 3/3] fix(observability): make VM standalone install not
 require Grafana CRDs

The VM HelmRelease had defaultDashboards.grafanaOperator.enabled set to
true, which generates GrafanaDashboard resources. When installed on its
own (e.g. `task install-prometheus-crds install-victoria-metrics`) the
grafana-operator CRDs are not present and Helm fails with
"no matches for kind GrafanaDashboard in version
grafana.integreatly.org/v1beta1".

Flip the default to false so per-component installs succeed, and patch
it back to true in the root observability kustomization so the full
stack continues to ship dashboards via the operator.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 components/observability/kustomization.yaml       | 15 +++++++++++++++
 .../victoria-metrics/victoria-metrics-hr.yaml     |  5 ++++-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/components/observability/kustomization.yaml b/components/observability/kustomization.yaml
index 36d7401..eb98a82 100644
--- a/components/observability/kustomization.yaml
+++ b/components/observability/kustomization.yaml
@@ -7,3 +7,18 @@ resources:
   - loki
   - tempo
   - grafana
+# When the full observability stack is installed together, grafana-operator is
+# present, so flip the VM chart's grafana-dashboard generation back on. The
+# per-component (composable) VM install leaves this at false so it does not
+# depend on the Grafana CRDs.
+patches:
+  - target:
+      group: helm.toolkit.fluxcd.io
+      version: v2
+      kind: HelmRelease
+      name: vm
+      namespace: flux-system
+    patch: |
+      - op: replace
+        path: /spec/values/defaultDashboards/grafanaOperator/enabled
+        value: true
diff --git a/components/observability/victoria-metrics/victoria-metrics-hr.yaml b/components/observability/victoria-metrics/victoria-metrics-hr.yaml
index 05ef53d..12f69cd 100644
--- a/components/observability/victoria-metrics/victoria-metrics-hr.yaml
+++ b/components/observability/victoria-metrics/victoria-metrics-hr.yaml
@@ -25,9 +25,12 @@ spec:
   values:
     nameOverride: "vm"
     # Enable Grafana Operator integration
+    # Default is false so the VM chart can install standalone without the
+    # grafana-operator CRDs. The root observability kustomization patches this
+    # back to true when the full stack is installed together.
     defaultDashboards:
       grafanaOperator:
-        enabled: true
+        enabled: false
         spec:
           instanceSelector:
             matchLabels: