diff --git a/Taskfile.yml b/Taskfile.yml index 48b45eb..6387fdb 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -107,7 +107,15 @@ tasks: - echo " install-cert-manager Deploy cert-manager + CSI driver" - echo " install-kyverno Deploy Kyverno policy engine" - echo " install-envoy-gateway-operator Deploy Envoy Gateway" - - echo " install-observability [OPTIONAL] Deploy telemetry stack" + - echo "" + - echo "📊 Observability (composable):" + - echo " install-observability [OPTIONAL] Deploy full telemetry stack" + - echo " install-prometheus-crds Prometheus Operator CRDs" + - echo " install-victoria-metrics Victoria Metrics (vmagent + vmsingle)" + - echo " install-otel-collector OpenTelemetry Operator + Collector" + - echo " install-grafana Grafana Operator + instance + datasources" + - echo " install-loki Loki for log aggregation" + - echo " install-tempo Tempo for distributed tracing" - echo "" - echo "🔨 CI/Development:" - echo " kind-load-image Load images into KIND" @@ -407,26 +415,16 @@ tasks: - echo " To find the actual NodePort assignments, run:" - echo " kubectl get svc -n envoy-gateway-system -l app.kubernetes.io/name=envoy" - install-observability: - desc: "[OPTIONAL] Deploy complete observability stack (Victoria Metrics, Loki, Tempo, Grafana, OTel)" + install-prometheus-crds: + desc: "Deploy Prometheus Operator CRDs" silent: true cmds: - task: ensure-repo - - echo "➡️ Reconciling Observability Stack …" - - kustomize build {{.REPO_DIR}}/components/observability | kubectl apply -f - - - echo "⏳ Waiting for Grafana Operator HelmRelease …" - - kubectl -n flux-system wait helmrelease/grafana-operator --for=condition=Ready --timeout={{.WAIT_TIMEOUT}} - - echo "⏳ Waiting for Victoria Metrics Stack HelmRelease …" - - kubectl -n flux-system wait helmrelease/vm --for=condition=Ready --timeout={{.WAIT_TIMEOUT}} - - echo "⏳ Waiting for Loki HelmRelease …" - - kubectl -n flux-system wait helmrelease/loki --for=condition=Ready --timeout={{.WAIT_TIMEOUT}} - - echo "⏳ Waiting for Tempo HelmRelease …" - - kubectl -n flux-system wait helmrelease/tempo --for=condition=Ready --timeout={{.WAIT_TIMEOUT}} - - echo "⏳ Waiting for OpenTelemetry Operator HelmRelease …" - - kubectl -n flux-system wait helmrelease/opentelemetry-operator --for=condition=Ready --timeout={{.WAIT_TIMEOUT}} + - echo "➡️ Reconciling Prometheus Operator CRDs …" + - kustomize build {{.REPO_DIR}}/components/observability/prometheus-crds | kubectl apply --server-side --force-conflicts -f - - | set -euo pipefail - echo "⏳ Waiting for Prometheus CRDs to be Established…" + echo "⏳ Waiting for Prometheus CRDs to be Established …" for crd in \ servicemonitors.monitoring.coreos.com \ podmonitors.monitoring.coreos.com \ @@ -436,22 +434,106 @@ tasks: alertmanagers.monitoring.coreos.com \ thanosrulers.monitoring.coreos.com do - echo "$crd" + echo " $crd" kubectl wait --for=condition=Established "crd/${crd}" --timeout=120s || true done - - echo "⏳ Waiting for vmagent & vmsingle to be Ready (best-effort)…" - - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmagent -n telemetry-system --timeout=120s || true - - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmsingle -n telemetry-system --timeout=120s || true + - echo "✅ Prometheus Operator CRDs are ready" + + install-victoria-metrics: + desc: "Deploy Victoria Metrics (vmagent + vmsingle)" + silent: true + deps: [install-prometheus-crds] + cmds: + - task: ensure-repo + - echo "➡️ Reconciling Victoria Metrics …" + - kustomize build {{.REPO_DIR}}/components/observability/victoria-metrics | kubectl apply -f - + - echo "⏳ Waiting for Victoria Metrics Stack HelmRelease …" + - kubectl -n flux-system wait helmrelease/vm --for=condition=Ready --timeout={{.WAIT_TIMEOUT}} + - echo "⏳ Waiting for vmagent & vmsingle to be Ready …" + - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmagent -n victoria-metrics-system --timeout={{.WAIT_TIMEOUT}} || true + - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmsingle -n victoria-metrics-system --timeout={{.WAIT_TIMEOUT}} || true + - echo "✅ Victoria Metrics is ready" + + install-otel-collector: + desc: "Deploy OpenTelemetry Operator + Collector" + silent: true + cmds: + - task: ensure-repo + - echo "➡️ Reconciling OpenTelemetry Operator …" + - kustomize build {{.REPO_DIR}}/components/observability/otel-collector | kubectl apply -f - + - echo "⏳ Waiting for OpenTelemetry Operator HelmRelease …" + - kubectl -n flux-system wait helmrelease/opentelemetry-operator --for=condition=Ready --timeout={{.WAIT_TIMEOUT}} - echo "⏳ Waiting for OpenTelemetryCollector CRD to be Established …" - kubectl wait --for=condition=Established crd/opentelemetrycollectors.opentelemetry.io --timeout={{.WAIT_TIMEOUT}} - - echo "➡️ Applying OpenTelemetryCollector CR …" - - kubectl apply -f {{.REPO_DIR}}/components/observability/otel-collector/opentelemetry-collector.yaml + - | + echo "⏳ Waiting for OTel webhook to be ready …" + sleep 10 + - echo "➡️ Applying OpenTelemetryCollector CR (with retry for webhook readiness) …" + - | + set -euo pipefail + for i in 1 2 3 4 5; do + if kubectl apply -f {{.REPO_DIR}}/components/observability/otel-collector/opentelemetry-collector.yaml 2>/dev/null; then + echo " CR applied successfully" + break + fi + echo " Attempt $i failed, retrying in 10s …" + sleep 10 + done - echo "⏳ Waiting for OTel Collector DaemonSet …" - - kubectl -n telemetry-system rollout status daemonset/otel-collector-collector --timeout={{.WAIT_TIMEOUT}} + - kubectl -n otel-collector-system rollout status daemonset/otel-collector-collector --timeout={{.WAIT_TIMEOUT}} + - echo "✅ OpenTelemetry Collector is ready" + + install-grafana: + desc: "Deploy Grafana Operator + instance + datasources" + silent: true + deps: [install-victoria-metrics] + cmds: + - task: ensure-repo + - echo "➡️ Reconciling Grafana Operator …" + - kustomize build {{.REPO_DIR}}/components/observability/grafana | kubectl apply -f - + - echo "⏳ Waiting for Grafana Operator HelmRelease …" + - kubectl -n flux-system wait helmrelease/grafana-operator --for=condition=Ready --timeout={{.WAIT_TIMEOUT}} - echo "➡️ Applying Grafana Instance (after Operator CRDs are ready) …" - - kubectl apply -f {{.REPO_DIR}}/components/observability/grafana-instance.yaml + - kubectl apply -f {{.REPO_DIR}}/components/observability/grafana/grafana-instance.yaml - echo "➡️ Applying Grafana Datasources …" - - kustomize build {{.REPO_DIR}}/components/observability/datasources | kubectl apply -f - + - kustomize build {{.REPO_DIR}}/components/observability/grafana/datasources | kubectl apply -f - + - echo "✅ Grafana is ready" + - echo "" + - 'echo "📊 Access Grafana at: http://localhost:30000"' + - 'echo " Username: admin"' + - 'echo " Password: datum123"' + + install-loki: + desc: "Deploy Loki for log aggregation" + silent: true + cmds: + - task: ensure-repo + - echo "➡️ Reconciling Loki …" + - kustomize build {{.REPO_DIR}}/components/observability/loki | kubectl apply -f - + - echo "⏳ Waiting for Loki HelmRelease …" + - kubectl -n flux-system wait helmrelease/loki --for=condition=Ready --timeout={{.WAIT_TIMEOUT}} + - echo "✅ Loki is ready" + + install-tempo: + desc: "Deploy Tempo for distributed tracing" + silent: true + cmds: + - task: ensure-repo + - echo "➡️ Reconciling Tempo …" + - kustomize build {{.REPO_DIR}}/components/observability/tempo | kubectl apply -f - + - echo "⏳ Waiting for Tempo HelmRelease …" + - kubectl -n flux-system wait helmrelease/tempo --for=condition=Ready --timeout={{.WAIT_TIMEOUT}} + - echo "✅ Tempo is ready" + + install-observability: + desc: "[OPTIONAL] Deploy complete observability stack (Victoria Metrics, Loki, Tempo, Grafana, OTel)" + silent: true + cmds: + - task: install-victoria-metrics + - task: install-otel-collector + - task: install-loki + - task: install-tempo + - task: install-grafana - echo "✅ Observability stack is ready" - echo "" - 'echo "📊 Access Grafana at: http://localhost:30000"' diff --git a/components/observability/README.md b/components/observability/README.md index a683d3d..5a285c4 100644 --- a/components/observability/README.md +++ b/components/observability/README.md @@ -4,20 +4,25 @@ Optional observability stack for test infrastructure, providing metrics, logs, a ## Overview -This component is an **optional add-on** deployed after core infrastructure. It provides comprehensive telemetry capabilities using Grafana, Victoria Metrics, Loki, and Tempo. +This component is an **optional add-on** deployed after core infrastructure. It provides comprehensive telemetry capabilities using Grafana, Victoria Metrics, Loki, Tempo, and the OpenTelemetry Collector. -## Components +The stack is split into composable subcomponents — each deploys into its own namespace so it can be installed, upgraded, or removed independently (`kubectl delete ns -system` cleanly uninstalls). -- **Grafana**: Visualization and dashboards -- **Victoria Metrics**: Metrics collection and storage -- **Loki**: Log aggregation and storage -- **Tempo**: Distributed tracing storage -- **Promtail**: Log collection agent +## Subcomponents + +| Component | Namespace | Purpose | +| ----------------- | ------------------------- | ------------------------------------- | +| `prometheus-crds` | (cluster-scoped) | Prometheus Operator CRDs | +| `victoria-metrics`| `victoria-metrics-system` | Metrics collection and storage | +| `otel-collector` | `otel-collector-system` | OpenTelemetry DaemonSet collector | +| `loki` | `loki-system` | Log aggregation and storage | +| `tempo` | `tempo-system` | Distributed tracing storage | +| `grafana` | `grafana-system` | Visualization, dashboards, datasources| ## Access - **Grafana UI**: Available at NodePort 30000 (admin/datum123) -- **Default Datasources**: Victoria Metrics (metrics), Loki (logs), Tempo (traces) +- **Default Datasources**: Victoria Metrics (metrics), Loki (logs), Tempo (traces), Alertmanager ## Prerequisites @@ -25,18 +30,35 @@ Core test infrastructure must be running before deploying observability componen ## Deployment -Deploy using the `install-observability` task target: +Deploy the whole stack: ```bash task install-observability ``` -This deploys all components and configures datasources automatically. +Or install components individually: + +```bash +task install-prometheus-crds +task install-victoria-metrics +task install-otel-collector +task install-loki +task install-tempo +task install-grafana +``` ## Removal -To remove the observability stack: +To remove an individual component: + +```bash +kubectl delete namespace -system +``` + +To remove the whole stack: ```bash -kubectl delete namespace observability +for ns in grafana-system tempo-system loki-system otel-collector-system victoria-metrics-system; do + kubectl delete namespace "$ns" +done ``` diff --git a/components/observability/datasources/alertmanager-datasource.yaml b/components/observability/grafana/datasources/alertmanager-datasource.yaml similarity index 73% rename from components/observability/datasources/alertmanager-datasource.yaml rename to components/observability/grafana/datasources/alertmanager-datasource.yaml index fae8cf3..6c2ac3e 100644 --- a/components/observability/datasources/alertmanager-datasource.yaml +++ b/components/observability/grafana/datasources/alertmanager-datasource.yaml @@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaDatasource metadata: name: alertmanager - namespace: telemetry-system + namespace: grafana-system spec: instanceSelector: matchLabels: @@ -11,7 +11,7 @@ spec: name: Alertmanager type: alertmanager access: proxy - url: http://vmalertmanager-telemetry-system-vm.telemetry-system.svc.cluster.local:9093 + url: http://vmalertmanager-victoria-metrics-system-vm.victoria-metrics-system.svc.cluster.local:9093 isDefault: true editable: true jsonData: diff --git a/components/observability/datasources/kustomization.yaml b/components/observability/grafana/datasources/kustomization.yaml similarity index 100% rename from components/observability/datasources/kustomization.yaml rename to components/observability/grafana/datasources/kustomization.yaml diff --git a/components/observability/datasources/loki-datasource.yaml b/components/observability/grafana/datasources/loki-datasource.yaml similarity index 83% rename from components/observability/datasources/loki-datasource.yaml rename to components/observability/grafana/datasources/loki-datasource.yaml index e1e5f21..803dd2e 100644 --- a/components/observability/datasources/loki-datasource.yaml +++ b/components/observability/grafana/datasources/loki-datasource.yaml @@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaDatasource metadata: name: loki-datasource - namespace: telemetry-system + namespace: grafana-system labels: dashboards: "grafana" spec: @@ -13,7 +13,7 @@ spec: name: Loki type: loki access: proxy - url: http://telemetry-system-loki.telemetry-system.svc.cluster.local:3100 + url: http://loki-system-loki.loki-system.svc.cluster.local:3100 isDefault: false editable: true jsonData: diff --git a/components/observability/datasources/tempo-datasource.yaml b/components/observability/grafana/datasources/tempo-datasource.yaml similarity index 91% rename from components/observability/datasources/tempo-datasource.yaml rename to components/observability/grafana/datasources/tempo-datasource.yaml index db66274..bc7ecb3 100644 --- a/components/observability/datasources/tempo-datasource.yaml +++ b/components/observability/grafana/datasources/tempo-datasource.yaml @@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaDatasource metadata: name: tempo-datasource - namespace: telemetry-system + namespace: grafana-system labels: dashboards: "grafana" spec: @@ -13,7 +13,7 @@ spec: name: Tempo type: tempo access: proxy - url: http://telemetry-system-tempo.telemetry-system.svc.cluster.local:3100 + url: http://tempo-system-tempo.tempo-system.svc.cluster.local:3100 uid: tempo-datasource isDefault: false editable: true diff --git a/components/observability/datasources/vm-datasource.yaml b/components/observability/grafana/datasources/vm-datasource.yaml similarity index 76% rename from components/observability/datasources/vm-datasource.yaml rename to components/observability/grafana/datasources/vm-datasource.yaml index 7dbb59d..61777c0 100644 --- a/components/observability/datasources/vm-datasource.yaml +++ b/components/observability/grafana/datasources/vm-datasource.yaml @@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaDatasource metadata: name: victoria-metrics-datasource - namespace: telemetry-system + namespace: grafana-system labels: dashboards: "grafana" spec: @@ -13,7 +13,7 @@ spec: name: VictoriaMetrics type: prometheus access: proxy - url: http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428 + url: http://vmsingle-victoria-metrics-system-vm.victoria-metrics-system.svc.cluster.local:8428 uid: victoria-metrics-datasource isDefault: true editable: true diff --git a/components/observability/grafana-instance.yaml b/components/observability/grafana/grafana-instance.yaml similarity index 97% rename from components/observability/grafana-instance.yaml rename to components/observability/grafana/grafana-instance.yaml index 852559f..32e1ec5 100644 --- a/components/observability/grafana-instance.yaml +++ b/components/observability/grafana/grafana-instance.yaml @@ -2,7 +2,7 @@ apiVersion: grafana.integreatly.org/v1beta1 kind: Grafana metadata: name: grafana - namespace: telemetry-system + namespace: grafana-system labels: dashboards: "grafana" spec: diff --git a/components/observability/grafana-operator-hr.yaml b/components/observability/grafana/grafana-operator-hr.yaml similarity index 94% rename from components/observability/grafana-operator-hr.yaml rename to components/observability/grafana/grafana-operator-hr.yaml index 40bfd16..135a3f1 100644 --- a/components/observability/grafana-operator-hr.yaml +++ b/components/observability/grafana/grafana-operator-hr.yaml @@ -6,7 +6,7 @@ metadata: spec: interval: 15m timeout: 5m - targetNamespace: telemetry-system + targetNamespace: grafana-system chart: spec: chart: grafana-operator diff --git a/components/observability/grafana/helm-repositories.yaml b/components/observability/grafana/helm-repositories.yaml new file mode 100644 index 0000000..83fa999 --- /dev/null +++ b/components/observability/grafana/helm-repositories.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: grafana + namespace: flux-system +spec: + interval: 12h + url: https://grafana.github.io/helm-charts/ diff --git a/components/observability/grafana/kustomization.yaml b/components/observability/grafana/kustomization.yaml new file mode 100644 index 0000000..57d0c20 --- /dev/null +++ b/components/observability/grafana/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - helm-repositories.yaml + - grafana-operator-hr.yaml + - grafana-instance.yaml + - datasources diff --git a/components/observability/grafana/namespace.yaml b/components/observability/grafana/namespace.yaml new file mode 100644 index 0000000..a2b112f --- /dev/null +++ b/components/observability/grafana/namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: grafana-system + labels: + name: grafana-system + dashboards: "grafana" diff --git a/components/observability/kustomization.yaml b/components/observability/kustomization.yaml index 945898a..eb98a82 100644 --- a/components/observability/kustomization.yaml +++ b/components/observability/kustomization.yaml @@ -1,15 +1,24 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - namespace.yaml - - helm-repositories.yaml - - grafana-operator-hr.yaml - - victoria-metrics-hr.yaml - - loki-hr.yaml - - tempo-hr.yaml - - otel-collector/helm-repository.yaml - - otel-collector/helm-release-operator.yaml - - otel-collector/opentelemetry-collector.yaml - # Prometheus Operator CRDs - - https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.81.0/stripped-down-crds.yaml - + - prometheus-crds + - victoria-metrics + - otel-collector + - loki + - tempo + - grafana +# When the full observability stack is installed together, grafana-operator is +# present, so flip the VM chart's grafana-dashboard generation back on. The +# per-component (composable) VM install leaves this at false so it does not +# depend on the Grafana CRDs. +patches: + - target: + group: helm.toolkit.fluxcd.io + version: v2 + kind: HelmRelease + name: vm + namespace: flux-system + patch: | + - op: replace + path: /spec/values/defaultDashboards/grafanaOperator/enabled + value: true diff --git a/components/observability/loki/helm-repositories.yaml b/components/observability/loki/helm-repositories.yaml new file mode 100644 index 0000000..40716b5 --- /dev/null +++ b/components/observability/loki/helm-repositories.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: loki-charts + namespace: flux-system +spec: + interval: 12h + url: https://grafana.github.io/helm-charts/ diff --git a/components/observability/loki/kustomization.yaml b/components/observability/loki/kustomization.yaml new file mode 100644 index 0000000..c0854db --- /dev/null +++ b/components/observability/loki/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - helm-repositories.yaml + - loki-hr.yaml diff --git a/components/observability/loki-hr.yaml b/components/observability/loki/loki-hr.yaml similarity index 96% rename from components/observability/loki-hr.yaml rename to components/observability/loki/loki-hr.yaml index 6c78137..f31800b 100644 --- a/components/observability/loki-hr.yaml +++ b/components/observability/loki/loki-hr.yaml @@ -6,14 +6,14 @@ metadata: spec: interval: 15m timeout: 10m - targetNamespace: telemetry-system + targetNamespace: loki-system chart: spec: chart: loki version: "6.16.0" sourceRef: kind: HelmRepository - name: grafana + name: loki-charts namespace: flux-system install: createNamespace: false diff --git a/components/observability/loki/namespace.yaml b/components/observability/loki/namespace.yaml new file mode 100644 index 0000000..de84677 --- /dev/null +++ b/components/observability/loki/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: loki-system + labels: + name: loki-system diff --git a/components/observability/namespace.yaml b/components/observability/namespace.yaml deleted file mode 100644 index 17def55..0000000 --- a/components/observability/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: telemetry-system - labels: - name: telemetry-system - dashboards: "grafana" \ No newline at end of file diff --git a/components/observability/otel-collector/helm-release-operator.yaml b/components/observability/otel-collector/helm-release-operator.yaml index 428169b..e134346 100644 --- a/components/observability/otel-collector/helm-release-operator.yaml +++ b/components/observability/otel-collector/helm-release-operator.yaml @@ -5,7 +5,7 @@ metadata: namespace: flux-system spec: interval: 5m - targetNamespace: telemetry-system + targetNamespace: otel-collector-system chart: spec: chart: opentelemetry-operator diff --git a/components/observability/otel-collector/kustomization.yaml b/components/observability/otel-collector/kustomization.yaml index 133b3ad..0ef3709 100644 --- a/components/observability/otel-collector/kustomization.yaml +++ b/components/observability/otel-collector/kustomization.yaml @@ -1,6 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - namespace.yaml - helm-repository.yaml - helm-release-operator.yaml - opentelemetry-collector.yaml diff --git a/components/observability/otel-collector/namespace.yaml b/components/observability/otel-collector/namespace.yaml new file mode 100644 index 0000000..7ded824 --- /dev/null +++ b/components/observability/otel-collector/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: otel-collector-system + labels: + name: otel-collector-system diff --git a/components/observability/otel-collector/opentelemetry-collector.yaml b/components/observability/otel-collector/opentelemetry-collector.yaml index cc24973..00044c5 100644 --- a/components/observability/otel-collector/opentelemetry-collector.yaml +++ b/components/observability/otel-collector/opentelemetry-collector.yaml @@ -2,7 +2,7 @@ apiVersion: opentelemetry.io/v1beta1 kind: OpenTelemetryCollector metadata: name: otel-collector - namespace: telemetry-system + namespace: otel-collector-system spec: managementState: managed mode: daemonset @@ -117,15 +117,15 @@ spec: exporters: otlphttp/tempo: - endpoint: http://telemetry-system-tempo.telemetry-system.svc.cluster.local:4318 + endpoint: http://tempo-system-tempo.tempo-system.svc.cluster.local:4318 tls: insecure: true prometheusremotewrite: - endpoint: http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428/api/v1/write + endpoint: http://vmsingle-victoria-metrics-system-vm.victoria-metrics-system.svc.cluster.local:8428/api/v1/write tls: insecure: true otlphttp/loki: - endpoint: http://telemetry-system-loki.telemetry-system.svc.cluster.local:3100/otlp + endpoint: http://loki-system-loki.loki-system.svc.cluster.local:3100/otlp tls: insecure: true diff --git a/components/observability/prometheus-crds/kustomization.yaml b/components/observability/prometheus-crds/kustomization.yaml new file mode 100644 index 0000000..d644887 --- /dev/null +++ b/components/observability/prometheus-crds/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + # Prometheus Operator CRDs (cluster-scoped, no namespace required) + - https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.81.0/stripped-down-crds.yaml diff --git a/components/observability/tempo/helm-repositories.yaml b/components/observability/tempo/helm-repositories.yaml new file mode 100644 index 0000000..9180816 --- /dev/null +++ b/components/observability/tempo/helm-repositories.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: tempo-charts + namespace: flux-system +spec: + interval: 12h + url: https://grafana.github.io/helm-charts/ diff --git a/components/observability/tempo/kustomization.yaml b/components/observability/tempo/kustomization.yaml new file mode 100644 index 0000000..15865a2 --- /dev/null +++ b/components/observability/tempo/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - helm-repositories.yaml + - tempo-hr.yaml diff --git a/components/observability/tempo/namespace.yaml b/components/observability/tempo/namespace.yaml new file mode 100644 index 0000000..43194c4 --- /dev/null +++ b/components/observability/tempo/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: tempo-system + labels: + name: tempo-system diff --git a/components/observability/tempo-hr.yaml b/components/observability/tempo/tempo-hr.yaml similarity index 97% rename from components/observability/tempo-hr.yaml rename to components/observability/tempo/tempo-hr.yaml index ae04e8b..d86ae2d 100644 --- a/components/observability/tempo-hr.yaml +++ b/components/observability/tempo/tempo-hr.yaml @@ -6,14 +6,14 @@ metadata: spec: interval: 15m timeout: 10m - targetNamespace: telemetry-system + targetNamespace: tempo-system chart: spec: chart: tempo version: "1.10.3" sourceRef: kind: HelmRepository - name: grafana + name: tempo-charts namespace: flux-system install: createNamespace: false diff --git a/components/observability/helm-repositories.yaml b/components/observability/victoria-metrics/helm-repositories.yaml similarity index 50% rename from components/observability/helm-repositories.yaml rename to components/observability/victoria-metrics/helm-repositories.yaml index 2678f3a..2fe274a 100644 --- a/components/observability/helm-repositories.yaml +++ b/components/observability/victoria-metrics/helm-repositories.yaml @@ -7,12 +7,3 @@ metadata: spec: interval: 12h url: https://victoriametrics.github.io/helm-charts/ ---- -apiVersion: source.toolkit.fluxcd.io/v1 -kind: HelmRepository -metadata: - name: grafana - namespace: flux-system -spec: - interval: 12h - url: https://grafana.github.io/helm-charts/ diff --git a/components/observability/victoria-metrics/kustomization.yaml b/components/observability/victoria-metrics/kustomization.yaml new file mode 100644 index 0000000..5159172 --- /dev/null +++ b/components/observability/victoria-metrics/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - helm-repositories.yaml + - victoria-metrics-hr.yaml diff --git a/components/observability/victoria-metrics/namespace.yaml b/components/observability/victoria-metrics/namespace.yaml new file mode 100644 index 0000000..84aee69 --- /dev/null +++ b/components/observability/victoria-metrics/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: victoria-metrics-system + labels: + name: victoria-metrics-system diff --git a/components/observability/victoria-metrics-hr.yaml b/components/observability/victoria-metrics/victoria-metrics-hr.yaml similarity index 82% rename from components/observability/victoria-metrics-hr.yaml rename to components/observability/victoria-metrics/victoria-metrics-hr.yaml index fa8cd66..12f69cd 100644 --- a/components/observability/victoria-metrics-hr.yaml +++ b/components/observability/victoria-metrics/victoria-metrics-hr.yaml @@ -6,7 +6,7 @@ metadata: spec: interval: 15m timeout: 10m - targetNamespace: telemetry-system + targetNamespace: victoria-metrics-system chart: spec: chart: victoria-metrics-k8s-stack @@ -25,14 +25,17 @@ spec: values: nameOverride: "vm" # Enable Grafana Operator integration + # Default is false so the VM chart can install standalone without the + # grafana-operator CRDs. The root observability kustomization patches this + # back to true when the full stack is installed together. defaultDashboards: grafanaOperator: - enabled: true + enabled: false spec: instanceSelector: matchLabels: dashboards: "grafana" - allowCrossNamespaceImport: false + allowCrossNamespaceImport: true # Victoria Metrics Single configuration vmsingle: @@ -84,13 +87,13 @@ spec: evaluationInterval: "15s" # Configure datasource for querying metrics datasource: - url: "http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428" + url: "http://vmsingle-victoria-metrics-system-vm.victoria-metrics-system.svc.cluster.local:8428" # Configure notifier to use Alertmanager notifier: - url: "http://vmalertmanager-telemetry-system-vm.telemetry-system.svc.cluster.local:9093" + url: "http://vmalertmanager-victoria-metrics-system-vm.victoria-metrics-system.svc.cluster.local:9093" # Enable remote write for storing alert state remoteWrite: - url: "http://vmsingle-telemetry-system-vm.telemetry-system.svc.cluster.local:8428/api/v1/write" + url: "http://vmsingle-victoria-metrics-system-vm.victoria-metrics-system.svc.cluster.local:8428/api/v1/write" # Alertmanager - enabled for test environment alertmanager: