Stream aggregate Hubble traffic to a public relay

.github/workflows/ci.yaml

@@ -21,7 +21,9 @@ jobs:
           curl -sSL "https://github.com/yannh/kubeconform/releases/download/${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz" \
             | sudo tar xz -C /usr/local/bin kubeconform
 
-      # values.yaml files are helm inputs, not manifests, so they are excluded.
+      # values.yaml files are Helm inputs, not manifests, so they are excluded.
+      # image/ and relay/ are workload source trees that Argo's directory
+      # sources do not recurse into; their own workflows validate them.
       # CRD kinds are skipped: the default schema registry has no schema for
       # CustomResourceDefinition itself. CRs are validated against the datree
       # CRD catalog (argoproj.io, metallb.io, bitnami.com, ...).
@@ -35,6 +37,7 @@ jobs:
             -ignore-filename-pattern '.*values\.yaml' \
             -ignore-filename-pattern '.*\.md' \
             -ignore-filename-pattern '.*\.pem' \
+            -ignore-filename-pattern '.*/(image|relay)/.*' \
             kubernetes/
 
   helm-template:

.github/workflows/hubble-traffic-stream.yaml

@@ -0,0 +1,104 @@
+name: Hubble traffic stream
+
+on:
+  pull_request:
+    paths:
+      - "kubernetes/apps/showcase/hubble-traffic-stream/**"
+      - "kubernetes/clusters/devata/hubble-traffic-stream.yaml"
+      - ".github/workflows/hubble-traffic-stream.yaml"
+  push:
+    branches: [main]
+    paths:
+      - "kubernetes/apps/showcase/hubble-traffic-stream/**"
+      - ".github/workflows/hubble-traffic-stream.yaml"
+  workflow_dispatch: {}
+
+env:
+  IMAGE: ghcr.io/pragalvaxfrez/hubble-traffic-streamer
+  VERSION: 0.1.0
+  RELAY_DIR: kubernetes/apps/showcase/hubble-traffic-stream/relay
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  verify-producer:
+    name: verify producer
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: 1.26.5
+          cache-dependency-path: kubernetes/apps/showcase/hubble-traffic-stream/image/go.mod
+      - name: Test
+        working-directory: kubernetes/apps/showcase/hubble-traffic-stream/image
+        run: go test ./... && go vet ./...
+
+  verify-relay:
+    name: verify relay
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: ${{ env.RELAY_DIR }}/package-lock.json
+      - name: Install
+        working-directory: ${{ env.RELAY_DIR }}
+        run: npm ci
+      - name: Test and build
+        working-directory: ${{ env.RELAY_DIR }}
+        run: npm test && npm run check
+
+  publish-producer:
+    name: publish producer image
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+    needs: [verify-producer, verify-relay]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/build-push-action@v6
+        with:
+          context: kubernetes/apps/showcase/hubble-traffic-stream/image
+          push: true
+          tags: ${{ env.IMAGE }}:${{ env.VERSION }}
+
+  deploy-relay:
+    name: deploy relay
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+    needs: [verify-producer, verify-relay]
+    runs-on: ubuntu-latest
+    environment: hubble-traffic-relay
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: ${{ env.RELAY_DIR }}/package-lock.json
+      - name: Install
+        working-directory: ${{ env.RELAY_DIR }}
+        run: npm ci
+      - name: Deploy Worker
+        working-directory: ${{ env.RELAY_DIR }}
+        env:
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+        run: npx wrangler deploy
+      - name: Install producer credential
+        working-directory: ${{ env.RELAY_DIR }}
+        env:
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          PRODUCER_TOKEN: ${{ secrets.HUBBLE_RELAY_PRODUCER_TOKEN }}
+        run: printf '%s' "$PRODUCER_TOKEN" | npx wrangler secret put PRODUCER_TOKEN
+      - name: Verify relay health
+        run: curl --fail --silent --show-error --retry 10 --retry-all-errors --retry-delay 3 https://telemetry.pragalva.me/health

README.md

@@ -18,7 +18,7 @@ Talos machine configuration inputs are version controlled here and applied out o
 
 ## Architecture
 
-The cluster sits on a home LAN behind CGNAT. Desired state comes in through Argo CD's outbound pull, while the public snapshot leaves through a publisher with read-only cluster access and write access to a separate data repository.
+The cluster sits on a home LAN behind CGNAT. Desired state comes in through Argo CD's outbound pull. Public evidence leaves through narrow outbound publishers: an hourly durable snapshot and a closed aggregate traffic stream.
 
 ```mermaid
 flowchart LR
@@ -28,10 +28,12 @@ flowchart LR
         devata["devata<br/>Talos + Kubernetes"]
     end
     snap["devata-snapshot<br/>observed state"]
+    relay["Cloudflare relay<br/>aggregate traffic only"]
     hub["pragalva.me/homelab"]
     lab -->|"Argo CD pulls"| devata
     operator -->|"applies talos/"| devata
     devata -->|"publisher pushes"| snap
+    devata -->|"aggregate rates push"| relay
     snap --> hub
 ```
 
@@ -50,7 +52,7 @@ The **sandbox** in [`lab-experiments/`](./lab-experiments) is where things get b
 - **`talos/` is reviewed but not reconciled.** It contains reusable patches, non-secret volume documents, and Image Factory schematics. Rendered machine configurations contain cluster trust material and remain outside Git.
 - **`kubernetes/` contains authoritative cluster state.** Bootstrap is applied once by hand; the root and child Applications reconcile cluster composition, platform components, and workloads with self-heal enabled. Pruning is enabled except where a component's recovery boundary requires otherwise, such as Argo CD managing its own installation.
 - **`lab-experiments/` is non-authoritative.** Nothing in the sandbox is referenced by the GitOps root or child Applications.
-- **Public state is allowlisted.** The snapshot publisher reads through a dedicated read-only ServiceAccount, builds a new document from approved fields, validates it against a closed schema, and can write only to `devata-snapshot`.
+- **Public state is allowlisted.** The snapshot publisher builds a validated document from approved fields and can write only to `devata-snapshot`. The traffic streamer has no inbound route or Kubernetes API token and can send only two aggregate Prometheus rates to its authenticated relay.
 
 ## Repository map
 

docs/decisions/0002-outbound-aggregate-traffic-relay.md

@@ -0,0 +1,23 @@
+# 0002 - Outbound aggregate traffic relay
+
+- Status: accepted
+- Date: 2026-07-31
+
+## Context
+
+The hourly public snapshot carries six hours of aggregate Hubble flow and drop rates. It is durable and safe when Devata or its uplink is unavailable, but it cannot drive a near-live chart. Hubble Relay and Prometheus are LAN-only, and exposing either would reveal a larger interface than the public chart needs.
+
+## Decision
+
+Run a single GitOps-managed producer in Devata. It queries only the two aggregate Prometheus rates and pushes a closed three-field payload outbound every five seconds. It has no Service or inbound route.
+
+Use a Cloudflare Worker with one SQLite-backed Durable Object as the public relay. The relay authenticates writes, validates the closed payload, rejects replays and excessive writes, retains six hours, and fans accepted samples out through hibernating WebSockets. The browser remains a read-only consumer. The hourly snapshot remains the fallback and historical proof surface.
+
+## Consequences
+
+- Nothing accepts unsolicited inbound traffic on the home network.
+- The public contract cannot grow by accidentally forwarding Prometheus labels or Hubble flow fields.
+- Five-second publishing adds one small always-on workload and about 17,280 relay writes per day.
+- The relay depends on Cloudflare Workers, Durable Objects, DNS, and one producer credential.
+- A relay outage makes the stream stale; it does not erase or fabricate the hourly snapshot.
+- The portfolio integration is a separate change after the producer and relay are verified live.

docs/decisions/README.md

@@ -7,3 +7,4 @@ The format is lightweight, after [Michael Nygard's ADRs](https://cognitect.com/b
 ## Index
 
 - [0001 - Repository structure](./0001-repository-structure.md)
+- [0002 - Outbound aggregate traffic relay](./0002-outbound-aggregate-traffic-relay.md)

kubernetes/apps/README.md

@@ -4,4 +4,4 @@ The workloads that run on top of the platform. Each app is its own folder, with
 
 New workloads are added as new folders here, which is what lets the cluster grow without touching the platform layer.
 
-- [`showcase/`](./showcase) the snapshot publisher and future apps beside it.
+- [`showcase/`](./showcase) public evidence producers, including durable snapshots and aggregate traffic streams.

kubernetes/apps/showcase/README.md

@@ -1,5 +1,5 @@
 # showcase
 
-The public face of the cluster. The first inhabitant is the snapshot publisher, the workload that takes the cluster state and surfaces it on the public homelab site.
+The public face of the cluster. The snapshot publisher provides durable hourly state, while the Hubble traffic streamer sends a closed aggregate payload to the public relay for near-live presentation.
 
-Lands under its own follow up issue. Future showcase apps sit beside it here.
+These workloads expose evidence without opening their internal data sources to the internet. Future showcase apps sit beside them here.

kubernetes/apps/showcase/hubble-traffic-stream/README.md

@@ -0,0 +1,78 @@
+# Hubble traffic stream
+
+This workload sends two aggregate Hubble rates from Devata to a public relay every five seconds. It exists only to make the portfolio traffic chart near-live. The hourly `snapshot.json` traffic block remains the durable fallback.
+
+The producer queries the same Prometheus series as the Devata overview dashboard, creates a new three-field document, and makes one authenticated outbound HTTPS request. It has no Service, listening port, Kubernetes API token, or inbound policy allowance.
+
+Six consecutive collection or delivery failures terminate the process so Kubernetes and Argo CD expose a broken stream instead of reporting a healthy Deployment that only logs errors.
+
+```json
+{
+  "timestamp": "2026-07-31T10:12:05Z",
+  "flowsPerSecond": 146.4,
+  "dropsPerSecond": 0.048
+}
+```
+
+The relay at `telemetry.pragalva.me` runs as a Cloudflare Worker backed by one SQLite Durable Object. It rejects unknown fields, stale or future timestamps, replays, writes faster than one every three seconds, and invalid bearer credentials. Accepted rows expire after six hours. Browsers read `/v1/history` and receive new samples through a hibernating WebSocket at `/v1/stream`.
+
+## Trust boundary
+
+- Prometheus stays reachable only inside the cluster.
+- The producer reads one fixed aggregate query and transforms the response into the closed payload above.
+- A Cilium policy permits only DNS for the relay name, Prometheus on TCP 9090, and `telemetry.pragalva.me` on TCP 443.
+- The producer credential is a SealedSecret in Git and a masked GitHub environment secret used to configure the Worker.
+- The relay accepts writes only with that credential. Browser endpoints are read-only and allow the portfolio origin.
+- Individual flows, labels, identities, addresses, ports, and node names never enter the outbound document.
+
+## Repository contents
+
+| Path | Responsibility |
+| --- | --- |
+| `deployment.yaml` | Single non-root producer; `Recreate` prevents overlapping writers during rollout |
+| `network-policy.yaml` | Deny inbound and allow only the required Prometheus, DNS, and relay egress |
+| `sealedsecret-producer-token.yaml` | Encrypted producer credential |
+| `image/` | Dependency-free Go producer and tests |
+| `relay/` | Worker, Durable Object, closed-contract tests, and deployment configuration |
+
+Subdirectories are source code, not Kubernetes manifests. Argo CD's directory source applies only the YAML files at this directory's top level.
+
+## First deployment
+
+The child Application is intentionally created without automated sync. Before merging, configure the `hubble-traffic-relay` GitHub environment with `CLOUDFLARE_ACCOUNT_ID`, `CLOUDFLARE_API_TOKEN`, and `HUBBLE_RELAY_PRODUCER_TOKEN`. The API token needs Workers Scripts edit and Workers Routes edit access for the `pragalva.me` zone.
+
+After merge, the workflow deploys the relay and producer image. Confirm the new GHCR package is public, then stage the workload:
+
+```sh
+curl -fsS https://telemetry.pragalva.me/health | jq .
+argocd app sync hubble-traffic-stream
+kubectl rollout status deployment/hubble-traffic-streamer -n showcase
+kubectl logs deployment/hubble-traffic-streamer -n showcase -f
+```
+
+Verify the public contract and inspect it for prohibited fields:
+
+```sh
+curl -fsS https://telemetry.pragalva.me/v1/history | tee /tmp/hubble-history.json | jq .
+grep -Ei 'pod|service|namespace|node|ip|port|label|192\.168|10\.|172\.' /tmp/hubble-history.json || echo clean
+```
+
+Send a request with a bad credential and expect `401`:
+
+```sh
+curl -sS -o /dev/null -w '%{http_code}\n' \
+  -H 'Authorization: Bearer rejected' \
+  -H 'Content-Type: application/json' \
+  --data '{"timestamp":"2026-07-31T10:12:05Z","flowsPerSecond":1,"dropsPerSecond":0}' \
+  https://telemetry.pragalva.me/v1/samples
+```
+
+After the live payload, rejection, retention, and egress policy are proven, add automated prune and self-heal to the child Application. Update the portfolio only after that gate.
+
+## Credential rotation
+
+Generate one token without writing it to disk. Seal it to the fixed Secret name and namespace using `kubernetes/infra/controllers/sealed-secrets/pub-cert.pem`, and set the same value as the `HUBBLE_RELAY_PRODUCER_TOKEN` environment secret. Deploy the Worker secret first, then merge the SealedSecret update. The producer retries safely during the brief rejection window.
+
+## Rollback
+
+Remove the child Application from `kubernetes/clusters/devata/` and let Argo prune the producer. Delete the Worker after the pod is gone. The portfolio continues to use the hourly traffic block in `snapshot.json`; no inbound cluster route is involved.

kubernetes/apps/showcase/hubble-traffic-stream/deployment.yaml

@@ -0,0 +1,54 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hubble-traffic-streamer
+  namespace: showcase
+  labels:
+    app.kubernetes.io/name: hubble-traffic-streamer
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: hubble-traffic-streamer
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: hubble-traffic-streamer
+    spec:
+      serviceAccountName: hubble-traffic-streamer
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 10
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: stream
+          image: ghcr.io/pragalvaxfrez/hubble-traffic-streamer:0.1.0
+          env:
+            - name: PROMETHEUS_URL
+              value: http://kps-kube-prometheus-stack-prometheus.monitoring.svc:9090/api/v1/query
+            - name: RELAY_URL
+              value: https://telemetry.pragalva.me/v1/samples
+            - name: INTERVAL
+              value: 5s
+            - name: PRODUCER_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: hubble-traffic-producer
+                  key: token
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+            limits:
+              memory: 64Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]

kubernetes/apps/showcase/hubble-traffic-stream/image/Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.26.5-alpine AS build
+
+WORKDIR /src
+COPY go.mod main.go ./
+RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/hubble-traffic-streamer .
+
+FROM gcr.io/distroless/static-debian12:nonroot
+
+LABEL org.opencontainers.image.source="https://github.com/PragalvaXFREZ/lab"
+LABEL org.opencontainers.image.description="Outbound-only aggregate Hubble traffic producer for devata"
+
+COPY --from=build /out/hubble-traffic-streamer /hubble-traffic-streamer
+ENTRYPOINT ["/hubble-traffic-streamer"]

kubernetes/apps/showcase/hubble-traffic-stream/image/go.mod

@@ -0,0 +1,3 @@
+module github.com/PragalvaXFREZ/lab/kubernetes/apps/showcase/hubble-traffic-stream/image
+
+go 1.23