feat: Zod validation, metrics tests, monitoring assets, smoke alias

README.md

@@ -46,3 +46,13 @@ npm run dev
 ```bash
 npm test
 ```
+
+## Smoke test
+
+Run a health-check smoke test against a running server:
+
+```bash
+npm run smoke
+```
+
+`smoke:health` is also available as a named alias for the startup health check. Both run the same script.

deploy/monitoring/grafana/dashboards/agent-loop.json

@@ -0,0 +1,59 @@
+{
+  "title": "NeuroWealth — Agent Loop",
+  "uid": "nw-agent-loop",
+  "schemaVersion": 38,
+  "refresh": "30s",
+  "time": { "from": "now-3h", "to": "now" },
+  "panels": [
+    {
+      "id": 1,
+      "title": "Agent Loop Status",
+      "type": "stat",
+      "gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 },
+      "targets": [{ "expr": "agent_loop_status" }],
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            { "type": "value", "options": { "0": { "text": "STOPPED", "color": "red" }, "1": { "text": "RUNNING", "color": "green" }, "2": { "text": "DEGRADED", "color": "yellow" } } }
+          ]
+        }
+      }
+    },
+    {
+      "id": 2,
+      "title": "Heartbeat Age (s)",
+      "type": "gauge",
+      "gridPos": { "x": 6, "y": 0, "w": 6, "h": 4 },
+      "targets": [{ "expr": "time() - agent_loop_heartbeat_timestamp" }],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "min": 0,
+          "max": 600,
+          "thresholds": { "steps": [{ "value": 0, "color": "green" }, { "value": 120, "color": "yellow" }, { "value": 300, "color": "red" }] }
+        }
+      }
+    },
+    {
+      "id": 3,
+      "title": "Rebalance Checks",
+      "type": "timeseries",
+      "gridPos": { "x": 0, "y": 4, "w": 12, "h": 6 },
+      "targets": [{ "expr": "rate(agent_rebalance_checks_total[5m])", "legendFormat": "{{status}}" }]
+    },
+    {
+      "id": 4,
+      "title": "Rebalances Triggered",
+      "type": "timeseries",
+      "gridPos": { "x": 12, "y": 4, "w": 12, "h": 6 },
+      "targets": [{ "expr": "rate(agent_rebalances_triggered_total[5m])", "legendFormat": "triggered" }]
+    },
+    {
+      "id": 5,
+      "title": "Snapshot Duration P95 (s)",
+      "type": "timeseries",
+      "gridPos": { "x": 0, "y": 10, "w": 24, "h": 6 },
+      "targets": [{ "expr": "histogram_quantile(0.95, rate(agent_snapshot_duration_seconds_bucket[5m]))", "legendFormat": "p95" }]
+    }
+  ]
+}

deploy/monitoring/grafana/dashboards/dlq.json

@@ -0,0 +1,42 @@
+{
+  "title": "NeuroWealth — DLQ",
+  "uid": "nw-dlq",
+  "schemaVersion": 38,
+  "refresh": "30s",
+  "time": { "from": "now-6h", "to": "now" },
+  "panels": [
+    {
+      "id": 1,
+      "title": "DLQ Size",
+      "type": "stat",
+      "gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 },
+      "targets": [{ "expr": "dlq_size" }],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": { "steps": [{ "value": 0, "color": "green" }, { "value": 20, "color": "yellow" }, { "value": 50, "color": "red" }] }
+        }
+      }
+    },
+    {
+      "id": 2,
+      "title": "DLQ Size Over Time",
+      "type": "timeseries",
+      "gridPos": { "x": 0, "y": 4, "w": 24, "h": 6 },
+      "targets": [{ "expr": "dlq_size", "legendFormat": "dlq size" }]
+    },
+    {
+      "id": 3,
+      "title": "DLQ Retry Rate",
+      "type": "timeseries",
+      "gridPos": { "x": 0, "y": 10, "w": 12, "h": 6 },
+      "targets": [{ "expr": "rate(dlq_retry_total[5m])", "legendFormat": "{{status}}" }]
+    },
+    {
+      "id": 4,
+      "title": "Cursor Lag (ledgers)",
+      "type": "timeseries",
+      "gridPos": { "x": 12, "y": 10, "w": 12, "h": 6 },
+      "targets": [{ "expr": "cursor_lag_ledgers", "legendFormat": "lag" }]
+    }
+  ]
+}

deploy/monitoring/grafana/dashboards/latency.json

@@ -0,0 +1,44 @@
+{
+  "title": "NeuroWealth — Latency",
+  "uid": "nw-latency",
+  "schemaVersion": 38,
+  "refresh": "30s",
+  "time": { "from": "now-3h", "to": "now" },
+  "panels": [
+    {
+      "id": 1,
+      "title": "HTTP P50 Latency (s)",
+      "type": "timeseries",
+      "gridPos": { "x": 0, "y": 0, "w": 12, "h": 6 },
+      "targets": [{ "expr": "histogram_quantile(0.50, rate(http_request_duration_seconds_bucket[5m]))", "legendFormat": "p50" }]
+    },
+    {
+      "id": 2,
+      "title": "HTTP P95 Latency (s)",
+      "type": "timeseries",
+      "gridPos": { "x": 12, "y": 0, "w": 12, "h": 6 },
+      "targets": [{ "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))", "legendFormat": "p95" }]
+    },
+    {
+      "id": 3,
+      "title": "DB Operation P95 Latency (s)",
+      "type": "timeseries",
+      "gridPos": { "x": 0, "y": 6, "w": 12, "h": 6 },
+      "targets": [{ "expr": "histogram_quantile(0.95, rate(db_operation_duration_seconds_bucket[5m]))", "legendFormat": "{{operation}}" }]
+    },
+    {
+      "id": 4,
+      "title": "Event Processing P95 Duration (s)",
+      "type": "timeseries",
+      "gridPos": { "x": 12, "y": 6, "w": 12, "h": 6 },
+      "targets": [{ "expr": "histogram_quantile(0.95, rate(events_processing_duration_seconds_bucket[5m]))", "legendFormat": "{{event_type}}" }]
+    },
+    {
+      "id": 5,
+      "title": "Analytics API P95 Latency (s)",
+      "type": "timeseries",
+      "gridPos": { "x": 0, "y": 12, "w": 24, "h": 6 },
+      "targets": [{ "expr": "histogram_quantile(0.95, rate(analytics_request_duration_seconds_bucket[5m]))", "legendFormat": "{{endpoint}}" }]
+    }
+  ]
+}

deploy/monitoring/grafana/dashboards/system-overview.json

@@ -0,0 +1,89 @@
+{
+  "title": "NeuroWealth — System Overview",
+  "uid": "nw-system-overview",
+  "schemaVersion": 38,
+  "refresh": "30s",
+  "time": { "from": "now-1h", "to": "now" },
+  "panels": [
+    {
+      "id": 1,
+      "title": "Agent Loop Status",
+      "type": "stat",
+      "gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 },
+      "targets": [{ "expr": "agent_loop_status", "legendFormat": "status" }],
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] } },
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            { "type": "value", "options": { "0": { "text": "STOPPED", "color": "red" }, "1": { "text": "RUNNING", "color": "green" }, "2": { "text": "DEGRADED", "color": "yellow" } } }
+          ]
+        }
+      }
+    },
+    {
+      "id": 2,
+      "title": "Heartbeat Age (s)",
+      "type": "stat",
+      "gridPos": { "x": 4, "y": 0, "w": 4, "h": 4 },
+      "targets": [{ "expr": "time() - agent_loop_heartbeat_timestamp", "legendFormat": "age" }],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "thresholds": { "steps": [{ "value": 0, "color": "green" }, { "value": 120, "color": "yellow" }, { "value": 300, "color": "red" }] }
+        }
+      }
+    },
+    {
+      "id": 3,
+      "title": "Cursor Lag (ledgers)",
+      "type": "stat",
+      "gridPos": { "x": 8, "y": 0, "w": 4, "h": 4 },
+      "targets": [{ "expr": "cursor_lag_ledgers", "legendFormat": "lag" }],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": { "steps": [{ "value": 0, "color": "green" }, { "value": 50, "color": "yellow" }, { "value": 100, "color": "red" }] }
+        }
+      }
+    },
+    {
+      "id": 4,
+      "title": "DLQ Size",
+      "type": "stat",
+      "gridPos": { "x": 12, "y": 0, "w": 4, "h": 4 },
+      "targets": [{ "expr": "dlq_size", "legendFormat": "dlq" }],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": { "steps": [{ "value": 0, "color": "green" }, { "value": 20, "color": "yellow" }, { "value": 50, "color": "red" }] }
+        }
+      }
+    },
+    {
+      "id": 5,
+      "title": "HTTP Request Rate",
+      "type": "timeseries",
+      "gridPos": { "x": 0, "y": 4, "w": 12, "h": 6 },
+      "targets": [{ "expr": "rate(http_requests_total[1m])", "legendFormat": "{{method}} {{route}} {{status_code}}" }]
+    },
+    {
+      "id": 6,
+      "title": "HTTP P95 Latency (s)",
+      "type": "timeseries",
+      "gridPos": { "x": 12, "y": 4, "w": 12, "h": 6 },
+      "targets": [{ "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))", "legendFormat": "p95" }]
+    },
+    {
+      "id": 7,
+      "title": "Event Processing Rate",
+      "type": "timeseries",
+      "gridPos": { "x": 0, "y": 10, "w": 12, "h": 6 },
+      "targets": [{ "expr": "rate(events_processed_total[1m])", "legendFormat": "{{event_type}} {{status}}" }]
+    },
+    {
+      "id": 8,
+      "title": "Failure Rate",
+      "type": "timeseries",
+      "gridPos": { "x": 12, "y": 10, "w": 12, "h": 6 },
+      "targets": [{ "expr": "rate(failures_total[1m])", "legendFormat": "{{component}} {{error_type}}" }]
+    }
+  ]
+}

deploy/monitoring/grafana/provisioning/dashboards.yaml

@@ -0,0 +1,8 @@
+apiVersion: 1
+providers:
+  - name: neurowealth
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    options:
+      path: /etc/grafana/dashboards

deploy/monitoring/grafana/provisioning/datasources.yaml

@@ -0,0 +1,9 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    jsonData:
+      timeInterval: "15s"

deploy/monitoring/prometheus/alert-rules.yaml

@@ -0,0 +1,105 @@
+groups:
+  - name: neurowealth_critical
+    interval: 30s
+    rules:
+      - alert: AgentLoopStopped
+        expr: agent_loop_status == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Agent loop is not running"
+          description: "Agent loop has been stopped for more than 1 minute"
+
+      - alert: AgentLoopHeartbeatStale
+        expr: time() - agent_loop_heartbeat_timestamp > 300
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Agent loop heartbeat is stale"
+          description: "Agent loop heartbeat not updated for 5 minutes"
+
+      - alert: CursorLagCritical
+        expr: cursor_lag_ledgers > 100
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Event cursor lag critical"
+          description: "Cursor lag is {{ $value }} ledgers (> 100)"
+
+      - alert: DLQSizeCritical
+        expr: dlq_size > 50
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Dead Letter Queue critically large"
+          description: "DLQ has {{ $value }} events (> 50)"
+
+      - alert: HighFailureRate
+        expr: rate(failures_total[5m]) > 10
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "High failure rate detected"
+          description: "Failure rate is {{ $value }}/min (> 10)"
+
+  - name: neurowealth_warning
+    interval: 30s
+    rules:
+      - alert: AgentLoopDegraded
+        expr: agent_loop_status == 2
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Agent loop in degraded state"
+          description: "Agent loop has been degraded for more than 5 minutes"
+
+      - alert: CursorLagWarning
+        expr: cursor_lag_ledgers > 50
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Event cursor lag elevated"
+          description: "Cursor lag is {{ $value }} ledgers (> 50)"
+
+      - alert: DLQSizeWarning
+        expr: dlq_size > 20
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Dead Letter Queue growing"
+          description: "DLQ has {{ $value }} events (> 20)"
+
+      - alert: EventProcessingSlow
+        expr: histogram_quantile(0.95, rate(events_processing_duration_seconds_bucket[5m])) > 2
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Event processing slow"
+          description: "P95 event processing duration is {{ $value }}s (> 2s)"
+
+      - alert: DatabaseOperationsSlow
+        expr: histogram_quantile(0.95, rate(db_operation_duration_seconds_bucket[5m])) > 1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Database operations slow"
+          description: "P95 DB operation duration is {{ $value }}s (> 1s)"
+
+      - alert: HTTPRequestsSlow
+        expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 5
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "HTTP requests slow"
+          description: "P95 HTTP request duration is {{ $value }}s (> 5s)"