fix(blockchain): resolve issues #917 #933 #934 #937 — reliability har…

performance/config/alerts.yaml

@@ -284,3 +284,36 @@ groups:
           summary: "Email queue depth is high"
           description: "Email queue depth is {{ $value }}, exceeding 100 for more than 5 minutes"
           runbook_url: "https://docs.predictiq.com/runbooks/email-queue-depth-high"
+
+  - name: blockchain_watch_map
+    interval: 30s
+    rules:
+      - alert: WatchedTxCountHigh
+        expr: watched_tx_count > 8000
+        for: 5m
+        labels:
+          severity: warning
+          component: blockchain
+        annotations:
+          summary: "Watched transaction map is above 80% capacity"
+          description: >
+            watched_tx_count is {{ $value }}, which exceeds 80% of the default
+            10 000-entry cap (WATCHED_TX_MAX_SIZE). New watch registrations will
+            be rejected with 503 when the map is full. Consider increasing
+            WATCHED_TX_MAX_SIZE or reducing WATCHED_TX_TTL_SECS.
+          runbook_url: "https://docs.predictiq.com/runbooks/watched-tx-count-high"
+
+      - alert: WatchedTxCountCritical
+        expr: watched_tx_count >= 10000
+        for: 1m
+        labels:
+          severity: critical
+          component: blockchain
+        annotations:
+          summary: "Watched transaction map is full — new registrations are being rejected"
+          description: >
+            watched_tx_count is {{ $value }}, which equals or exceeds the
+            configured cap (WATCHED_TX_MAX_SIZE). All new watch registration
+            requests are returning 503 Service Unavailable. Immediate action
+            required: increase WATCHED_TX_MAX_SIZE or reduce WATCHED_TX_TTL_SECS.
+          runbook_url: "https://docs.predictiq.com/runbooks/watched-tx-count-high"

services/api/.env.example

@@ -30,6 +30,16 @@ PREDICTIQ_CONTRACT_ID=predictiq_contract
 # Set explicitly when using a custom network or to guard against misconfiguration.
 # STELLAR_NETWORK_PASSPHRASE=Test SDF Network ; September 2015
 
+# TTL for watched-transaction map entries (seconds). Default: 1800 (30 min).
+# WATCHED_TX_TTL_SECS=1800
+
+# Maximum watched-transaction map entries. New registrations return 503 when full.
+# WATCHED_TX_MAX_SIZE=10000
+
+# Set to "production" to make the Stellar RPC startup probe fail-fast (exit 1)
+# on passphrase mismatch or unreachable RPC. Logs a warning in all other envs.
+# PREDICTIQ_ENV=production
+
 # Contract storage key schema (v1 defaults shown).
 # Override these when a network uses different key naming conventions.
 # Templates that require a per-record ID must contain the literal "{id}".

services/api/README.md

@@ -75,6 +75,9 @@ The following Prometheus gauges are exported on `/metrics` and updated on each s
 | `BLOCKCHAIN_NETWORK` | `testnet` | Network to connect to: `testnet`, `mainnet`, or `custom` |
 | `BLOCKCHAIN_RPC_URL` | _(network default)_ | Soroban RPC endpoint |
 | `STELLAR_NETWORK_PASSPHRASE` | _(network default)_ | Expected network passphrase; validated against the RPC node at startup |
+| `WATCHED_TX_TTL_SECS` | `1800` | TTL (seconds) for entries in the in-memory watched-transaction map. Entries older than this are evicted on the next write regardless of finalization status. Applied to the `expires_at` column of the `watched_transactions` DB table too. |
+| `WATCHED_TX_MAX_SIZE` | `10000` | Maximum number of transaction hashes that may be tracked simultaneously. When the cap is reached, new `GET /api/v1/blockchain/tx/:hash` registrations return `503 Service Unavailable`. |
+| `PREDICTIQ_ENV` | _(empty)_ | Set to `production` to make the Stellar RPC reachability startup probe fail-fast with `exit(1)` on failure. In all other environments only a warning is logged. |
 
 Expected passphrases per `BLOCKCHAIN_NETWORK`:
 
@@ -86,8 +89,27 @@ Expected passphrases per `BLOCKCHAIN_NETWORK`:
 
 At startup the API queries the RPC node's `getNetwork` endpoint. If the returned passphrase does not match the configured `STELLAR_NETWORK_PASSPHRASE`, the service rejects startup with a fatal error. This prevents silently signing transactions for the wrong network.
 
+In production (`PREDICTIQ_ENV=production`) a mismatch or unreachable RPC causes `process::exit(1)`.  In development environments only a warning is logged and the process continues.
+
 To disable validation entirely (e.g. for a local custom network without a fixed passphrase), leave `STELLAR_NETWORK_PASSPHRASE` unset.
 
+### Health endpoints
+
+| Endpoint | Description |
+|---|---|
+| `GET /health` | Liveness probe — checks Redis, DB, and email queue worker status |
+| `GET /health/ready` | Readiness probe — validates the Stellar RPC endpoint is reachable and returns the expected network passphrase. Returns `200 OK` with `{ "ready": true, "stellar_rpc": "ok" }` on success, or `503 Service Unavailable` with `{ "ready": false, "stellar_rpc": "unreachable" }` on failure. Use this for Kubernetes `readinessProbe` configuration. |
+
+### Watched-transaction metrics
+
+The following Prometheus gauge is exported on `/metrics`:
+
+| Metric | Description |
+|---|---|
+| `watched_tx_count` | Current number of transaction hashes being monitored in the in-memory watch map |
+
+An alert (`WatchedTxCountHigh`) fires when `watched_tx_count` exceeds 8 000 (80% of the default 10 000 cap). A critical alert (`WatchedTxCountCritical`) fires when the map is full and new registrations are being rejected.
+
 See `DATABASE.md` for database-specific configuration.
 
 ## HMAC Key Rotation

services/api/database/migrations/019_create_watched_transactions.sql

@@ -0,0 +1,23 @@
+-- Migration 019: watched_transactions table
+--
+-- Persists the set of transaction hashes being monitored so the watch-map
+-- state survives API restarts.  The UNIQUE constraint on tx_hash enforces the
+-- deduplication invariant at the database layer (#937).  The expires_at column
+-- allows server-side TTL expiry to be driven by WATCHED_TX_TTL_SECS (#933).
+--
+-- This table is append-only during normal operation; a periodic cleanup job
+-- (or a simple DELETE WHERE expires_at < NOW()) is responsible for pruning
+-- expired rows.
+
+CREATE TABLE IF NOT EXISTS watched_transactions (
+    id          BIGSERIAL    PRIMARY KEY,
+    tx_hash     VARCHAR(128) NOT NULL,
+    watched_at  TIMESTAMPTZ  NOT NULL DEFAULT NOW(),
+    expires_at  TIMESTAMPTZ  NOT NULL,
+
+    CONSTRAINT uq_watched_transactions_tx_hash UNIQUE (tx_hash)
+);
+
+-- Index to speed up TTL cleanup queries.
+CREATE INDEX IF NOT EXISTS idx_watched_transactions_expires_at
+    ON watched_transactions (expires_at);

services/api/database/migrations/rollbacks/019_create_watched_transactions_down.sql

@@ -0,0 +1,2 @@
+-- Rollback 019: drop watched_transactions table
+DROP TABLE IF EXISTS watched_transactions;