diff --git a/deploy/k8s/deployment.yaml b/deploy/k8s/deployment.yaml index 3549ae7..9cc9500 100644 --- a/deploy/k8s/deployment.yaml +++ b/deploy/k8s/deployment.yaml @@ -79,9 +79,11 @@ spec: timeoutSeconds: 5 failureThreshold: 3 readinessProbe: - httpGet: - path: /health/ready - port: http + exec: + command: + - /bin/sh + - -c + - "wget --spider -q --header=\"X-Internal-Token: $INTERNAL_SERVICE_TOKEN\" http://localhost:3001/health/deep" initialDelaySeconds: 15 periodSeconds: 10 timeoutSeconds: 5 diff --git a/package-lock.json b/package-lock.json index ad14fc1..a1a5899 100644 --- a/package-lock.json +++ b/package-lock.json @@ -705,7 +705,7 @@ "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", "deprecated": "Use @eslint/config-array instead", "dev": true, - "license": "Apache-2.0", + "license": "MIT", "dependencies": { "@humanwhocodes/object-schema": "^2.0.3", "debug": "^4.3.1", @@ -1274,7 +1274,7 @@ "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz", "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==", "dev": true, - "license": "MIT", + "license": "ISC", "dependencies": { "expect": "^29.7.0", "jest-snapshot": "^29.7.0" @@ -2962,6 +2962,10 @@ "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", "dev": true, "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, "engines": { "node": "18 || 20 || >=22" } @@ -3630,7 +3634,10 @@ "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true, - "license": "MIT" + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } }, "node_modules/deepmerge": { "version": "4.3.1", @@ -3813,7 +3820,7 @@ "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", "dev": true, - "license": "MIT", + "license": "BSD-2-Clause", "dependencies": { "is-arrayish": "^0.2.1" } @@ -4268,6 +4275,15 @@ "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", "license": "MIT", "dependencies": { "@jest/expect-utils": "^29.7.0", @@ -4483,7 +4499,28 @@ "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", "dev": true, - "license": "MIT", + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", + "dev": true, + "license": "MIT" + }, + "node_modules/fastq": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", + "dev": true, + "license": "ISC", "dependencies": { "flat-cache": "^3.0.4" }, @@ -4496,6 +4533,15 @@ "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bser": "2.1.1" + } + }, + "node_modules/feaxios": { + "version": "0.0.23", + "resolved": "https://registry.npmjs.org/feaxios/-/feaxios-0.0.23.tgz", + "integrity": "sha512-eghR0A21fvbkcQBgZuMfQhrXxJzC0GNUGC9fXhBge33D+mFDTwl0aJ35zoQQn575BhyjQitRc5N4f+L4cP708g==", "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" @@ -4681,6 +4727,13 @@ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, "engines": { "node": ">= 0.6" } @@ -4849,7 +4902,7 @@ "slash": "^3.0.0" }, "engines": { - "node": ">=10" + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -4975,9 +5028,8 @@ "engines": { "node": ">= 0.8" }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" + "optionalDependencies": { + "uglify-js": "^3.1.4" } }, "node_modules/human-signals": { @@ -5008,6 +5060,9 @@ "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", "dev": true, "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, "engines": { "node": ">= 4" } @@ -5489,6 +5544,12 @@ "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], "license": "MIT", "dependencies": { "@sinclair/typebox": "^0.27.8" @@ -5861,7 +5922,10 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=8" + } }, "node_modules/jest-config/node_modules/brace-expansion": { "version": "1.1.15", @@ -5936,7 +6000,7 @@ "semver": "^6.3.0" }, "engines": { - "node": ">=8" + "node": "*" } }, "node_modules/jest-config/node_modules/jest-haste-map": { @@ -6282,6 +6346,12 @@ "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], "license": "MIT", "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" @@ -6576,6 +6646,9 @@ } ], "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, "engines": { "node": ">=8" } @@ -6783,7 +6856,7 @@ "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.10.tgz", "integrity": "sha512-MTBk/3jGLNB2tVxv6uLlFh1iu64iYOQ2PbdOSK3NW8JZsmlaOh2q6sdtKowBhfw8QFLmYNzTW4/oK4uATIi6ZA==", "dev": true, - "license": "MIT" + "license": "ISC" }, "node_modules/jest-runner/node_modules/babel-plugin-istanbul": { "version": "6.1.1", @@ -6799,7 +6872,7 @@ "test-exclude": "^6.0.0" }, "engines": { - "node": ">=8" + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, "node_modules/jest-runner/node_modules/ci-info": { @@ -6814,6 +6887,9 @@ } ], "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, "engines": { "node": ">=8" } @@ -6824,8 +6900,25 @@ "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true, "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "babel-plugin-istanbul": "^6.1.1", + "chalk": "^4.0.0", + "convert-source-map": "^2.0.0", + "fast-json-stable-stringify": "^2.1.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "micromatch": "^4.0.4", + "pirates": "^4.0.4", + "slash": "^3.0.0", + "write-file-atomic": "^4.0.2" + }, "engines": { - "node": ">=8" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-runner/node_modules/istanbul-lib-instrument": { @@ -6876,7 +6969,21 @@ "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", "dev": true, - "license": "MIT", + "license": "MIT" + }, + "node_modules/jest-runtime/node_modules/babel-plugin-istanbul": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", + "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@istanbuljs/load-nyc-config": "^1.0.0", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-instrument": "^5.0.4", + "test-exclude": "^6.0.0" + }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } @@ -7059,10 +7166,12 @@ "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", "dev": true, - "license": "MIT", + "license": "ISC", "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" } }, "node_modules/jest-runtime/node_modules/@sinclair/typebox": { @@ -7220,7 +7329,14 @@ "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", "dev": true, - "license": "MIT", + "license": "MIT" + }, + "node_modules/jest-snapshot/node_modules/babel-plugin-istanbul": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", + "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", + "dev": true, + "license": "BSD-3-Clause", "dependencies": { "@jest/types": "^29.6.3", "@types/node": "*", @@ -7238,6 +7354,12 @@ "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], "license": "MIT", "dependencies": { "@types/node": "*", @@ -7284,7 +7406,7 @@ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", "dev": true, - "license": "MIT", + "license": "BSD-3-Clause", "dependencies": { "has-flag": "^4.0.0" }, @@ -7455,7 +7577,11 @@ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true, - "license": "MIT", + "license": "ISC", + "dependencies": { + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.7" + }, "engines": { "node": ">=8" } @@ -7696,6 +7822,39 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/jest-watcher": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz", + "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "jest-util": "^29.7.0", + "string-length": "^4.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-watcher/node_modules/@jest/schemas": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", + "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, "node_modules/jest-watcher/node_modules/@jest/types": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", @@ -7733,8 +7892,11 @@ } ], "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, "engines": { - "node": ">=8" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-watcher/node_modules/jest-util": { @@ -9050,6 +9212,18 @@ "node": ">=8" } }, + "node_modules/smol-toml": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.7.0.tgz", + "integrity": "sha512-aqVvWoyO21L23mb+drl4RmMXbf6N7FdHjAhTRA9ZBL7apWBgfWC16KjrASI+1p9GAroljyMHj6fK67i0UiTNvQ==", + "license": "BSD-3-Clause", + "engines": { + "node": ">= 18" + }, + "funding": { + "url": "https://github.com/sponsors/cyyynthia" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -9543,7 +9717,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -9567,6 +9741,18 @@ "node": ">=0.8.0" } }, + "node_modules/uint8array-extras": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/uint8array-extras/-/uint8array-extras-1.5.0.tgz", + "integrity": "sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/undefsafe": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", diff --git a/package.json b/package.json index 440a40a..325c14f 100644 --- a/package.json +++ b/package.json @@ -33,15 +33,21 @@ "npm": ">=9.0.0" }, "dependencies": { + "@prisma/client": "^7.8.0", + "@stellar/stellar-sdk": "^16.0.1", + "@types/node-cron": "^3.0.11", + "@types/supertest": "^7.2.0", "cors": "^2.8.5", "dotenv": "^16.3.1", "express": "^4.18.2", "ioredis": "^5.11.1" }, "devDependencies": { + "@aws-sdk/client-ssm": "^3.1075.0", "@types/cors": "^2.8.17", "@types/express": "^4.17.21", "@types/jest": "^29.5.10", + "@types/jsonwebtoken": "^9.0.10", "@types/node": "^20.10.6", "@typescript-eslint/eslint-plugin": "^6.16.0", "@typescript-eslint/parser": "^6.16.0", diff --git a/src/agent/loop.ts b/src/agent/loop.ts index f0ab294..5151d72 100644 --- a/src/agent/loop.ts +++ b/src/agent/loop.ts @@ -10,7 +10,7 @@ import { executeRebalanceIfNeeded, getThresholds, logAgentAction } from './route import { captureAllUserBalances, cleanupOldSnapshots } from './snapshotter'; import db from '../db'; import { - updateAgentHeartbeat, + updateAgentHeartbeat as metricsUpdateAgentHeartbeat, updateAgentStatus, recordRebalanceCheck, recordRebalanceTriggered, @@ -24,10 +24,16 @@ let lastRebalanceAt: Date | null = null; let currentProtocol: string | null = null; let currentApy: number | null = null; let lastError: string | null = null; +let lastTickAt: Date | null = null; // Store cron job references for cleanup const cronJobs: ScheduledTask[] = []; +function updateAgentHeartbeat(): void { + lastTickAt = new Date(); + metricsUpdateAgentHeartbeat(); +} + /** * Get current agent status */ @@ -40,6 +46,7 @@ export function getAgentStatus() { nextScheduledCheck: getNextCheckTime(), lastError, healthStatus: determineHealthStatus(), + lastTickAt, }; } diff --git a/src/routes/health.ts b/src/routes/health.ts index a1ff4c8..9c3c332 100644 --- a/src/routes/health.ts +++ b/src/routes/health.ts @@ -1,5 +1,11 @@ import { Router, Request, Response } from 'express' import { getReadiness } from '../config/readiness' +import db from '../db' +import { getResilientClient } from '../stellar/client' +import twilio from 'twilio' +import { config } from '../config' +import { getAgentStatus } from '../agent/loop' +import pkg from '../../package.json' const router = Router() @@ -7,7 +13,7 @@ router.get('/', (req: Request, res: Response) => { res.status(200).json({ status: 'ok', timestamp: new Date().toISOString(), - version: '1.0.0', + version: pkg.version || '1.0.0', environment: process.env.NODE_ENV || 'development' }) }) @@ -25,4 +31,135 @@ router.get('/ready', (req: Request, res: Response) => { }) }) +const withTimeout = (promise: Promise, timeoutMs: number, name: string): Promise => { + return Promise.race([ + promise, + new Promise((_, reject) => + setTimeout(() => reject(new Error(`${name} check timed out after ${timeoutMs}ms`)), timeoutMs) + ) + ]) +} + +router.get('/deep', async (req: Request, res: Response) => { + const token = req.headers['x-internal-token'] || req.headers['authorization']?.replace('Bearer ', '') + const expectedToken = process.env.INTERNAL_SERVICE_TOKEN + + if (!expectedToken || token !== expectedToken) { + return res.status(401).json({ error: 'Unauthorized' }) + } + + const startDb = Date.now() + const checkDatabase = async (): Promise<{ status: 'healthy' | 'degraded' | 'unhealthy'; latencyMs: number; error?: string }> => { + try { + await withTimeout(db.$queryRaw`SELECT 1`, 3000, 'database') + return { status: 'healthy', latencyMs: Date.now() - startDb } + } catch (error: any) { + return { status: 'unhealthy', latencyMs: Date.now() - startDb, error: error.message } + } + } + + const startStellar = Date.now() + const checkStellarRpc = async (): Promise<{ status: 'healthy' | 'degraded' | 'unhealthy'; latencyMs: number; ledger?: number; error?: string }> => { + try { + const client = getResilientClient() + const latestLedger = await withTimeout( + client.execute(server => server.getLatestLedger(), 'health-check'), + 3000, + 'stellarRpc' + ) + return { status: 'healthy', latencyMs: Date.now() - startStellar, ledger: latestLedger.sequence } + } catch (error: any) { + return { status: 'unhealthy', latencyMs: Date.now() - startStellar, error: error.message } + } + } + + const startTwilio = Date.now() + const checkTwilio = async (): Promise<{ status: 'healthy' | 'degraded' | 'unhealthy'; latencyMs: number; error?: string }> => { + try { + const sid = config.whatsapp.twilioSid + const twilioToken = config.whatsapp.twilioToken + if (!sid || !twilioToken) { + throw new Error('Twilio credentials not configured') + } + const client = twilio(sid, twilioToken) + await withTimeout( + client.api.accounts(sid).fetch(), + 3000, + 'twilio' + ) + return { status: 'healthy', latencyMs: Date.now() - startTwilio } + } catch (error: any) { + return { status: 'unhealthy', latencyMs: Date.now() - startTwilio, error: error.message } + } + } + + const checkAgentLoop = () => { + try { + const status = getAgentStatus() + const TICK_INTERVAL_MS = 30 * 60 * 1000 // 30 minutes + let checkStatus: 'healthy' | 'degraded' | 'unhealthy' = 'healthy' + + if (!status.isRunning) { + checkStatus = 'unhealthy' + } else if (!status.lastTickAt) { + checkStatus = 'unhealthy' + } else { + const timeSinceLastTick = Date.now() - new Date(status.lastTickAt).getTime() + if (timeSinceLastTick > 2 * TICK_INTERVAL_MS) { + checkStatus = 'unhealthy' + } else if (status.lastError || status.healthStatus === 'degraded') { + checkStatus = 'degraded' + } + } + + return { + status: checkStatus, + lastTickAt: status.lastTickAt ? status.lastTickAt.toISOString() : null + } + } catch (error: any) { + return { status: 'unhealthy' as const, lastTickAt: null, error: error.message } + } + } + + const [dbResult, stellarResult, twilioResult] = await Promise.all([ + checkDatabase(), + checkStellarRpc(), + checkTwilio() + ]) + const agentResult = checkAgentLoop() + + const hasUnhealthy = + dbResult.status === 'unhealthy' || + stellarResult.status === 'unhealthy' || + twilioResult.status === 'unhealthy' || + agentResult.status === 'unhealthy' + + const hasDegraded = + dbResult.status === 'degraded' || + stellarResult.status === 'degraded' || + twilioResult.status === 'degraded' || + agentResult.status === 'degraded' + + let overallStatus: 'healthy' | 'degraded' | 'unhealthy' = 'healthy' + if (hasUnhealthy) { + overallStatus = 'unhealthy' + } else if (hasDegraded) { + overallStatus = 'degraded' + } + + const statusCode = overallStatus === 'unhealthy' ? 503 : 200 + + return res.status(statusCode).json({ + status: overallStatus, + version: pkg.version || '1.0.0', + uptime: Math.floor(process.uptime()), + checks: { + database: dbResult, + stellarRpc: stellarResult, + twilio: twilioResult, + agentLoop: agentResult + } + }) +}) + export default router diff --git a/tests/unit/routes/health.test.ts b/tests/unit/routes/health.test.ts new file mode 100644 index 0000000..07d528a --- /dev/null +++ b/tests/unit/routes/health.test.ts @@ -0,0 +1,217 @@ +process.env.NODE_ENV = 'test' +process.env.STELLAR_NETWORK = 'testnet' +process.env.STELLAR_RPC_URL = 'https://soroban-testnet.stellar.org' +process.env.STELLAR_AGENT_SECRET_KEY = 'S' + 'A'.repeat(55) +process.env.VAULT_CONTRACT_ID = 'C' + 'A'.repeat(55) +process.env.USDC_TOKEN_ADDRESS = 'C' + 'B'.repeat(55) +process.env.ANTHROPIC_API_KEY = 'sk-ant-test-key' +process.env.DATABASE_URL = 'postgresql://localhost:5432/test' +process.env.JWT_SEED = '0'.repeat(64) +process.env.WALLET_ENCRYPTION_KEY = '0'.repeat(64) +process.env.TWILIO_AUTH_TOKEN = '0'.repeat(32) +process.env.TWILIO_ACCOUNT_SID = 'AC' + '0'.repeat(32) +process.env.INTERNAL_SERVICE_TOKEN = 'test-internal-token' + +import request from 'supertest' +import express from 'express' +import healthRouter from '../../../src/routes/health' + +const app = express() +app.use('/health', healthRouter) + +// Mock the dependencies +const mockQueryRaw = jest.fn() +jest.mock('../../../src/db', () => ({ + __esModule: true, + default: { + $queryRaw: (...args: any[]) => mockQueryRaw(...args) + } +})) + +const mockStellarExecute = jest.fn() +jest.mock('../../../src/stellar/client', () => ({ + __esModule: true, + getResilientClient: () => ({ + execute: mockStellarExecute + }) +})) + +const mockTwilioFetch = jest.fn() +const mockTwilioClient = { + api: { + accounts: () => ({ + fetch: mockTwilioFetch + }) + } +} +jest.mock('twilio', () => { + return jest.fn().mockImplementation(() => mockTwilioClient) +}) + +const mockGetAgentStatus = jest.fn() +jest.mock('../../../src/agent/loop', () => ({ + __esModule: true, + getAgentStatus: () => mockGetAgentStatus() +})) + +describe('GET /health/deep', () => { + const token = 'test-internal-token' + + beforeEach(() => { + jest.clearAllMocks() + process.env.INTERNAL_SERVICE_TOKEN = token + process.env.TWILIO_ACCOUNT_SID = 'AC123' + process.env.TWILIO_AUTH_TOKEN = 'token123' + }) + + it('should return 401 if unauthorized', async () => { + const res = await request(app).get('/health/deep') + expect(res.status).toBe(401) + }) + + it('should return 200 and healthy status when all dependencies are healthy', async () => { + mockQueryRaw.mockResolvedValue([1]) + mockStellarExecute.mockResolvedValue({ sequence: 12345 }) + mockTwilioFetch.mockResolvedValue({}) + mockGetAgentStatus.mockReturnValue({ + isRunning: true, + lastTickAt: new Date(), + lastError: null, + healthStatus: 'healthy' + }) + + const res = await request(app) + .get('/health/deep') + .set('X-Internal-Token', token) + + expect(res.status).toBe(200) + expect(res.body.status).toBe('healthy') + expect(res.body.checks.database.status).toBe('healthy') + expect(res.body.checks.stellarRpc.status).toBe('healthy') + expect(res.body.checks.stellarRpc.ledger).toBe(12345) + expect(res.body.checks.twilio.status).toBe('healthy') + expect(res.body.checks.agentLoop.status).toBe('healthy') + }) + + it('should return 503 if database check fails', async () => { + mockQueryRaw.mockRejectedValue(new Error('DB Connection Failed')) + mockStellarExecute.mockResolvedValue({ sequence: 12345 }) + mockTwilioFetch.mockResolvedValue({}) + mockGetAgentStatus.mockReturnValue({ + isRunning: true, + lastTickAt: new Date(), + lastError: null, + healthStatus: 'healthy' + }) + + const res = await request(app) + .get('/health/deep') + .set('X-Internal-Token', token) + + expect(res.status).toBe(503) + expect(res.body.status).toBe('unhealthy') + expect(res.body.checks.database.status).toBe('unhealthy') + expect(res.body.checks.database.error).toBe('DB Connection Failed') + }) + + it('should return 503 if stellar rpc check fails', async () => { + mockQueryRaw.mockResolvedValue([1]) + mockStellarExecute.mockRejectedValue(new Error('RPC Error')) + mockTwilioFetch.mockResolvedValue({}) + mockGetAgentStatus.mockReturnValue({ + isRunning: true, + lastTickAt: new Date(), + lastError: null, + healthStatus: 'healthy' + }) + + const res = await request(app) + .get('/health/deep') + .set('X-Internal-Token', token) + + expect(res.status).toBe(503) + expect(res.body.status).toBe('unhealthy') + expect(res.body.checks.stellarRpc.status).toBe('unhealthy') + }) + + it('should return 503 if twilio check fails', async () => { + mockQueryRaw.mockResolvedValue([1]) + mockStellarExecute.mockResolvedValue({ sequence: 12345 }) + mockTwilioFetch.mockRejectedValue(new Error('Twilio Auth Error')) + mockGetAgentStatus.mockReturnValue({ + isRunning: true, + lastTickAt: new Date(), + lastError: null, + healthStatus: 'healthy' + }) + + const res = await request(app) + .get('/health/deep') + .set('X-Internal-Token', token) + + expect(res.status).toBe(503) + expect(res.body.status).toBe('unhealthy') + expect(res.body.checks.twilio.status).toBe('unhealthy') + }) + + it('should return 503 if agent loop has not ticked within 2x interval', async () => { + mockQueryRaw.mockResolvedValue([1]) + mockStellarExecute.mockResolvedValue({ sequence: 12345 }) + mockTwilioFetch.mockResolvedValue({}) + const oldDate = new Date(Date.now() - 65 * 60 * 1000) + mockGetAgentStatus.mockReturnValue({ + isRunning: true, + lastTickAt: oldDate, + lastError: null, + healthStatus: 'healthy' + }) + + const res = await request(app) + .get('/health/deep') + .set('X-Internal-Token', token) + + expect(res.status).toBe(503) + expect(res.body.status).toBe('unhealthy') + expect(res.body.checks.agentLoop.status).toBe('unhealthy') + }) + + it('should return 200 and degraded if agent loop is degraded but no unhealthy dependencies', async () => { + mockQueryRaw.mockResolvedValue([1]) + mockStellarExecute.mockResolvedValue({ sequence: 12345 }) + mockTwilioFetch.mockResolvedValue({}) + mockGetAgentStatus.mockReturnValue({ + isRunning: true, + lastTickAt: new Date(), + lastError: 'Some warning', + healthStatus: 'degraded' + }) + + const res = await request(app) + .get('/health/deep') + .set('X-Internal-Token', token) + + expect(res.status).toBe(200) + expect(res.body.status).toBe('degraded') + expect(res.body.checks.agentLoop.status).toBe('degraded') + }) + + it('should timeout individual dependency checks after 3s', async () => { + mockQueryRaw.mockImplementation(() => new Promise(resolve => setTimeout(resolve, 5000))) + mockStellarExecute.mockResolvedValue({ sequence: 12345 }) + mockTwilioFetch.mockResolvedValue({}) + mockGetAgentStatus.mockReturnValue({ + isRunning: true, + lastTickAt: new Date(), + lastError: null, + healthStatus: 'healthy' + }) + + const res = await request(app) + .get('/health/deep') + .set('X-Internal-Token', token) + + expect(res.status).toBe(503) + expect(res.body.checks.database.status).toBe('unhealthy') + expect(res.body.checks.database.error).toContain('timed out') + }) +})