diff --git a/Dockerfile.cli b/Dockerfile.cli index ec314d6e..68931064 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -1,6 +1,8 @@ # Use the official Node.js as the base image FROM node:22.15.0-bullseye-slim +RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/* + # Set the working directory WORKDIR /app @@ -9,7 +11,8 @@ COPY package*.json ./ COPY packages/ ./packages/ # Install dependencies -RUN npm ci +RUN npm ci --ignore-scripts +RUN npm rebuild sqlite3 RUN npm run build COPY scripts ./scripts/ diff --git a/Dockerfile.explorer b/Dockerfile.explorer index 8a5760af..4316b268 100644 --- a/Dockerfile.explorer +++ b/Dockerfile.explorer @@ -1,6 +1,8 @@ # Use the official Node.js as the base image FROM node:22.15.0-bullseye-slim +RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/* + # Set the working directory WORKDIR /app @@ -9,7 +11,8 @@ COPY package*.json ./ COPY packages/ ./packages/ # Install dependencies -RUN npm ci +RUN npm ci --ignore-scripts +RUN npm rebuild sqlite3 RUN npm run build COPY services/explorer ./explorer/ diff --git a/Dockerfile.gatekeeper b/Dockerfile.gatekeeper index f2071e30..8ce0321f 100644 --- a/Dockerfile.gatekeeper +++ b/Dockerfile.gatekeeper @@ -1,6 +1,8 @@ # Use the official Node.js as the base image FROM node:22.15.0-bullseye-slim +RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/* + # Set the working directory WORKDIR /app @@ -9,7 +11,8 @@ COPY package*.json ./ COPY packages/ ./packages/ # Install dependencies -RUN npm ci +RUN npm ci --ignore-scripts +RUN npm rebuild sqlite3 RUN npm run build COPY services/gatekeeper ./gatekeeper/ diff --git a/Dockerfile.hyperswarm b/Dockerfile.hyperswarm index 4f528b5d..98ef847f 100644 --- a/Dockerfile.hyperswarm +++ b/Dockerfile.hyperswarm @@ -12,7 +12,8 @@ COPY package*.json ./ COPY packages/ ./packages/ # Install dependencies -RUN npm ci +RUN npm ci --ignore-scripts +RUN npm rebuild sqlite3 RUN npm run build COPY services/mediators/hyperswarm ./hyperswarm/ diff --git a/Dockerfile.keymaster b/Dockerfile.keymaster index 0999bca8..101f9eee 100644 --- a/Dockerfile.keymaster +++ b/Dockerfile.keymaster @@ -1,6 +1,8 @@ # Use the official Node.js as the base image FROM node:22.15.0-bullseye-slim +RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/* + # Set the working directory WORKDIR /app @@ -9,7 +11,8 @@ COPY package*.json ./ COPY packages/ ./packages/ # Install dependencies -RUN npm ci +RUN npm ci --ignore-scripts +RUN npm rebuild sqlite3 RUN npm run build COPY services/keymaster ./keymaster/ diff --git a/Dockerfile.react-wallet b/Dockerfile.react-wallet index 209618a0..f4b19530 100644 --- a/Dockerfile.react-wallet +++ b/Dockerfile.react-wallet @@ -1,6 +1,8 @@ # Use the official Node.js as the base image FROM node:22.15.0-bullseye-slim +RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/* + # Set the working directory WORKDIR /app @@ -9,7 +11,8 @@ COPY package*.json ./ COPY packages/ ./packages/ # Install dependencies -RUN npm ci +RUN npm ci --ignore-scripts +RUN npm rebuild sqlite3 RUN npm run build COPY apps/react-wallet ./apps/react-wallet/ diff --git a/Dockerfile.satoshi b/Dockerfile.satoshi index 5cb46a6b..a1247391 100644 --- a/Dockerfile.satoshi +++ b/Dockerfile.satoshi @@ -1,6 +1,8 @@ # Use the official Node.js as the base image FROM node:22.15.0-bullseye-slim +RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/* + # Set the working directory WORKDIR /app @@ -9,7 +11,8 @@ COPY package*.json ./ COPY packages/ ./packages/ # Install dependencies -RUN npm ci +RUN npm ci --ignore-scripts +RUN npm rebuild sqlite3 RUN npm run build COPY services/mediators/satoshi ./satoshi/ diff --git a/Dockerfile.satoshi-inscription b/Dockerfile.satoshi-inscription index db8661dc..3b1a0918 100644 --- a/Dockerfile.satoshi-inscription +++ b/Dockerfile.satoshi-inscription @@ -1,6 +1,8 @@ # Use the official Node.js as the base image FROM node:22.15.0-bullseye-slim +RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/* + # Set the working directory WORKDIR /app @@ -9,7 +11,8 @@ COPY package*.json ./ COPY packages/ ./packages/ # Install dependencies -RUN npm ci +RUN npm ci --ignore-scripts +RUN npm rebuild sqlite3 RUN npm run build COPY services/mediators/satoshi-inscription ./satoshi/ diff --git a/Dockerfile.search-server b/Dockerfile.search-server index 9aeaa3cd..0191b9d8 100644 --- a/Dockerfile.search-server +++ b/Dockerfile.search-server @@ -1,6 +1,8 @@ # Use the official Node.js as the base image FROM node:22.15.0-bullseye-slim +RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/* + # Set the working directory WORKDIR /app @@ -10,7 +12,8 @@ COPY packages/ ./packages/ COPY services/search-server ./search-server/ # Install dependencies -RUN npm ci +RUN npm ci --ignore-scripts +RUN npm rebuild sqlite3 RUN npm run build # Make sure dir is owned by user who will build @@ -28,4 +31,3 @@ EXPOSE 4002 # Run... CMD ["node", "dist/index.js"] - diff --git a/docker-compose.yml b/docker-compose.yml index 633eb646..e396a998 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,13 @@ services: - KC_GATEKEEPER_GC_INTERVAL=${KC_GATEKEEPER_GC_INTERVAL} - KC_GATEKEEPER_STATUS_INTERVAL=${KC_GATEKEEPER_STATUS_INTERVAL} - KC_GATEKEEPER_SERVE_CLIENT=${KC_GATEKEEPER_SERVE_CLIENT} + - KC_GATEKEEPER_TRUST_PROXY=${KC_GATEKEEPER_TRUST_PROXY} + - KC_GATEKEEPER_RATE_LIMIT_ENABLED=${KC_GATEKEEPER_RATE_LIMIT_ENABLED} + - KC_GATEKEEPER_RATE_LIMIT_WINDOW_VALUE=${KC_GATEKEEPER_RATE_LIMIT_WINDOW_VALUE} + - KC_GATEKEEPER_RATE_LIMIT_WINDOW_UNIT=${KC_GATEKEEPER_RATE_LIMIT_WINDOW_UNIT} + - KC_GATEKEEPER_RATE_LIMIT_MAX_REQUESTS=${KC_GATEKEEPER_RATE_LIMIT_MAX_REQUESTS} + - KC_GATEKEEPER_RATE_LIMIT_WHITELIST=${KC_GATEKEEPER_RATE_LIMIT_WHITELIST} + - KC_GATEKEEPER_RATE_LIMIT_SKIP_PATHS=${KC_GATEKEEPER_RATE_LIMIT_SKIP_PATHS} - KC_LOG_LEVEL=${KC_LOG_LEVEL} - KC_MONGODB_URL=mongodb://mongodb:27017 - KC_REDIS_URL=redis://redis:6379 @@ -65,13 +72,20 @@ services: - KC_KEYMASTER_PORT=4226 - KC_GATEKEEPER_URL=http://gatekeeper:4224 - KC_DISABLE_SEARCH=${KC_DISABLE_SEARCH:-false} - - KC_SEARCH_URL=http://search-server:4002 + - KC_SEARCH_URL=http://search-server:${SEARCH_SERVER_PORT:-4002} - KC_NODE_ID=${KC_NODE_ID} - KC_KEYMASTER_DB=${KC_KEYMASTER_DB} - KC_ENCRYPTED_PASSPHRASE=${KC_ENCRYPTED_PASSPHRASE} - KC_WALLET_CACHE=${KC_WALLET_CACHE} - KC_DEFAULT_REGISTRY=${KC_DEFAULT_REGISTRY} - KC_KEYMASTER_SERVE_CLIENT=${KC_KEYMASTER_SERVE_CLIENT} + - KC_KEYMASTER_TRUST_PROXY=${KC_KEYMASTER_TRUST_PROXY} + - KC_KEYMASTER_RATE_LIMIT_ENABLED=${KC_KEYMASTER_RATE_LIMIT_ENABLED} + - KC_KEYMASTER_RATE_LIMIT_WINDOW_VALUE=${KC_KEYMASTER_RATE_LIMIT_WINDOW_VALUE} + - KC_KEYMASTER_RATE_LIMIT_WINDOW_UNIT=${KC_KEYMASTER_RATE_LIMIT_WINDOW_UNIT} + - KC_KEYMASTER_RATE_LIMIT_MAX_REQUESTS=${KC_KEYMASTER_RATE_LIMIT_MAX_REQUESTS} + - KC_KEYMASTER_RATE_LIMIT_WHITELIST=${KC_KEYMASTER_RATE_LIMIT_WHITELIST} + - KC_KEYMASTER_RATE_LIMIT_SKIP_PATHS=${KC_KEYMASTER_RATE_LIMIT_SKIP_PATHS} - KC_LOG_LEVEL=${KC_LOG_LEVEL} - KC_MONGODB_URL=mongodb://mongodb:27017 - KC_REDIS_URL=redis://redis:6379 @@ -344,7 +358,7 @@ services: environment: - VITE_EXPLORER_PORT=4000 - VITE_GATEKEEPER_URL=http://localhost:4224 - - VITE_SEARCH_SERVER=http://localhost:4002 + - VITE_SEARCH_SERVER=http://localhost:${SEARCH_SERVER_PORT:-4002} - VITE_OPERATION_NETWORKS=hyperswarm,local,TFTC,TBTC - KC_LOG_LEVEL=${KC_LOG_LEVEL} ports: @@ -358,16 +372,23 @@ services: dockerfile: Dockerfile.search-server image: keychainmdip/search-server environment: - - SEARCH_SERVER_PORT=4002 - - SEARCH_SERVER_GATEKEEPER_URL=http://gatekeeper:4224 - - SEARCH_SERVER_REFRESH_INTERVAL_MS=5000 - - SEARCH_SERVER_DB=sqlite + - SEARCH_SERVER_PORT=${SEARCH_SERVER_PORT:-4002} + - SEARCH_SERVER_GATEKEEPER_URL=${SEARCH_SERVER_GATEKEEPER_URL:-http://gatekeeper:4224} + - SEARCH_SERVER_REFRESH_INTERVAL_MS=${SEARCH_SERVER_REFRESH_INTERVAL_MS:-5000} + - SEARCH_SERVER_DB=${SEARCH_SERVER_DB:-sqlite} + - SEARCH_SERVER_TRUST_PROXY=${SEARCH_SERVER_TRUST_PROXY:-false} + - SEARCH_SERVER_RATE_LIMIT_ENABLED=${SEARCH_SERVER_RATE_LIMIT_ENABLED:-false} + - SEARCH_SERVER_RATE_LIMIT_WINDOW_VALUE=${SEARCH_SERVER_RATE_LIMIT_WINDOW_VALUE:-1} + - SEARCH_SERVER_RATE_LIMIT_WINDOW_UNIT=${SEARCH_SERVER_RATE_LIMIT_WINDOW_UNIT:-minute} + - SEARCH_SERVER_RATE_LIMIT_MAX_REQUESTS=${SEARCH_SERVER_RATE_LIMIT_MAX_REQUESTS:-600} + - SEARCH_SERVER_RATE_LIMIT_WHITELIST=${SEARCH_SERVER_RATE_LIMIT_WHITELIST:-} + - SEARCH_SERVER_RATE_LIMIT_SKIP_PATHS=${SEARCH_SERVER_RATE_LIMIT_SKIP_PATHS:-/api/v1/ready} - KC_LOG_LEVEL=${KC_LOG_LEVEL} volumes: - ./data:/app/search-server/data user: "${KC_UID}:${KC_GID}" ports: - - "4002:4002" + - "${SEARCH_SERVER_PORT:-4002}:${SEARCH_SERVER_PORT:-4002}" depends_on: - gatekeeper @@ -380,7 +401,7 @@ services: - VITE_PORT=${KC_REACT_WALLET_PORT:-4228} - VITE_GATEKEEPER_URL=http://gatekeeper:4224 - VITE_KEYMASTER_URL=http://keymaster:4226 - - VITE_SEARCH_SERVER=http://search-server:4002 + - VITE_SEARCH_SERVER=http://search-server:${SEARCH_SERVER_PORT:-4002} user: "${KC_UID}:${KC_GID}" ports: - "${KC_REACT_WALLET_PORT:-4228}:${KC_REACT_WALLET_PORT:-4228}" diff --git a/sample.env b/sample.env index c68ce785..886ac61d 100644 --- a/sample.env +++ b/sample.env @@ -17,6 +17,13 @@ KC_GATEKEEPER_MAX_OP_BYTES=262144 KC_GATEKEEPER_GC_INTERVAL=60 KC_GATEKEEPER_STATUS_INTERVAL=1 KC_GATEKEEPER_SERVE_CLIENT=true +KC_GATEKEEPER_TRUST_PROXY=false # Trust proxy headers. Set if behind reverse proxy/load balancer +KC_GATEKEEPER_RATE_LIMIT_ENABLED=false # Whether the rate limiter is enabled +KC_GATEKEEPER_RATE_LIMIT_WINDOW_VALUE=1 # The number of seconds, minutes or hours the limit applies to +KC_GATEKEEPER_RATE_LIMIT_WINDOW_UNIT=minute # second, minute, hour +KC_GATEKEEPER_RATE_LIMIT_MAX_REQUESTS=600 +KC_GATEKEEPER_RATE_LIMIT_WHITELIST= # Whitelist as CSV (127.0.0.1,10.0.0.0/8,2001:db8::/32) +KC_GATEKEEPER_RATE_LIMIT_SKIP_PATHS=/api/v1/ready # API paths to skip rate limiter on # Keymaster KC_KEYMASTER_PORT=4226 @@ -25,14 +32,29 @@ KC_ENCRYPTED_PASSPHRASE= KC_WALLET_CACHE=false KC_DEFAULT_REGISTRY=hyperswarm KC_KEYMASTER_SERVE_CLIENT=true +KC_KEYMASTER_TRUST_PROXY=false # Trust proxy headers. Set if behind reverse proxy/load balancer +KC_KEYMASTER_RATE_LIMIT_ENABLED=false # Whether the rate limiter is enabled +KC_KEYMASTER_RATE_LIMIT_WINDOW_VALUE=1 # The number of seconds, minutes or hours the limit applies to +KC_KEYMASTER_RATE_LIMIT_WINDOW_UNIT=minute # second, minute, hour +KC_KEYMASTER_RATE_LIMIT_MAX_REQUESTS=600 # Number of requests per window +KC_KEYMASTER_RATE_LIMIT_WHITELIST= # Whitelist as CSV (127.0.0.1,10.0.0.0/8,2001:db8::/32) +KC_KEYMASTER_RATE_LIMIT_SKIP_PATHS=/api/v1/ready # API paths to skip rate limiter on # React-Wallet KC_REACT_WALLET_PORT=4228 -# CLI -KC_GATEKEEPER_URL=http://localhost:4224 -KC_KEYMASTER_URL=http://localhost:4226 -KC_SEARCH_URL=http://localhost:4002 +# Search Server +SEARCH_SERVER_PORT=4002 +SEARCH_SERVER_GATEKEEPER_URL=http://gatekeeper:4224 +SEARCH_SERVER_REFRESH_INTERVAL_MS=5000 +SEARCH_SERVER_DB=sqlite +SEARCH_SERVER_TRUST_PROXY=false +SEARCH_SERVER_RATE_LIMIT_ENABLED=false +SEARCH_SERVER_RATE_LIMIT_WINDOW_VALUE=1 +SEARCH_SERVER_RATE_LIMIT_WINDOW_UNIT=minute +SEARCH_SERVER_RATE_LIMIT_MAX_REQUESTS=600 +SEARCH_SERVER_RATE_LIMIT_WHITELIST= +SEARCH_SERVER_RATE_LIMIT_SKIP_PATHS=/api/v1/ready # Hyperswarm KC_HYPR_EXPORT_INTERVAL=2 # Seconds between export-loop ticks. integer >= 1. diff --git a/services/gatekeeper/server/README.md b/services/gatekeeper/server/README.md index f6e48a12..f1538ed2 100644 --- a/services/gatekeeper/server/README.md +++ b/services/gatekeeper/server/README.md @@ -8,15 +8,22 @@ Operations come from Keymaster clients such as end-user wallets and network medi ## Environment variables -| variable | default | description | -| ------------------------------- | ---------| ---------------------------------------------------------------------- | -| `KC_GATEKEEPER_PORT` | 4224 | Service port | -| `KC_GATEKEEPER_DB` | redis | DID database adapter, must be `redis`, `json`, `mongodb`, or `sqlite` | -| `KC_GATEKEEPER_DID_PREFIX` | did:test | Default prefix assigned to DIDs created | -| `KC_IPFS_ENABLE` | true | Enable IPFS storage for opids and CAS endpoints | -| `KC_GATEKEEPER_GC_INTERVAL` | 15 | The number of minutes between garbage collection cycles (0 to disable) | -| `KC_GATEKEEPER_STATUS_INTERVAL` | 5 | The number of minutes between logging status updates (0 to disable) | -| `KC_LOG_LEVEL` | info | Log level: `debug`, `info`, `warn`, `error` | +| variable | default | description | +| --- | --- | --- | +| `KC_GATEKEEPER_PORT` | 4224 | Service port | +| `KC_GATEKEEPER_DB` | redis | DID database adapter, must be `redis`, `json`, `mongodb`, or `sqlite` | +| `KC_GATEKEEPER_DID_PREFIX` | did:test | Default prefix assigned to DIDs created | +| `KC_IPFS_ENABLE` | true | Enable IPFS storage for opids and CAS endpoints | +| `KC_GATEKEEPER_GC_INTERVAL` | 15 | The number of minutes between garbage collection cycles (0 to disable) | +| `KC_GATEKEEPER_STATUS_INTERVAL` | 5 | The number of minutes between logging status updates (0 to disable) | +| `KC_GATEKEEPER_TRUST_PROXY` | false | If true, trust upstream proxy headers when determining client IP (`req.ip`) | +| `KC_GATEKEEPER_RATE_LIMIT_ENABLED` | false | Enable API rate limiting | +| `KC_GATEKEEPER_RATE_LIMIT_WINDOW_VALUE` | 1 | Time window size for rate limiting | +| `KC_GATEKEEPER_RATE_LIMIT_WINDOW_UNIT` | minute | Time unit for rate limiting window: `second`, `minute`, or `hour` | +| `KC_GATEKEEPER_RATE_LIMIT_MAX_REQUESTS` | 600 | Max requests allowed per client during one window | +| `KC_GATEKEEPER_RATE_LIMIT_WHITELIST` | (empty) | Comma-separated IP/CIDR list to bypass limits | +| `KC_GATEKEEPER_RATE_LIMIT_SKIP_PATHS` | /api/v1/ready | Comma-separated API paths excluded from limits | +| `KC_LOG_LEVEL` | info | Log level: `debug`, `info`, `warn`, `error` | ## IPFS disabled mode diff --git a/services/gatekeeper/server/package-lock.json b/services/gatekeeper/server/package-lock.json index 168ff9ae..34b815db 100644 --- a/services/gatekeeper/server/package-lock.json +++ b/services/gatekeeper/server/package-lock.json @@ -1,23 +1,22 @@ { "name": "gatekeeper-server", - "version": "1.4.0-beta.2", + "version": "1.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "gatekeeper-server", - "version": "1.4.0-beta.2", + "version": "1.4.0", "license": "MIT", "dependencies": { "cors": "^2.8.5", "dotenv": "^16.4.5", "express": "^4.21.0", - "morgan": "^1.10.0" + "express-rate-limit": "^8.2.1" }, "devDependencies": { "@types/cors": "^2.8.17", "@types/express": "^5.0.1", - "@types/morgan": "^1.9.9", "@types/node": "^22.13.16", "typescript": "^5.8.2" } @@ -85,15 +84,6 @@ "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", "dev": true }, - "node_modules/@types/morgan": { - "version": "1.9.9", - "resolved": "https://registry.npmjs.org/@types/morgan/-/morgan-1.9.9.tgz", - "integrity": "sha512-iRYSDKVaC6FkGSpEVVIvrRGw0DfJMiQzIn3qr2G5B3C//AWkulhXgaBd7tS9/J79GWSYMTHGs7PfI5b3Y8m+RQ==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/node": { "version": "22.13.16", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.16.tgz", @@ -153,22 +143,6 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, - "node_modules/basic-auth": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", - "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", - "dependencies": { - "safe-buffer": "5.1.2" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/basic-auth/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, "node_modules/body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", @@ -406,6 +380,24 @@ "url": "https://opencollective.com/express" } }, + "node_modules/express-rate-limit": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.2.1.tgz", + "integrity": "sha512-PCZEIEIxqwhzw4KF0n7QF4QqruVTcF73O5kFKUnGOyjbCCgizBBiFaYpd/fnBLUMPw/BWw9OsiN7GgrNYr7j6g==", + "license": "MIT", + "dependencies": { + "ip-address": "10.0.1" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, "node_modules/finalhandler": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", @@ -551,6 +543,15 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/ip-address": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.0.1.tgz", + "integrity": "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -613,33 +614,6 @@ "node": ">= 0.6" } }, - "node_modules/morgan": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.1.tgz", - "integrity": "sha512-223dMRJtI/l25dJKWpgij2cMtywuG/WiUKXdvwfbhGKBhy1puASqXwFzmWZ7+K73vUPoR7SS2Qz2cI/g9MKw0A==", - "license": "MIT", - "dependencies": { - "basic-auth": "~2.0.1", - "debug": "2.6.9", - "depd": "~2.0.0", - "on-finished": "~2.3.0", - "on-headers": "~1.1.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/morgan/node_modules/on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -683,15 +657,6 @@ "node": ">= 0.8" } }, - "node_modules/on-headers": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz", - "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", diff --git a/services/gatekeeper/server/package.json b/services/gatekeeper/server/package.json index 6fe5dc2e..b9438ed4 100644 --- a/services/gatekeeper/server/package.json +++ b/services/gatekeeper/server/package.json @@ -14,7 +14,8 @@ "dependencies": { "cors": "^2.8.5", "dotenv": "^16.4.5", - "express": "^4.21.0" + "express": "^4.21.0", + "express-rate-limit": "^8.2.1" }, "devDependencies": { "@types/cors": "^2.8.17", diff --git a/services/gatekeeper/server/src/config.js b/services/gatekeeper/server/src/config.js index 657f3452..cc4308c4 100644 --- a/services/gatekeeper/server/src/config.js +++ b/services/gatekeeper/server/src/config.js @@ -2,6 +2,63 @@ import dotenv from 'dotenv'; dotenv.config(); +const DEFAULT_RATE_LIMIT_SKIP_PATHS = ['/api/v1/ready']; + +function parseBoolean(value, defaultValue) { + if (value === undefined) { + return defaultValue; + } + + const normalized = value.trim().toLowerCase(); + + if (normalized === 'true') { + return true; + } + + if (normalized === 'false') { + return false; + } + + return defaultValue; +} + +function parsePositiveInteger(value, defaultValue) { + const parsed = Number.parseInt(value ?? '', 10); + + if (Number.isInteger(parsed) && parsed > 0) { + return parsed; + } + + return defaultValue; +} + +function parseWindowUnit(value) { + const normalized = (value ?? '').trim().toLowerCase(); + + if (normalized === 'second' || normalized === 'seconds') { + return 'second'; + } + + if (normalized === 'hour' || normalized === 'hours') { + return 'hour'; + } + + return 'minute'; +} + +function parseCsv(value) { + if (!value) { + return []; + } + + return value + .split(',') + .map(item => item.trim()) + .filter(Boolean); +} + +const configuredSkipPaths = parseCsv(process.env.KC_GATEKEEPER_RATE_LIMIT_SKIP_PATHS); + const config = { port: process.env.KC_GATEKEEPER_PORT ? parseInt(process.env.KC_GATEKEEPER_PORT) : 4224, db: process.env.KC_GATEKEEPER_DB || 'redis', @@ -15,6 +72,13 @@ const config = { maxOpBytes: process.env.KC_GATEKEEPER_MAX_OP_BYTES ? parseInt(process.env.KC_GATEKEEPER_MAX_OP_BYTES) : undefined, gcInterval: process.env.KC_GATEKEEPER_GC_INTERVAL ? parseInt(process.env.KC_GATEKEEPER_GC_INTERVAL) : 15, statusInterval: process.env.KC_GATEKEEPER_STATUS_INTERVAL ? parseInt(process.env.KC_GATEKEEPER_STATUS_INTERVAL) : 5, + gatekeeperTrustProxy: parseBoolean(process.env.KC_GATEKEEPER_TRUST_PROXY, false), + rateLimitEnabled: parseBoolean(process.env.KC_GATEKEEPER_RATE_LIMIT_ENABLED, false), + rateLimitWindowValue: parsePositiveInteger(process.env.KC_GATEKEEPER_RATE_LIMIT_WINDOW_VALUE, 1), + rateLimitWindowUnit: parseWindowUnit(process.env.KC_GATEKEEPER_RATE_LIMIT_WINDOW_UNIT), + rateLimitMaxRequests: parsePositiveInteger(process.env.KC_GATEKEEPER_RATE_LIMIT_MAX_REQUESTS, 600), + rateLimitWhitelist: parseCsv(process.env.KC_GATEKEEPER_RATE_LIMIT_WHITELIST), + rateLimitSkipPaths: configuredSkipPaths.length > 0 ? configuredSkipPaths : DEFAULT_RATE_LIMIT_SKIP_PATHS, }; export default config; diff --git a/services/gatekeeper/server/src/gatekeeper-api.ts b/services/gatekeeper/server/src/gatekeeper-api.ts index 77d6ced2..d937d013 100644 --- a/services/gatekeeper/server/src/gatekeeper-api.ts +++ b/services/gatekeeper/server/src/gatekeeper-api.ts @@ -1,8 +1,10 @@ import express from 'express'; import cors from 'cors'; +import { BlockList, isIP } from 'net'; import path from 'path'; import { fileURLToPath } from 'url'; import { EventEmitter } from 'events'; +import rateLimit from 'express-rate-limit'; import Gatekeeper from '@mdip/gatekeeper'; import DbJsonCache from '@mdip/gatekeeper/db/json-cache'; @@ -18,6 +20,11 @@ import config from './config.js'; EventEmitter.defaultMaxListeners = 100; const log = childLogger({ service: 'gatekeeper-server' }); +const rateLimitWindowUnits = { + second: 1000, + minute: 60 * 1000, + hour: 60 * 60 * 1000, +} as const; function logRequest(req: express.Request, res: express.Response, next: express.NextFunction): void { const startTime = process.hrtime.bigint(); @@ -92,6 +99,135 @@ const startTime = new Date(); const app = express(); const v1router = express.Router(); +if (config.gatekeeperTrustProxy) { + app.set('trust proxy', true); +} + +function normalizeIp(ip: string): string { + const withoutZone = ip.split('%')[0]; + + if (withoutZone === '::1') { + return '127.0.0.1'; + } + + if (withoutZone.startsWith('::ffff:')) { + return withoutZone.slice(7); + } + + return withoutZone; +} + +function detectIpFamily(ip: string): 'ipv4' | 'ipv6' | null { + const version = isIP(ip); + + if (version === 4) { + return 'ipv4'; + } + + if (version === 6) { + return 'ipv6'; + } + + return null; +} + +function createWhitelistBlockList(whitelist: string[]): BlockList { + const blockList = new BlockList(); + + for (const entry of whitelist) { + const [rawAddress, rawPrefixLength] = entry.split('/'); + const address = normalizeIp(rawAddress); + const family = detectIpFamily(address); + + if (!family) { + log.warn(`Ignoring invalid rate limit whitelist entry: '${entry}'`); + continue; + } + + if (rawPrefixLength !== undefined) { + const prefixLength = Number.parseInt(rawPrefixLength, 10); + + if (!Number.isInteger(prefixLength)) { + log.warn(`Ignoring invalid rate limit CIDR entry: '${entry}'`); + continue; + } + + try { + blockList.addSubnet(address, prefixLength, family); + } + catch { + log.warn(`Ignoring invalid rate limit CIDR entry: '${entry}'`); + } + continue; + } + + try { + blockList.addAddress(address, family); + } + catch { + log.warn(`Ignoring invalid rate limit whitelist entry: '${entry}'`); + } + } + + return blockList; +} + +function shouldSkipRateLimitPath(req: express.Request): boolean { + const pathOnly = req.originalUrl.split('?')[0]; + + return config.rateLimitSkipPaths.some((skipPath: string) => + pathOnly === skipPath || pathOnly.startsWith(`${skipPath}/`)); +} + +const whitelistBlockList = createWhitelistBlockList(config.rateLimitWhitelist); +const rateLimitWindowUnit = config.rateLimitWindowUnit as keyof typeof rateLimitWindowUnits; +const rateLimitWindowMs = config.rateLimitWindowValue * (rateLimitWindowUnits[rateLimitWindowUnit] ?? rateLimitWindowUnits.minute); + +const apiRateLimiter = config.rateLimitEnabled + ? rateLimit({ + windowMs: rateLimitWindowMs, + limit: config.rateLimitMaxRequests, + statusCode: 429, + message: { error: 'Too many requests' }, + standardHeaders: 'draft-7', + legacyHeaders: false, + skip: (req) => { + if (req.method === 'OPTIONS') { + return true; + } + + if (shouldSkipRateLimitPath(req)) { + return true; + } + + if (config.rateLimitWhitelist.length === 0) { + return false; + } + + const candidates = [req.ip, req.socket.remoteAddress] + .filter((ip): ip is string => typeof ip === 'string' && ip.length > 0); + + for (const candidate of candidates) { + const normalizedIp = normalizeIp(candidate); + const family = detectIpFamily(normalizedIp); + + if (family && whitelistBlockList.check(normalizedIp, family)) { + return true; + } + } + + return false; + }, + }) + : null; + +if (config.rateLimitEnabled) { + log.info(`Rate limiting enabled: ${config.rateLimitMaxRequests} requests per ${config.rateLimitWindowValue} ${config.rateLimitWindowUnit}(s)`); +} +else { + log.info('Rate limiting disabled'); +} + app.use(cors()); app.options('*', cors()); @@ -2077,6 +2213,10 @@ v1router.post('/block/:registry', async (req, res) => { } }); +if (apiRateLimiter) { + app.use('/api', apiRateLimiter); +} + app.use('/api/v1', v1router); app.use('/api', (req, res) => { diff --git a/services/keymaster/server/README.md b/services/keymaster/server/README.md index c204aefe..adc5bbaf 100644 --- a/services/keymaster/server/README.md +++ b/services/keymaster/server/README.md @@ -6,12 +6,19 @@ This service is also useful when clients share a wallet, such as the `kc` CLI an ## Environment variables -| variable | default | description | -| --------------------- | ---------------------- | ----------------------------- | -| `KC_GATEKEEPER_URL` | http://localhost:4224 | MDIP gatekeeper service URL | -| `KC_KEYMASTER_PORT` | 4226 | Service port | -| `KC_KEYMASTER_DB` | json | Wallet database adapter, must be `redis`, `json`, `mongodb`, or `sqlite` | -| `KC_ENCRYPTED_PASSPHRASE` | (no default) | If specified, the wallet will be encrypted and decrypted with this passphrase | -| `KC_WALLET_CACHE` | false | Use wallet cache to increase performance (but understand security implications) | -| `KC_DEFAULT_REGISTRY` | hyperswarm | Default registry to use when creating DIDs | -| `KC_LOG_LEVEL` | info | Log level: `debug`, `info`, `warn`, `error` | +| variable | default | description | +| --- | --- | --- | +| `KC_GATEKEEPER_URL` | http://localhost:4224 | MDIP gatekeeper service URL | +| `KC_KEYMASTER_PORT` | 4226 | Service port | +| `KC_KEYMASTER_DB` | json | Wallet database adapter, must be `redis`, `json`, `mongodb`, or `sqlite` | +| `KC_ENCRYPTED_PASSPHRASE` | (no default) | If specified, the wallet will be encrypted and decrypted with this passphrase | +| `KC_WALLET_CACHE` | false | Use wallet cache to increase performance (but understand security implications) | +| `KC_DEFAULT_REGISTRY` | hyperswarm | Default registry to use when creating DIDs | +| `KC_KEYMASTER_TRUST_PROXY` | false | If true, trust upstream proxy headers when determining client IP (`req.ip`) | +| `KC_KEYMASTER_RATE_LIMIT_ENABLED` | false | Enable API rate limiting | +| `KC_KEYMASTER_RATE_LIMIT_WINDOW_VALUE` | 1 | Time window size for rate limiting | +| `KC_KEYMASTER_RATE_LIMIT_WINDOW_UNIT` | minute | Time unit for rate limiting window: `second`, `minute`, or `hour` | +| `KC_KEYMASTER_RATE_LIMIT_MAX_REQUESTS` | 600 | Max requests allowed per client during one window | +| `KC_KEYMASTER_RATE_LIMIT_WHITELIST` | (empty) | Comma-separated IP/CIDR list to bypass limits | +| `KC_KEYMASTER_RATE_LIMIT_SKIP_PATHS` | /api/v1/ready | Comma-separated API paths excluded from limits | +| `KC_LOG_LEVEL` | info | Log level: `debug`, `info`, `warn`, `error` | diff --git a/services/keymaster/server/package-lock.json b/services/keymaster/server/package-lock.json index 2e37e42b..f983104d 100644 --- a/services/keymaster/server/package-lock.json +++ b/services/keymaster/server/package-lock.json @@ -10,7 +10,8 @@ "license": "MIT", "dependencies": { "dotenv": "^16.4.5", - "express": "^4.21.0" + "express": "^4.21.0", + "express-rate-limit": "^8.2.1" }, "devDependencies": { "@types/express": "^5.0.3", @@ -357,6 +358,24 @@ "url": "https://opencollective.com/express" } }, + "node_modules/express-rate-limit": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.2.1.tgz", + "integrity": "sha512-PCZEIEIxqwhzw4KF0n7QF4QqruVTcF73O5kFKUnGOyjbCCgizBBiFaYpd/fnBLUMPw/BWw9OsiN7GgrNYr7j6g==", + "license": "MIT", + "dependencies": { + "ip-address": "10.0.1" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, "node_modules/finalhandler": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", @@ -502,6 +521,15 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/ip-address": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.0.1.tgz", + "integrity": "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", diff --git a/services/keymaster/server/package.json b/services/keymaster/server/package.json index 1db2ba17..d96641a4 100644 --- a/services/keymaster/server/package.json +++ b/services/keymaster/server/package.json @@ -13,7 +13,8 @@ "license": "MIT", "dependencies": { "dotenv": "^16.4.5", - "express": "^4.21.0" + "express": "^4.21.0", + "express-rate-limit": "^8.2.1" }, "devDependencies": { "@types/express": "^5.0.3", diff --git a/services/keymaster/server/src/config.js b/services/keymaster/server/src/config.js index ebd8fef7..b7e69aee 100644 --- a/services/keymaster/server/src/config.js +++ b/services/keymaster/server/src/config.js @@ -2,6 +2,63 @@ import dotenv from 'dotenv'; dotenv.config(); +const DEFAULT_RATE_LIMIT_SKIP_PATHS = ['/api/v1/ready']; + +function parseBoolean(value, defaultValue) { + if (value === undefined) { + return defaultValue; + } + + const normalized = value.trim().toLowerCase(); + + if (normalized === 'true') { + return true; + } + + if (normalized === 'false') { + return false; + } + + return defaultValue; +} + +function parsePositiveInteger(value, defaultValue) { + const parsed = Number.parseInt(value ?? '', 10); + + if (Number.isInteger(parsed) && parsed > 0) { + return parsed; + } + + return defaultValue; +} + +function parseWindowUnit(value) { + const normalized = (value ?? '').trim().toLowerCase(); + + if (normalized === 'second' || normalized === 'seconds') { + return 'second'; + } + + if (normalized === 'hour' || normalized === 'hours') { + return 'hour'; + } + + return 'minute'; +} + +function parseCsv(value) { + if (!value) { + return []; + } + + return value + .split(',') + .map(item => item.trim()) + .filter(Boolean); +} + +const configuredSkipPaths = parseCsv(process.env.KC_KEYMASTER_RATE_LIMIT_SKIP_PATHS); + const config = { gatekeeperURL: process.env.KC_GATEKEEPER_URL || 'http://localhost:4224', searchURL: process.env.KC_SEARCH_URL || 'http://localhost:4002', @@ -11,7 +68,14 @@ const config = { db: process.env.KC_KEYMASTER_DB || 'json', keymasterPassphrase: process.env.KC_ENCRYPTED_PASSPHRASE || '', walletCache: process.env.KC_WALLET_CACHE ? process.env.KC_WALLET_CACHE === 'true' : false, - defaultRegistry: process.env.KC_DEFAULT_REGISTRY + defaultRegistry: process.env.KC_DEFAULT_REGISTRY, + keymasterTrustProxy: parseBoolean(process.env.KC_KEYMASTER_TRUST_PROXY, false), + rateLimitEnabled: parseBoolean(process.env.KC_KEYMASTER_RATE_LIMIT_ENABLED, false), + rateLimitWindowValue: parsePositiveInteger(process.env.KC_KEYMASTER_RATE_LIMIT_WINDOW_VALUE, 1), + rateLimitWindowUnit: parseWindowUnit(process.env.KC_KEYMASTER_RATE_LIMIT_WINDOW_UNIT), + rateLimitMaxRequests: parsePositiveInteger(process.env.KC_KEYMASTER_RATE_LIMIT_MAX_REQUESTS, 600), + rateLimitWhitelist: parseCsv(process.env.KC_KEYMASTER_RATE_LIMIT_WHITELIST), + rateLimitSkipPaths: configuredSkipPaths.length > 0 ? configuredSkipPaths : DEFAULT_RATE_LIMIT_SKIP_PATHS, }; export default config; diff --git a/services/keymaster/server/src/keymaster-api.ts b/services/keymaster/server/src/keymaster-api.ts index 25999478..5bb8ef68 100644 --- a/services/keymaster/server/src/keymaster-api.ts +++ b/services/keymaster/server/src/keymaster-api.ts @@ -1,6 +1,8 @@ import express from 'express'; +import { BlockList, isIP } from 'net'; import path from 'path'; import { fileURLToPath } from 'url'; +import rateLimit from 'express-rate-limit'; import GatekeeperClient from '@mdip/gatekeeper/client'; import Keymaster from '@mdip/keymaster'; @@ -19,6 +21,11 @@ import config from './config.js'; const app = express(); const v1router = express.Router(); const log = childLogger({ service: 'keymaster-server' }); +const rateLimitWindowUnits = { + second: 1000, + minute: 60 * 1000, + hour: 60 * 60 * 1000, +} as const; function logRequest(req: express.Request, res: express.Response, next: express.NextFunction): void { const startTime = process.hrtime.bigint(); @@ -45,6 +52,135 @@ function logRequest(req: express.Request, res: express.Response, next: express.N next(); } +if (config.keymasterTrustProxy) { + app.set('trust proxy', true); +} + +function normalizeIp(ip: string): string { + const withoutZone = ip.split('%')[0]; + + if (withoutZone === '::1') { + return '127.0.0.1'; + } + + if (withoutZone.startsWith('::ffff:')) { + return withoutZone.slice(7); + } + + return withoutZone; +} + +function detectIpFamily(ip: string): 'ipv4' | 'ipv6' | null { + const version = isIP(ip); + + if (version === 4) { + return 'ipv4'; + } + + if (version === 6) { + return 'ipv6'; + } + + return null; +} + +function createWhitelistBlockList(whitelist: string[]): BlockList { + const blockList = new BlockList(); + + for (const entry of whitelist) { + const [rawAddress, rawPrefixLength] = entry.split('/'); + const address = normalizeIp(rawAddress); + const family = detectIpFamily(address); + + if (!family) { + log.warn(`Ignoring invalid rate limit whitelist entry: '${entry}'`); + continue; + } + + if (rawPrefixLength !== undefined) { + const prefixLength = Number.parseInt(rawPrefixLength, 10); + + if (!Number.isInteger(prefixLength)) { + log.warn(`Ignoring invalid rate limit CIDR entry: '${entry}'`); + continue; + } + + try { + blockList.addSubnet(address, prefixLength, family); + } + catch { + log.warn(`Ignoring invalid rate limit CIDR entry: '${entry}'`); + } + continue; + } + + try { + blockList.addAddress(address, family); + } + catch { + log.warn(`Ignoring invalid rate limit whitelist entry: '${entry}'`); + } + } + + return blockList; +} + +function shouldSkipRateLimitPath(req: express.Request): boolean { + const pathOnly = req.originalUrl.split('?')[0]; + + return config.rateLimitSkipPaths.some((skipPath: string) => + pathOnly === skipPath || pathOnly.startsWith(`${skipPath}/`)); +} + +const whitelistBlockList = createWhitelistBlockList(config.rateLimitWhitelist); +const rateLimitWindowUnit = config.rateLimitWindowUnit as keyof typeof rateLimitWindowUnits; +const rateLimitWindowMs = config.rateLimitWindowValue * (rateLimitWindowUnits[rateLimitWindowUnit] ?? rateLimitWindowUnits.minute); + +const apiRateLimiter = config.rateLimitEnabled + ? rateLimit({ + windowMs: rateLimitWindowMs, + limit: config.rateLimitMaxRequests, + statusCode: 429, + message: { error: 'Too many requests' }, + standardHeaders: 'draft-7', + legacyHeaders: false, + skip: (req) => { + if (req.method === 'OPTIONS') { + return true; + } + + if (shouldSkipRateLimitPath(req)) { + return true; + } + + if (config.rateLimitWhitelist.length === 0) { + return false; + } + + const candidates = [req.ip, req.socket.remoteAddress] + .filter((ip): ip is string => typeof ip === 'string' && ip.length > 0); + + for (const candidate of candidates) { + const normalizedIp = normalizeIp(candidate); + const family = detectIpFamily(normalizedIp); + + if (family && whitelistBlockList.check(normalizedIp, family)) { + return true; + } + } + + return false; + }, + }) + : null; + +if (config.rateLimitEnabled) { + log.info(`Rate limiting enabled: ${config.rateLimitMaxRequests} requests per ${config.rateLimitWindowValue} ${config.rateLimitWindowUnit}(s)`); +} +else { + log.info('Rate limiting disabled'); +} + app.use(logRequest); app.use(express.json()); @@ -6088,6 +6224,10 @@ v1router.post('/notices/refresh', async (req, res) => { } }); +if (apiRateLimiter) { + app.use('/api', apiRateLimiter); +} + app.use('/api/v1', v1router); app.use('/api', (req, res) => { diff --git a/services/search-server/README.md b/services/search-server/README.md index 140a2fe7..fd9ca204 100644 --- a/services/search-server/README.md +++ b/services/search-server/README.md @@ -31,6 +31,20 @@ SEARCH_SERVER_GATEKEEPER_URL=http://localhost:4224 # How often (in ms) to poll Gatekeeper for new or updated DIDs. SEARCH_SERVER_REFRESH_INTERVAL_MS=5000 +# Database backend (sqlite or memory) +SEARCH_SERVER_DB=sqlite + +# Trust proxy headers when determining req.ip +SEARCH_SERVER_TRUST_PROXY=false + +# API rate limiting +SEARCH_SERVER_RATE_LIMIT_ENABLED=false +SEARCH_SERVER_RATE_LIMIT_WINDOW_VALUE=1 +SEARCH_SERVER_RATE_LIMIT_WINDOW_UNIT=minute +SEARCH_SERVER_RATE_LIMIT_MAX_REQUESTS=600 +SEARCH_SERVER_RATE_LIMIT_WHITELIST= +SEARCH_SERVER_RATE_LIMIT_SKIP_PATHS=/api/v1/ready + # Logging KC_LOG_LEVEL=info ``` diff --git a/services/search-server/package-lock.json b/services/search-server/package-lock.json index 29ce918e..15968a5c 100644 --- a/services/search-server/package-lock.json +++ b/services/search-server/package-lock.json @@ -1,16 +1,17 @@ { "name": "search-server", - "version": "1.3.0", + "version": "1.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "search-server", - "version": "1.3.0", + "version": "1.4.0", "dependencies": { "cors": "^2.8.5", "dotenv": "^16.5.0", - "express": "^5.1.0" + "express": "^5.1.0", + "express-rate-limit": "^8.2.1" }, "devDependencies": { "@types/cors": "^2.8.13", @@ -399,6 +400,24 @@ "url": "https://opencollective.com/express" } }, + "node_modules/express-rate-limit": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.2.1.tgz", + "integrity": "sha512-PCZEIEIxqwhzw4KF0n7QF4QqruVTcF73O5kFKUnGOyjbCCgizBBiFaYpd/fnBLUMPw/BWw9OsiN7GgrNYr7j6g==", + "license": "MIT", + "dependencies": { + "ip-address": "10.0.1" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, "node_modules/express/node_modules/mime-db": { "version": "1.54.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", @@ -557,6 +576,15 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/ip-address": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.0.1.tgz", + "integrity": "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", diff --git a/services/search-server/package.json b/services/search-server/package.json index 382d4f4b..9b7dd1dd 100644 --- a/services/search-server/package.json +++ b/services/search-server/package.json @@ -9,7 +9,8 @@ "dependencies": { "cors": "^2.8.5", "dotenv": "^16.5.0", - "express": "^5.1.0" + "express": "^5.1.0", + "express-rate-limit": "^8.2.1" }, "devDependencies": { "@types/cors": "^2.8.13", diff --git a/services/search-server/sample.env b/services/search-server/sample.env index 1a56b7e3..00699621 100644 --- a/services/search-server/sample.env +++ b/services/search-server/sample.env @@ -2,4 +2,11 @@ SEARCH_SERVER_PORT=4002 SEARCH_SERVER_GATEKEEPER_URL=http://localhost:4224 SEARCH_SERVER_REFRESH_INTERVAL_MS=5000 SEARCH_SERVER_DB=sqlite +SEARCH_SERVER_TRUST_PROXY=false +SEARCH_SERVER_RATE_LIMIT_ENABLED=false +SEARCH_SERVER_RATE_LIMIT_WINDOW_VALUE=1 +SEARCH_SERVER_RATE_LIMIT_WINDOW_UNIT=minute +SEARCH_SERVER_RATE_LIMIT_MAX_REQUESTS=600 +SEARCH_SERVER_RATE_LIMIT_WHITELIST= +SEARCH_SERVER_RATE_LIMIT_SKIP_PATHS=/api/v1/ready KC_LOG_LEVEL=info diff --git a/services/search-server/src/config.ts b/services/search-server/src/config.ts new file mode 100644 index 00000000..6eceb19c --- /dev/null +++ b/services/search-server/src/config.ts @@ -0,0 +1,77 @@ +import dotenv from "dotenv"; + +dotenv.config(); + +const DEFAULT_RATE_LIMIT_SKIP_PATHS = ['/api/v1/ready']; + +function parseBoolean(value: string | undefined, defaultValue: boolean): boolean { + if (value === undefined) { + return defaultValue; + } + + const normalized = value.trim().toLowerCase(); + + if (normalized === 'true') { + return true; + } + + if (normalized === 'false') { + return false; + } + + return defaultValue; +} + +function parsePositiveInteger(value: string | undefined, defaultValue: number): number { + const parsed = Number.parseInt(value ?? '', 10); + + if (Number.isInteger(parsed) && parsed > 0) { + return parsed; + } + + return defaultValue; +} + +function parseWindowUnit(value: string | undefined): 'second' | 'minute' | 'hour' { + const normalized = (value ?? '').trim().toLowerCase(); + + if (normalized === 'second' || normalized === 'seconds') { + return 'second'; + } + + if (normalized === 'hour' || normalized === 'hours') { + return 'hour'; + } + + return 'minute'; +} + +function parseCsv(value: string | undefined): string[] { + if (!value) { + return []; + } + + return value + .split(',') + .map(item => item.trim()) + .filter(Boolean); +} + +const configuredSkipPaths = parseCsv(process.env.SEARCH_SERVER_RATE_LIMIT_SKIP_PATHS); + +const config = { + port: parsePositiveInteger(process.env.SEARCH_SERVER_PORT, 4002), + gatekeeperURL: process.env.SEARCH_SERVER_GATEKEEPER_URL || 'http://localhost:4224', + refreshIntervalMs: parsePositiveInteger(process.env.SEARCH_SERVER_REFRESH_INTERVAL_MS, 5000), + db: process.env.SEARCH_SERVER_DB || 'sqlite', + trustProxy: parseBoolean(process.env.SEARCH_SERVER_TRUST_PROXY, false), + jsonLimit: '2mb', + rateLimitEnabled: parseBoolean(process.env.SEARCH_SERVER_RATE_LIMIT_ENABLED, false), + rateLimitWindowValue: parsePositiveInteger(process.env.SEARCH_SERVER_RATE_LIMIT_WINDOW_VALUE, 1), + rateLimitWindowUnit: parseWindowUnit(process.env.SEARCH_SERVER_RATE_LIMIT_WINDOW_UNIT), + rateLimitMaxRequests: parsePositiveInteger(process.env.SEARCH_SERVER_RATE_LIMIT_MAX_REQUESTS, 600), + rateLimitWhitelist: parseCsv(process.env.SEARCH_SERVER_RATE_LIMIT_WHITELIST), + rateLimitSkipPaths: configuredSkipPaths.length > 0 ? configuredSkipPaths : DEFAULT_RATE_LIMIT_SKIP_PATHS, +}; + +export default config; diff --git a/services/search-server/src/index.ts b/services/search-server/src/index.ts index 9474875f..314c0938 100644 --- a/services/search-server/src/index.ts +++ b/services/search-server/src/index.ts @@ -1,26 +1,103 @@ import express from "express"; import cors from "cors"; -import dotenv from "dotenv"; +import { BlockList, isIP } from "net"; +import rateLimit from "express-rate-limit"; import GatekeeperClient from "@mdip/gatekeeper/client"; import DIDsSQLite from "./db/sqlite.js"; import DIDsDbMemory from './db/json-memory.js'; import DidIndexer from "./DidIndexer.js"; import {DIDsDb} from "./types.js"; import { childLogger } from "@mdip/common/logger"; +import config from "./config.js"; -dotenv.config(); const log = childLogger({ service: 'search-server' }); +const rateLimitWindowUnits = { + second: 1000, + minute: 60 * 1000, + hour: 60 * 60 * 1000, +} as const; -async function main() { - const { - SEARCH_SERVER_PORT = 4002, - SEARCH_SERVER_GATEKEEPER_URL = 'http://localhost:4224', - SEARCH_SERVER_REFRESH_INTERVAL_MS = 5000, - SEARCH_SERVER_DB = 'sqlite', - } = process.env; +function normalizeIp(ip: string): string { + const withoutZone = ip.split('%')[0]; + + if (withoutZone === '::1') { + return '127.0.0.1'; + } + + if (withoutZone.startsWith('::ffff:')) { + return withoutZone.slice(7); + } + + return withoutZone; +} + +function detectIpFamily(ip: string): 'ipv4' | 'ipv6' | null { + const version = isIP(ip); + + if (version === 4) { + return 'ipv4'; + } + + if (version === 6) { + return 'ipv6'; + } + + return null; +} + +function createWhitelistBlockList(whitelist: string[]): BlockList { + const blockList = new BlockList(); + + for (const entry of whitelist) { + const [rawAddress, rawPrefixLength] = entry.split('/'); + const address = normalizeIp(rawAddress); + const family = detectIpFamily(address); + + if (!family) { + log.warn(`Ignoring invalid rate limit whitelist entry: '${entry}'`); + continue; + } + + if (rawPrefixLength !== undefined) { + const prefixLength = Number.parseInt(rawPrefixLength, 10); + + if (!Number.isInteger(prefixLength)) { + log.warn(`Ignoring invalid rate limit CIDR entry: '${entry}'`); + continue; + } + + try { + blockList.addSubnet(address, prefixLength, family); + } + catch { + log.warn(`Ignoring invalid rate limit CIDR entry: '${entry}'`); + } + continue; + } + try { + blockList.addAddress(address, family); + } + catch { + log.warn(`Ignoring invalid rate limit whitelist entry: '${entry}'`); + } + } + + return blockList; +} + +function shouldSkipRateLimitPath(req: express.Request, skipPaths: string[]): boolean { + const pathOnly = req.originalUrl.split('?')[0]; + + return skipPaths.some(skipPath => + pathOnly === skipPath || pathOnly.startsWith(`${skipPath}/`)); +} + +async function main() { const app = express(); const v1router = express.Router(); + const whitelistBlockList = createWhitelistBlockList(config.rateLimitWhitelist); + const rateLimitWindowMs = config.rateLimitWindowValue * rateLimitWindowUnits[config.rateLimitWindowUnit]; const corsOptions = { origin: '*', // Origin needs to be specified with credentials true @@ -28,12 +105,61 @@ async function main() { optionsSuccessStatus: 200 // Some legacy browsers choke on 204 }; + if (config.trustProxy) { + app.set('trust proxy', true); + } + + const apiRateLimiter = config.rateLimitEnabled + ? rateLimit({ + windowMs: rateLimitWindowMs, + limit: config.rateLimitMaxRequests, + statusCode: 429, + message: { error: 'Too many requests' }, + standardHeaders: 'draft-7', + legacyHeaders: false, + skip: (req) => { + if (req.method === 'OPTIONS') { + return true; + } + + if (shouldSkipRateLimitPath(req, config.rateLimitSkipPaths)) { + return true; + } + + if (config.rateLimitWhitelist.length === 0) { + return false; + } + + const candidates = [req.ip, req.socket.remoteAddress] + .filter((ip): ip is string => typeof ip === 'string' && ip.length > 0); + + for (const candidate of candidates) { + const normalizedIp = normalizeIp(candidate); + const family = detectIpFamily(normalizedIp); + + if (family && whitelistBlockList.check(normalizedIp, family)) { + return true; + } + } + + return false; + }, + }) + : null; + + if (config.rateLimitEnabled) { + log.info(`Rate limiting enabled: ${config.rateLimitMaxRequests} requests per ${config.rateLimitWindowValue} ${config.rateLimitWindowUnit}(s)`); + } + else { + log.info('Rate limiting disabled'); + } + app.use(cors(corsOptions)); - app.use(express.json({ limit: '2mb' })); + app.use(express.json({ limit: config.jsonLimit })); let didDb: DIDsDb; - if (SEARCH_SERVER_DB === 'sqlite') { + if (config.db === 'sqlite') { didDb = await DIDsSQLite.create(); } else { didDb = new DIDsDbMemory(); @@ -41,14 +167,14 @@ async function main() { const gatekeeper = new GatekeeperClient(); await gatekeeper.connect({ - url: SEARCH_SERVER_GATEKEEPER_URL, + url: config.gatekeeperURL, waitUntilReady: true, intervalSeconds: 5, chatty: true, }); const indexer = new DidIndexer(gatekeeper, didDb, { - intervalMs: Number(SEARCH_SERVER_REFRESH_INTERVAL_MS), + intervalMs: config.refreshIntervalMs, }); // Let's not await here, we will continue and start @@ -107,9 +233,13 @@ async function main() { } }); + if (apiRateLimiter) { + app.use('/api', apiRateLimiter); + } + app.use('/api/v1', v1router); - const port = Number(SEARCH_SERVER_PORT) || 4002; + const port = config.port; const server = app.listen(port, () => { log.info(`Listening on port ${port}`); }); diff --git a/tests/cli-tests/generate_test_env.sh b/tests/cli-tests/generate_test_env.sh index 1b3e3ae1..76014874 100755 --- a/tests/cli-tests/generate_test_env.sh +++ b/tests/cli-tests/generate_test_env.sh @@ -31,6 +31,13 @@ KC_GATEKEEPER_REGISTRIES=local KC_GATEKEEPER_GC_INTERVAL=60 KC_GATEKEEPER_STATUS_INTERVAL=1 KC_GATEKEEPER_SERVE_CLIENT=true +KC_GATEKEEPER_TRUST_PROXY=false +KC_GATEKEEPER_RATE_LIMIT_ENABLED=false +KC_GATEKEEPER_RATE_LIMIT_WINDOW_VALUE=1 +KC_GATEKEEPER_RATE_LIMIT_WINDOW_UNIT=minute +KC_GATEKEEPER_RATE_LIMIT_MAX_REQUESTS=600 +KC_GATEKEEPER_RATE_LIMIT_WHITELIST= +KC_GATEKEEPER_RATE_LIMIT_SKIP_PATHS=/api/v1/ready # Keymaster KC_KEYMASTER_PORT=4226 @@ -40,10 +47,26 @@ KC_KEYMASTER_SERVE_CLIENT=true KC_WALLET_CACHE=false KC_DEFAULT_REGISTRY=local KC_DISABLE_SEARCH=true - -# CLI -KC_GATEKEEPER_URL=http://localhost:4224 -KC_KEYMASTER_URL=http://localhost:4226 +KC_KEYMASTER_TRUST_PROXY=false +KC_KEYMASTER_RATE_LIMIT_ENABLED=false +KC_KEYMASTER_RATE_LIMIT_WINDOW_VALUE=1 +KC_KEYMASTER_RATE_LIMIT_WINDOW_UNIT=minute +KC_KEYMASTER_RATE_LIMIT_MAX_REQUESTS=600 +KC_KEYMASTER_RATE_LIMIT_WHITELIST= +KC_KEYMASTER_RATE_LIMIT_SKIP_PATHS=/api/v1/ready + +# Search Server +SEARCH_SERVER_PORT=4002 +SEARCH_SERVER_GATEKEEPER_URL=http://gatekeeper:4224 +SEARCH_SERVER_REFRESH_INTERVAL_MS=5000 +SEARCH_SERVER_DB=sqlite +SEARCH_SERVER_TRUST_PROXY=false +SEARCH_SERVER_RATE_LIMIT_ENABLED=true +SEARCH_SERVER_RATE_LIMIT_WINDOW_VALUE=1 +SEARCH_SERVER_RATE_LIMIT_WINDOW_UNIT=minute +SEARCH_SERVER_RATE_LIMIT_MAX_REQUESTS=600 +SEARCH_SERVER_RATE_LIMIT_WHITELIST= +SEARCH_SERVER_RATE_LIMIT_SKIP_PATHS=/api/v1/ready # Hyperswarm KC_HYPR_EXPORT_INTERVAL=2