feat: add rate limiter to APIs

Dockerfile.cli

@@ -1,6 +1,8 @@
 # Use the official Node.js as the base image
 FROM node:22.15.0-bullseye-slim
 
+RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/*
+
 # Set the working directory
 WORKDIR /app
 
@@ -9,7 +11,8 @@ COPY package*.json ./
 COPY packages/ ./packages/
 
 # Install dependencies
-RUN npm ci
+RUN npm ci --ignore-scripts
+RUN npm rebuild sqlite3
 RUN npm run build
 
 COPY scripts ./scripts/

Dockerfile.explorer

@@ -1,6 +1,8 @@
 # Use the official Node.js as the base image
 FROM node:22.15.0-bullseye-slim
 
+RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/*
+
 # Set the working directory
 WORKDIR /app
 
@@ -9,7 +11,8 @@ COPY package*.json ./
 COPY packages/ ./packages/
 
 # Install dependencies
-RUN npm ci
+RUN npm ci --ignore-scripts
+RUN npm rebuild sqlite3
 RUN npm run build
 
 COPY services/explorer ./explorer/

Dockerfile.gatekeeper

@@ -1,6 +1,8 @@
 # Use the official Node.js as the base image
 FROM node:22.15.0-bullseye-slim
 
+RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/*
+
 # Set the working directory
 WORKDIR /app
 
@@ -9,7 +11,8 @@ COPY package*.json ./
 COPY packages/ ./packages/
 
 # Install dependencies
-RUN npm ci
+RUN npm ci --ignore-scripts
+RUN npm rebuild sqlite3
 RUN npm run build
 
 COPY services/gatekeeper ./gatekeeper/

Dockerfile.hyperswarm

@@ -12,7 +12,8 @@ COPY package*.json ./
 COPY packages/ ./packages/
 
 # Install dependencies
-RUN npm ci
+RUN npm ci --ignore-scripts
+RUN npm rebuild sqlite3
 RUN npm run build
 
 COPY services/mediators/hyperswarm ./hyperswarm/

Dockerfile.keymaster

@@ -1,6 +1,8 @@
 # Use the official Node.js as the base image
 FROM node:22.15.0-bullseye-slim
 
+RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/*
+
 # Set the working directory
 WORKDIR /app
 
@@ -9,7 +11,8 @@ COPY package*.json ./
 COPY packages/ ./packages/
 
 # Install dependencies
-RUN npm ci
+RUN npm ci --ignore-scripts
+RUN npm rebuild sqlite3
 RUN npm run build
 
 COPY services/keymaster ./keymaster/

Dockerfile.react-wallet

@@ -1,6 +1,8 @@
 # Use the official Node.js as the base image
 FROM node:22.15.0-bullseye-slim
 
+RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/*
+
 # Set the working directory
 WORKDIR /app
 
@@ -9,7 +11,8 @@ COPY package*.json ./
 COPY packages/ ./packages/
 
 # Install dependencies
-RUN npm ci
+RUN npm ci --ignore-scripts
+RUN npm rebuild sqlite3
 RUN npm run build
 
 COPY apps/react-wallet ./apps/react-wallet/

Dockerfile.satoshi

@@ -1,6 +1,8 @@
 # Use the official Node.js as the base image
 FROM node:22.15.0-bullseye-slim
 
+RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/*
+
 # Set the working directory
 WORKDIR /app
 
@@ -9,7 +11,8 @@ COPY package*.json ./
 COPY packages/ ./packages/
 
 # Install dependencies
-RUN npm ci
+RUN npm ci --ignore-scripts
+RUN npm rebuild sqlite3
 RUN npm run build
 
 COPY services/mediators/satoshi ./satoshi/

Dockerfile.satoshi-inscription

@@ -1,6 +1,8 @@
 # Use the official Node.js as the base image
 FROM node:22.15.0-bullseye-slim
 
+RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/*
+
 # Set the working directory
 WORKDIR /app
 
@@ -9,7 +11,8 @@ COPY package*.json ./
 COPY packages/ ./packages/
 
 # Install dependencies
-RUN npm ci
+RUN npm ci --ignore-scripts
+RUN npm rebuild sqlite3
 RUN npm run build
 
 COPY services/mediators/satoshi-inscription ./satoshi/

Dockerfile.search-server

@@ -1,6 +1,8 @@
 # Use the official Node.js as the base image
 FROM node:22.15.0-bullseye-slim
 
+RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/*
+
 # Set the working directory
 WORKDIR /app
 
@@ -10,7 +12,8 @@ COPY packages/ ./packages/
 COPY services/search-server ./search-server/
 
 # Install dependencies
-RUN npm ci
+RUN npm ci --ignore-scripts
+RUN npm rebuild sqlite3
 RUN npm run build
 
 # Make sure dir is owned by user who will build
@@ -28,4 +31,3 @@ EXPOSE 4002
 
 # Run...
 CMD ["node", "dist/index.js"]
-

docker-compose.yml

@@ -40,6 +40,13 @@ services:
       - KC_GATEKEEPER_GC_INTERVAL=${KC_GATEKEEPER_GC_INTERVAL}
       - KC_GATEKEEPER_STATUS_INTERVAL=${KC_GATEKEEPER_STATUS_INTERVAL}
       - KC_GATEKEEPER_SERVE_CLIENT=${KC_GATEKEEPER_SERVE_CLIENT}
+      - KC_GATEKEEPER_TRUST_PROXY=${KC_GATEKEEPER_TRUST_PROXY}
+      - KC_GATEKEEPER_RATE_LIMIT_ENABLED=${KC_GATEKEEPER_RATE_LIMIT_ENABLED}
+      - KC_GATEKEEPER_RATE_LIMIT_WINDOW_VALUE=${KC_GATEKEEPER_RATE_LIMIT_WINDOW_VALUE}
+      - KC_GATEKEEPER_RATE_LIMIT_WINDOW_UNIT=${KC_GATEKEEPER_RATE_LIMIT_WINDOW_UNIT}
+      - KC_GATEKEEPER_RATE_LIMIT_MAX_REQUESTS=${KC_GATEKEEPER_RATE_LIMIT_MAX_REQUESTS}
+      - KC_GATEKEEPER_RATE_LIMIT_WHITELIST=${KC_GATEKEEPER_RATE_LIMIT_WHITELIST}
+      - KC_GATEKEEPER_RATE_LIMIT_SKIP_PATHS=${KC_GATEKEEPER_RATE_LIMIT_SKIP_PATHS}
       - KC_LOG_LEVEL=${KC_LOG_LEVEL}
       - KC_MONGODB_URL=mongodb://mongodb:27017
       - KC_REDIS_URL=redis://redis:6379
@@ -65,13 +72,20 @@ services:
       - KC_KEYMASTER_PORT=4226
       - KC_GATEKEEPER_URL=http://gatekeeper:4224
       - KC_DISABLE_SEARCH=${KC_DISABLE_SEARCH:-false}
-      - KC_SEARCH_URL=http://search-server:4002
+      - KC_SEARCH_URL=http://search-server:${SEARCH_SERVER_PORT:-4002}
       - KC_NODE_ID=${KC_NODE_ID}
       - KC_KEYMASTER_DB=${KC_KEYMASTER_DB}
       - KC_ENCRYPTED_PASSPHRASE=${KC_ENCRYPTED_PASSPHRASE}
       - KC_WALLET_CACHE=${KC_WALLET_CACHE}
       - KC_DEFAULT_REGISTRY=${KC_DEFAULT_REGISTRY}
       - KC_KEYMASTER_SERVE_CLIENT=${KC_KEYMASTER_SERVE_CLIENT}
+      - KC_KEYMASTER_TRUST_PROXY=${KC_KEYMASTER_TRUST_PROXY}
+      - KC_KEYMASTER_RATE_LIMIT_ENABLED=${KC_KEYMASTER_RATE_LIMIT_ENABLED}
+      - KC_KEYMASTER_RATE_LIMIT_WINDOW_VALUE=${KC_KEYMASTER_RATE_LIMIT_WINDOW_VALUE}
+      - KC_KEYMASTER_RATE_LIMIT_WINDOW_UNIT=${KC_KEYMASTER_RATE_LIMIT_WINDOW_UNIT}
+      - KC_KEYMASTER_RATE_LIMIT_MAX_REQUESTS=${KC_KEYMASTER_RATE_LIMIT_MAX_REQUESTS}
+      - KC_KEYMASTER_RATE_LIMIT_WHITELIST=${KC_KEYMASTER_RATE_LIMIT_WHITELIST}
+      - KC_KEYMASTER_RATE_LIMIT_SKIP_PATHS=${KC_KEYMASTER_RATE_LIMIT_SKIP_PATHS}
       - KC_LOG_LEVEL=${KC_LOG_LEVEL}
       - KC_MONGODB_URL=mongodb://mongodb:27017
       - KC_REDIS_URL=redis://redis:6379
@@ -344,7 +358,7 @@ services:
     environment:
       - VITE_EXPLORER_PORT=4000
       - VITE_GATEKEEPER_URL=http://localhost:4224
-      - VITE_SEARCH_SERVER=http://localhost:4002
+      - VITE_SEARCH_SERVER=http://localhost:${SEARCH_SERVER_PORT:-4002}
       - VITE_OPERATION_NETWORKS=hyperswarm,local,TFTC,TBTC
       - KC_LOG_LEVEL=${KC_LOG_LEVEL}
     ports:
@@ -358,16 +372,23 @@ services:
       dockerfile: Dockerfile.search-server
     image: keychainmdip/search-server
     environment:
-      - SEARCH_SERVER_PORT=4002
-      - SEARCH_SERVER_GATEKEEPER_URL=http://gatekeeper:4224
-      - SEARCH_SERVER_REFRESH_INTERVAL_MS=5000
-      - SEARCH_SERVER_DB=sqlite
+      - SEARCH_SERVER_PORT=${SEARCH_SERVER_PORT:-4002}
+      - SEARCH_SERVER_GATEKEEPER_URL=${SEARCH_SERVER_GATEKEEPER_URL:-http://gatekeeper:4224}
+      - SEARCH_SERVER_REFRESH_INTERVAL_MS=${SEARCH_SERVER_REFRESH_INTERVAL_MS:-5000}
+      - SEARCH_SERVER_DB=${SEARCH_SERVER_DB:-sqlite}
+      - SEARCH_SERVER_TRUST_PROXY=${SEARCH_SERVER_TRUST_PROXY:-false}
+      - SEARCH_SERVER_RATE_LIMIT_ENABLED=${SEARCH_SERVER_RATE_LIMIT_ENABLED:-false}
+      - SEARCH_SERVER_RATE_LIMIT_WINDOW_VALUE=${SEARCH_SERVER_RATE_LIMIT_WINDOW_VALUE:-1}
+      - SEARCH_SERVER_RATE_LIMIT_WINDOW_UNIT=${SEARCH_SERVER_RATE_LIMIT_WINDOW_UNIT:-minute}
+      - SEARCH_SERVER_RATE_LIMIT_MAX_REQUESTS=${SEARCH_SERVER_RATE_LIMIT_MAX_REQUESTS:-600}
+      - SEARCH_SERVER_RATE_LIMIT_WHITELIST=${SEARCH_SERVER_RATE_LIMIT_WHITELIST:-}
+      - SEARCH_SERVER_RATE_LIMIT_SKIP_PATHS=${SEARCH_SERVER_RATE_LIMIT_SKIP_PATHS:-/api/v1/ready}
       - KC_LOG_LEVEL=${KC_LOG_LEVEL}
     volumes:
       - ./data:/app/search-server/data
     user: "${KC_UID}:${KC_GID}"
     ports:
-      - "4002:4002"
+      - "${SEARCH_SERVER_PORT:-4002}:${SEARCH_SERVER_PORT:-4002}"
     depends_on:
       - gatekeeper
 
@@ -380,7 +401,7 @@ services:
       - VITE_PORT=${KC_REACT_WALLET_PORT:-4228}
       - VITE_GATEKEEPER_URL=http://gatekeeper:4224
       - VITE_KEYMASTER_URL=http://keymaster:4226
-      - VITE_SEARCH_SERVER=http://search-server:4002
+      - VITE_SEARCH_SERVER=http://search-server:${SEARCH_SERVER_PORT:-4002}
     user: "${KC_UID}:${KC_GID}"
     ports:
       - "${KC_REACT_WALLET_PORT:-4228}:${KC_REACT_WALLET_PORT:-4228}"

sample.env

@@ -17,6 +17,13 @@ KC_GATEKEEPER_MAX_OP_BYTES=262144
 KC_GATEKEEPER_GC_INTERVAL=60
 KC_GATEKEEPER_STATUS_INTERVAL=1
 KC_GATEKEEPER_SERVE_CLIENT=true
+KC_GATEKEEPER_TRUST_PROXY=false # Trust proxy headers. Set if behind reverse proxy/load balancer
+KC_GATEKEEPER_RATE_LIMIT_ENABLED=false # Whether the rate limiter is enabled
+KC_GATEKEEPER_RATE_LIMIT_WINDOW_VALUE=1 # The number of seconds, minutes or hours the limit applies to
+KC_GATEKEEPER_RATE_LIMIT_WINDOW_UNIT=minute # second, minute, hour
+KC_GATEKEEPER_RATE_LIMIT_MAX_REQUESTS=600
+KC_GATEKEEPER_RATE_LIMIT_WHITELIST= # Whitelist as CSV (127.0.0.1,10.0.0.0/8,2001:db8::/32)
+KC_GATEKEEPER_RATE_LIMIT_SKIP_PATHS=/api/v1/ready # API paths to skip rate limiter on
 
 # Keymaster
 KC_KEYMASTER_PORT=4226
@@ -25,14 +32,29 @@ KC_ENCRYPTED_PASSPHRASE=
 KC_WALLET_CACHE=false
 KC_DEFAULT_REGISTRY=hyperswarm
 KC_KEYMASTER_SERVE_CLIENT=true
+KC_KEYMASTER_TRUST_PROXY=false # Trust proxy headers. Set if behind reverse proxy/load balancer
+KC_KEYMASTER_RATE_LIMIT_ENABLED=false # Whether the rate limiter is enabled
+KC_KEYMASTER_RATE_LIMIT_WINDOW_VALUE=1 # The number of seconds, minutes or hours the limit applies to
+KC_KEYMASTER_RATE_LIMIT_WINDOW_UNIT=minute # second, minute, hour
+KC_KEYMASTER_RATE_LIMIT_MAX_REQUESTS=600 # Number of requests per window
+KC_KEYMASTER_RATE_LIMIT_WHITELIST= # Whitelist as CSV (127.0.0.1,10.0.0.0/8,2001:db8::/32)
+KC_KEYMASTER_RATE_LIMIT_SKIP_PATHS=/api/v1/ready # API paths to skip rate limiter on
 
 # React-Wallet
 KC_REACT_WALLET_PORT=4228
 
-# CLI
-KC_GATEKEEPER_URL=http://localhost:4224
-KC_KEYMASTER_URL=http://localhost:4226
-KC_SEARCH_URL=http://localhost:4002
+# Search Server
+SEARCH_SERVER_PORT=4002
+SEARCH_SERVER_GATEKEEPER_URL=http://gatekeeper:4224
+SEARCH_SERVER_REFRESH_INTERVAL_MS=5000
+SEARCH_SERVER_DB=sqlite
+SEARCH_SERVER_TRUST_PROXY=false
+SEARCH_SERVER_RATE_LIMIT_ENABLED=false
+SEARCH_SERVER_RATE_LIMIT_WINDOW_VALUE=1
+SEARCH_SERVER_RATE_LIMIT_WINDOW_UNIT=minute
+SEARCH_SERVER_RATE_LIMIT_MAX_REQUESTS=600
+SEARCH_SERVER_RATE_LIMIT_WHITELIST=
+SEARCH_SERVER_RATE_LIMIT_SKIP_PATHS=/api/v1/ready
 
 # Hyperswarm
 KC_HYPR_EXPORT_INTERVAL=2 # Seconds between export-loop ticks. integer >= 1.

services/gatekeeper/server/README.md

@@ -8,15 +8,22 @@ Operations come from Keymaster clients such as end-user wallets and network medi
 
 ## Environment variables
 
-| variable                        | default  | description                                                            |
-| ------------------------------- | ---------| ---------------------------------------------------------------------- |
-| `KC_GATEKEEPER_PORT`            | 4224     | Service port                                                           |
-| `KC_GATEKEEPER_DB`              | redis    | DID database adapter, must be `redis`, `json`, `mongodb`, or `sqlite`  |
-| `KC_GATEKEEPER_DID_PREFIX`      | did:test | Default prefix assigned to DIDs created                                |
-| `KC_IPFS_ENABLE`                | true     | Enable IPFS storage for opids and CAS endpoints                        |
-| `KC_GATEKEEPER_GC_INTERVAL`     | 15       | The number of minutes between garbage collection cycles (0 to disable) |
-| `KC_GATEKEEPER_STATUS_INTERVAL` |  5       | The number of minutes between logging status updates (0 to disable)    |
-| `KC_LOG_LEVEL`                  | info     | Log level: `debug`, `info`, `warn`, `error`                            |
+| variable | default | description |
+| --- | --- | --- |
+| `KC_GATEKEEPER_PORT` | 4224 | Service port |
+| `KC_GATEKEEPER_DB` | redis | DID database adapter, must be `redis`, `json`, `mongodb`, or `sqlite` |
+| `KC_GATEKEEPER_DID_PREFIX` | did:test | Default prefix assigned to DIDs created |
+| `KC_IPFS_ENABLE` | true | Enable IPFS storage for opids and CAS endpoints |
+| `KC_GATEKEEPER_GC_INTERVAL` | 15 | The number of minutes between garbage collection cycles (0 to disable) |
+| `KC_GATEKEEPER_STATUS_INTERVAL` | 5 | The number of minutes between logging status updates (0 to disable) |
+| `KC_GATEKEEPER_TRUST_PROXY` | false | If true, trust upstream proxy headers when determining client IP (`req.ip`) |
+| `KC_GATEKEEPER_RATE_LIMIT_ENABLED` | false | Enable API rate limiting |
+| `KC_GATEKEEPER_RATE_LIMIT_WINDOW_VALUE` | 1 | Time window size for rate limiting |
+| `KC_GATEKEEPER_RATE_LIMIT_WINDOW_UNIT` | minute | Time unit for rate limiting window: `second`, `minute`, or `hour` |
+| `KC_GATEKEEPER_RATE_LIMIT_MAX_REQUESTS` | 600 | Max requests allowed per client during one window |
+| `KC_GATEKEEPER_RATE_LIMIT_WHITELIST` | (empty) | Comma-separated IP/CIDR list to bypass limits |
+| `KC_GATEKEEPER_RATE_LIMIT_SKIP_PATHS` | /api/v1/ready | Comma-separated API paths excluded from limits |
+| `KC_LOG_LEVEL` | info | Log level: `debug`, `info`, `warn`, `error` |
 
 ## IPFS disabled mode