diff --git a/data/.gitignore b/data/.gitignore index ad80caea5..fa49d4da7 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -3,6 +3,7 @@ ipfs/ mongodb/ redis/ +postgres/ tesseract/ tftc/ tbtc/ diff --git a/docker-compose.yml b/docker-compose.yml index 633eb6469..524e15f47 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,23 @@ services: ports: - 127.0.0.1:6379:6379 + postgres: + image: postgres:17-alpine + environment: + - POSTGRES_DB=${KC_POSTGRES_DB:-mdip} + - POSTGRES_USER=${KC_POSTGRES_USER:-mdip} + - POSTGRES_PASSWORD=${KC_POSTGRES_PASSWORD:-mdip} + volumes: + - ./data/postgres:/var/lib/postgresql/data + ports: + - 127.0.0.1:${KC_POSTGRES_PORT:-5432}:5432 + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U ${KC_POSTGRES_USER:-mdip} -d ${KC_POSTGRES_DB:-mdip}" ] + interval: 5s + timeout: 5s + retries: 10 + start_period: 10s + ipfs: profiles: - ipfs @@ -43,6 +60,7 @@ services: - KC_LOG_LEVEL=${KC_LOG_LEVEL} - KC_MONGODB_URL=mongodb://mongodb:27017 - KC_REDIS_URL=redis://redis:6379 + - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip} - KC_IPFS_ENABLE=${KC_IPFS_ENABLE} - KC_IPFS_URL=http://ipfs:5001/api/v0 - KC_IPFS_CLUSTER_URL=${KC_IPFS_CLUSTER_URL} @@ -55,6 +73,7 @@ services: depends_on: - mongodb - redis + - postgres keymaster: build: @@ -75,6 +94,7 @@ services: - KC_LOG_LEVEL=${KC_LOG_LEVEL} - KC_MONGODB_URL=mongodb://mongodb:27017 - KC_REDIS_URL=redis://redis:6379 + - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip} volumes: - ./data:/app/keymaster/data user: "${KC_UID}:${KC_GID}" @@ -84,6 +104,7 @@ services: - gatekeeper - redis - mongodb + - postgres - search-server hypr-mediator: @@ -99,6 +120,8 @@ services: - KC_NODE_ID=${KC_NODE_ID} - KC_NODE_NAME=${KC_NODE_NAME} - KC_MDIP_PROTOCOL=${KC_MDIP_PROTOCOL} + - KC_HYPR_DB=${KC_HYPR_DB:-sqlite} + - KC_HYPR_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip} - KC_HYPR_EXPORT_INTERVAL=${KC_HYPR_EXPORT_INTERVAL} - KC_HYPR_NEGENTROPY_ENABLE=${KC_HYPR_NEGENTROPY_ENABLE} - KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT=${KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT} @@ -130,6 +153,7 @@ services: - KC_KEYMASTER_URL=http://keymaster:4226 - KC_MONGODB_URL=mongodb://mongodb:27017 - KC_REDIS_URL=redis://redis:6379 + - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip} - KC_NODE_ID=${KC_NODE_ID} - KC_SAT_CHAIN=TFTC - KC_SAT_NETWORK=testnet @@ -173,6 +197,7 @@ services: - KC_KEYMASTER_URL=http://keymaster:4226 - KC_MONGODB_URL=mongodb://mongodb:27017 - KC_REDIS_URL=redis://redis:6379 + - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip} - KC_NODE_ID=${KC_NODE_ID} - KC_SAT_CHAIN=BTC - KC_SAT_NETWORK=bitcoin @@ -214,6 +239,7 @@ services: - KC_KEYMASTER_URL=http://keymaster:4226 - KC_MONGODB_URL=mongodb://mongodb:27017 - KC_REDIS_URL=redis://redis:6379 + - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip} - KC_NODE_ID=${KC_NODE_ID} - KC_SAT_CHAIN=TBTC - KC_SAT_NETWORK=testnet @@ -257,6 +283,7 @@ services: - KC_KEYMASTER_URL=http://keymaster:4226 - KC_MONGODB_URL=mongodb://mongodb:27017 - KC_REDIS_URL=redis://redis:6379 + - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip} - KC_NODE_ID=${KC_NODE_ID} - KC_SAT_CHAIN=Signet - KC_SAT_NETWORK=testnet @@ -293,6 +320,7 @@ services: - KC_KEYMASTER_URL=http://keymaster:4226 - KC_MONGODB_URL=mongodb://mongodb:27017 - KC_REDIS_URL=redis://redis:6379 + - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip} - KC_NODE_ID=${KC_NODE_ID} - KC_SAT_CHAIN=Signet - KC_SAT_NETWORK=testnet @@ -361,7 +389,8 @@ services: - SEARCH_SERVER_PORT=4002 - SEARCH_SERVER_GATEKEEPER_URL=http://gatekeeper:4224 - SEARCH_SERVER_REFRESH_INTERVAL_MS=5000 - - SEARCH_SERVER_DB=sqlite + - SEARCH_SERVER_DB=${KC_SEARCH_SERVER_DB:-sqlite} + - SEARCH_SERVER_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip} - KC_LOG_LEVEL=${KC_LOG_LEVEL} volumes: - ./data:/app/search-server/data diff --git a/package-lock.json b/package-lock.json index 66a1cbae9..cc12f4773 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8400,6 +8400,18 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/pg": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.18.0.tgz", + "integrity": "sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, "node_modules/@types/resolve": { "version": "1.20.2", "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.2.tgz", @@ -22129,6 +22141,95 @@ "node": ">=8" } }, + "node_modules/pg": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.19.0.tgz", + "integrity": "sha512-QIcLGi508BAHkQ3pJNptsFz5WQMlpGbuBGBaIaXsWK8mel2kQ/rThYI+DbgjUvZrIr7MiuEuc9LcChJoEZK1xQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.11.0", + "pg-pool": "^3.12.0", + "pg-protocol": "^1.12.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.3.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.11.0.tgz", + "integrity": "sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.12.0.tgz", + "integrity": "sha512-eIJ0DES8BLaziFHW7VgJEBPi5hg3Nyng5iKpYtj3wbcAUV9A1wLgWiY7ajf/f/oO1wfxt83phXPY8Emztg7ITg==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.12.0.tgz", + "integrity": "sha512-uOANXNRACNdElMXJ0tPz6RBM0XQ61nONGAwlt8da5zs/iUOOCLBQOHSXnrC6fMsvtjxbOJrZZl5IScGv+7mpbg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -22277,6 +22378,45 @@ "node": ">=4" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prebuild-install": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", @@ -26893,7 +27033,6 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.4" @@ -27227,11 +27366,13 @@ "dotenv": "^16.4.5", "ioredis": "^5.4.1", "mongodb": "^6.5.0", + "pg": "^8.16.3", "sqlite": "^5.1.1", "sqlite3": "^5.1.7" }, "devDependencies": { - "@rollup/plugin-json": "^6.1.0" + "@rollup/plugin-json": "^6.1.0", + "@types/pg": "^8.15.5" } }, "packages/inscription": { @@ -27282,11 +27423,13 @@ "image-size": "^2.0.1", "ioredis": "^5.4.1", "mongodb": "^6.5.0", + "pg": "^8.16.3", "sqlite": "^5.1.1", "sqlite3": "^5.1.7" }, "devDependencies": { - "@rollup/plugin-json": "^6.1.0" + "@rollup/plugin-json": "^6.1.0", + "@types/pg": "^8.15.5" } }, "packages/keymaster/node_modules/image-size": { diff --git a/packages/gatekeeper/README.md b/packages/gatekeeper/README.md index 0af9b9de3..737ceb6ba 100644 --- a/packages/gatekeeper/README.md +++ b/packages/gatekeeper/README.md @@ -17,8 +17,9 @@ The library must be configured by calling the start function with one of the sup - JSON - @mdip/gatekeeper/db/json - JSON with memory cache - @mdip/gatekeeper/db/json-cache - sqlite - @mdip/gatekeeper/db/sqlite -- mongodb - @mdip/gatekeeper/db/mongodb +- mongodb - @mdip/gatekeeper/db/mongo - redis - @mdip/gatekeeper/db/redis +- postgres - @mdip/gatekeeper/db/postgres ```js // Import using subpaths diff --git a/packages/gatekeeper/package.json b/packages/gatekeeper/package.json index fd5bcb5e3..0878d6e50 100644 --- a/packages/gatekeeper/package.json +++ b/packages/gatekeeper/package.json @@ -58,6 +58,11 @@ "require": "./dist/cjs/db/redis.cjs", "types": "./dist/types/db/redis.d.ts" }, + "./db/postgres": { + "import": "./dist/esm/db/postgres.js", + "require": "./dist/cjs/db/postgres.cjs", + "types": "./dist/types/db/postgres.d.ts" + }, "./db/mongo": { "import": "./dist/esm/db/mongo.js", "require": "./dist/cjs/db/mongo.cjs", @@ -87,6 +92,9 @@ "db/redis": [ "./dist/types/db/redis.d.ts" ], + "db/postgres": [ + "./dist/types/db/postgres.d.ts" + ], "db/mongo": [ "./dist/types/db/mongo.d.ts" ], @@ -113,6 +121,7 @@ "dotenv": "^16.4.5", "ioredis": "^5.4.1", "mongodb": "^6.5.0", + "pg": "^8.16.3", "sqlite": "^5.1.1", "sqlite3": "^5.1.7" }, @@ -121,6 +130,7 @@ "url": "git+https://github.com/KeychainMDIP/kc.git" }, "devDependencies": { + "@types/pg": "^8.15.5", "@rollup/plugin-json": "^6.1.0" } } diff --git a/packages/gatekeeper/rollup.cjs.config.js b/packages/gatekeeper/rollup.cjs.config.js index 22e4bb14e..18976ba56 100644 --- a/packages/gatekeeper/rollup.cjs.config.js +++ b/packages/gatekeeper/rollup.cjs.config.js @@ -20,6 +20,7 @@ const config = { 'db/json-memory': 'dist/esm/db/json-memory.js', 'db/sqlite': 'dist/esm/db/sqlite.js', 'db/redis': 'dist/esm/db/redis.js', + 'db/postgres': 'dist/esm/db/postgres.js', 'db/mongo': 'dist/esm/db/mongo.js' }, output: { diff --git a/packages/gatekeeper/src/db/postgres.ts b/packages/gatekeeper/src/db/postgres.ts new file mode 100644 index 000000000..2db2f8757 --- /dev/null +++ b/packages/gatekeeper/src/db/postgres.ts @@ -0,0 +1,386 @@ +import { Pool, PoolClient } from 'pg'; +import { InvalidDIDError } from '@mdip/common/errors'; +import { childLogger } from '@mdip/common/logger'; +import { GatekeeperDb, GatekeeperEvent, Operation, BlockId, BlockInfo } from '../types.js'; + +interface EventsRow { + events: GatekeeperEvent[] | string | null; +} + +interface QueueRow { + ops: Operation[] | string | null; +} + +interface BlockRow { + hash: string; + height: number | string; + time: number | string; +} + +interface LengthRow { + length: number | string; +} + +const POSTGRES_NOT_STARTED_ERROR = 'Postgres DB not started. Call start() first.'; +const log = childLogger({ service: 'gatekeeper-db', module: 'postgres' }); + +export default class DbPostgres implements GatekeeperDb { + private readonly dbName: string; + private readonly url: string; + private pool: Pool | null; + + constructor(dbName: string) { + this.dbName = dbName; + this.url = process.env.KC_POSTGRES_URL || 'postgresql://mdip:mdip@localhost:5432/mdip'; + this.pool = null; + } + + private getPool(): Pool { + if (!this.pool) { + throw new Error(POSTGRES_NOT_STARTED_ERROR); + } + return this.pool; + } + + private async withTx(fn: (client: PoolClient) => Promise): Promise { + const pool = this.getPool(); + const client = await pool.connect(); + + try { + await client.query('BEGIN'); + const result = await fn(client); + await client.query('COMMIT'); + return result; + } catch (error) { + try { + await client.query('ROLLBACK'); + } catch { + // Ignore rollback errors and rethrow original error. + } + throw error; + } finally { + client.release(); + } + } + + private splitSuffix(did: string): string { + if (!did) { + throw new InvalidDIDError(); + } + const suffix = did.split(':').pop(); + if (!suffix) { + throw new InvalidDIDError(); + } + return suffix; + } + + private parseArray(value: unknown): T[] { + if (value == null) { + return []; + } + + if (typeof value === 'string') { + const parsed = JSON.parse(value); + if (!Array.isArray(parsed)) { + throw new Error('Expected JSON array'); + } + return parsed as T[]; + } + + if (!Array.isArray(value)) { + throw new Error('Expected array value'); + } + + return value as T[]; + } + + private toNumber(value: number | string): number { + if (typeof value === 'number') { + return value; + } + return parseInt(value, 10); + } + + async start(): Promise { + if (this.pool) { + return; + } + + this.pool = new Pool({ connectionString: this.url }); + + const pool = this.getPool(); + await pool.query(` + CREATE TABLE IF NOT EXISTS gatekeeper_dids ( + namespace TEXT NOT NULL, + id TEXT NOT NULL, + events JSONB NOT NULL DEFAULT '[]'::jsonb, + PRIMARY KEY (namespace, id) + ); + + CREATE TABLE IF NOT EXISTS gatekeeper_queue ( + namespace TEXT NOT NULL, + id TEXT NOT NULL, + ops JSONB NOT NULL DEFAULT '[]'::jsonb, + PRIMARY KEY (namespace, id) + ); + + CREATE TABLE IF NOT EXISTS gatekeeper_blocks ( + namespace TEXT NOT NULL, + registry TEXT NOT NULL, + hash TEXT NOT NULL, + height INTEGER NOT NULL, + time INTEGER NOT NULL, + txns INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (namespace, registry, hash), + UNIQUE (namespace, registry, height) + ); + + CREATE INDEX IF NOT EXISTS idx_gatekeeper_blocks_ns_registry_height + ON gatekeeper_blocks (namespace, registry, height DESC); + `); + } + + async stop(): Promise { + if (this.pool) { + await this.pool.end(); + this.pool = null; + } + } + + async resetDb(): Promise { + await this.withTx(async client => { + await client.query('DELETE FROM gatekeeper_dids WHERE namespace = $1', [this.dbName]); + await client.query('DELETE FROM gatekeeper_queue WHERE namespace = $1', [this.dbName]); + await client.query('DELETE FROM gatekeeper_blocks WHERE namespace = $1', [this.dbName]); + }); + } + + async addEvent(did: string, event: GatekeeperEvent): Promise { + const id = this.splitSuffix(did); + const pool = this.getPool(); + + const result = await pool.query( + `INSERT INTO gatekeeper_dids (namespace, id, events) + VALUES ($1, $2, $3::jsonb) + ON CONFLICT (namespace, id) + DO UPDATE SET events = COALESCE(gatekeeper_dids.events, '[]'::jsonb) || EXCLUDED.events`, + [this.dbName, id, JSON.stringify([event])] + ); + + return result.rowCount ?? 0; + } + + async setEvents(did: string, events: GatekeeperEvent[]): Promise { + const id = this.splitSuffix(did); + const pool = this.getPool(); + + const result = await pool.query( + `INSERT INTO gatekeeper_dids (namespace, id, events) + VALUES ($1, $2, $3::jsonb) + ON CONFLICT (namespace, id) + DO UPDATE SET events = EXCLUDED.events`, + [this.dbName, id, JSON.stringify(events)] + ); + + return result.rowCount ?? 0; + } + + async getEvents(did: string): Promise { + const pool = this.getPool(); + + try { + const id = this.splitSuffix(did); + const result = await pool.query( + `SELECT events + FROM gatekeeper_dids + WHERE namespace = $1 AND id = $2 + LIMIT 1`, + [this.dbName, id] + ); + + if (result.rowCount === 0) { + return []; + } + + return this.parseArray(result.rows[0].events); + } catch { + return []; + } + } + + async deleteEvents(did: string): Promise { + const id = this.splitSuffix(did); + const pool = this.getPool(); + + const result = await pool.query( + `DELETE FROM gatekeeper_dids + WHERE namespace = $1 AND id = $2`, + [this.dbName, id] + ); + + return result.rowCount ?? 0; + } + + async getAllKeys(): Promise { + const pool = this.getPool(); + const result = await pool.query<{ id: string }>( + `SELECT id + FROM gatekeeper_dids + WHERE namespace = $1`, + [this.dbName] + ); + + return result.rows.map(row => row.id); + } + + async queueOperation(registry: string, op: Operation): Promise { + const pool = this.getPool(); + + const result = await pool.query( + `INSERT INTO gatekeeper_queue (namespace, id, ops) + VALUES ($1, $2, $3::jsonb) + ON CONFLICT (namespace, id) + DO UPDATE SET ops = COALESCE(gatekeeper_queue.ops, '[]'::jsonb) || EXCLUDED.ops + RETURNING jsonb_array_length(ops) AS length`, + [this.dbName, registry, JSON.stringify([op])] + ); + + if (result.rowCount === 0) { + return 0; + } + + return this.toNumber(result.rows[0].length); + } + + async getQueue(registry: string): Promise { + const pool = this.getPool(); + + try { + const result = await pool.query( + `SELECT ops + FROM gatekeeper_queue + WHERE namespace = $1 AND id = $2 + LIMIT 1`, + [this.dbName, registry] + ); + + if (result.rowCount === 0) { + return []; + } + + return this.parseArray(result.rows[0].ops); + } catch { + return []; + } + } + + async clearQueue(registry: string, batch: Operation[]): Promise { + const hashes = new Set( + batch.map(op => op.signature?.hash).filter((hash): hash is string => !!hash) + ); + + if (hashes.size === 0) { + return true; + } + + try { + await this.withTx(async client => { + const current = await client.query( + `SELECT ops + FROM gatekeeper_queue + WHERE namespace = $1 AND id = $2 + FOR UPDATE`, + [this.dbName, registry] + ); + + const oldQueue = current.rowCount === 0 + ? [] + : this.parseArray(current.rows[0].ops); + + const newQueue = oldQueue.filter(op => !hashes.has(op.signature?.hash || '')); + + await client.query( + `INSERT INTO gatekeeper_queue (namespace, id, ops) + VALUES ($1, $2, $3::jsonb) + ON CONFLICT (namespace, id) + DO UPDATE SET ops = EXCLUDED.ops`, + [this.dbName, registry, JSON.stringify(newQueue)] + ); + }); + + return true; + } catch (error) { + log.error({ error }, 'Postgres clearQueue error'); + return false; + } + } + + async addBlock(registry: string, blockInfo: BlockInfo): Promise { + const pool = this.getPool(); + + try { + await pool.query( + `INSERT INTO gatekeeper_blocks (namespace, registry, hash, height, time, txns) + VALUES ($1, $2, $3, $4, $5, 0) + ON CONFLICT (namespace, registry, hash) + DO UPDATE SET + height = EXCLUDED.height, + time = EXCLUDED.time, + txns = EXCLUDED.txns`, + [this.dbName, registry, blockInfo.hash, blockInfo.height, blockInfo.time] + ); + + return true; + } catch { + return false; + } + } + + async getBlock(registry: string, blockId?: BlockId): Promise { + const pool = this.getPool(); + + try { + let result; + + if (blockId === undefined) { + result = await pool.query( + `SELECT hash, height, time + FROM gatekeeper_blocks + WHERE namespace = $1 AND registry = $2 + ORDER BY height DESC + LIMIT 1`, + [this.dbName, registry] + ); + } else if (typeof blockId === 'number') { + result = await pool.query( + `SELECT hash, height, time + FROM gatekeeper_blocks + WHERE namespace = $1 AND registry = $2 AND height = $3 + LIMIT 1`, + [this.dbName, registry, blockId] + ); + } else { + result = await pool.query( + `SELECT hash, height, time + FROM gatekeeper_blocks + WHERE namespace = $1 AND registry = $2 AND hash = $3 + LIMIT 1`, + [this.dbName, registry, blockId] + ); + } + + if (result.rowCount === 0) { + return null; + } + + const row = result.rows[0]; + return { + hash: row.hash, + height: this.toNumber(row.height), + time: this.toNumber(row.time), + }; + } catch { + return null; + } + } +} diff --git a/packages/gatekeeper/src/node.ts b/packages/gatekeeper/src/node.ts index e120a3e54..583830cca 100644 --- a/packages/gatekeeper/src/node.ts +++ b/packages/gatekeeper/src/node.ts @@ -3,5 +3,6 @@ export { default } from './gatekeeper.js'; export { default as DbJson } from './db/json.js'; export { default as DbJsonCache } from './db/json-cache.js'; export { default as DbMongo } from './db/mongo.js'; +export { default as DbPostgres } from './db/postgres.js'; export { default as DbRedis } from './db/redis.js'; export { default as DbSqlite } from './db/sqlite.js'; diff --git a/packages/keymaster/package.json b/packages/keymaster/package.json index 04947512f..24d89fd19 100644 --- a/packages/keymaster/package.json +++ b/packages/keymaster/package.json @@ -68,6 +68,11 @@ "require": "./dist/cjs/db/sqlite.cjs", "types": "./dist/types/db/sqlite.d.ts" }, + "./wallet/postgres": { + "import": "./dist/esm/db/postgres.js", + "require": "./dist/cjs/db/postgres.cjs", + "types": "./dist/types/db/postgres.d.ts" + }, "./wallet/cache": { "import": "./dist/esm/db/cache.js", "require": "./dist/cjs/db/cache.cjs", @@ -118,6 +123,9 @@ "wallet/sqlite": [ "./dist/types/db/sqlite.d.ts" ], + "wallet/postgres": [ + "./dist/types/db/postgres.d.ts" + ], "wallet/cache": [ "./dist/types/db/cache.d.ts" ], @@ -151,6 +159,7 @@ "image-size": "^2.0.1", "ioredis": "^5.4.1", "mongodb": "^6.5.0", + "pg": "^8.16.3", "sqlite": "^5.1.1", "sqlite3": "^5.1.7" }, @@ -159,6 +168,7 @@ "url": "git+https://github.com/KeychainMDIP/kc.git" }, "devDependencies": { + "@types/pg": "^8.15.5", "@rollup/plugin-json": "^6.1.0" } } diff --git a/packages/keymaster/rollup.cjs.config.js b/packages/keymaster/rollup.cjs.config.js index e28916b94..1be7d312d 100644 --- a/packages/keymaster/rollup.cjs.config.js +++ b/packages/keymaster/rollup.cjs.config.js @@ -22,6 +22,7 @@ const config = { 'db/redis': 'dist/esm/db/redis.js', 'db/mongo': 'dist/esm/db/mongo.js', 'db/sqlite': 'dist/esm/db/sqlite.js', + 'db/postgres': 'dist/esm/db/postgres.js', 'db/cache': 'dist/esm/db/cache.js', 'db/web': 'dist/esm/db/web.js', 'db/chrome': 'dist/esm/db/chrome.js', diff --git a/packages/keymaster/src/db/postgres.ts b/packages/keymaster/src/db/postgres.ts new file mode 100644 index 000000000..db7e981d5 --- /dev/null +++ b/packages/keymaster/src/db/postgres.ts @@ -0,0 +1,119 @@ +import { Pool } from 'pg'; +import { StoredWallet, WalletBase } from '../types.js'; + +interface WalletRow { + data: StoredWallet | string; +} + +export default class WalletPostgres implements WalletBase { + private readonly url: string; + private readonly walletKey: string; + private pool: Pool | null; + + public static async create(walletKey: string = 'wallet'): Promise { + const wallet = new WalletPostgres(walletKey); + await wallet.connect(); + return wallet; + } + + constructor(walletKey: string = 'wallet') { + this.url = process.env.KC_POSTGRES_URL || 'postgresql://mdip:mdip@localhost:5432/mdip'; + this.walletKey = walletKey; + this.pool = null; + } + + async connect(): Promise { + if (this.pool) { + return; + } + + this.pool = new Pool({ connectionString: this.url }); + await this.pool.query(` + CREATE TABLE IF NOT EXISTS wallet ( + id TEXT PRIMARY KEY, + data JSONB NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + `); + } + + async disconnect(): Promise { + if (this.pool) { + await this.pool.end(); + this.pool = null; + } + } + + private getPool(): Pool { + if (!this.pool) { + throw new Error('Postgres is not connected. Call connect() first or use WalletPostgres.create().'); + } + + return this.pool; + } + + async saveWallet(wallet: StoredWallet, overwrite: boolean = false): Promise { + await this.connect(); + const pool = this.getPool(); + const client = await pool.connect(); + + try { + await client.query('BEGIN'); + + const existing = await client.query( + 'SELECT id FROM wallet WHERE id = $1 FOR UPDATE', + [this.walletKey] + ); + + if (existing.rowCount && !overwrite) { + await client.query('COMMIT'); + return false; + } + + await client.query( + `INSERT INTO wallet (id, data, updated_at) + VALUES ($1, $2::jsonb, NOW()) + ON CONFLICT (id) + DO UPDATE SET data = EXCLUDED.data, updated_at = NOW()`, + [this.walletKey, JSON.stringify(wallet)] + ); + + await client.query('COMMIT'); + return true; + } + catch (error) { + try { + await client.query('ROLLBACK'); + } + catch { + // Ignore rollback errors and rethrow original error. + } + throw error; + } + finally { + client.release(); + } + } + + async loadWallet(): Promise { + await this.connect(); + const pool = this.getPool(); + + const result = await pool.query( + 'SELECT data FROM wallet WHERE id = $1 LIMIT 1', + [this.walletKey] + ); + + if (result.rowCount === 0) { + return null; + } + + const data = result.rows[0].data; + + if (typeof data === 'string') { + return JSON.parse(data); + } + + return data; + } +} diff --git a/packages/keymaster/src/node.ts b/packages/keymaster/src/node.ts index fad8a596d..d88d14789 100644 --- a/packages/keymaster/src/node.ts +++ b/packages/keymaster/src/node.ts @@ -4,3 +4,4 @@ export { default as WalletJson } from './db/json.js'; export { default as WalletRedis } from './db/redis.js'; export { default as WalletMongo } from './db/mongo.js'; export { default as WalletSQLite } from './db/sqlite.js'; +export { default as WalletPostgres } from './db/postgres.js'; diff --git a/sample.env b/sample.env index c68ce785f..5d2686623 100644 --- a/sample.env +++ b/sample.env @@ -2,8 +2,7 @@ KC_UID=1000 KC_GID=1000 KC_DEBUG=false -KC_LOG_LEVEL=info -# Levels: trace, debug, info, warn, error, fatal, silent +KC_LOG_LEVEL=info # Levels: trace, debug, info, warn, error, fatal, silent KC_NODE_ID=mynodeID KC_NODE_NAME=mynodeName @@ -26,6 +25,16 @@ KC_WALLET_CACHE=false KC_DEFAULT_REGISTRY=hyperswarm KC_KEYMASTER_SERVE_CLIENT=true +# Search server +KC_SEARCH_SERVER_DB=sqlite # sqlite | postgres + +# Postgres +KC_POSTGRES_PORT=5432 +KC_POSTGRES_DB=mdip +KC_POSTGRES_USER=mdip +KC_POSTGRES_PASSWORD=mdip +KC_POSTGRES_URL=postgresql://mdip:mdip@localhost:5432/mdip + # React-Wallet KC_REACT_WALLET_PORT=4228 @@ -35,6 +44,8 @@ KC_KEYMASTER_URL=http://localhost:4226 KC_SEARCH_URL=http://localhost:4002 # Hyperswarm +KC_HYPR_DB=sqlite # Sync store backend for hyperswarm mediator: sqlite | postgres +KC_HYPR_POSTGRES_URL=postgresql://mdip:mdip@localhost:5432/mdip KC_HYPR_EXPORT_INTERVAL=2 # Seconds between export-loop ticks. integer >= 1. KC_MDIP_PROTOCOL=/MDIP/v1.0-public KC_HYPR_LEGACY_SYNC_ENABLE=true # Enables legacy sync for peers without negentropy. true|false. @@ -61,7 +72,7 @@ KC_BTC_FEE_FALLBACK_SAT_BYTE=10 KC_BTC_FEE_MAX=0.00010000 KC_BTC_RBF_ENABLED=true KC_BTC_REIMPORT=true -KC_BTC_DB=json +KC_BTC_DB=json # json | sqlite | mongodb | redis | postgres # Bitcoin testnet4 mediator KC_TBTC_HOST=localhost @@ -79,7 +90,7 @@ KC_TBTC_FEE_FALLBACK_SAT_BYTE=10 KC_TBTC_FEE_MAX=0.00200000 KC_TBTC_RBF_ENABLED=false KC_TBTC_REIMPORT=true -KC_TBTC_DB=json +KC_TBTC_DB=json # json | sqlite | mongodb | redis | postgres # Feathercoin testnet mediator KC_TFTC_HOST=localhost @@ -95,7 +106,7 @@ KC_TFTC_FEE_FALLBACK_SAT_BYTE=10 KC_TFTC_FEE_MAX=0.00200000 KC_TFTC_RBF_ENABLED=false KC_TFTC_REIMPORT=true -KC_TFTC_DB=json +KC_TFTC_DB=json # json | sqlite | mongodb | redis | postgres # BTC signet mediator KC_SIGNET_HOST=localhost @@ -111,7 +122,7 @@ KC_SIGNET_FEE_FALLBACK_SAT_BYTE=10 KC_SIGNET_FEE_MAX=0.00003000 KC_SIGNET_RBF_ENABLED=false KC_SIGNET_REIMPORT=true -KC_SIGNET_DB=json +KC_SIGNET_DB=json # json | sqlite | mongodb | redis | postgres # BTC signet inscribed mediator KC_SIGNET_INS_HOST=localhost @@ -127,7 +138,7 @@ KC_SIGNET_INS_FEE_FALLBACK_SAT_BYTE=10 KC_SIGNET_INS_FEE_MAX=0.00200000 KC_SIGNET_INS_RBF_ENABLED=false KC_SIGNET_INS_REIMPORT=true -KC_SIGNET_INS_DB=json +KC_SIGNET_INS_DB=json # json | sqlite | mongodb | redis | postgres # IPFS mediator KC_IPFS_ENABLE=true diff --git a/services/gatekeeper/server/README.md b/services/gatekeeper/server/README.md index f6e48a128..2df0386fe 100644 --- a/services/gatekeeper/server/README.md +++ b/services/gatekeeper/server/README.md @@ -11,7 +11,7 @@ Operations come from Keymaster clients such as end-user wallets and network medi | variable | default | description | | ------------------------------- | ---------| ---------------------------------------------------------------------- | | `KC_GATEKEEPER_PORT` | 4224 | Service port | -| `KC_GATEKEEPER_DB` | redis | DID database adapter, must be `redis`, `json`, `mongodb`, or `sqlite` | +| `KC_GATEKEEPER_DB` | redis | DID database adapter, must be `redis`, `json`, `mongodb`, `sqlite`, or `postgres` | | `KC_GATEKEEPER_DID_PREFIX` | did:test | Default prefix assigned to DIDs created | | `KC_IPFS_ENABLE` | true | Enable IPFS storage for opids and CAS endpoints | | `KC_GATEKEEPER_GC_INTERVAL` | 15 | The number of minutes between garbage collection cycles (0 to disable) | diff --git a/services/gatekeeper/server/src/gatekeeper-api.ts b/services/gatekeeper/server/src/gatekeeper-api.ts index 77d6ced21..fd5d3299c 100644 --- a/services/gatekeeper/server/src/gatekeeper-api.ts +++ b/services/gatekeeper/server/src/gatekeeper-api.ts @@ -9,6 +9,7 @@ import DbJsonCache from '@mdip/gatekeeper/db/json-cache'; import DbRedis from '@mdip/gatekeeper/db/redis'; import DbSqlite from '@mdip/gatekeeper/db/sqlite'; import DbMongo from '@mdip/gatekeeper/db/mongo'; +import DbPostgres from '@mdip/gatekeeper/db/postgres'; import { CheckDIDsResult, ResolveDIDOptions, Operation } from '@mdip/gatekeeper/types'; import KuboClient from '@mdip/ipfs/kubo'; import { childLogger } from '@mdip/common/logger'; @@ -50,6 +51,7 @@ const db = (() => { case 'sqlite': return new DbSqlite(dbName); case 'mongodb': return new DbMongo(dbName); case 'redis': return new DbRedis(dbName); + case 'postgres': return new DbPostgres(dbName); case 'json': case 'json-cache': return new DbJsonCache(dbName); default: return null; diff --git a/services/keymaster/server/README.md b/services/keymaster/server/README.md index c204aefe5..9995c15c7 100644 --- a/services/keymaster/server/README.md +++ b/services/keymaster/server/README.md @@ -10,7 +10,7 @@ This service is also useful when clients share a wallet, such as the `kc` CLI an | --------------------- | ---------------------- | ----------------------------- | | `KC_GATEKEEPER_URL` | http://localhost:4224 | MDIP gatekeeper service URL | | `KC_KEYMASTER_PORT` | 4226 | Service port | -| `KC_KEYMASTER_DB` | json | Wallet database adapter, must be `redis`, `json`, `mongodb`, or `sqlite` | +| `KC_KEYMASTER_DB` | json | Wallet database adapter, must be `redis`, `json`, `mongodb`, `sqlite`, or `postgres` | | `KC_ENCRYPTED_PASSPHRASE` | (no default) | If specified, the wallet will be encrypted and decrypted with this passphrase | | `KC_WALLET_CACHE` | false | Use wallet cache to increase performance (but understand security implications) | | `KC_DEFAULT_REGISTRY` | hyperswarm | Default registry to use when creating DIDs | diff --git a/services/keymaster/server/src/keymaster-api.ts b/services/keymaster/server/src/keymaster-api.ts index 25999478e..fb91728a0 100644 --- a/services/keymaster/server/src/keymaster-api.ts +++ b/services/keymaster/server/src/keymaster-api.ts @@ -10,6 +10,7 @@ import WalletJson from '@mdip/keymaster/wallet/json'; import WalletRedis from '@mdip/keymaster/wallet/redis'; import WalletMongo from '@mdip/keymaster/wallet/mongo'; import WalletSQLite from '@mdip/keymaster/wallet/sqlite'; +import WalletPostgres from '@mdip/keymaster/wallet/postgres'; import WalletCache from '@mdip/keymaster/wallet/cache'; import CipherNode from '@mdip/cipher/node'; import { InvalidParameterError } from '@mdip/common/errors'; @@ -6143,6 +6144,8 @@ async function initWallet() { wallet = await WalletMongo.create(); } else if (config.db === 'sqlite') { wallet = await WalletSQLite.create(); + } else if (config.db === 'postgres') { + wallet = await WalletPostgres.create(); } else if (config.db === 'json') { wallet = new WalletJson(); } else { diff --git a/services/mediators/hyperswarm/README.md b/services/mediators/hyperswarm/README.md index 0e187c106..3bf995794 100644 --- a/services/mediators/hyperswarm/README.md +++ b/services/mediators/hyperswarm/README.md @@ -45,6 +45,8 @@ The mediator emits periodic structured sync metrics in `connectionLoop` includin | `KC_NODE_ID ` | (no default) | Keymaster node agent name | | `KC_NODE_NAME` | anon | Human-readable name for the node | | `KC_MDIP_PROTOCOL` | /MDIP/v1.0-public | MDIP network topic to join | +| `KC_HYPR_DB` | sqlite | Sync-store backend (`sqlite` or `postgres`) | +| `KC_HYPR_POSTGRES_URL` | postgresql://mdip:mdip@localhost:5432/mdip | Postgres DSN used when `KC_HYPR_DB=postgres` | | `KC_HYPR_EXPORT_INTERVAL` | 2 | Seconds between export cycles | | `KC_HYPR_NEGENTROPY_FRAME_SIZE_LIMIT` | 0 | Negentropy frame-size limit in KB (0 or >= 4) | | `KC_HYPR_NEGENTROPY_WINDOW_DAYS` | 30 | Reconciliation window size in days for full-sync chunking | @@ -67,6 +69,9 @@ Set `KC_IPFS_ENABLE=false` to run the mediator without IPFS or Keymaster integra The mediator now includes a sync-store abstraction in `src/db/` with: - `SqliteOperationSyncStore` for persistent ordered storage +- `PostgresOperationSyncStore` for persistent ordered storage - `InMemoryOperationSyncStore` for tests -The SQLite implementation uses a fixed data path under `data/hyperswarm` (relative to the mediator working directory), with an index on `(ts, id)` to use SQLite's native B-tree ordering for range queries. +SQLite uses a fixed data path under `data/hyperswarm` (relative to the mediator working directory), with an index on `(ts, id)` to use SQLite's native B-tree ordering for range queries. + +Postgres uses `hyperswarm_sync_operations` with a B-tree index on `(ts, id)` to preserve the same deterministic keyset ordering required by negentropy window rebuilds and cursor pagination. diff --git a/services/mediators/hyperswarm/package-lock.json b/services/mediators/hyperswarm/package-lock.json index 3120fe2ce..5c7c829b5 100644 --- a/services/mediators/hyperswarm/package-lock.json +++ b/services/mediators/hyperswarm/package-lock.json @@ -16,6 +16,7 @@ "express": "^4.21.0", "graceful-goodbye": "^1.3.0", "hyperswarm": "^4.7.14", + "pg": "^8.19.0", "sqlite": "^5.1.1", "sqlite3": "^5.1.7" }, @@ -1805,6 +1806,134 @@ "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "license": "MIT" }, + "node_modules/pg": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.19.0.tgz", + "integrity": "sha512-QIcLGi508BAHkQ3pJNptsFz5WQMlpGbuBGBaIaXsWK8mel2kQ/rThYI+DbgjUvZrIr7MiuEuc9LcChJoEZK1xQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.11.0", + "pg-pool": "^3.12.0", + "pg-protocol": "^1.12.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.3.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.11.0.tgz", + "integrity": "sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.12.0.tgz", + "integrity": "sha512-eIJ0DES8BLaziFHW7VgJEBPi5hg3Nyng5iKpYtj3wbcAUV9A1wLgWiY7ajf/f/oO1wfxt83phXPY8Emztg7ITg==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.12.0.tgz", + "integrity": "sha512-uOANXNRACNdElMXJ0tPz6RBM0XQ61nONGAwlt8da5zs/iUOOCLBQOHSXnrC6fMsvtjxbOJrZZl5IScGv+7mpbg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prebuild-install": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", @@ -2293,6 +2422,15 @@ } } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sqlite": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-5.1.1.tgz", @@ -2633,6 +2771,15 @@ "resolved": "https://registry.npmjs.org/xache/-/xache-1.2.1.tgz", "integrity": "sha512-igRS6jPreJ54ABdzhh4mCDXcz+XMaWO2q1ABRV2yWYuk29jlp8VT7UBdCqNkX7rpYBbXsebVVKkwIuYZjyZNqA==" }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", diff --git a/services/mediators/hyperswarm/package.json b/services/mediators/hyperswarm/package.json index 607197098..f11ed3422 100644 --- a/services/mediators/hyperswarm/package.json +++ b/services/mediators/hyperswarm/package.json @@ -19,6 +19,7 @@ "express": "^4.21.0", "graceful-goodbye": "^1.3.0", "hyperswarm": "^4.7.14", + "pg": "^8.19.0", "sqlite": "^5.1.1", "sqlite3": "^5.1.7" }, diff --git a/services/mediators/hyperswarm/src/config.js b/services/mediators/hyperswarm/src/config.js index f7659a22e..a3f76ba08 100644 --- a/services/mediators/hyperswarm/src/config.js +++ b/services/mediators/hyperswarm/src/config.js @@ -46,6 +46,20 @@ function parseBooleanEnv(varName, defaultValue) { throw new Error(`Invalid ${varName}; expected true or false`); } +function parseSyncDbEnv() { + const raw = process.env.KC_HYPR_DB; + if (raw == null || raw === '') { + return 'sqlite'; + } + + const normalized = raw.trim().toLowerCase(); + if (normalized === 'sqlite' || normalized === 'postgres') { + return normalized; + } + + throw new Error('Invalid KC_HYPR_DB; expected sqlite or postgres'); +} + const config = { debug: process.env.KC_DEBUG ? process.env.KC_DEBUG === 'true' : false, gatekeeperURL: process.env.KC_GATEKEEPER_URL || 'http://localhost:4224', @@ -63,6 +77,10 @@ const config = { negentropyMaxRoundsPerSession: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_MAX_ROUNDS_PER_SESSION', 64), negentropyIntervalSeconds: parsePositiveIntEnv('KC_HYPR_NEGENTROPY_INTERVAL', 300), legacySyncEnabled: parseBooleanEnv('KC_HYPR_LEGACY_SYNC_ENABLE', true), + db: parseSyncDbEnv(), + postgresURL: process.env.KC_HYPR_POSTGRES_URL + || process.env.KC_POSTGRES_URL + || 'postgresql://mdip:mdip@localhost:5432/mdip', }; if (!config.negentropyEnabled && !config.legacySyncEnabled) { diff --git a/services/mediators/hyperswarm/src/db/postgres.ts b/services/mediators/hyperswarm/src/db/postgres.ts new file mode 100644 index 000000000..23891fee0 --- /dev/null +++ b/services/mediators/hyperswarm/src/db/postgres.ts @@ -0,0 +1,244 @@ +import { Pool, PoolClient } from 'pg'; +import { Operation } from '@mdip/gatekeeper/types'; +import { childLogger } from '@mdip/common/logger'; +import { OperationSyncStore, SyncOperationRecord, SyncStoreListOptions } from './types.js'; + +interface SyncRow { + id: string; + ts: number | string; + operation_json: Operation | string; + inserted_at: number | string; +} + +interface CountRow { + count: number | string; +} + +const POSTGRES_NOT_STARTED_ERROR = 'Sync Postgres DB not open. Call start() first.'; +const log = childLogger({ service: 'hyperswarm-sync-db', module: 'postgres' }); + +export default class PostgresOperationSyncStore implements OperationSyncStore { + private readonly url: string; + private pool: Pool | null; + private _lock: Promise = Promise.resolve(); + + constructor( + url: string = process.env.KC_HYPR_POSTGRES_URL + || process.env.KC_POSTGRES_URL + || 'postgresql://mdip:mdip@localhost:5432/mdip' + ) { + this.url = url; + this.pool = null; + } + + private getPool(): Pool { + if (!this.pool) { + throw new Error(POSTGRES_NOT_STARTED_ERROR); + } + + return this.pool; + } + + private runExclusive(fn: () => Promise | T): Promise { + const run = async () => await fn(); + const chained = this._lock.then(run, run); + this._lock = chained.then(() => undefined, () => undefined); + return chained; + } + + private async withTx(fn: (client: PoolClient) => Promise): Promise { + const client = await this.getPool().connect(); + + try { + await client.query('BEGIN'); + const result = await fn(client); + await client.query('COMMIT'); + return result; + } catch (error) { + try { + await client.query('ROLLBACK'); + } catch {} + throw error; + } finally { + client.release(); + } + } + + async start(): Promise { + if (this.pool) { + return; + } + + this.pool = new Pool({ connectionString: this.url }); + + const pool = this.getPool(); + await pool.query(` + CREATE TABLE IF NOT EXISTS hyperswarm_sync_operations ( + id TEXT COLLATE "C" PRIMARY KEY, + ts BIGINT NOT NULL, + operation_json JSONB NOT NULL, + inserted_at BIGINT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_hypr_sync_operations_ts_id + ON hyperswarm_sync_operations (ts ASC, id ASC); + `); + } + + async stop(): Promise { + if (this.pool) { + await this.pool.end(); + this.pool = null; + } + } + + async reset(): Promise { + this.getPool(); + + await this.runExclusive(async () => { + await this.withTx(async (client) => { + await client.query('DELETE FROM hyperswarm_sync_operations'); + }); + }); + } + + async upsertMany(records: Array | SyncOperationRecord>): Promise { + this.getPool(); + + if (!Array.isArray(records) || records.length === 0) { + return 0; + } + + return this.runExclusive(async () => { + try { + return await this.withTx(async (client) => { + let inserted = 0; + const now = Date.now(); + + for (const record of records) { + const insertedAt = 'insertedAt' in record ? record.insertedAt : now; + const response = await client.query( + `INSERT INTO hyperswarm_sync_operations(id, ts, operation_json, inserted_at) + VALUES($1, $2, $3::jsonb, $4) + ON CONFLICT(id) DO NOTHING`, + [record.id, record.ts, JSON.stringify(record.operation), insertedAt] + ); + + inserted += response.rowCount ?? 0; + } + + return inserted; + }); + } catch (error) { + log.error({ error }, 'upsertMany failed'); + throw error; + } + }); + } + + async getByIds(ids: string[]): Promise { + if (!Array.isArray(ids) || ids.length === 0) { + return []; + } + + const result = await this.getPool().query( + `SELECT id, ts, operation_json, inserted_at + FROM hyperswarm_sync_operations + WHERE id = ANY($1::text[])`, + [ids] + ); + + const byId = new Map(result.rows.map(row => [row.id, this.mapRow(row)])); + return ids + .map(id => byId.get(id)) + .filter((item): item is SyncOperationRecord => !!item); + } + + async iterateSorted(options: SyncStoreListOptions = {}): Promise { + const limit = options.limit ?? 1000; + const after = options.after; + const fromTs = options.fromTs; + const toTs = options.toTs; + + const params: Array = []; + const predicates: string[] = []; + + if (after) { + const afterTsParam = params.push(after.ts); + const afterIdParam = params.push(after.id); + predicates.push(`(ts > $${afterTsParam} OR (ts = $${afterTsParam} AND id > $${afterIdParam}))`); + } + + if (typeof fromTs === 'number') { + const fromTsParam = params.push(fromTs); + predicates.push(`ts >= $${fromTsParam}`); + } + + if (typeof toTs === 'number') { + const toTsParam = params.push(toTs); + predicates.push(`ts <= $${toTsParam}`); + } + + const where = predicates.length > 0 + ? `WHERE ${predicates.join(' AND ')}` + : ''; + + const limitParam = params.push(limit); + + const result = await this.getPool().query( + `SELECT id, ts, operation_json, inserted_at + FROM hyperswarm_sync_operations + ${where} + ORDER BY ts ASC, id ASC + LIMIT $${limitParam}`, + params + ); + + return result.rows.map(row => this.mapRow(row)); + } + + async has(id: string): Promise { + const result = await this.getPool().query<{ id: string }>( + `SELECT id + FROM hyperswarm_sync_operations + WHERE id = $1 + LIMIT 1`, + [id] + ); + + return result.rowCount !== 0; + } + + async count(): Promise { + const result = await this.getPool().query( + 'SELECT COUNT(*) AS count FROM hyperswarm_sync_operations' + ); + + if (result.rowCount === 0) { + return 0; + } + + return this.toNumber(result.rows[0].count); + } + + private mapRow(row: SyncRow): SyncOperationRecord { + const operation = typeof row.operation_json === 'string' + ? JSON.parse(row.operation_json) as Operation + : row.operation_json; + + return { + id: row.id, + ts: this.toNumber(row.ts), + operation, + insertedAt: this.toNumber(row.inserted_at), + }; + } + + private toNumber(value: number | string): number { + if (typeof value === 'number') { + return value; + } + + return Number.parseInt(value, 10); + } +} diff --git a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts index 2c40974b9..3fcdcbcf7 100644 --- a/services/mediators/hyperswarm/src/hyperswarm-mediator.ts +++ b/services/mediators/hyperswarm/src/hyperswarm-mediator.ts @@ -14,6 +14,7 @@ import { childLogger } from '@mdip/common/logger'; import config from './config.js'; import type { OperationSyncStore } from './db/types.js'; import SqliteOperationSyncStore from './db/sqlite.js'; +import PostgresOperationSyncStore from './db/postgres.js'; import NegentropyAdapter, { type NegentropyWindowStats, type ReconciliationWindow, @@ -229,7 +230,15 @@ const gatekeeper = new GatekeeperClient(); const keymaster = new KeymasterClient(); const ipfs = new KuboClient(); const cipher = new CipherNode(); -let syncStore: OperationSyncStore = new SqliteOperationSyncStore(); +function createConfiguredSyncStore(): OperationSyncStore { + if (config.db === 'postgres') { + return new PostgresOperationSyncStore(config.postgresURL); + } + + return new SqliteOperationSyncStore(); +} + +let syncStore: OperationSyncStore = createConfiguredSyncStore(); let negentropyAdapter: NegentropyAdapter | null = null; let adapterChangeSeq = 0; let adapterBuiltSeq = -1; @@ -1959,6 +1968,7 @@ const networkID = Buffer.from(hash).toString('hex'); const topic = Buffer.from(b4a.from(networkID, 'hex')); async function main(): Promise { + log.info({ db: config.db }, 'sync-store backend selected'); await syncStore.start(); await gatekeeper.connect({ diff --git a/services/mediators/satoshi-inscription/README.md b/services/mediators/satoshi-inscription/README.md index cf68f66e5..00da8b9e0 100644 --- a/services/mediators/satoshi-inscription/README.md +++ b/services/mediators/satoshi-inscription/README.md @@ -29,5 +29,5 @@ The mediator has two responsibilities: | `KC_SAT_RBF_ENABLED` | false | Whether Replace-By-Fee is enabled | | `KC_SAT_START_BLOCK` | 0 | Blockchain scan starting block index | | `KC_SAT_REIMPORT` | true | Whether to reimport all discovered batches on startup | -| `KC_SAT_DB` | json | Database adapter, must be `redis`, `json`, `mongodb`, or `sqlite` | +| `KC_SAT_DB` | json | Database adapter, must be `redis`, `json`, `mongodb`, `sqlite`, or `postgres` | | `KC_LOG_LEVEL` | info | Log level: `debug`, `info`, `warn`, `error` | diff --git a/services/mediators/satoshi-inscription/package-lock.json b/services/mediators/satoshi-inscription/package-lock.json index 981a0b963..85b01a884 100644 --- a/services/mediators/satoshi-inscription/package-lock.json +++ b/services/mediators/satoshi-inscription/package-lock.json @@ -1,12 +1,12 @@ { "name": "inscription-mediator", - "version": "1.4.0-beta.5", + "version": "1.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "inscription-mediator", - "version": "1.4.0-beta.5", + "version": "1.4.0", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -17,12 +17,13 @@ "express": "^4.21.0", "ioredis": "^5.5.0", "mongodb": "^6.13.0", - "morgan": "^1.10.0", + "pg": "^8.16.3", "sqlite": "^5.1.1", "sqlite3": "^5.1.7", "tiny-secp256k1": "^2.2.4" }, "devDependencies": { + "@types/pg": "^8.15.5", "patch-package": "^8.0.1" } }, @@ -160,6 +161,28 @@ "node": ">= 6" } }, + "node_modules/@types/node": { + "version": "25.3.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.3.3.tgz", + "integrity": "sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@types/pg": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.18.0.tgz", + "integrity": "sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, "node_modules/@types/webidl-conversions": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", @@ -409,22 +432,6 @@ } ] }, - "node_modules/basic-auth": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", - "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", - "dependencies": { - "safe-buffer": "5.1.2" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/basic-auth/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, "node_modules/bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -2401,33 +2408,6 @@ "whatwg-url": "^14.1.0 || ^13.0.0" } }, - "node_modules/morgan": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.1.tgz", - "integrity": "sha512-223dMRJtI/l25dJKWpgij2cMtywuG/WiUKXdvwfbhGKBhy1puASqXwFzmWZ7+K73vUPoR7SS2Qz2cI/g9MKw0A==", - "license": "MIT", - "dependencies": { - "basic-auth": "~2.0.1", - "debug": "2.6.9", - "depd": "~2.0.0", - "on-finished": "~2.3.0", - "on-headers": "~1.1.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/morgan/node_modules/on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -2650,15 +2630,6 @@ "node": ">= 0.8" } }, - "node_modules/on-headers": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz", - "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -2781,6 +2752,95 @@ "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==", "license": "MIT" }, + "node_modules/pg": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.19.0.tgz", + "integrity": "sha512-QIcLGi508BAHkQ3pJNptsFz5WQMlpGbuBGBaIaXsWK8mel2kQ/rThYI+DbgjUvZrIr7MiuEuc9LcChJoEZK1xQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.11.0", + "pg-pool": "^3.12.0", + "pg-protocol": "^1.12.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.3.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.11.0.tgz", + "integrity": "sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.12.0.tgz", + "integrity": "sha512-eIJ0DES8BLaziFHW7VgJEBPi5hg3Nyng5iKpYtj3wbcAUV9A1wLgWiY7ajf/f/oO1wfxt83phXPY8Emztg7ITg==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.12.0.tgz", + "integrity": "sha512-uOANXNRACNdElMXJ0tPz6RBM0XQ61nONGAwlt8da5zs/iUOOCLBQOHSXnrC6fMsvtjxbOJrZZl5IScGv+7mpbg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -2794,6 +2854,45 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prebuild-install": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", @@ -3310,6 +3409,15 @@ "memory-pager": "^1.0.2" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sprintf-js": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", @@ -3641,6 +3749,13 @@ "node": ">=14.0.0" } }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, + "license": "MIT" + }, "node_modules/unique-filename": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", @@ -3828,6 +3943,15 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", diff --git a/services/mediators/satoshi-inscription/package.json b/services/mediators/satoshi-inscription/package.json index b78a18ddd..2eb53a47f 100644 --- a/services/mediators/satoshi-inscription/package.json +++ b/services/mediators/satoshi-inscription/package.json @@ -19,11 +19,13 @@ "express": "^4.21.0", "ioredis": "^5.5.0", "mongodb": "^6.13.0", + "pg": "^8.16.3", "sqlite": "^5.1.1", "sqlite3": "^5.1.7", "tiny-secp256k1": "^2.2.4" }, "devDependencies": { + "@types/pg": "^8.15.5", "patch-package": "^8.0.1" } } diff --git a/services/mediators/satoshi-inscription/src/config.ts b/services/mediators/satoshi-inscription/src/config.ts index 6dd059a40..9c2d0c735 100644 --- a/services/mediators/satoshi-inscription/src/config.ts +++ b/services/mediators/satoshi-inscription/src/config.ts @@ -4,7 +4,7 @@ dotenv.config(); export type NetworkName = 'bitcoin' | 'testnet' | 'regtest'; export type ChainName = 'BTC' | 'TBTC' | 'Signet' | 'TFTC'; -export type SatoshiDB = 'json' | 'sqlite' | 'mongodb' | 'redis'; +export type SatoshiDB = 'json' | 'sqlite' | 'mongodb' | 'redis' | 'postgres'; export interface AppConfig { nodeID?: string; @@ -70,6 +70,8 @@ function toDB(name: string | undefined): SatoshiDB { return 'mongodb'; case 'redis': return 'redis'; + case 'postgres': + return 'postgres'; default: throw new Error(`Unsupported DB "${name}"`); } diff --git a/services/mediators/satoshi-inscription/src/db/postgres.ts b/services/mediators/satoshi-inscription/src/db/postgres.ts new file mode 100644 index 000000000..d1713763a --- /dev/null +++ b/services/mediators/satoshi-inscription/src/db/postgres.ts @@ -0,0 +1,98 @@ +import { Pool } from 'pg'; +import { MediatorDb } from '../types.js'; +import AbstractDB from './abstract-db.js'; + +interface MediatorStateRow { + data: MediatorDb | string; +} + +export default class JsonPostgres extends AbstractDB { + private readonly url: string; + private readonly mediator: string; + private readonly registry: string; + private pool: Pool | null; + + static async create(registry: string): Promise { + const json = new JsonPostgres(registry); + await json.connect(); + return json; + } + + constructor(registry: string) { + super(); + this.url = process.env.KC_POSTGRES_URL || 'postgresql://mdip:mdip@localhost:5432/mdip'; + this.mediator = 'satoshi-inscription'; + this.registry = registry; + this.pool = null; + } + + async connect(): Promise { + if (this.pool) { + return; + } + + this.pool = new Pool({ connectionString: this.url }); + + await this.pool.query(` + CREATE TABLE IF NOT EXISTS satoshi_mediator_state ( + mediator TEXT NOT NULL, + registry TEXT NOT NULL, + data JSONB NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (mediator, registry) + ) + `); + } + + async disconnect(): Promise { + if (this.pool) { + await this.pool.end(); + this.pool = null; + } + } + + private getPool(): Pool { + if (!this.pool) { + throw new Error('Postgres client not connected. Call connect() first.'); + } + + return this.pool; + } + + async saveDb(data: MediatorDb): Promise { + const pool = this.getPool(); + + await pool.query( + `INSERT INTO satoshi_mediator_state (mediator, registry, data, updated_at) + VALUES ($1, $2, $3::jsonb, NOW()) + ON CONFLICT (mediator, registry) + DO UPDATE SET data = EXCLUDED.data, updated_at = NOW()`, + [this.mediator, this.registry, JSON.stringify(data)] + ); + + return true; + } + + async loadDb(): Promise { + const pool = this.getPool(); + const result = await pool.query( + `SELECT data + FROM satoshi_mediator_state + WHERE mediator = $1 AND registry = $2 + LIMIT 1`, + [this.mediator, this.registry] + ); + + if (result.rowCount === 0) { + return null; + } + + const rowData = result.rows[0].data; + + if (typeof rowData === 'string') { + return JSON.parse(rowData) as MediatorDb; + } + + return rowData; + } +} diff --git a/services/mediators/satoshi-inscription/src/inscription-mediator.ts b/services/mediators/satoshi-inscription/src/inscription-mediator.ts index bdca4e1a2..5b114132c 100644 --- a/services/mediators/satoshi-inscription/src/inscription-mediator.ts +++ b/services/mediators/satoshi-inscription/src/inscription-mediator.ts @@ -8,6 +8,7 @@ import JsonFile from './db/jsonfile.js'; import JsonRedis from './db/redis.js'; import JsonMongo from './db/mongo.js'; import JsonSQLite from './db/sqlite.js'; +import JsonPostgres from './db/postgres.js'; import config from './config.js'; import { GatekeeperEvent, Operation } from '@mdip/gatekeeper/types'; import Inscription from '@mdip/inscription'; @@ -845,6 +846,9 @@ async function main() { else if (config.db === 'sqlite') { jsonPersister = await JsonSQLite.create(REGISTRY); } + else if (config.db === 'postgres') { + jsonPersister = await JsonPostgres.create(REGISTRY); + } else { jsonPersister = jsonFile; } diff --git a/services/mediators/satoshi/README.md b/services/mediators/satoshi/README.md index 38bed0e50..d96d7a9db 100644 --- a/services/mediators/satoshi/README.md +++ b/services/mediators/satoshi/README.md @@ -29,5 +29,5 @@ The mediator has two responsibilities: | `KC_SAT_RBF_ENABLED` | false | Whether Replace-By-Fee is enabled | | `KC_SAT_START_BLOCK` | 0 | Blockchain scan starting block index | | `KC_SAT_REIMPORT` | true | Whether to reimport all discovered batches on startup | -| `KC_SAT_DB` | json | Database adapter, must be `redis`, `json`, `mongodb`, or `sqlite` | +| `KC_SAT_DB` | json | Database adapter, must be `redis`, `json`, `mongodb`, `sqlite`, or `postgres` | | `KC_LOG_LEVEL` | info | Log level: `debug`, `info`, `warn`, `error` | diff --git a/services/mediators/satoshi/package-lock.json b/services/mediators/satoshi/package-lock.json index 889a4d402..fa06cc91f 100644 --- a/services/mediators/satoshi/package-lock.json +++ b/services/mediators/satoshi/package-lock.json @@ -1,12 +1,12 @@ { "name": "satoshi-mediator", - "version": "1.4.0-beta.5", + "version": "1.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "satoshi-mediator", - "version": "1.4.0-beta.5", + "version": "1.4.0", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -15,11 +15,12 @@ "express": "^4.21.0", "ioredis": "^5.5.0", "mongodb": "^6.13.0", - "morgan": "^1.10.0", + "pg": "^8.16.3", "sqlite": "^5.1.1", "sqlite3": "^5.1.7" }, "devDependencies": { + "@types/pg": "^8.15.5", "patch-package": "^8.0.1" } }, @@ -136,6 +137,28 @@ "node": ">= 6" } }, + "node_modules/@types/node": { + "version": "25.3.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.3.3.tgz", + "integrity": "sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@types/pg": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.18.0.tgz", + "integrity": "sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, "node_modules/@types/webidl-conversions": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", @@ -372,22 +395,6 @@ } ] }, - "node_modules/basic-auth": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", - "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", - "dependencies": { - "safe-buffer": "5.1.2" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/basic-auth/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, "node_modules/bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -2255,33 +2262,6 @@ "whatwg-url": "^14.1.0 || ^13.0.0" } }, - "node_modules/morgan": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.1.tgz", - "integrity": "sha512-223dMRJtI/l25dJKWpgij2cMtywuG/WiUKXdvwfbhGKBhy1puASqXwFzmWZ7+K73vUPoR7SS2Qz2cI/g9MKw0A==", - "license": "MIT", - "dependencies": { - "basic-auth": "~2.0.1", - "debug": "2.6.9", - "depd": "~2.0.0", - "on-finished": "~2.3.0", - "on-headers": "~1.1.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/morgan/node_modules/on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -2500,15 +2480,6 @@ "node": ">= 0.8" } }, - "node_modules/on-headers": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz", - "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -2630,6 +2601,95 @@ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, + "node_modules/pg": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.19.0.tgz", + "integrity": "sha512-QIcLGi508BAHkQ3pJNptsFz5WQMlpGbuBGBaIaXsWK8mel2kQ/rThYI+DbgjUvZrIr7MiuEuc9LcChJoEZK1xQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.11.0", + "pg-pool": "^3.12.0", + "pg-protocol": "^1.12.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.3.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.11.0.tgz", + "integrity": "sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.12.0.tgz", + "integrity": "sha512-eIJ0DES8BLaziFHW7VgJEBPi5hg3Nyng5iKpYtj3wbcAUV9A1wLgWiY7ajf/f/oO1wfxt83phXPY8Emztg7ITg==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.12.0.tgz", + "integrity": "sha512-uOANXNRACNdElMXJ0tPz6RBM0XQ61nONGAwlt8da5zs/iUOOCLBQOHSXnrC6fMsvtjxbOJrZZl5IScGv+7mpbg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -2643,6 +2703,45 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prebuild-install": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", @@ -3147,6 +3246,15 @@ "memory-pager": "^1.0.2" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sprintf-js": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", @@ -3439,6 +3547,13 @@ "node": ">= 0.6" } }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, + "license": "MIT" + }, "node_modules/unique-filename": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", @@ -3575,6 +3690,15 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", diff --git a/services/mediators/satoshi/package.json b/services/mediators/satoshi/package.json index b082f0384..a906c6425 100644 --- a/services/mediators/satoshi/package.json +++ b/services/mediators/satoshi/package.json @@ -17,10 +17,12 @@ "express": "^4.21.0", "ioredis": "^5.5.0", "mongodb": "^6.13.0", + "pg": "^8.16.3", "sqlite": "^5.1.1", "sqlite3": "^5.1.7" }, "devDependencies": { + "@types/pg": "^8.15.5", "patch-package": "^8.0.1" } } diff --git a/services/mediators/satoshi/src/config.ts b/services/mediators/satoshi/src/config.ts index 6dd059a40..9c2d0c735 100644 --- a/services/mediators/satoshi/src/config.ts +++ b/services/mediators/satoshi/src/config.ts @@ -4,7 +4,7 @@ dotenv.config(); export type NetworkName = 'bitcoin' | 'testnet' | 'regtest'; export type ChainName = 'BTC' | 'TBTC' | 'Signet' | 'TFTC'; -export type SatoshiDB = 'json' | 'sqlite' | 'mongodb' | 'redis'; +export type SatoshiDB = 'json' | 'sqlite' | 'mongodb' | 'redis' | 'postgres'; export interface AppConfig { nodeID?: string; @@ -70,6 +70,8 @@ function toDB(name: string | undefined): SatoshiDB { return 'mongodb'; case 'redis': return 'redis'; + case 'postgres': + return 'postgres'; default: throw new Error(`Unsupported DB "${name}"`); } diff --git a/services/mediators/satoshi/src/db/postgres.ts b/services/mediators/satoshi/src/db/postgres.ts new file mode 100644 index 000000000..21b6f5bc0 --- /dev/null +++ b/services/mediators/satoshi/src/db/postgres.ts @@ -0,0 +1,98 @@ +import { Pool } from 'pg'; +import { MediatorDb } from '../types.js'; +import AbstractDB from './abstract-db.js'; + +interface MediatorStateRow { + data: MediatorDb | string; +} + +export default class JsonPostgres extends AbstractDB { + private readonly url: string; + private readonly mediator: string; + private readonly registry: string; + private pool: Pool | null; + + static async create(registry: string): Promise { + const json = new JsonPostgres(registry); + await json.connect(); + return json; + } + + constructor(registry: string) { + super(); + this.url = process.env.KC_POSTGRES_URL || 'postgresql://mdip:mdip@localhost:5432/mdip'; + this.mediator = 'satoshi'; + this.registry = registry; + this.pool = null; + } + + async connect(): Promise { + if (this.pool) { + return; + } + + this.pool = new Pool({ connectionString: this.url }); + + await this.pool.query(` + CREATE TABLE IF NOT EXISTS satoshi_mediator_state ( + mediator TEXT NOT NULL, + registry TEXT NOT NULL, + data JSONB NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (mediator, registry) + ) + `); + } + + async disconnect(): Promise { + if (this.pool) { + await this.pool.end(); + this.pool = null; + } + } + + private getPool(): Pool { + if (!this.pool) { + throw new Error('Postgres client not connected. Call connect() first.'); + } + + return this.pool; + } + + async saveDb(data: MediatorDb): Promise { + const pool = this.getPool(); + + await pool.query( + `INSERT INTO satoshi_mediator_state (mediator, registry, data, updated_at) + VALUES ($1, $2, $3::jsonb, NOW()) + ON CONFLICT (mediator, registry) + DO UPDATE SET data = EXCLUDED.data, updated_at = NOW()`, + [this.mediator, this.registry, JSON.stringify(data)] + ); + + return true; + } + + async loadDb(): Promise { + const pool = this.getPool(); + const result = await pool.query( + `SELECT data + FROM satoshi_mediator_state + WHERE mediator = $1 AND registry = $2 + LIMIT 1`, + [this.mediator, this.registry] + ); + + if (result.rowCount === 0) { + return null; + } + + const rowData = result.rows[0].data; + + if (typeof rowData === 'string') { + return JSON.parse(rowData) as MediatorDb; + } + + return rowData; + } +} diff --git a/services/mediators/satoshi/src/satoshi-mediator.ts b/services/mediators/satoshi/src/satoshi-mediator.ts index 4d4693ab3..59a466eb3 100644 --- a/services/mediators/satoshi/src/satoshi-mediator.ts +++ b/services/mediators/satoshi/src/satoshi-mediator.ts @@ -5,6 +5,7 @@ import JsonFile from './db/jsonfile.js'; import JsonRedis from './db/redis.js'; import JsonMongo from './db/mongo.js'; import JsonSQLite from './db/sqlite.js'; +import JsonPostgres from './db/postgres.js'; import config from './config.js'; import { isValidDID } from '@mdip/ipfs/utils'; import { MediatorDb, MediatorDbInterface, DiscoveredItem, BlockVerbosity } from './types.js'; @@ -632,6 +633,9 @@ async function main() { else if (config.db === 'sqlite') { jsonPersister = await JsonSQLite.create(REGISTRY); } + else if (config.db === 'postgres') { + jsonPersister = await JsonPostgres.create(REGISTRY); + } else { jsonPersister = jsonFile; } diff --git a/services/search-server/README.md b/services/search-server/README.md index 140a2fe7a..3a1d659df 100644 --- a/services/search-server/README.md +++ b/services/search-server/README.md @@ -31,6 +31,13 @@ SEARCH_SERVER_GATEKEEPER_URL=http://localhost:4224 # How often (in ms) to poll Gatekeeper for new or updated DIDs. SEARCH_SERVER_REFRESH_INTERVAL_MS=5000 +# Database adapter: sqlite | postgres | memory +SEARCH_SERVER_DB=sqlite + +# Used when SEARCH_SERVER_DB=postgres +# Falls back to KC_POSTGRES_URL when unset +SEARCH_SERVER_POSTGRES_URL=postgresql://mdip:mdip@localhost:5432/mdip + # Logging KC_LOG_LEVEL=info ``` diff --git a/services/search-server/package-lock.json b/services/search-server/package-lock.json index 29ce918ef..2eed78094 100644 --- a/services/search-server/package-lock.json +++ b/services/search-server/package-lock.json @@ -1,16 +1,17 @@ { "name": "search-server", - "version": "1.3.0", + "version": "1.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "search-server", - "version": "1.3.0", + "version": "1.4.0", "dependencies": { "cors": "^2.8.5", "dotenv": "^16.5.0", - "express": "^5.1.0" + "express": "^5.1.0", + "pg": "^8.19.0" }, "devDependencies": { "@types/cors": "^2.8.13", @@ -664,6 +665,134 @@ "node": ">=16" } }, + "node_modules/pg": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.19.0.tgz", + "integrity": "sha512-QIcLGi508BAHkQ3pJNptsFz5WQMlpGbuBGBaIaXsWK8mel2kQ/rThYI+DbgjUvZrIr7MiuEuc9LcChJoEZK1xQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.11.0", + "pg-pool": "^3.12.0", + "pg-protocol": "^1.12.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.3.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.11.0.tgz", + "integrity": "sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.12.0.tgz", + "integrity": "sha512-eIJ0DES8BLaziFHW7VgJEBPi5hg3Nyng5iKpYtj3wbcAUV9A1wLgWiY7ajf/f/oO1wfxt83phXPY8Emztg7ITg==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.12.0.tgz", + "integrity": "sha512-uOANXNRACNdElMXJ0tPz6RBM0XQ61nONGAwlt8da5zs/iUOOCLBQOHSXnrC6fMsvtjxbOJrZZl5IScGv+7mpbg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -878,6 +1007,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -965,6 +1103,15 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } } } } diff --git a/services/search-server/package.json b/services/search-server/package.json index 382d4f4ba..ccd85d60f 100644 --- a/services/search-server/package.json +++ b/services/search-server/package.json @@ -9,7 +9,8 @@ "dependencies": { "cors": "^2.8.5", "dotenv": "^16.5.0", - "express": "^5.1.0" + "express": "^5.1.0", + "pg": "^8.19.0" }, "devDependencies": { "@types/cors": "^2.8.13", diff --git a/services/search-server/sample.env b/services/search-server/sample.env index 1a56b7e39..e11594f5e 100644 --- a/services/search-server/sample.env +++ b/services/search-server/sample.env @@ -2,4 +2,5 @@ SEARCH_SERVER_PORT=4002 SEARCH_SERVER_GATEKEEPER_URL=http://localhost:4224 SEARCH_SERVER_REFRESH_INTERVAL_MS=5000 SEARCH_SERVER_DB=sqlite +SEARCH_SERVER_POSTGRES_URL=postgresql://mdip:mdip@localhost:5432/mdip KC_LOG_LEVEL=info diff --git a/services/search-server/src/db/postgres.ts b/services/search-server/src/db/postgres.ts new file mode 100644 index 000000000..b1da73ced --- /dev/null +++ b/services/search-server/src/db/postgres.ts @@ -0,0 +1,298 @@ +import { Pool } from 'pg'; +import { DIDsDb } from '../types.js'; + +interface ConfigRow { + value: string; +} + +interface DocRow { + doc: object | string; +} + +interface DidRow { + did: string; +} + +export default class Postgres implements DIDsDb { + private readonly url: string; + private pool: Pool | null = null; + private static readonly ARRAY_WILDCARD_END = /\[\*]$/; + private static readonly ARRAY_WILDCARD_MID = /\[\*]\./; + + static async create(url: string): Promise { + const db = new Postgres(url); + await db.connect(); + return db; + } + + constructor(url: string) { + this.url = url; + } + + async connect(): Promise { + if (this.pool) { + return; + } + + this.pool = new Pool({ connectionString: this.url }); + + await this.pool.query(` + CREATE TABLE IF NOT EXISTS did_docs ( + did TEXT PRIMARY KEY, + doc JSONB NOT NULL + ); + + CREATE TABLE IF NOT EXISTS config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + `); + } + + async disconnect(): Promise { + if (this.pool) { + await this.pool.end(); + this.pool = null; + } + } + + async loadUpdatedAfter(): Promise { + const pool = this.getPool(); + const result = await pool.query( + 'SELECT value FROM config WHERE key = $1 LIMIT 1', + ['updated_after'] + ); + + if (result.rowCount === 0) { + return null; + } + + return result.rows[0].value; + } + + async saveUpdatedAfter(timestamp: string): Promise { + const pool = this.getPool(); + await pool.query( + `INSERT INTO config (key, value) VALUES ('updated_after', $1) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, + [timestamp] + ); + } + + async storeDID(did: string, doc: object): Promise { + const pool = this.getPool(); + await pool.query( + `INSERT INTO did_docs (did, doc) VALUES ($1, $2::jsonb) + ON CONFLICT (did) DO UPDATE SET doc = EXCLUDED.doc`, + [did, JSON.stringify(doc)] + ); + } + + async getDID(did: string): Promise { + const pool = this.getPool(); + const result = await pool.query( + 'SELECT doc FROM did_docs WHERE did = $1 LIMIT 1', + [did] + ); + + if (result.rowCount === 0) { + return null; + } + + const { doc } = result.rows[0]; + if (typeof doc === 'string') { + return JSON.parse(doc); + } + + return doc; + } + + async searchDocs(q: string): Promise { + const pool = this.getPool(); + const result = await pool.query( + `SELECT did + FROM did_docs + WHERE doc::text LIKE '%' || $1 || '%'`, + [q] + ); + + return result.rows.map(row => row.did); + } + + async queryDocs(where: Record): Promise { + const pool = this.getPool(); + + const entry = Object.entries(where)[0] as [string, any] | undefined; + if (!entry) { + return []; + } + + const [rawPath, cond] = entry; + if (typeof cond !== 'object' || !Array.isArray(cond.$in)) { + throw new Error('Only {$in:[…]} supported'); + } + + const list = cond.$in as unknown[]; + if (list.length === 0) { + return []; + } + + const isKeyWildcard = rawPath.endsWith('.*'); + const isValueWildcard = rawPath.includes('.*.'); + const isArrayTail = Postgres.ARRAY_WILDCARD_END.test(rawPath); + const isArrayMid = Postgres.ARRAY_WILDCARD_MID.test(rawPath); + + let result; + + if (isArrayTail) { + const basePath = this.toPathTokens(rawPath.replace(Postgres.ARRAY_WILDCARD_END, '')); + result = await pool.query( + `SELECT DISTINCT d.did + FROM did_docs d + JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof(d.doc #> $1::text[]) = 'array' THEN d.doc #> $1::text[] + ELSE '[]'::jsonb + END + ) AS elem(value) ON TRUE + WHERE EXISTS ( + SELECT 1 + FROM unnest($2::text[]) AS expected(value) + WHERE elem.value = expected.value::jsonb + )`, + [basePath, this.toJsonLiterals(list)] + ); + } else if (isArrayMid) { + const [prefix, suffix] = rawPath.split('[*].'); + const basePath = this.toPathTokens(prefix); + const suffixPath = this.toPathTokens(suffix); + result = await pool.query( + `SELECT DISTINCT d.did + FROM did_docs d + JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof(d.doc #> $1::text[]) = 'array' THEN d.doc #> $1::text[] + ELSE '[]'::jsonb + END + ) AS elem(value) ON TRUE + WHERE EXISTS ( + SELECT 1 + FROM unnest($3::text[]) AS expected(value) + WHERE elem.value #> $2::text[] = expected.value::jsonb + )`, + [basePath, suffixPath, this.toJsonLiterals(list)] + ); + } else if (isKeyWildcard) { + const basePath = this.toPathTokens(rawPath.slice(0, -2)); + result = await pool.query( + `SELECT DISTINCT d.did + FROM did_docs d + JOIN LATERAL jsonb_each( + CASE + WHEN jsonb_typeof(d.doc #> $1::text[]) = 'object' THEN d.doc #> $1::text[] + ELSE '{}'::jsonb + END + ) AS member(key, value) ON TRUE + WHERE member.key = ANY($2::text[])`, + [basePath, list.map(value => String(value))] + ); + } else if (isValueWildcard) { + const [prefix, suffix] = rawPath.split('.*.'); + const basePath = this.toPathTokens(prefix); + const suffixPath = this.toPathTokens(suffix); + result = await pool.query( + `SELECT DISTINCT d.did + FROM did_docs d + JOIN LATERAL jsonb_each( + CASE + WHEN jsonb_typeof(d.doc #> $1::text[]) = 'object' THEN d.doc #> $1::text[] + ELSE '{}'::jsonb + END + ) AS member(key, value) ON TRUE + WHERE EXISTS ( + SELECT 1 + FROM unnest($3::text[]) AS expected(value) + WHERE member.value #> $2::text[] = expected.value::jsonb + )`, + [basePath, suffixPath, this.toJsonLiterals(list)] + ); + } else { + const path = this.toPathTokens(rawPath); + result = await pool.query( + `SELECT DISTINCT did + FROM did_docs + WHERE EXISTS ( + SELECT 1 + FROM unnest($2::text[]) AS expected(value) + WHERE did_docs.doc #> $1::text[] = expected.value::jsonb + )`, + [path, this.toJsonLiterals(list)] + ); + } + + return result.rows.map(row => row.did); + } + + async wipeDb(): Promise { + const pool = this.getPool(); + await pool.query('DELETE FROM did_docs'); + await pool.query('DELETE FROM config'); + } + + private getPool(): Pool { + if (!this.pool) { + throw new Error('Postgres DB not connected'); + } + + return this.pool; + } + + private toJsonLiterals(values: unknown[]): string[] { + return values.map((value) => { + if (value === undefined) { + return 'null'; + } + + const encoded = JSON.stringify(value); + return encoded === undefined ? 'null' : encoded; + }); + } + + private toPathTokens(path: string): string[] { + const normalized = this.normalizePath(path); + if (!normalized) { + return []; + } + + const tokens: string[] = []; + const re = /([^[.\]]+)|\[(\d+)]/g; + let match: RegExpExecArray | null = re.exec(normalized); + + while (match) { + if (match[1]) { + tokens.push(match[1]); + } else if (match[2]) { + tokens.push(match[2]); + } + match = re.exec(normalized); + } + + return tokens; + } + + private normalizePath(path: string): string { + if (!path || path === '$') { + return ''; + } + + if (path.startsWith('$.')) { + return path.slice(2); + } + + if (path.startsWith('$')) { + return path.slice(1).replace(/^\./, ''); + } + + return path; + } +} diff --git a/services/search-server/src/index.ts b/services/search-server/src/index.ts index 9474875f5..e5be3215e 100644 --- a/services/search-server/src/index.ts +++ b/services/search-server/src/index.ts @@ -4,6 +4,7 @@ import dotenv from "dotenv"; import GatekeeperClient from "@mdip/gatekeeper/client"; import DIDsSQLite from "./db/sqlite.js"; import DIDsDbMemory from './db/json-memory.js'; +import DIDsPostgres from './db/postgres.js'; import DidIndexer from "./DidIndexer.js"; import {DIDsDb} from "./types.js"; import { childLogger } from "@mdip/common/logger"; @@ -17,6 +18,8 @@ async function main() { SEARCH_SERVER_GATEKEEPER_URL = 'http://localhost:4224', SEARCH_SERVER_REFRESH_INTERVAL_MS = 5000, SEARCH_SERVER_DB = 'sqlite', + SEARCH_SERVER_POSTGRES_URL, + KC_POSTGRES_URL, } = process.env; const app = express(); @@ -35,6 +38,12 @@ async function main() { if (SEARCH_SERVER_DB === 'sqlite') { didDb = await DIDsSQLite.create(); + } else if (SEARCH_SERVER_DB === 'postgres') { + const postgresUrl = SEARCH_SERVER_POSTGRES_URL + || KC_POSTGRES_URL + || 'postgresql://mdip:mdip@localhost:5432/mdip'; + + didDb = await DIDsPostgres.create(postgresUrl); } else { didDb = new DIDsDbMemory(); } diff --git a/start-node-ci b/start-node-ci index acc3aa502..ef43ad06c 100755 --- a/start-node-ci +++ b/start-node-ci @@ -2,7 +2,7 @@ set -e PROFILE_ARGS=() -SERVICES=(cli gatekeeper keymaster redis mongodb search-server) +SERVICES=(cli gatekeeper keymaster redis mongodb postgres search-server) if [ -f .env ]; then set -a diff --git a/tests/hyperswarm/bootstrap.test.ts b/tests/hyperswarm/bootstrap.test.ts index 8d4406bdc..494642cc7 100644 --- a/tests/hyperswarm/bootstrap.test.ts +++ b/tests/hyperswarm/bootstrap.test.ts @@ -29,6 +29,20 @@ function makeDid(index: number): string { } describe('bootstrapSyncStoreIfEmpty', () => { + it('throws when driftThresholdPct is outside [0, 1]', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + + const gatekeeper = { + getDIDs: jest.fn(async () => []), + exportBatch: jest.fn(async () => []), + }; + + await expect( + bootstrapSyncStoreIfEmpty(store, gatekeeper, { driftThresholdPct: -0.001 }) + ).rejects.toThrow('Invalid driftThresholdPct; expected a number between 0 and 1'); + }); + it('skips rebuild when store drift is within configured percentage tolerance', async () => { const store = new InMemoryOperationSyncStore(); await store.start(); @@ -66,6 +80,7 @@ describe('bootstrapSyncStoreIfEmpty', () => { }]); const opA = makeOperation('a', '2026-02-10T10:00:00.000Z'); + // eslint-disable-next-line sonarjs/no-duplicate-string const opB = makeOperation('b', '2026-02-10T11:00:00.000Z'); const gatekeeper = { getDIDs: jest.fn(async () => [makeDid(1), makeDid(2)]), @@ -122,6 +137,46 @@ describe('bootstrapSyncStoreIfEmpty', () => { await expect(bootstrapSyncStoreIfEmpty(store, gatekeeper)).rejects.toThrow('boom'); }); + it('includes non-Error export failures in batch error messages', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + + const gatekeeper = { + getDIDs: jest.fn(async () => [makeDid(1)]), + exportBatch: jest.fn(async () => { + throw 'explode'; + }), + }; + + await expect(bootstrapSyncStoreIfEmpty(store, gatekeeper)) + .rejects + .toThrow('bootstrap exportBatch failed for DID batch 1/1 (1 dids): explode'); + }); + + it('normalizes DID list from mixed DID docs and strings before batching', async () => { + const store = new InMemoryOperationSyncStore(); + await store.start(); + + const opA = makeOperation('a', '2026-02-10T10:00:00.000Z'); + const opB = makeOperation('b', '2026-02-10T11:00:00.000Z'); + const gatekeeper = { + getDIDs: jest.fn(async () => [ + makeDid(1), + { didDocument: { id: makeDid(2) } }, + { didDocument: { id: '' } }, + { didDocument: {} }, + makeDid(1), + ]), + exportBatch: jest.fn(async () => [makeEvent(opA), makeEvent(opB)]), + }; + + const result = await bootstrapSyncStoreIfEmpty(store, gatekeeper as any); + expect(result.skipped).toBe(false); + expect(result.mapped).toBe(2); + expect(gatekeeper.exportBatch).toHaveBeenCalledTimes(1); + expect(gatekeeper.exportBatch).toHaveBeenCalledWith([makeDid(1), makeDid(2)]); + }); + it('handles empty/invalid export payload without upserting', async () => { const store = new InMemoryOperationSyncStore(); await store.start(); diff --git a/tests/hyperswarm/config.test.ts b/tests/hyperswarm/config.test.ts index 52aa8769b..c68158549 100644 --- a/tests/hyperswarm/config.test.ts +++ b/tests/hyperswarm/config.test.ts @@ -89,4 +89,41 @@ describe('hyperswarm config', () => { }) ).rejects.toThrow('Invalid KC_HYPR_NEGENTROPY_ENABLE; expected true or false'); }); + + it('defaults sync DB to sqlite when KC_HYPR_DB is empty', async () => { + process.env.KC_HYPR_DB = ''; + + const config = await importConfigIsolated(); + expect(config.db).toBe('sqlite'); + }); + + it('accepts postgres sync DB and uses service postgres URL before shared URL', async () => { + process.env.KC_HYPR_DB = 'postgres'; + process.env.KC_HYPR_POSTGRES_URL = 'postgresql://hypr-user:hypr-pass@db:5432/hypr'; + // eslint-disable-next-line sonarjs/no-duplicate-string + process.env.KC_POSTGRES_URL = 'postgresql://shared-user:shared-pass@db:5432/shared'; + + const config = await importConfigIsolated(); + expect(config.db).toBe('postgres'); + expect(config.postgresURL).toBe('postgresql://hypr-user:hypr-pass@db:5432/hypr'); + }); + + it('uses shared KC_POSTGRES_URL when KC_HYPR_POSTGRES_URL is not set', async () => { + process.env.KC_HYPR_DB = 'postgres'; + process.env.KC_HYPR_POSTGRES_URL = ''; + process.env.KC_POSTGRES_URL = 'postgresql://shared-user:shared-pass@db:5432/shared'; + + const config = await importConfigIsolated(); + expect(config.postgresURL).toBe('postgresql://shared-user:shared-pass@db:5432/shared'); + }); + + it('throws on invalid KC_HYPR_DB values', async () => { + process.env.KC_HYPR_DB = 'redis'; + + await expect( + jest.isolateModulesAsync(async () => { + await import(CONFIG_PATH); + }) + ).rejects.toThrow('Invalid KC_HYPR_DB; expected sqlite or postgres'); + }); }); diff --git a/tests/hyperswarm/negentropy-adapter.test.ts b/tests/hyperswarm/negentropy-adapter.test.ts index 2b61f8407..26f93db1f 100644 --- a/tests/hyperswarm/negentropy-adapter.test.ts +++ b/tests/hyperswarm/negentropy-adapter.test.ts @@ -97,6 +97,21 @@ describe('NegentropyAdapter', () => { expect(stats.durationMs).toBeGreaterThanOrEqual(0); }); + it('sets wantUint8ArrayOutput on the underlying negentropy instance', async () => { + const store = new InMemoryOperationSyncStore(); + await seedStore(store, [ + { id: h('a'), ts: 1000, op: makeOp('a', '2026-02-09T10:00:00.000Z') }, + ]); + + const adapter = await NegentropyAdapter.create({ + syncStore: store, + frameSizeLimit: 0, + wantUint8ArrayOutput: true, + }); + + expect((adapter as any).ne.wantUint8ArrayOutput).toBe(true); + }); + it('reconciles two stores and reports have/need ids', async () => { const storeA = new InMemoryOperationSyncStore(); const storeB = new InMemoryOperationSyncStore(); @@ -348,6 +363,76 @@ describe('NegentropyAdapter', () => { expect(session.windows[0].rounds).toBe(1); }); + it('returns cloned session stats snapshots', async () => { + const nowTs = toEpochSeconds('2026-02-10T00:00:00.000Z'); + const storeA = new InMemoryOperationSyncStore(); + const storeB = new InMemoryOperationSyncStore(); + + await seedStore(storeA, [ + // eslint-disable-next-line sonarjs/no-duplicate-string + { id: h('a'), ts: nowTs - 100, op: makeOp('a', '2026-02-09T23:58:20.000Z') }, + ]); + await seedStore(storeB, [ + { id: h('a'), ts: nowTs - 100, op: makeOp('a', '2026-02-09T23:58:20.000Z') }, + ]); + + const adapterA = await NegentropyAdapter.create({ + syncStore: storeA, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + const adapterB = await NegentropyAdapter.create({ + syncStore: storeB, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + + await adapterA.runWindowedSessionWithPeer(adapterB, { nowTs }); + const snapshotA = adapterA.getLastSessionStats(); + expect(snapshotA).not.toBeNull(); + + snapshotA!.windows[0].windowName = 'mutated'; + const snapshotB = adapterA.getLastSessionStats(); + expect(snapshotB).not.toBeNull(); + expect(snapshotB!.windows[0].windowName).not.toBe('mutated'); + }); + + it('completes a window when peer responds with null', async () => { + const nowTs = toEpochSeconds('2026-02-10T00:00:00.000Z'); + const storeA = new InMemoryOperationSyncStore(); + const storeB = new InMemoryOperationSyncStore(); + + await seedStore(storeA, [ + { id: h('a'), ts: nowTs - 100, op: makeOp('a', '2026-02-09T23:58:20.000Z') }, + ]); + await seedStore(storeB, [ + { id: h('a'), ts: nowTs - 100, op: makeOp('a', '2026-02-09T23:58:20.000Z') }, + ]); + + const adapterA = await NegentropyAdapter.create({ + syncStore: storeA, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + const adapterB = await NegentropyAdapter.create({ + syncStore: storeB, + frameSizeLimit: 0, + deferInitialBuild: true, + }); + + (adapterB as any).respond = async () => null; + + const session = await adapterA.runWindowedSessionWithPeer(adapterB, { + nowTs, + maxRoundsPerSession: 3, + }); + + expect(session.windowCount).toBeGreaterThan(0); + expect(session.windows[0].rounds).toBe(1); + expect(session.windows[0].completed).toBe(true); + expect(session.windows[0].cappedByRounds).toBe(false); + }); + it('throws for invalid runWindowedSessionWithPeer maxRoundsPerSession option', async () => { const storeA = new InMemoryOperationSyncStore(); const storeB = new InMemoryOperationSyncStore(); diff --git a/tests/hyperswarm/sync-store.test.ts b/tests/hyperswarm/sync-store.test.ts index 4c7048526..404afbf70 100644 --- a/tests/hyperswarm/sync-store.test.ts +++ b/tests/hyperswarm/sync-store.test.ts @@ -148,6 +148,12 @@ describe('SqliteOperationSyncStore', () => { await expect(store.count()).rejects.toThrow('Call start() first'); }); + it('throws from internal transaction helper when DB is not started', async () => { + const store = new SqliteOperationSyncStore(); + + await expect((store as any).withTx(async () => undefined)).rejects.toThrow('Call start() first'); + }); + it('returns early for empty inputs', async () => { const store = new SqliteOperationSyncStore('operations.db', path.join(tmpRoot, 'data/hyperswarm')); await store.start();