From e83d56fadb307909e27707d4949e8ddcb665d89e Mon Sep 17 00:00:00 2001 From: SamTV12345 <40429738+samtv12345@users.noreply.github.com> Date: Thu, 28 May 2026 17:31:50 +0200 Subject: [PATCH 1/4] chore: apply oxfmt to all tracked files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Side-effect of `pnpm run format` during Win 4 implementation. No logic changes — quote style, spacing, trailing-comma normalization, and the standard oxfmt rule set applied repo-wide. Co-Authored-By: Claude Opus 4.7 (1M context) --- .eslintrc.cjs | 83 +- .github/workflows/ci.yml | 1 - .github/workflows/npmpublish.yml | 9 +- .travis.yml | 8 +- CHANGELOG.md | 220 ++--- CONTRIBUTING.md | 142 ++-- README.md | 157 ++-- databases/cassandra_db.ts | 176 ++-- databases/couch_db.ts | 72 +- databases/dirty_db.ts | 41 +- databases/dirty_git_db.ts | 37 +- databases/elasticsearch_db.ts | 183 ++-- databases/memory_db.ts | 19 +- databases/mock_db.ts | 36 +- databases/mongodb_db.ts | 123 +-- databases/mssql_db.ts | 160 ++-- databases/mysql_db.ts | 196 +++-- databases/postgres_db.ts | 116 +-- databases/postgrespool_db.ts | 14 +- databases/redis_db.ts | 54 +- databases/rethink_db.ts | 91 +- databases/rusty_db.ts | 115 ++- databases/sqlite_db.ts | 58 +- databases/surrealdb_db.ts | 308 +++---- docker-compose.yml | 84 +- index.ts | 118 +-- lib/AbstractDatabase.ts | 16 +- lib/logging.ts | 32 +- package.json | 132 +-- pnpm-workspace.yaml | 2 +- rolldown.config.mjs | 14 +- test/cassandra/test.cassandra.spec.ts | 41 +- test/couch/test.couch.spec.ts | 91 +- test/dirty/test.dirty.spec.ts | 10 +- test/elasticsearch/test.elasticsearch.spec.ts | 364 ++++---- test/lib/databases.ts | 84 +- test/lib/test_lib.ts | 789 +++++++++--------- test/memory/test.memory.spec.ts | 19 +- test/memory/test_getSub.spec.ts | 24 +- test/memory/test_memory.spec.ts | 30 +- test/memory/test_tojson.spec.ts | 32 +- test/mock/test_bulk.spec.ts | 73 +- test/mock/test_findKeys.spec.ts | 40 +- test/mock/test_flush.spec.ts | 55 +- test/mock/test_lru.spec.ts | 158 ++-- test/mock/test_setSub.spec.ts | 14 +- test/mongodb/test.spec.ts | 28 +- test/mysql/test.mysql.spec.ts | 49 +- test/mysql/test_mysql.spec.ts | 149 ++-- test/postgres/test.postgresql.spec.ts | 122 ++- test/redis/test.redis.spec.ts | 35 +- test/rethinkdb/rethinkdb.spec.ts | 35 +- test/rusty/test.rusty.spec.ts | 10 +- test/rusty/test_rusty.spec.ts | 228 +++-- test/sqlite/test.sqlite.spec.ts | 10 +- test/surrealdb/test.surrealdb.spec.ts | 80 +- tsconfig.json | 24 +- vitest.config.ts | 30 +- 58 files changed, 2802 insertions(+), 2639 deletions(-) diff --git a/.eslintrc.cjs b/.eslintrc.cjs index dfba7b77..01cfb868 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -1,64 +1,55 @@ -'use strict'; +"use strict"; // This is a workaround for https://github.com/eslint/eslint/issues/3458 -require('eslint-config-etherpad/patch/modern-module-resolution'); +require("eslint-config-etherpad/patch/modern-module-resolution"); module.exports = { parserOptions: { - project: ['./tsconfig.json'], + project: ["./tsconfig.json"], }, root: true, - extends: 'etherpad/node', + extends: "etherpad/node", rules: { - 'mocha/no-exports': 'off', - '@typescript-eslint/no-unsafe-call': 'off', - 'max-len': 'off', - '@typescript-eslint/restrict-template-expressions': 'off', - '@typescript-eslint/no-unsafe-member-access': 'off', - 'n/no-missing-import': 'off', - 'strict': 'off', - '@typescript-eslint/no-unsafe-return': 'off', - '@typescript-eslint/no-explicit-any': 'off', - '@typescript-eslint/no-unsafe-argument': 'off', - '@typescript-eslint/no-unsafe-assignment': 'off', - 'prefer-arrow/prefer-arrow-functions': 'off', - '@typescript-eslint/await-thenable': 'off', - '@typescript-eslint/brace-style': 'off', - '@typescript-eslint/comma-spacing': 'off', - '@typescript-eslint/consistent-type-assertions': 'off', - '@typescript-eslint/consistent-type-definitions': 'off', - '@typescript-eslint/default-param-last': 'off', - '@typescript-eslint/dot-notation': 'off', - '@typescript-eslint/explicit-function-return-type': 'off', - '@typescript-eslint/explicit-member-accessibility': 'off', - 'func-call-spacing': 'off', - '@typescript-eslint/no-floating-promises': 'off', - 'camelcase': 'off', - 'n/no-unpublished-import': 'off', - 'n/no-unpublished-require': 'off', - 'no-unused-vars': 'off', - '@typescript-eslint/no-unused-vars': 'off', - '@typescript-eslint/ban-ts-comment': 'off', - '@typescript-eslint/restrict-plus-operands': 'off', + "mocha/no-exports": "off", + "@typescript-eslint/no-unsafe-call": "off", + "max-len": "off", + "@typescript-eslint/restrict-template-expressions": "off", + "@typescript-eslint/no-unsafe-member-access": "off", + "n/no-missing-import": "off", + strict: "off", + "@typescript-eslint/no-unsafe-return": "off", + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-unsafe-argument": "off", + "@typescript-eslint/no-unsafe-assignment": "off", + "prefer-arrow/prefer-arrow-functions": "off", + "@typescript-eslint/await-thenable": "off", + "@typescript-eslint/brace-style": "off", + "@typescript-eslint/comma-spacing": "off", + "@typescript-eslint/consistent-type-assertions": "off", + "@typescript-eslint/consistent-type-definitions": "off", + "@typescript-eslint/default-param-last": "off", + "@typescript-eslint/dot-notation": "off", + "@typescript-eslint/explicit-function-return-type": "off", + "@typescript-eslint/explicit-member-accessibility": "off", + "func-call-spacing": "off", + "@typescript-eslint/no-floating-promises": "off", + camelcase: "off", + "n/no-unpublished-import": "off", + "n/no-unpublished-require": "off", + "no-unused-vars": "off", + "@typescript-eslint/no-unused-vars": "off", + "@typescript-eslint/ban-ts-comment": "off", + "@typescript-eslint/restrict-plus-operands": "off", }, overrides: [ { - files: [ - 'lib/**/*', - 'databases/**/*', - 'tests/**/*', - ], - extends: 'etherpad/tests/backend', + files: ["lib/**/*", "databases/**/*", "tests/**/*"], + extends: "etherpad/tests/backend", overrides: [ { - files: [ - 'lib/**/*', - 'databases/**/*', - 'tests/**/*', - ], - + files: ["lib/**/*", "databases/**/*", "tests/**/*"], }, ], }, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 589d25ff..c074fc86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,4 +45,3 @@ jobs: - name: Build run: pnpm run build - diff --git a/.github/workflows/npmpublish.yml b/.github/workflows/npmpublish.yml index 764ab030..9a4d2688 100644 --- a/.github/workflows/npmpublish.yml +++ b/.github/workflows/npmpublish.yml @@ -16,7 +16,8 @@ jobs: # PR ends up red even when only a single integration container is flaky. fail-fast: false matrix: - db: [couch, dirty, elasticsearch, mongo, mysql, redis, mock, sqlite, memory, rusty, surrealdb] + db: + [couch, dirty, elasticsearch, mongo, mysql, redis, mock, sqlite, memory, rusty, surrealdb] steps: - uses: actions/checkout@v6 @@ -54,7 +55,7 @@ jobs: publish-npm: if: github.event_name == 'push' needs: - - test + - test runs-on: ubuntu-latest # OIDC trusted publishing: npm exchanges the GitHub-issued `id-token` # for a short-lived publish credential at the registry, so there is no @@ -76,7 +77,7 @@ jobs: - uses: actions/setup-node@v6 with: node-version: 24 - registry-url: 'https://registry.npmjs.org' + registry-url: "https://registry.npmjs.org" - name: Install pnpm uses: pnpm/action-setup@v5 @@ -100,7 +101,7 @@ jobs: # This is required if the package has a prepare script that uses something # in dependencies or devDependencies. This is also needed for bumping the # version. - - run: pnpm install --frozen-lockfile # Workaround based on https://github.com/pnpm/pnpm/issues/3141 + - run: pnpm install --frozen-lockfile # Workaround based on https://github.com/pnpm/pnpm/issues/3141 - name: Bump version (patch) run: | diff --git a/.travis.yml b/.travis.yml index 3dfc2cf2..bdbc2521 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,12 +21,12 @@ before_install: - mysql -e "ALTER DATABASE ueberdb CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;" - mysql -e "grant CREATE,ALTER,SELECT,INSERT,UPDATE,DELETE on ueberdb.* to 'ueberdb'@'localhost';" - mysql -e "CREATE TABLE \`store\` (\`key\` varchar(100) COLLATE utf8mb4_bin NOT NULL DEFAULT '',\`value\` longtext COLLATE utf8mb4_bin NOT NULL,PRIMARY KEY (\`key\`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;" ueberdb -# - cp /etc/rethinkdb/default.conf.sample /etc/rethinkdb/instances.d/instance1.conf + # - cp /etc/rethinkdb/default.conf.sample /etc/rethinkdb/instances.d/instance1.conf -#addons: -# rethinkdb: '2.3.4' + #addons: + # rethinkdb: '2.3.4' -# Brings in procedure required for creating large data set + # Brings in procedure required for creating large data set - mysql ueberdb < test/lib/mysql.sql - mysql ueberdb -e 'CALL generate_data();'; - psql -c 'create database ueberdb;' -U postgres diff --git a/CHANGELOG.md b/CHANGELOG.md index c5b01859..2787057d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,224 +4,226 @@ Security fix: - * `getSub()` now returns `null` when it encounters a non-"own" property - (including `__proto__`) or any non-object while walking the given property - path. This should make it easier to avoid accidental prototype pollution - vulnerabilities. +- `getSub()` now returns `null` when it encounters a non-"own" property + (including `__proto__`) or any non-object while walking the given property + path. This should make it easier to avoid accidental prototype pollution + vulnerabilities. ## v4.0.0 Compatibility changes: - * `redis`: The `socket` and `client_options` settings, deprecated since - v1.3.1, have been removed. - * `redis`: The client configuration object has changed with the new version of - the `redis` client library. See the [`redis` client library - documentation](https://github.com/redis/node-redis/blob/redis%404.1.0/docs/client-configuration.md) - for details. +- `redis`: The `socket` and `client_options` settings, deprecated since + v1.3.1, have been removed. +- `redis`: The client configuration object has changed with the new version of + the `redis` client library. See the [`redis` client library + documentation](https://github.com/redis/node-redis/blob/redis%404.1.0/docs/client-configuration.md) + for details. Bug fixes: - * `redis`: Several `findKeys()` fixes. +- `redis`: Several `findKeys()` fixes. Updated database dependencies: - * `redis`: Updated `redis` from 3.1.2 to 4.1.0. +- `redis`: Updated `redis` from 3.1.2 to 4.1.0. ## v3.0.2 Security fix: - * `getSub()` now returns `null` when it encounters a non-"own" property - (including `__proto__`) or any non-object while walking the given property - path. This should make it easier to avoid accidental prototype pollution - vulnerabilities. +- `getSub()` now returns `null` when it encounters a non-"own" property + (including `__proto__`) or any non-object while walking the given property + path. This should make it easier to avoid accidental prototype pollution + vulnerabilities. ## v3.0.1 Bug fixes: - * Fixed `findKeys()` calls containing special regular expression characters - (applicable to the database drivers that use the glob-to-regex helper - function). +- Fixed `findKeys()` calls containing special regular expression characters + (applicable to the database drivers that use the glob-to-regex helper + function). ## v3.0.0 Compatibility changes: - * Minimum supported Node.js version is now 14.15.0. - * `elasticsearch`: New index name and mapping (schema). To automatically copy - existing data to the new index when the ueberdb client is initialized, set - the `migrate_to_newer_schema` option to `true`. - * As mentioned in the v2.2.0 changes, passing callbacks to the database - methods is deprecated. Use the returned Promises instead. - * `postgrespool`: As mentioned in the v1.4.15 changes, `postgrespool` is - deprecated. Use `postgres` instead. - * `redis`: As mentioned in the v1.3.1 changes, the `socket` and - `client_options` settings are deprecated. Pass the [client options - object](https://www.npmjs.com/package/redis/v/3.1.2#options-object-properties) - directly. +- Minimum supported Node.js version is now 14.15.0. +- `elasticsearch`: New index name and mapping (schema). To automatically copy + existing data to the new index when the ueberdb client is initialized, set + the `migrate_to_newer_schema` option to `true`. +- As mentioned in the v2.2.0 changes, passing callbacks to the database + methods is deprecated. Use the returned Promises instead. +- `postgrespool`: As mentioned in the v1.4.15 changes, `postgrespool` is + deprecated. Use `postgres` instead. +- `redis`: As mentioned in the v1.3.1 changes, the `socket` and + `client_options` settings are deprecated. Pass the [client options + object](https://www.npmjs.com/package/redis/v/3.1.2#options-object-properties) + directly. Bug fixes: - * `elasticsearch`: Rewrote driver to fix numerous bugs and modernize the code. +- `elasticsearch`: Rewrote driver to fix numerous bugs and modernize the code. Updated database dependencies: - * `couch`: Updated `nano` to 10.0.0. - * `dirty_git`: Updated `simple-git` to 3.7.1. - * `elasticsearch`: Switched the client library from the deprecated - `elasticsearch` to `@elastic/elasticsearch` version 7.17.0. - * `postgres`: Updated `pg` to 8.7.3. - * `sqlite`: Updated `sqlite3` to 5.0.6. +- `couch`: Updated `nano` to 10.0.0. +- `dirty_git`: Updated `simple-git` to 3.7.1. +- `elasticsearch`: Switched the client library from the deprecated + `elasticsearch` to `@elastic/elasticsearch` version 7.17.0. +- `postgres`: Updated `pg` to 8.7.3. +- `sqlite`: Updated `sqlite3` to 5.0.6. ## v2.2.4 Security fix: - * `getSub()` now returns `null` when it encounters a non-"own" property - (including `__proto__`) or any non-object while walking the given property - path. This should make it easier to avoid accidental prototype pollution - vulnerabilities. +- `getSub()` now returns `null` when it encounters a non-"own" property + (including `__proto__`) or any non-object while walking the given property + path. This should make it easier to avoid accidental prototype pollution + vulnerabilities. ## v2.2.0 Compatibility changes: - * Passing callbacks to the database methods is deprecated; use the returned - Promises instead. +- Passing callbacks to the database methods is deprecated; use the returned + Promises instead. New features: - * Database methods now return a Promise if a callback is not provided. +- Database methods now return a Promise if a callback is not provided. Bug fixes: - * A call to `flush()` immediately after a call to `set()`, `setSub()`, or - `remove()` (within the same ECMAScript macro- or microtask) now flushes the - new write operation. - * Fixed a bug where `findKeys()` would return stale results when write - buffering is enabled and writes are pending. - * `couch`: Rewrote driver to fix numerous bugs. +- A call to `flush()` immediately after a call to `set()`, `setSub()`, or + `remove()` (within the same ECMAScript macro- or microtask) now flushes the + new write operation. +- Fixed a bug where `findKeys()` would return stale results when write + buffering is enabled and writes are pending. +- `couch`: Rewrote driver to fix numerous bugs. ## v2.1.1 Security fix: - * Fix `setSub()` prototype pollution vulnerability. +- Fix `setSub()` prototype pollution vulnerability. ## v2.1.0 - * `memory`: New `data` setting that allows users to supply the backing Map - object (rather than create a new Map). +- `memory`: New `data` setting that allows users to supply the backing Map + object (rather than create a new Map). Updated database dependencies: - * `dirty_git`: Updated `simple-git` to 3.6.0. - * `mssql`: Updated `mssql` to 8.1.0. +- `dirty_git`: Updated `simple-git` to 3.6.0. +- `mssql`: Updated `mssql` to 8.1.0. ## v2.0.0 -* When saving an object that has a `.toJSON()` method, the value returned from +- When saving an object that has a `.toJSON()` method, the value returned from that method is saved to the database instead of the object itself. This matches [the behavior of `JSON.stringify()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#tojson_behavior). The `.toJSON()` method is used even if the chosen database driver never actually converts anything to JSON. -* New `memory` database driver that stores values in memory only. +- New `memory` database driver that stores values in memory only. ## v1.4.19 Updated database (and other) dependencies: -* `mongodb`: Updated `mongodb` to 3.7.3. -* `mssql`: Updated `mssql` to 7.3.0. -* `dirty_git`: Updated `simple-git` to 2.47.0. + +- `mongodb`: Updated `mongodb` to 3.7.3. +- `mssql`: Updated `mssql` to 7.3.0. +- `dirty_git`: Updated `simple-git` to 2.47.0. ## v1.4.16 -* `postgres`: You can now provide a connection string instead of a settings +- `postgres`: You can now provide a connection string instead of a settings object. For example: ```javascript - const db = new ueberdb.Database('postgres', 'postgres://user:password@host/dbname'); + const db = new ueberdb.Database("postgres", "postgres://user:password@host/dbname"); ``` ## v1.4.15 -* `postgres`, `postgrespool`: The `postgrespool` database driver was renamed to +- `postgres`, `postgrespool`: The `postgrespool` database driver was renamed to `postgres`, replacing the old `postgres` driver. The old `postgrespool` name is still usable, but is deprecated. For users of the old `postgres` driver, this change increases the number of concurrent database connections. You may need to increase your configured connection limit. -* `sqlite`: Updated `sqlite3` to 5.0.2. +- `sqlite`: Updated `sqlite3` to 5.0.2. ## v1.4.14 Updated dependencies: -* `cassandra`: Updated `cassandra-driver` to 4.6.3. -* `couch`: Updated `nano` to 9.0.3. -* `dirty`: Updated `dirty` to 1.1.3. -* `dirty_git`: Updated `simple-git` to 2.45.0. -* `mongodb`: Updated `mongodb` to 3.6.11. -* `mssql`: Updated `mssql` to 7.2.1. -* `postgres`, `postgrespool`: Updated `pg` to 8.7.1. + +- `cassandra`: Updated `cassandra-driver` to 4.6.3. +- `couch`: Updated `nano` to 9.0.3. +- `dirty`: Updated `dirty` to 1.1.3. +- `dirty_git`: Updated `simple-git` to 2.45.0. +- `mongodb`: Updated `mongodb` to 3.6.11. +- `mssql`: Updated `mssql` to 7.2.1. +- `postgres`, `postgrespool`: Updated `pg` to 8.7.1. ## v1.4.13 -* `mongodb`: The `dbName` setting has been renamed to `database` for consistency +- `mongodb`: The `dbName` setting has been renamed to `database` for consistency with other database drivers. The `dbName` setting will continue to work (for backwards compatibility), but it is deprecated and is ignored if `database` is set. -* `mongodb`: The `database` (formerly `dbName`) setting is now optional. If it +- `mongodb`: The `database` (formerly `dbName`) setting is now optional. If it is not specified, the database name embedded in the `url` setting is used. ## v1.4.8 -* `redis`: Updated `redis` dependency to 3.1.2. +- `redis`: Updated `redis` dependency to 3.1.2. ## v1.4.7 -* Each write operation in a bulk write batch is now retried if the bulk write +- Each write operation in a bulk write batch is now retried if the bulk write fails. -* Fixed write metrics for `setSub()` read failures. +- Fixed write metrics for `setSub()` read failures. ## v1.4.6 -* `mysql`: Use a connection pool to improve performance and simplify the code. +- `mysql`: Use a connection pool to improve performance and simplify the code. ## v1.4.5 -* `mysql`: Reconnect on fatal error. -* `mysql`: Log MySQL errors. +- `mysql`: Reconnect on fatal error. +- `mysql`: Log MySQL errors. ## v1.4.4 -* New experimental setting to limit the number of operations written at a time +- New experimental setting to limit the number of operations written at a time when flushing outstanding writes. -* `mysql`: Bulk writes are limited to 100 changes at a time to avoid query +- `mysql`: Bulk writes are limited to 100 changes at a time to avoid query timeouts. -* `mysql`: Raised default cache size from 500 entries to 10000. +- `mysql`: Raised default cache size from 500 entries to 10000. ## v1.4.2 -* Refined the experimental read and write metrics. +- Refined the experimental read and write metrics. ## v1.4.1 -* The two callback arguments in `remove()`, `set()`, and `setSub()` have +- The two callback arguments in `remove()`, `set()`, and `setSub()` have changed: Instead of a callback that is called after the write is buffered and another callback that is called after the write is committed, both callbacks are now called after the write is committed. Futhermore, the second callback argument is now deprecated. -* Modernized record locking. -* Experimental metrics for reads, writes, and locking. +- Modernized record locking. +- Experimental metrics for reads, writes, and locking. ## v1.3.2 -* `dirty`: Updated `dirty` dependency. +- `dirty`: Updated `dirty` dependency. ## v1.3.1 -* `redis`: The database config object is now passed directly to the `redis` +- `redis`: The database config object is now passed directly to the `redis` package. For details, see https://www.npmjs.com/package/redis/v/3.0.2#options-object-properties. Old-style settings objects (where the `redis` options are in the @@ -229,75 +231,75 @@ Updated dependencies: ## v1.2.9 -* `dirty`: Workaround for a bug in the upstream `dirty` driver. +- `dirty`: Workaround for a bug in the upstream `dirty` driver. ## v1.2.7 -* `redis`: Experimental support for passing the settings object directly to the +- `redis`: Experimental support for passing the settings object directly to the `redis` package. ## v1.2.6 -* `redis`: Fixed "Callback was already called" exception during init. +- `redis`: Fixed "Callback was already called" exception during init. ## v1.2.5 -* All: Fixed a major bug introduced in v1.1.10 that caused `setSub()` to +- All: Fixed a major bug introduced in v1.1.10 that caused `setSub()` to silently discard changes. -* All: Fixed a bug that prevented cache entries from being marked as most +- All: Fixed a bug that prevented cache entries from being marked as most recently used. ## v1.2.4 -* `mssql`: Updated `mssql` dependency. -* `dirty_git`: Updated `simple-git` dependency. -* `sqlite`: Updated `sqlite3` dependency. +- `mssql`: Updated `mssql` dependency. +- `dirty_git`: Updated `simple-git` dependency. +- `sqlite`: Updated `sqlite3` dependency. ## v1.2.3 -* `mssql`: Updated `mssql` dependency. +- `mssql`: Updated `mssql` dependency. ## v1.2.2 -* All: Fixed minor `setSub()` corner cases. +- All: Fixed minor `setSub()` corner cases. ## v1.2.1 -* New `flush()` method. -* The `doShutdown()` method is deprecated. Use `flush()` instead. -* The `close()` method now flushes unwritten entries before closing the database +- New `flush()` method. +- The `doShutdown()` method is deprecated. Use `flush()` instead. +- The `close()` method now flushes unwritten entries before closing the database connection. -* Bug fix: `null`/`undefined` is no longer cached if there is an error reading +- Bug fix: `null`/`undefined` is no longer cached if there is an error reading from the database. ## v1.1.10 -* Major performance improvement: The caching logic was rewritten with much more +- Major performance improvement: The caching logic was rewritten with much more efficient algorithms. Also: Scans for entries to evict is performed less often. Depending on your workload you might observe a slight memory usage increase. ## v1.1.7 -* `mysql` dependency bumped to 7.0.0-alpha4 to avoid a security vulnerability in +- `mysql` dependency bumped to 7.0.0-alpha4 to avoid a security vulnerability in one of its indirect dependencies. ## v1.1.6 -* Bug fix: When write buffering is disabled, reads of keys with values that were +- Bug fix: When write buffering is disabled, reads of keys with values that were changed but not yet written to the underlying database used to return the previous value. Now the updated value is returned. -* Minor performance improvement: Setting a key to the same value no longer +- Minor performance improvement: Setting a key to the same value no longer triggers a database write. ## v1.1.5 -* Minor performance improvement: Debug log message strings are no longer +- Minor performance improvement: Debug log message strings are no longer generated if debug logging is not enabled. ## v1.1.1 -* The `database()` constructor is deprecated; use `Database()` instead. +- The `database()` constructor is deprecated; use `Database()` instead. ## Older diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6be662c3..f102dc4e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,103 +1,115 @@ # Contributor Guidelines + (Please talk to people on the mailing list before you change this page, see our section on [how to get in touch](https://github.com/ether/etherpad-lite#get-in-touch)) ## Pull requests -* the commit series in the PR should be _linear_ (it **should not contain merge commits**). This is necessary because we want to be able to [bisect](https://en.wikipedia.org/wiki/Bisection_(software_engineering)) bugs easily. Rewrite history/perform a rebase if necessary -* PRs should be issued against the **develop** branch: we never pull directly into **master** -* PRs **should not have conflicts** with develop. If there are, please resolve them rebasing and force-pushing -* when preparing your PR, please make sure that you have included the relevant **changes to the documentation** (preferably with usage examples) -* contain meaningful and detailed **commit messages** in the form: +- the commit series in the PR should be _linear_ (it **should not contain merge commits**). This is necessary because we want to be able to [bisect]() bugs easily. Rewrite history/perform a rebase if necessary +- PRs should be issued against the **develop** branch: we never pull directly into **master** +- PRs **should not have conflicts** with develop. If there are, please resolve them rebasing and force-pushing +- when preparing your PR, please make sure that you have included the relevant **changes to the documentation** (preferably with usage examples) +- contain meaningful and detailed **commit messages** in the form: + ``` submodule: description longer description of the change you have made, eventually mentioning the number of the issue that is being fixed, in the form: Fixes #someIssueNumber ``` -* if the PR is a **bug fix**: - * the first commit in the series must be a test that shows the failure - * subsequent commits will fix the bug and make the test pass - * the final commit message should include the text `Fixes: #xxx` to link it to its bug report -* think about stability: code has to be backwards compatible as much as possible. Always **assume your code will be run with an older version of the DB/config file** -* if you want to remove a feature, **deprecate it instead**: - * write an issue with your deprecation plan - * output a `WARN` in the log informing that the feature is going to be removed - * remove the feature in the next version -* if you want to add a new feature, put it under a **feature flag**: - * once the new feature has reached a minimal level of stability, do a PR for it, so it can be integrated early - * expose a mechanism for enabling/disabling the feature - * the new feature should be **disabled** by default. With the feature disabled, the code path should be exactly the same as before your contribution. This is a __necessary condition__ for early integration -* think of the PR not as something that __you wrote__, but as something that __someone else is going to read__. The commit series in the PR should tell a novice developer the story of your thoughts when developing it + +- if the PR is a **bug fix**: + - the first commit in the series must be a test that shows the failure + - subsequent commits will fix the bug and make the test pass + - the final commit message should include the text `Fixes: #xxx` to link it to its bug report +- think about stability: code has to be backwards compatible as much as possible. Always **assume your code will be run with an older version of the DB/config file** +- if you want to remove a feature, **deprecate it instead**: + - write an issue with your deprecation plan + - output a `WARN` in the log informing that the feature is going to be removed + - remove the feature in the next version +- if you want to add a new feature, put it under a **feature flag**: + - once the new feature has reached a minimal level of stability, do a PR for it, so it can be integrated early + - expose a mechanism for enabling/disabling the feature + - the new feature should be **disabled** by default. With the feature disabled, the code path should be exactly the same as before your contribution. This is a **necessary condition** for early integration +- think of the PR not as something that **you wrote**, but as something that **someone else is going to read**. The commit series in the PR should tell a novice developer the story of your thoughts when developing it ## How to write a bug report -* Please be polite, we all are humans and problems can occur. -* Please add as much information as possible, for example - * client os(s) and version(s) - * browser(s) and version(s), is the problem reproducible on different clients - * special environments like firewalls or antivirus - * host os and version - * npm and nodejs version - * Logfiles if available - * steps to reproduce - * what you expected to happen - * what actually happened -* Please format logfiles and code examples with markdown see github Markdown help below the issue textarea for more information. +- Please be polite, we all are humans and problems can occur. +- Please add as much information as possible, for example + - client os(s) and version(s) + - browser(s) and version(s), is the problem reproducible on different clients + - special environments like firewalls or antivirus + - host os and version + - npm and nodejs version + - Logfiles if available + - steps to reproduce + - what you expected to happen + - what actually happened +- Please format logfiles and code examples with markdown see github Markdown help below the issue textarea for more information. ## General goals of UeberDB + To make sure everybody is going in the same direction: -* easy to install for admins and easy to use for people -* easy to integrate into other apps, but also usable as standalone -* lightweight and scalable -* extensible, as much functionality should be extendable with plugins so changes don't have to be done in core. + +- easy to install for admins and easy to use for people +- easy to integrate into other apps, but also usable as standalone +- lightweight and scalable +- extensible, as much functionality should be extendable with plugins so changes don't have to be done in core. ## How to work with git? -* Don't work in your master branch. -* Make a new branch for every feature you're working on. (This ensures that you can work you can do lots of small, independent pull requests instead of one big one with complete different features) -* Don't use the online edit function of github (this only creates ugly and not working commits!) -* Try to make clean commits that are easy readable (including descriptive commit messages!) -* Test before you push. Sounds easy, it isn't! -* Don't check in stuff that gets generated during build or runtime -* Make small pull requests that are easy to review but make sure they do add value by themselves / individually + +- Don't work in your master branch. +- Make a new branch for every feature you're working on. (This ensures that you can work you can do lots of small, independent pull requests instead of one big one with complete different features) +- Don't use the online edit function of github (this only creates ugly and not working commits!) +- Try to make clean commits that are easy readable (including descriptive commit messages!) +- Test before you push. Sounds easy, it isn't! +- Don't check in stuff that gets generated during build or runtime +- Make small pull requests that are easy to review but make sure they do add value by themselves / individually ## Coding style -* Do write comments. (You don't have to comment every line, but if you come up with something that's a bit complex/weird, just leave a comment. Bear in mind that you will probably leave the project at some point and that other people will read your code. Undocumented huge amounts of code are worthless!) -* Never ever use tabs -* Indentation: JS/CSS: 2 spaces; HTML: 4 spaces -* Don't overengineer. Don't try to solve any possible problem in one step, but try to solve problems as easy as possible and improve the solution over time! -* Do generalize sooner or later! (if an old solution, quickly hacked together, poses more problems than it solves today, refactor it!) -* Keep it compatible. Do not introduce changes to the public API, db schema or configurations too lightly. Don't make incompatible changes without good reasons! -* If you do make changes, document them! (see below) -* Use protocol independent urls "//" + +- Do write comments. (You don't have to comment every line, but if you come up with something that's a bit complex/weird, just leave a comment. Bear in mind that you will probably leave the project at some point and that other people will read your code. Undocumented huge amounts of code are worthless!) +- Never ever use tabs +- Indentation: JS/CSS: 2 spaces; HTML: 4 spaces +- Don't overengineer. Don't try to solve any possible problem in one step, but try to solve problems as easy as possible and improve the solution over time! +- Do generalize sooner or later! (if an old solution, quickly hacked together, poses more problems than it solves today, refactor it!) +- Keep it compatible. Do not introduce changes to the public API, db schema or configurations too lightly. Don't make incompatible changes without good reasons! +- If you do make changes, document them! (see below) +- Use protocol independent urls "//" ## Branching model / git workflow + see git flow http://nvie.com/posts/a-successful-git-branching-model/ ### `master` branch -* the stable -* This is the branch everyone should use for production stuff + +- the stable +- This is the branch everyone should use for production stuff ### feature branches (in your own repos) -* these are the branches where you develop your features in -* If it's ready to go out, it will be merged into develop + +- these are the branches where you develop your features in +- If it's ready to go out, it will be merged into develop Over the time we pull features from feature branches into the develop branch. Every month we pull from develop into master. Bugs in master get fixed in hotfix branches. These branches will get merged into master AND develop. There should never be commits in master that aren't in develop ## Testing + Front-end tests are found in the `tests/frontend/` folder in the repository. Run them by pointing your browser to `/tests/frontend`. Back-end tests can be run via `npm test`. ## Things you can help with -The Etherpad Foundation is much more than software. So if you aren't a developer then worry not, there is still a LOT you can do! A big part of what we do is community engagement. You can help in the following ways - * Triage bugs (applying labels) and confirming their existence - * Testing fixes (simply applying them and seeing if it fixes your issue or not) - Some git experience required - * Notifying large site admins of new releases - * Writing Changelogs for releases - * Creating Windows packages - * Creating releases - * Bumping dependencies periodically and checking they don't break anything - * Write proposals for grants - * Co-Author and Publish CVEs - * Work with SFC to maintain legal side of project +The Etherpad Foundation is much more than software. So if you aren't a developer then worry not, there is still a LOT you can do! A big part of what we do is community engagement. You can help in the following ways + +- Triage bugs (applying labels) and confirming their existence +- Testing fixes (simply applying them and seeing if it fixes your issue or not) - Some git experience required +- Notifying large site admins of new releases +- Writing Changelogs for releases +- Creating Windows packages +- Creating releases +- Bumping dependencies periodically and checking they don't break anything +- Write proposals for grants +- Co-Author and Publish CVEs +- Work with SFC to maintain legal side of project diff --git a/README.md b/README.md index 30cb0354..fdbc23b9 100644 --- a/README.md +++ b/README.md @@ -14,21 +14,22 @@ writes are done in a bulk. This can be turned off. ## Database Support -* Couch -* Dirty -* Elasticsearch -* Maria -* `memory`: An in-memory ephemeral database. -* Mongo -* MsSQL -* MySQL -* Postgres (single connection and with connection pool) -* Redis -* Rethink -* `rustydb` -* SQLite -* Surrealdb -* +- Couch +- Dirty +- Elasticsearch +- Maria +- `memory`: An in-memory ephemeral database. +- Mongo +- MsSQL +- MySQL +- Postgres (single connection and with connection pool) +- Redis +- Rethink +- `rustydb` +- SQLite +- Surrealdb +- + ## Install ``` @@ -40,24 +41,24 @@ npm install ueberdb2 ### Basic ```javascript -const ueberdb = require('ueberdb2'); +const ueberdb = require("ueberdb2"); (async () => { // mysql - const db = new ueberdb.Database('mysql', { - user: 'root', - host: 'localhost', - password: '', - database: 'store', - engine: 'InnoDB', + const db = new ueberdb.Database("mysql", { + user: "root", + host: "localhost", + password: "", + database: "store", + engine: "InnoDB", }); // dirty to file system //const db = new ueberdb.Database('dirty', {filename: 'var/dirty.db'}); await db.init(); try { - await db.set('valueA', {a: 1, b: 2}); - console.log('valueA is', await db.get('valueA')); + await db.set("valueA", { a: 1, b: 2 }); + console.log("valueA is", await db.get("valueA")); } finally { await db.close(); } @@ -67,19 +68,19 @@ const ueberdb = require('ueberdb2'); ### findKeys ```javascript -const ueberdb = require('ueberdb2'); +const ueberdb = require("ueberdb2"); (async () => { - const db = new ueberdb.Database('dirty', {filename: 'var/dirty.db'}); + const db = new ueberdb.Database("dirty", { filename: "var/dirty.db" }); await db.init(); try { await Promise.all([ - db.set('valueA', {a: 1, b: 2}), - db.set('valueA:h1', {a: 1, b: 2}), - db.set('valueA:h2', {a: 3, b: 4}), + db.set("valueA", { a: 1, b: 2 }), + db.set("valueA:h1", { a: 1, b: 2 }), + db.set("valueA:h2", { a: 3, b: 4 }), ]); // prints [ 'valueA:h1', 'valueA:h2' ] - console.log(await db.findKeys('valueA:*', null)); + console.log(await db.findKeys("valueA:*", null)); } finally { await db.close(); } @@ -95,18 +96,18 @@ sweep OOMed the host. `findKeysPaged()` walks the same keyspace in fixed-size pages using an exclusive `after` cursor: ```javascript -const ueberdb = require('ueberdb2'); +const ueberdb = require("ueberdb2"); (async () => { - const db = new ueberdb.Database('mysql', settings); + const db = new ueberdb.Database("mysql", settings); await db.init(); try { let after; let total = 0; while (true) { - const page = await db.findKeysPaged('sessionstorage:*', null, { + const page = await db.findKeysPaged("sessionstorage:*", null, { limit: 500, - ...(after != null ? {after} : {}), + ...(after != null ? { after } : {}), }); if (page.length === 0) break; total += page.length; @@ -130,8 +131,8 @@ Semantics: - `limit` must be a positive integer; non-positive or non-integer values throw. - Native implementations: **mysql** (ranged `BINARY \`key\` > ?`), - **postgres** (`key > $n`). All other backends fall back to - `findKeys() + JS-side slicing` via the cache layer — correct, but +**postgres** (`key > $n`). All other backends fall back to +`findKeys() + JS-side slicing` via the cache layer — correct, but defeats the OOM-mitigation purpose. PRs for native paged paths on other backends welcome. @@ -159,24 +160,24 @@ exist or if the given property path does not exist. Examples: ```javascript -(async () => { - await db.set(key, {prop1: {prop2: ['value']}}); +async () => { + await db.set(key, { prop1: { prop2: ["value"] } }); - const val1 = await db.getSub(key, ['prop1', 'prop2', '0']); - console.log('1.', val1); // prints "1. value" + const val1 = await db.getSub(key, ["prop1", "prop2", "0"]); + console.log("1.", val1); // prints "1. value" - const val2 = await db.getSub(key, ['prop1', 'prop2']); - console.log('2.', val2); // prints "2. [ 'value' ]" + const val2 = await db.getSub(key, ["prop1", "prop2"]); + console.log("2.", val2); // prints "2. [ 'value' ]" - const val3 = await db.getSub(key, ['prop1']); - console.log('3.', val3); // prints "3. { prop2: [ 'value' ] }" + const val3 = await db.getSub(key, ["prop1"]); + console.log("3.", val3); // prints "3. { prop2: [ 'value' ] }" const val4 = await db.getSub(key, []); - console.log('4.', val4); // prints "4. { prop1: { prop2: [ 'value' ] } }" + console.log("4.", val4); // prints "4. { prop1: { prop2: [ 'value' ] } }" - const val5 = await db.getSub(key, ['does', 'not', 'exist']); - console.log('5.', val5); // prints "5. null" or "5. undefined" -}); + const val5 = await db.getSub(key, ["does", "not", "exist"]); + console.log("5.", val5); // prints "5. null" or "5. undefined" +}; ``` #### `setSub` @@ -221,15 +222,14 @@ to the database driver (except for reads of written values that have not yet been committed to the database): ```javascript -const ueberdb = require('ueberdb2'); +const ueberdb = require("ueberdb2"); (async () => { - const db = new ueberdb.Database( - 'dirty', {filename: 'var/dirty.db'}, {cache: 0}); + const db = new ueberdb.Database("dirty", { filename: "var/dirty.db" }, { cache: 0 }); await db.init(); try { - await db.set('valueA', {a: 1, b: 2}); - const value = await db.get('valueA'); + await db.set("valueA", { a: 1, b: 2 }); + const value = await db.get("valueA"); console.log(JSON.stringify(value)); } finally { await db.close(); @@ -243,15 +243,14 @@ Set the `writeInterval` wrapper option to 0 to force writes to go directly to the database driver: ```javascript -const ueberdb = require('ueberdb2'); +const ueberdb = require("ueberdb2"); (async () => { - const db = new ueberdb.Database( - 'dirty', {filename: 'var/dirty.db'}, {writeInterval: 0}); + const db = new ueberdb.Database("dirty", { filename: "var/dirty.db" }, { writeInterval: 0 }); await db.init(); try { - await db.set('valueA', {a: 1, b: 2}); - const value = await db.get('valueA'); + await db.set("valueA", { a: 1, b: 2 }); + const value = await db.get("valueA"); console.log(JSON.stringify(value)); } finally { await db.close(); @@ -262,17 +261,17 @@ const ueberdb = require('ueberdb2'); ## Feature support | | Get | Set | findKeys | findKeysPaged | Remove | getSub | setSub | doBulk | CI Coverage | -|---------------|-----|-----|----------|---------------|--------|--------|--------|--------|-------------| -| cassandra | ✓ | ✓ | * | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| ------------- | --- | --- | -------- | ------------- | ------ | ------ | ------ | ------ | ----------- | +| cassandra | ✓ | ✓ | \* | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | couchdb | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | dirty | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | ✓ | | dirty_git | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | ✓ | -| elasticsearch | ✓ | ✓ | * | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| elasticsearch | ✓ | ✓ | \* | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | maria | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | mysql | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | postgres | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| redis | ✓ | ✓ | * | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| rethinkdb | ✓ | ✓ | * | ✓ | ✓ | ✓ | ✓ | ✓ | +| redis | ✓ | ✓ | \* | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| rethinkdb | ✓ | ✓ | \* | ✓ | ✓ | ✓ | ✓ | ✓ | | rustydb | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | sqlite | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | surrealdb | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | @@ -284,14 +283,14 @@ const ueberdb = require('ueberdb2'); The following characters should be avoided in keys `\^$.|?*+()[{` as they will cause findKeys to fail. -### findKeys database support* +### findKeys database support\* The following have limitations on findKeys -* redis (Only keys of the format \*:\*:\*) -* cassandra (Only keys of the format \*:\*:\*) -* elasticsearch (Only keys of the format \*:\*:\*) -* rethink (Currently doesn't work) +- redis (Only keys of the format \*:\*:\*) +- cassandra (Only keys of the format \*:\*:\*) +- elasticsearch (Only keys of the format \*:\*:\*) +- rethink (Currently doesn't work) For details on how it works please refer to the wiki: https://github.com/ether/UeberDB/wiki/findKeys-functionality @@ -341,7 +340,7 @@ If you enabled TLS on your Redis database (available since Redis 6.0) you will need to change your connections parameters, here is an example: ```javascript -const db = new ueberdb.Database('redis', {url: 'rediss://localhost'}); +const db = new ueberdb.Database("redis", { url: "rediss://localhost" }); ``` Do not provide a `host` value. @@ -371,21 +370,21 @@ environment variable `NODE_TLS_REJECT_UNAUTHORIZED = 0` and add the flag ## What's changed from UeberDB? -* Dropped broken databases: CrateDB, LevelDB, LMDB (probably a +- Dropped broken databases: CrateDB, LevelDB, LMDB (probably a breaking change for some people) -* Introduced CI. -* Introduced better testing. -* Fixed broken database clients IE Redis. -* Updated Depdendencies where possible. -* Tidied file structure. -* Improved documentation. -* Sensible name for software makes it clear that it's maintained by The Etherpad +- Introduced CI. +- Introduced better testing. +- Fixed broken database clients IE Redis. +- Updated Depdendencies where possible. +- Tidied file structure. +- Improved documentation. +- Sensible name for software makes it clear that it's maintained by The Etherpad Foundation. -* Make db.init await / async +- Make db.init await / async ### Dirty_Git Easter Egg. -* I suck at hiding Easter eggs.. +- I suck at hiding Easter eggs.. Dirty_git will `commit` and `push` to Git on every `set`. To use `git init` or `git clone` within your dirty database location and then set your upstream IE diff --git a/databases/cassandra_db.ts b/databases/cassandra_db.ts index e57c8100..0259cd4f 100644 --- a/databases/cassandra_db.ts +++ b/databases/cassandra_db.ts @@ -12,20 +12,19 @@ * limitations under the License. */ -import AbstractDatabase, {type Settings} from '../lib/AbstractDatabase'; -import {Client, types} from 'cassandra-driver'; -import type {ArrayOrObject, ValueCallback} from 'cassandra-driver'; +import AbstractDatabase, { type Settings } from "../lib/AbstractDatabase"; +import { Client, types } from "cassandra-driver"; +import type { ArrayOrObject, ValueCallback } from "cassandra-driver"; type ResultSet = types.ResultSet; - type Result = { rows: any[]; }; export type BulkObject = { - type: string - key:string - value?: string + type: string; + key: string; + value?: string; }; export default class Cassandra_db extends AbstractDatabase { @@ -42,15 +41,15 @@ export default class Cassandra_db extends AbstractDatabase { * the Cassandra driver. See https://github.com/datastax/nodejs-driver#logging for more * information */ - constructor(settings:Settings) { + constructor(settings: Settings) { super(settings); if (!settings.clientOptions) { - throw new Error('The Cassandra client options should be defined'); + throw new Error("The Cassandra client options should be defined"); } if (!settings.columnFamily) { - throw new Error('The Cassandra column family should be defined'); + throw new Error("The Cassandra column family should be defined"); } - this.settings = {database: settings.database}; + this.settings = { database: settings.database }; this.settings.clientOptions = settings.clientOptions; this.settings.columnFamily = settings.columnFamily; this.settings.logger = settings.logger; @@ -63,7 +62,7 @@ export default class Cassandra_db extends AbstractDatabase { * @param {Function} callback Standard callback method. * @param {Error} callback.err An error object (if any.) */ - init(callback: (arg: any)=>{}) { + init(callback: (arg: any) => {}) { // Create a client // eslint-disable-next-line @typescript-eslint/no-explicit-any this.client = new Client(this.settings.clientOptions as any); @@ -71,37 +70,38 @@ export default class Cassandra_db extends AbstractDatabase { // Pass on log messages if a logger has been configured if (this.settings.logger) { // eslint-disable-next-line @typescript-eslint/no-explicit-any - this.client.on('log', (...args: any[]) => (this.settings.logger as any)(...args)); + this.client.on("log", (...args: any[]) => (this.settings.logger as any)(...args)); } // Check whether our column family already exists and create it if necessary this.client.execute( - 'SELECT columnfamily_name FROM system.schema_columnfamilies WHERE keyspace_name = ?', - // eslint-disable-next-line @typescript-eslint/no-explicit-any - [(this.settings.clientOptions as any).keyspace], - (err, result) => { - if (err) { - return callback(err); - } + "SELECT columnfamily_name FROM system.schema_columnfamilies WHERE keyspace_name = ?", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + [(this.settings.clientOptions as any).keyspace], + (err, result) => { + if (err) { + return callback(err); + } - let isDefined = false; - const length = result.rows.length; - for (let i = 0; i < length; i++) { - if (result.rows[i].columnfamily_name === this.settings.columnFamily) { - isDefined = true; - break; - } + let isDefined = false; + const length = result.rows.length; + for (let i = 0; i < length; i++) { + if (result.rows[i].columnfamily_name === this.settings.columnFamily) { + isDefined = true; + break; } + } - if (isDefined) { - return callback(null); - } else { - const cql = - `CREATE COLUMNFAMILY "${this.settings.columnFamily}" ` + - '(key text PRIMARY KEY, data text)'; - this.client && this.client.execute(cql, callback); - } - }); + if (isDefined) { + return callback(null); + } else { + const cql = + `CREATE COLUMNFAMILY "${this.settings.columnFamily}" ` + + "(key text PRIMARY KEY, data text)"; + this.client && this.client.execute(cql, callback); + } + }, + ); } /** @@ -112,19 +112,20 @@ export default class Cassandra_db extends AbstractDatabase { * @param {Error} callback.err An error object, if any * @param {String} callback.value The value for the given key (if any) */ - get(key:string, callback: (err:Error | null, data?:any)=>{}) { + get(key: string, callback: (err: Error | null, data?: any) => {}) { const cql = `SELECT data FROM "${this.settings.columnFamily}" WHERE key = ?`; - this.client && this.client.execute(cql, [key], (err, result) => { - if (err) { - return callback(err); - } + this.client && + this.client.execute(cql, [key], (err, result) => { + if (err) { + return callback(err); + } - if (!result.rows || result.rows.length === 0) { - return callback(null, null); - } + if (!result.rows || result.rows.length === 0) { + return callback(null, null); + } - return callback(null, result.rows[0].data); - }); + return callback(null, result.rows[0].data); + }); } /** @@ -138,29 +139,30 @@ export default class Cassandra_db extends AbstractDatabase { * @param {Error} callback.err An error object, if any * @param {String[]} callback.keys An array of keys that match the specified filters */ - findKeys(key:string, notKey:string, callback: Function) { + findKeys(key: string, notKey: string, callback: Function) { let cql = null; if (!notKey) { // Get all the keys cql = `SELECT key FROM "${this.settings.columnFamily}"`; - this.client && this.client.execute(cql, (err: Error, result:Result) => { - if (err) { - return callback(err); - } + this.client && + this.client.execute(cql, (err: Error, result: Result) => { + if (err) { + return callback(err); + } - // Construct a regular expression based on the given key - const regex = new RegExp(`^${key.replace(/\*/g, '.*')}$`); + // Construct a regular expression based on the given key + const regex = new RegExp(`^${key.replace(/\*/g, ".*")}$`); - const keys:string[] = []; - result.rows.forEach((row) => { - if (regex.test(row.key)) { - keys.push(row.key); - } - }); + const keys: string[] = []; + result.rows.forEach((row) => { + if (regex.test(row.key)) { + keys.push(row.key); + } + }); - return callback(null, keys); - }); - } else if (notKey === '*:*:*') { + return callback(null, keys); + }); + } else if (notKey === "*:*:*") { // restrict key to format 'text:*' const matches = /^([^:]+):\*$/.exec(key); if (matches) { @@ -168,26 +170,25 @@ export default class Cassandra_db extends AbstractDatabase { // We can retrieve them from this column as we're duplicating them on .set/.remove cql = `SELECT * from "${this.settings.columnFamily}" WHERE key = ?`; this.client && - this.client - .execute(cql, [`ueberdb:keys:${matches[1]}`], (err, result) => { - if (err) { - return callback(err); - } + this.client.execute(cql, [`ueberdb:keys:${matches[1]}`], (err, result) => { + if (err) { + return callback(err); + } - if (!result.rows || result.rows.length === 0) { - return callback(null, []); - } + if (!result.rows || result.rows.length === 0) { + return callback(null, []); + } - const keys = result.rows.map((row) => row.data); - return callback(null, keys); - }); + const keys = result.rows.map((row) => row.data); + return callback(null, keys); + }); } else { const msg = - 'Cassandra db only supports key patterns like pad:* when notKey is set to *:*:*'; + "Cassandra db only supports key patterns like pad:* when notKey is set to *:*:*"; return callback(new Error(msg), null); } } else { - return callback(new Error('Cassandra db currently only supports *:*:* as notKey'), null); + return callback(new Error("Cassandra db currently only supports *:*:* as notKey"), null); } } @@ -199,8 +200,8 @@ export default class Cassandra_db extends AbstractDatabase { * @param {Function} callback Standard callback method * @param {Error} callback.err An error object, if any */ - set(key: string, value:string, callback:()=>{}) { - this.doBulk([{type: 'set', key, value}], callback); + set(key: string, value: string, callback: () => {}) { + this.doBulk([{ type: "set", key, value }], callback); } /** @@ -210,11 +211,10 @@ export default class Cassandra_db extends AbstractDatabase { * @param {Function} callback Standard callback method * @param {Error} callback.err An error object, if any */ - remove(key:string, callback: ValueCallback) { - this.doBulk([{type: 'remove', key}], callback); + remove(key: string, callback: ValueCallback) { + this.doBulk([{ type: "remove", key }], callback); } - /** * Performs multiple operations in one action * @@ -222,13 +222,13 @@ export default class Cassandra_db extends AbstractDatabase { * @param {Function} callback Standard callback method * @param {Error} callback.err An error object, if any */ - doBulk(bulk:BulkObject[], callback:ValueCallback) { - const queries:Array = []; + doBulk(bulk: BulkObject[], callback: ValueCallback) { + const queries: Array = []; bulk.forEach((operation) => { // We support finding keys of the form `test:*`. If anything matches, we will try and save // this const matches = /^([^:]+):([^:]+)$/.exec(operation.key); - if (operation.type === 'set') { + if (operation.type === "set") { queries.push({ query: `UPDATE "${this.settings.columnFamily}" SET data = ? WHERE key = ?`, params: [operation.value, operation.key], @@ -237,10 +237,10 @@ export default class Cassandra_db extends AbstractDatabase { if (matches) { queries.push({ query: `UPDATE "${this.settings.columnFamily}" SET data = ? WHERE key = ?`, - params: ['1', `ueberdb:keys:${matches[1]}`], + params: ["1", `ueberdb:keys:${matches[1]}`], }); } - } else if (operation.type === 'remove') { + } else if (operation.type === "remove") { queries.push({ query: `DELETE FROM "${this.settings.columnFamily}" WHERE key=?`, params: [operation.key], @@ -254,7 +254,7 @@ export default class Cassandra_db extends AbstractDatabase { } } }); - this.client && this.client.batch(queries, {prepare: true}, callback); + this.client && this.client.batch(queries, { prepare: true }, callback); } /** @@ -263,7 +263,7 @@ export default class Cassandra_db extends AbstractDatabase { * @param {Function} callback Standard callback method * @param {Error} callback.err Error object in case something goes wrong */ - close(callback: ()=>{}) { + close(callback: () => {}) { this.pool.shutdown(callback); } -}; +} diff --git a/databases/couch_db.ts b/databases/couch_db.ts index fa38cd6a..869a1977 100644 --- a/databases/couch_db.ts +++ b/databases/couch_db.ts @@ -14,9 +14,9 @@ * limitations under the License. */ -import AbstractDatabase, {type Settings} from '../lib/AbstractDatabase'; -import nano from 'nano'; -import type {BulkObject} from './cassandra_db'; +import AbstractDatabase, { type Settings } from "../lib/AbstractDatabase"; +import nano from "nano"; +import type { BulkObject } from "./cassandra_db"; export default class Couch_db extends AbstractDatabase { public db: nano.DocumentScope | null; @@ -32,7 +32,9 @@ export default class Couch_db extends AbstractDatabase { this.settings.json = false; } - get isAsync() { return true; } + get isAsync() { + return true; + } async init() { // nano 11 dropped support for requestDefaults.auth = {username, password}. @@ -62,43 +64,46 @@ export default class Couch_db extends AbstractDatabase { this.db = client.use(this.settings.database!); } - async get(key:string): Promise { + async get(key: string): Promise { let doc; try { if (this.db) { doc = await this.db.get(key); } - } catch (err:any) { + } catch (err: any) { if (err.statusCode === 404) return null; throw err; } - if (doc && 'value' in doc) { + if (doc && "value" in doc) { return doc.value as string; } - return ''; + return ""; } - async findKeys(key:string, notKey:string) { - const pfxLen = key.indexOf('*'); + async findKeys(key: string, notKey: string) { + const pfxLen = key.indexOf("*"); if (!this.db) { return; } const pfx = pfxLen < 0 ? key : key.slice(0, pfxLen); const results = await this.db.find({ selector: { - _id: pfxLen < 0 ? pfx : { - $gte: pfx, - // https://docs.couchdb.org/en/3.2.2/ddocs/views/collation.html#string-ranges - $lte: `${pfx}\ufff0`, - $regex: this.createFindRegex(key, notKey).source, - }, + _id: + pfxLen < 0 + ? pfx + : { + $gte: pfx, + // https://docs.couchdb.org/en/3.2.2/ddocs/views/collation.html#string-ranges + $lte: `${pfx}\ufff0`, + $regex: this.createFindRegex(key, notKey).source, + }, }, - fields: ['_id'], + fields: ["_id"], }); return results.docs.map((doc) => doc._id); } - async set(key:string, value:string) { + async set(key: string, value: string) { let doc; if (!this.db) { @@ -107,27 +112,29 @@ export default class Couch_db extends AbstractDatabase { try { doc = await this.db.get(key); - } catch (err:any) { + } catch (err: any) { if (err.statusCode !== 404) throw err; } await this.db.insert({ _id: key, // @ts-ignore value, - ...doc == null ? {} : { - _rev: doc._rev, - }, + ...(doc == null + ? {} + : { + _rev: doc._rev, + }), }); } - async remove(key:string) { + async remove(key: string) { let header; if (!this.db) { return; } try { header = await this.db.head(key); - } catch (err:any) { + } catch (err: any) { if (err.statusCode === 404) return; throw err; } @@ -136,30 +143,29 @@ export default class Couch_db extends AbstractDatabase { await this.db.destroy(key, etag); } - async doBulk(bulk:BulkObject[]) { + async doBulk(bulk: BulkObject[]) { if (!this.db) { return; } const keys = bulk.map((op) => op.key); - const revs:{[key:string]:any} = {}; + const revs: { [key: string]: any } = {}; // @ts-ignore - for (const {key, value} of (await this.db.fetchRevs({keys})).rows) { + for (const { key, value } of (await this.db.fetchRevs({ keys })).rows) { // couchDB will return error instead of value if key does not exist if (value != null) revs[key] = value.rev; } const setters = []; for (const item of bulk) { - const set = {_id: item.key, _rev: undefined, - _deleted: false, value: ''}; + const set = { _id: item.key, _rev: undefined, _deleted: false, value: "" }; if (revs[item.key] != null) set._rev = revs[item.key]; - if (item.type === 'set') set.value = item.value as string; - if (item.type === 'remove') set._deleted = true; + if (item.type === "set") set.value = item.value as string; + if (item.type === "remove") set._deleted = true; setters.push(set); } - await this.db.bulk({docs: setters}); + await this.db.bulk({ docs: setters }); } async close() { this.db = null; } -}; +} diff --git a/databases/dirty_db.ts b/databases/dirty_db.ts index 0a19f861..18f804ae 100644 --- a/databases/dirty_db.ts +++ b/databases/dirty_db.ts @@ -15,14 +15,14 @@ */ /* -* -* Fair warning that length may not provide the correct value upon load. -* See https://github.com/ether/etherpad-lite/pull/3984 -* -*/ + * + * Fair warning that length may not provide the correct value upon load. + * See https://github.com/ether/etherpad-lite/pull/3984 + * + */ -import AbstractDatabase, {type Settings} from '../lib/AbstractDatabase'; -import DirtyImport from 'dirty-ts'; +import AbstractDatabase, { type Settings } from "../lib/AbstractDatabase"; +import DirtyImport from "dirty-ts"; // dirty-ts is a CJS package that exposes the constructor as `module.exports.default` // with the `__esModule` marker. Under Node's ESM-to-CJS interop, the default @@ -30,18 +30,17 @@ import DirtyImport from 'dirty-ts'; // so unwrap `.default` if it's present. const Dirty: any = (DirtyImport as any).default ?? DirtyImport; -type DirtyDBCallback = (p?:any, keys?: string[])=>{}; - +type DirtyDBCallback = (p?: any, keys?: string[]) => {}; export default class extends AbstractDatabase { public db: any; - constructor(settings:Settings) { + constructor(settings: Settings) { super(settings); this.db = null; if (!settings || !settings.filename) { // @ts-ignore - settings = {filename: null}; + settings = { filename: null }; } this.settings = settings; @@ -52,21 +51,21 @@ export default class extends AbstractDatabase { this.settings.json = false; } - init(callback: ()=>{}) { + init(callback: () => {}) { this.db = new Dirty(this.settings.filename); - this.db.on('load', () => { + this.db.on("load", () => { callback(); }); } - get(key:string, callback:DirtyDBCallback) { + get(key: string, callback: DirtyDBCallback) { callback(null, this.db.get(key)); } - findKeys(key:string, notKey:string, callback:DirtyDBCallback) { - const keys:string[] = []; + findKeys(key: string, notKey: string, callback: DirtyDBCallback) { + const keys: string[] = []; const regex = this.createFindRegex(key, notKey); - this.db.forEach((key:string) => { + this.db.forEach((key: string) => { if (key.search(regex) !== -1) { keys.push(key); } @@ -74,17 +73,17 @@ export default class extends AbstractDatabase { callback(null, keys); } - set(key:string, value:string, callback:DirtyDBCallback) { + set(key: string, value: string, callback: DirtyDBCallback) { this.db.set(key, value, callback); } - remove(key:string, callback:DirtyDBCallback) { + remove(key: string, callback: DirtyDBCallback) { this.db.rm(key, callback); } - close(callback:DirtyDBCallback) { + close(callback: DirtyDBCallback) { this.db.close(); this.db = null; if (callback) callback(); } -}; +} diff --git a/databases/dirty_git_db.ts b/databases/dirty_git_db.ts index 18577275..62d1f1c4 100644 --- a/databases/dirty_git_db.ts +++ b/databases/dirty_git_db.ts @@ -1,6 +1,6 @@ -import AbstractDatabase, {type Settings} from '../lib/AbstractDatabase'; -import {dirname} from 'node:path' -import {simpleGit} from 'simple-git' +import AbstractDatabase, { type Settings } from "../lib/AbstractDatabase"; +import { dirname } from "node:path"; +import { simpleGit } from "simple-git"; /** * 2011 Peter 'Pita' Martischka * @@ -17,8 +17,7 @@ import {simpleGit} from 'simple-git' * limitations under the License. */ - -import DirtyImport from 'dirty-ts'; +import DirtyImport from "dirty-ts"; // dirty-ts is a CJS package that exposes the constructor as `module.exports.default` // with the `__esModule` marker. Under Node's ESM-to-CJS interop, the default @@ -45,21 +44,21 @@ export default class extends AbstractDatabase { this.settings.json = false; } - init(callback: ()=>void) { + init(callback: () => void) { this.db = new Dirty(this.settings.filename); - this.db.on('load', (err: Error) => { + this.db.on("load", (err: Error) => { callback(); }); } - get(key:string, callback: (err: string | any, value: string)=>void) { + get(key: string, callback: (err: string | any, value: string) => void) { callback(null, this.db.get(key)); } - findKeys(key:string, notKey:string, callback:(v:any, keys:string[])=>{}) { - const keys:string[] = []; + findKeys(key: string, notKey: string, callback: (v: any, keys: string[]) => {}) { + const keys: string[] = []; const regex = this.createFindRegex(key, notKey); - this.db.forEach((key:string, val:string) => { + this.db.forEach((key: string, val: string) => { if (key.search(regex) !== -1) { keys.push(key); } @@ -67,22 +66,22 @@ export default class extends AbstractDatabase { callback(null, keys); } - set(key:string, value: string, callback: ()=>{}) { + set(key: string, value: string, callback: () => {}) { this.db.set(key, value, callback); const databasePath = dirname(this.settings.filename!); simpleGit(databasePath) - .silent(true) - .add('./*.db') - .commit('Automated commit...') - .push(['-u', 'origin', 'master'], () => console.debug('Stored git commit')); + .silent(true) + .add("./*.db") + .commit("Automated commit...") + .push(["-u", "origin", "master"], () => console.debug("Stored git commit")); } - remove(key:string, callback:()=> {}) { + remove(key: string, callback: () => {}) { this.db.rm(key, callback); } - close(callback: ()=>void) { + close(callback: () => void) { this.db.close(); if (callback) callback(); } -}; +} diff --git a/databases/elasticsearch_db.ts b/databases/elasticsearch_db.ts index b3a2c54a..c31bf6a0 100644 --- a/databases/elasticsearch_db.ts +++ b/databases/elasticsearch_db.ts @@ -14,90 +14,118 @@ * limitations under the License. */ -import AbstractDatabase, {type Settings} from '../lib/AbstractDatabase'; -import assert from 'assert'; -import {equal} from 'assert'; -import {Buffer} from 'buffer'; -import {createHash} from 'crypto'; -import {Client} from '@elastic/elasticsearch'; -import type {estypes} from '@elastic/elasticsearch'; -import type {BulkObject} from './cassandra_db'; +import AbstractDatabase, { type Settings } from "../lib/AbstractDatabase"; +import assert from "assert"; +import { equal } from "assert"; +import { Buffer } from "buffer"; +import { createHash } from "crypto"; +import { Client } from "@elastic/elasticsearch"; +import type { estypes } from "@elastic/elasticsearch"; +import type { BulkObject } from "./cassandra_db"; -const schema = '2'; +const schema = "2"; -const keyToId = (key:string) => { +const keyToId = (key: string) => { const keyBuf = Buffer.from(key); - return keyBuf.length > 512 ? createHash('sha512').update(keyBuf).digest('hex') : key; + return keyBuf.length > 512 ? createHash("sha512").update(keyBuf).digest("hex") : key; }; const mappings: estypes.MappingTypeMapping = { // _id is expected to equal key, unless the UTF-8 encoded key is > 512 bytes, in which case it is // the hex-encoded sha512 hash of the UTF-8 encoded key. properties: { - key: {type: 'wildcard'}, // For findKeys, and because _id is limited to 512 bytes. - value: {type: 'object', enabled: false}, // Values should be opaque to Elasticsearch. + key: { type: "wildcard" }, // For findKeys, and because _id is limited to 512 bytes. + value: { type: "object", enabled: false }, // Values should be opaque to Elasticsearch. }, }; const legacyDocToSchema2Key = (index: string, id: string, type: unknown, v1BaseIndex?: string) => { - const legacyType = typeof type === 'string' && type !== '' && type !== '_doc' ? type : null; + const legacyType = typeof type === "string" && type !== "" && type !== "_doc" ? type : null; if (v1BaseIndex && index !== v1BaseIndex) { - const parts = index.slice(v1BaseIndex.length + 1).split('-'); + const parts = index.slice(v1BaseIndex.length + 1).split("-"); if (parts.length !== 2) { throw new Error(`unable to migrate records from index ${index} due to data ambiguity`); } - if (legacyType != null) return `${parts[0]}:${decodeURIComponent(legacyType)}:${parts[1]}:${id}`; - const idParts = id.split(':'); + if (legacyType != null) + return `${parts[0]}:${decodeURIComponent(legacyType)}:${parts[1]}:${id}`; + const idParts = id.split(":"); if (idParts.length !== 2) { throw new Error(`unable to migrate records from index ${index} due to data ambiguity`); } return `${parts[0]}:${idParts[0]}:${parts[1]}:${idParts[1]}`; } if (legacyType != null) return `${legacyType}:${id}`; - const idParts = id.split(':'); + const idParts = id.split(":"); if (idParts.length !== 2) { - throw new Error(`unable to migrate records from index ${index} due to missing legacy type metadata`); + throw new Error( + `unable to migrate records from index ${index} due to missing legacy type metadata`, + ); } return `${idParts[0]}:${idParts[1]}`; }; -const migrateToSchema2 = async (client: Client, v1BaseIndex: string | undefined, v2Index: string, logger: any) => { +const migrateToSchema2 = async ( + client: Client, + v1BaseIndex: string | undefined, + v2Index: string, + logger: any, +) => { let recordsMigratedLastLogged = 0; let recordsMigrated = 0; const totals = new Map(); - logger.info('Attempting elasticsearch record migration from schema v1 at base index ' + - `${v1BaseIndex} to schema v2 at index ${v2Index}...`); - const indices = await client.indices.get({index: [v1BaseIndex as string, `${v1BaseIndex}-*-*`]}); + logger.info( + "Attempting elasticsearch record migration from schema v1 at base index " + + `${v1BaseIndex} to schema v2 at index ${v2Index}...`, + ); + const indices = await client.indices.get({ + index: [v1BaseIndex as string, `${v1BaseIndex}-*-*`], + }); const scrollIds = new Map(); const q = []; try { for (const index of Object.keys(indices)) { - const res = await client.search({index, scroll: '10m'}); + const res = await client.search({ index, scroll: "10m" }); scrollIds.set(index, res._scroll_id); - q.push({index, res}); + q.push({ index, res }); } while (q.length) { - const {index, res: {hits: {hits, total: {value: total}}}}:any = q.shift(); + const { + index, + res: { + hits: { + hits, + total: { value: total }, + }, + }, + }: any = q.shift(); if (hits.length === 0) continue; totals.set(index, total); const body = []; - for (const {_id, _type, _source: {val}} of hits) { + for (const { + _id, + _type, + _source: { val }, + } of hits) { const key = legacyDocToSchema2Key(index, _id, _type, v1BaseIndex); - body.push({index: {_id: keyToId(key)}}, {key, value: JSON.parse(val)}); + body.push({ index: { _id: keyToId(key) } }, { key, value: JSON.parse(val) }); } - await client.bulk({index: v2Index, body}); + await client.bulk({ index: v2Index, body }); recordsMigrated += hits.length; if (Math.floor(recordsMigrated / 100) > Math.floor(recordsMigratedLastLogged / 100)) { const total = [...totals.values()].reduce((a, b) => a + b, 0); logger.info(`Migrated ${recordsMigrated} records out of ${total}`); recordsMigratedLastLogged = recordsMigrated; } - q.push( - {index, res: (await client.scroll({scroll: '5m', scroll_id: scrollIds.get(index)}))}); + q.push({ + index, + res: await client.scroll({ scroll: "5m", scroll_id: scrollIds.get(index) }), + }); } logger.info(`Finished migrating ${recordsMigrated} records`); } finally { - await Promise.all([...scrollIds.values()].map((scrollId) => client.clearScroll({scroll_id:scrollId}))); + await Promise.all( + [...scrollIds.values()].map((scrollId) => client.clearScroll({ scroll_id: scrollId })), + ); } }; @@ -105,27 +133,29 @@ export default class extends AbstractDatabase { public _client: any; public readonly _index: any; public _indexClean: boolean; - public readonly _q: {index: any}; - constructor(settings:Settings) { + public readonly _q: { index: any }; + constructor(settings: Settings) { super(settings); this._client = null; this.settings = { - host: '127.0.0.1', - port: '9200', - base_index: 'ueberes', + host: "127.0.0.1", + port: "9200", + base_index: "ueberes", migrate_to_newer_schema: false, // for a list of valid API values see: // https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/configuration.html#config-options - api: '7.6', - ...settings || {}, + api: "7.6", + ...(settings || {}), json: false, // Elasticsearch will do the JSON conversion as necessary. }; this._index = `${this.settings.base_index}_s${schema}`; - this._q = {index: this._index}; + this._q = { index: this._index }; this._indexClean = true; } - get isAsync() { return true; } + get isAsync() { + return true; + } async _refreshIndex() { if (this._indexClean) return; @@ -143,31 +173,34 @@ export default class extends AbstractDatabase { node: `http://${this.settings.host}:${this.settings.port}`, }); await client.ping(); - if (!(await client.indices.exists({index: this._index}))) { + if (!(await client.indices.exists({ index: this._index }))) { let tmpIndex; - const exists = await client.indices.exists({index: this.settings.base_index as string}); + const exists = await client.indices.exists({ index: this.settings.base_index as string }); if (exists && !this.settings.migrate_to_newer_schema) { throw new Error( - `Data exists under the legacy index (schema) named ${this.settings.base_index}. ` + - 'Set migrate_to_newer_schema to true to copy the existing data to a new index ' + - `named ${this._index}.`); + `Data exists under the legacy index (schema) named ${this.settings.base_index}. ` + + "Set migrate_to_newer_schema to true to copy the existing data to a new index " + + `named ${this._index}.`, + ); } let attempt = 0; while (true) { - tmpIndex = `${this._index}_${exists ? 'migrate_attempt_' : 'i'}${attempt++}`; - if (!(await client.indices.exists({index: tmpIndex}))) break; + tmpIndex = `${this._index}_${exists ? "migrate_attempt_" : "i"}${attempt++}`; + if (!(await client.indices.exists({ index: tmpIndex }))) break; } - await client.indices.create({index: tmpIndex, mappings: mappings}); + await client.indices.create({ index: tmpIndex, mappings: mappings }); if (exists) await migrateToSchema2(client, this.settings.base_index, tmpIndex, this.logger); - await client.indices.putAlias({index: tmpIndex, name: this._index}); + await client.indices.putAlias({ index: tmpIndex, name: this._index }); } - const indices = Object.values((await client.indices.get({index: this._index}))); + const indices = Object.values(await client.indices.get({ index: this._index })); equal(indices.length, 1); try { assert.deepEqual(indices[0].mappings, mappings); } catch (err) { - this.logger.warn(`Index ${this._index} mappings does not match expected; ` + - `attempting to use index anyway. Details: ${err}`); + this.logger.warn( + `Index ${this._index} mappings does not match expected; ` + + `attempting to use index anyway. Details: ${err}`, + ); } this._client = client; } @@ -177,8 +210,8 @@ export default class extends AbstractDatabase { * * @param {String} key Key */ - async get(key:string) { - const res = await this._client.get({...this._q, id: keyToId(key)}, {ignore: [404]}); + async get(key: string) { + const res = await this._client.get({ ...this._q, id: keyToId(key) }, { ignore: [404] }); if (!res.found) return null; return res._source.value; } @@ -187,23 +220,25 @@ export default class extends AbstractDatabase { * @param key Search key, which uses an asterisk (*) as the wild card. * @param notKey Used to filter the result set */ - async findKeys(key:string, notKey:string) { + async findKeys(key: string, notKey: string) { await this._refreshIndex(); const q = { ...this._q, body: { query: { bool: { - filter: {wildcard: {key: {value: key}}}, - ...notKey == null ? {} : { - must_not: {wildcard: {key: {value: notKey}}}, - }, + filter: { wildcard: { key: { value: key } } }, + ...(notKey == null + ? {} + : { + must_not: { wildcard: { key: { value: notKey } } }, + }), }, }, }, }; - const {hits:hits} = await this._client.search(q); - return hits.hits.map((h:{_source:{key:string}}) => h._source.key); + const { hits: hits } = await this._client.search(q); + return hits.hits.map((h: { _source: { key: string } }) => h._source.key); } /** @@ -212,9 +247,9 @@ export default class extends AbstractDatabase { * @param {String} key Record identifier. * @param {JSON|String} value The value to store in the database. */ - async set(key: string, value:string) { + async set(key: string, value: string) { this._indexClean = false; - await this._client.index({...this._q, id: keyToId(key), body: {key, value}}); + await this._client.index({ ...this._q, id: keyToId(key), body: { key, value } }); } /** @@ -225,9 +260,9 @@ export default class extends AbstractDatabase { * * @param {String} key Record identifier. */ - async remove(key:string) { + async remove(key: string) { this._indexClean = false; - await this._client.delete({...this._q, id: keyToId(key)}, {ignore: [404]}); + await this._client.delete({ ...this._q, id: keyToId(key) }, { ignore: [404] }); } /** @@ -245,24 +280,24 @@ export default class extends AbstractDatabase { const operations = []; - for (const {type, key, value} of bulk) { + for (const { type, key, value } of bulk) { this._indexClean = false; switch (type) { - case 'set': - operations.push({index: {_id: keyToId(key)}}); - operations.push({key, value}); + case "set": + operations.push({ index: { _id: keyToId(key) } }); + operations.push({ key, value }); break; - case 'remove': - operations.push({delete: {_id: keyToId(key)}}); + case "remove": + operations.push({ delete: { _id: keyToId(key) } }); break; default: } } - await this._client.bulk({...this._q, body: operations}); + await this._client.bulk({ ...this._q, body: operations }); } async close() { if (this._client != null) this._client.close(); this._client = null; } -}; +} diff --git a/databases/memory_db.ts b/databases/memory_db.ts index 2dd08b20..50b26080 100644 --- a/databases/memory_db.ts +++ b/databases/memory_db.ts @@ -1,9 +1,8 @@ -import AbstractDatabase, {type Settings} from '../lib/AbstractDatabase'; - +import AbstractDatabase, { type Settings } from "../lib/AbstractDatabase"; export default class MemoryDB extends AbstractDatabase { public _data: any; - constructor(settings:Settings) { + constructor(settings: Settings) { super(settings); this.settings = settings; settings.json = false; @@ -12,18 +11,20 @@ export default class MemoryDB extends AbstractDatabase { this._data = null; } - get isAsync() { return true; } + get isAsync() { + return true; + } close() { this._data = null; } - findKeys(key:string, notKey:string) { + findKeys(key: string, notKey: string) { const regex = this.createFindRegex(key, notKey); return [...this._data.keys()].filter((k) => regex.test(k)); } - get(key:string) { + get(key: string) { return this._data.get(key); } @@ -31,11 +32,11 @@ export default class MemoryDB extends AbstractDatabase { this._data = this.settings.data || new Map(); } - remove(key:string) { + remove(key: string) { this._data.delete(key); } - set(key:string, value:string) { + set(key: string, value: string) { this._data.set(key, value); } -}; +} diff --git a/databases/mock_db.ts b/databases/mock_db.ts index e2e454d6..86b6fe9a 100644 --- a/databases/mock_db.ts +++ b/databases/mock_db.ts @@ -1,11 +1,11 @@ -import type {Settings} from '../lib/AbstractDatabase'; +import type { Settings } from "../lib/AbstractDatabase"; -import events from 'events'; +import events from "events"; export default class extends events.EventEmitter { public settings: Settings; public mock: any; - constructor(settings:Settings) { + constructor(settings: Settings) { super(); this.settings = { writeInterval: 1, @@ -15,31 +15,31 @@ export default class extends events.EventEmitter { this.settings = settings; } - close(cb: ()=>{}) { - this.emit('close', cb); + close(cb: () => {}) { + this.emit("close", cb); } - doBulk(ops:string, cb: ()=>{}) { - this.emit('doBulk', ops, cb); + doBulk(ops: string, cb: () => {}) { + this.emit("doBulk", ops, cb); } - findKeys(key:string, notKey:string, cb:()=>{}) { - this.emit('findKeys', key, notKey, cb); + findKeys(key: string, notKey: string, cb: () => {}) { + this.emit("findKeys", key, notKey, cb); } - get(key:string, cb:()=>{}) { - this.emit('get', key, cb); + get(key: string, cb: () => {}) { + this.emit("get", key, cb); } - async init(cb:()=>{}) { - this.emit('init', cb()); + async init(cb: () => {}) { + this.emit("init", cb()); } - remove(key:string, cb:()=>{}) { - this.emit('remove', key, cb); + remove(key: string, cb: () => {}) { + this.emit("remove", key, cb); } - set(key:string, value:string, cb:()=>{}) { - this.emit('set', key, value, cb); + set(key: string, value: string, cb: () => {}) { + this.emit("set", key, value, cb); } -}; +} diff --git a/databases/mongodb_db.ts b/databases/mongodb_db.ts index 6bcd40d3..c98cba6c 100644 --- a/databases/mongodb_db.ts +++ b/databases/mongodb_db.ts @@ -14,25 +14,25 @@ * limitations under the License. */ -import AbstractDatabase, {type Settings} from '../lib/AbstractDatabase'; -import type {BulkObject} from './cassandra_db'; -import {MongoClient} from 'mongodb'; -import type {Collection, Db} from 'mongodb'; +import AbstractDatabase, { type Settings } from "../lib/AbstractDatabase"; +import type { BulkObject } from "./cassandra_db"; +import { MongoClient } from "mongodb"; +import type { Collection, Db } from "mongodb"; export default class extends AbstractDatabase { public interval: NodeJS.Timer | undefined; - public database: Db|undefined; - public client: MongoClient|undefined; - public collection: Collection|undefined; - constructor(settings:Settings) { + public database: Db | undefined; + public client: MongoClient | undefined; + public collection: Collection | undefined; + constructor(settings: Settings) { super(settings); this.settings = settings; - if (!this.settings.url) throw new Error('You must specify a mongodb url'); + if (!this.settings.url) throw new Error("You must specify a mongodb url"); // For backwards compatibility: if (this.settings.database == null) this.settings.database = this.settings.dbName; - if (!this.settings.collection) this.settings.collection = 'ueberdb'; + if (!this.settings.collection) this.settings.collection = "ueberdb"; } clearPing() { @@ -50,103 +50,106 @@ export default class extends AbstractDatabase { }, 10000); } - init(callback:Function) { - - MongoClient.connect(this.settings.url!).then((v)=>{ + init(callback: Function) { + MongoClient.connect(this.settings.url!) + .then((v) => { this.client = v; this.database = v.db(this.settings.database); - this.schedulePing(); + this.schedulePing(); this.collection = this.database.collection(this.settings.collection!); callback(null); - }) - .catch((v:Error)=>{ - callback(v); - }) - - + }) + .catch((v: Error) => { + callback(v); + }); } - get(key:string, callback:Function) { + get(key: string, callback: Function) { // @ts-ignore - this.collection!.findOne({_id: key}) - .then((v)=>{ - callback(null, v&&v.value); - }).catch(v=> { - console.log(v) - callback(v); - }) + this.collection!.findOne({ _id: key }) + .then((v) => { + callback(null, v && v.value); + }) + .catch((v) => { + console.log(v); + callback(v); + }); this.schedulePing(); } - findKeys(key:string, notKey:string, callback:Function) { + findKeys(key: string, notKey: string, callback: Function) { const selector = { - $and: [ - {_id: {$regex: `${key.replace(/\*/g, '')}`}}, - ], + $and: [{ _id: { $regex: `${key.replace(/\*/g, "")}` } }], }; if (notKey) { // @ts-ignore - selector.$and.push({_id: {$not: {$regex: `${notKey.replace(/\*/g, '')}`}}}); + selector.$and.push({ _id: { $not: { $regex: `${notKey.replace(/\*/g, "")}` } } }); } // @ts-ignore - this.collection!.find(selector).map((i: any) => i._id) - .toArray() - .then(r =>{ + this.collection!.find(selector) + .map((i: any) => i._id) + .toArray() + .then((r) => { callback(null, r); - }) - .catch(v=>callback(v)); - + }) + .catch((v) => callback(v)); this.schedulePing(); } - set(key:string, value:string, callback:Function) { + set(key: string, value: string, callback: Function) { if (key.length > 100) { - callback('Your Key can only be 100 chars'); + callback("Your Key can only be 100 chars"); } else { // @ts-ignore - this.collection!.updateMany({_id: key}, {$set: {value}}, {upsert: true}) - .then(()=>callback(null)) - .catch(v=>callback(v)); + this.collection!.updateMany({ _id: key }, { $set: { value } }, { upsert: true }) + .then(() => callback(null)) + .catch((v) => callback(v)); } this.schedulePing(); } - remove(key:string, callback:Function) { + remove(key: string, callback: Function) { // @ts-ignore - this.collection!.deleteOne({_id: key}, ) - .then(r =>callback(null,r) ) - .catch(v=>callback(v)); + this.collection!.deleteOne({ _id: key }) + .then((r) => callback(null, r)) + .catch((v) => callback(v)); this.schedulePing(); } - doBulk(bulk:BulkObject[], callback:Function) { + doBulk(bulk: BulkObject[], callback: Function) { const bulkMongo = this.collection!.initializeOrderedBulkOp(); for (const i in bulk) { - if (bulk[i].type === 'set') { - bulkMongo.find({_id: bulk[i].key}).upsert().updateOne({$set: {value: bulk[i].value}}); - } else if (bulk[i].type === 'remove') { - bulkMongo.find({_id: bulk[i].key}).deleteOne(); + if (bulk[i].type === "set") { + bulkMongo + .find({ _id: bulk[i].key }) + .upsert() + .updateOne({ $set: { value: bulk[i].value } }); + } else if (bulk[i].type === "remove") { + bulkMongo.find({ _id: bulk[i].key }).deleteOne(); } } - bulkMongo.execute().then((res:any) => { - callback(null, res); - }).catch((error:any) => { - callback(error); - }); + bulkMongo + .execute() + .then((res: any) => { + callback(null, res); + }) + .catch((error: any) => { + callback(error); + }); this.schedulePing(); } - close(callback:any) { + close(callback: any) { this.clearPing(); - this.client!.close().then(r =>callback(r)); + this.client!.close().then((r) => callback(r)); } } diff --git a/databases/mssql_db.ts b/databases/mssql_db.ts index 47d4b881..572ed890 100644 --- a/databases/mssql_db.ts +++ b/databases/mssql_db.ts @@ -20,20 +20,19 @@ * */ -import AbstractDatabase, {type Settings} from '../lib/AbstractDatabase'; -import async from 'async'; -import mssql from 'mssql'; -import type {ConnectionPool} from 'mssql'; -import type {BulkObject} from './cassandra_db'; +import AbstractDatabase, { type Settings } from "../lib/AbstractDatabase"; +import async from "async"; +import mssql from "mssql"; +import type { ConnectionPool } from "mssql"; +import type { BulkObject } from "./cassandra_db"; type RowResult = { - key: string; + key: string; }; - export default class MSSQL extends AbstractDatabase { public db: ConnectionPool | undefined; - constructor(settings:Settings) { + constructor(settings: Settings) { super(settings); settings = settings || {}; @@ -56,15 +55,15 @@ export default class MSSQL extends AbstractDatabase { this.settings.writeInterval = 0; } - init(callback:(err: any)=>{}) { + init(callback: (err: any) => {}) { const sqlCreate = - "IF OBJECT_ID(N'dbo.store', N'U') IS NULL" + - ' BEGIN' + - ' CREATE TABLE [store] (' + - ' [key] NVARCHAR(100) PRIMARY KEY,' + - ' [value] NTEXT NOT NULL' + - ' );' + - ' END'; + "IF OBJECT_ID(N'dbo.store', N'U') IS NULL" + + " BEGIN" + + " CREATE TABLE [store] (" + + " [key] NVARCHAR(100) PRIMARY KEY," + + " [value] NTEXT NOT NULL" + + " );" + + " END"; // @ts-ignore new mssql.ConnectionPool(this.settings).connect().then((pool) => { @@ -76,18 +75,18 @@ export default class MSSQL extends AbstractDatabase { callback(err); }); - this.db.on('error', (err) => { + this.db.on("error", (err) => { console.log(err); }); }); } - get(key:string, callback:(err?:Error, value?:string)=>{}) { + get(key: string, callback: (err?: Error, value?: string) => {}) { const request = new mssql.Request(this.db); - request.input('key', mssql.NVarChar(100), key); + request.input("key", mssql.NVarChar(100), key); - request.query('SELECT [value] FROM [store] WHERE [key] = @key', (err, results) => { + request.query("SELECT [value] FROM [store] WHERE [key] = @key", (err, results) => { let value = null; if (!err && results && results.rowsAffected[0] === 1) { @@ -99,24 +98,24 @@ export default class MSSQL extends AbstractDatabase { }); } - findKeys(key:string, notKey:string, callback:(err: Error | undefined, value:string[])=>{}) { + findKeys(key: string, notKey: string, callback: (err: Error | undefined, value: string[]) => {}) { const request = new mssql.Request(this.db); - let query = 'SELECT [key] FROM [store] WHERE [key] LIKE @key'; + let query = "SELECT [key] FROM [store] WHERE [key] LIKE @key"; // desired keys are key, e.g. pad:% - key = key.replace(/\*/g, '%'); + key = key.replace(/\*/g, "%"); - request.input('key', mssql.NVarChar(100), key); + request.input("key", mssql.NVarChar(100), key); if (notKey != null) { // not desired keys are notKey, e.g. %:%:% - notKey = notKey.replace(/\*/g, '%'); - request.input('notkey', mssql.NVarChar(100), notKey); - query += ' AND [key] NOT LIKE @notkey'; + notKey = notKey.replace(/\*/g, "%"); + request.input("notkey", mssql.NVarChar(100), notKey); + query += " AND [key] NOT LIKE @notkey"; } request.query(query, (err, results) => { - const value:string[] = []; + const value: string[] = []; if (!err && results && results.rowsAffected[0] > 0) { for (let i = 0; i < results.recordset.length; i++) { @@ -128,58 +127,59 @@ export default class MSSQL extends AbstractDatabase { }); } - set(key:string, value:string, callback: (val:string)=>{}) { + set(key: string, value: string, callback: (val: string) => {}) { const request = new mssql.Request(this.db); if (key.length > 100) { - callback('Your Key can only be 100 chars'); + callback("Your Key can only be 100 chars"); } else { const query = - 'MERGE [store] t USING (SELECT @key [key], @value [value]) s' + - ' ON t.[key] = s.[key]' + - ' WHEN MATCHED AND s.[value] IS NOT NULL THEN UPDATE SET t.[value] = s.[value]' + - ' WHEN NOT MATCHED THEN INSERT ([key], [value]) VALUES (s.[key], s.[value]);'; + "MERGE [store] t USING (SELECT @key [key], @value [value]) s" + + " ON t.[key] = s.[key]" + + " WHEN MATCHED AND s.[value] IS NOT NULL THEN UPDATE SET t.[value] = s.[value]" + + " WHEN NOT MATCHED THEN INSERT ([key], [value]) VALUES (s.[key], s.[value]);"; - request.input('key', mssql.NVarChar(100), key); - request.input('value', mssql.NText, value); + request.input("key", mssql.NVarChar(100), key); + request.input("value", mssql.NText, value); request.query(query, (err, info) => { - callback(err ? err.toString() : ''); + callback(err ? err.toString() : ""); }); } } - remove(key:string, callback:()=>{}) { + remove(key: string, callback: () => {}) { const request = new mssql.Request(this.db); - request.input('key', mssql.NVarChar(100), key); - request.query('DELETE FROM [store] WHERE [key] = @key', callback); + request.input("key", mssql.NVarChar(100), key); + request.query("DELETE FROM [store] WHERE [key] = @key", callback); } - doBulk(bulk: BulkObject[], callback:(err:any, results?: any)=>{}) { + doBulk(bulk: BulkObject[], callback: (err: any, results?: any) => {}) { const maxInserts = 100; const request = new mssql.Request(this.db); let firstReplace = true; let firstRemove = true; const replacements: string[] = []; - let removeSQL = 'DELETE FROM [store] WHERE [key] IN ('; + let removeSQL = "DELETE FROM [store] WHERE [key] IN ("; for (const i in bulk) { - if (bulk[i].type === 'set') { + if (bulk[i].type === "set") { if (firstReplace) { - replacements.push('BEGIN TRANSACTION;'); + replacements.push("BEGIN TRANSACTION;"); firstReplace = false; } else if (Number(i) % maxInserts === 0) { - replacements.push('\nCOMMIT TRANSACTION;\nBEGIN TRANSACTION;\n'); + replacements.push("\nCOMMIT TRANSACTION;\nBEGIN TRANSACTION;\n"); } replacements.push( - `MERGE [store] t USING (SELECT '${bulk[i].key}' [key], '${bulk[i].value}' [value]) s`, - 'ON t.[key] = s.[key]', - 'WHEN MATCHED AND s.[value] IS NOT NULL THEN UPDATE SET t.[value] = s.[value]', - 'WHEN NOT MATCHED THEN INSERT ([key], [value]) VALUES (s.[key], s.[value]);'); - } else if (bulk[i].type === 'remove') { + `MERGE [store] t USING (SELECT '${bulk[i].key}' [key], '${bulk[i].value}' [value]) s`, + "ON t.[key] = s.[key]", + "WHEN MATCHED AND s.[value] IS NOT NULL THEN UPDATE SET t.[value] = s.[value]", + "WHEN NOT MATCHED THEN INSERT ([key], [value]) VALUES (s.[key], s.[value]);", + ); + } else if (bulk[i].type === "remove") { if (!firstRemove) { - removeSQL += ','; + removeSQL += ","; } firstRemove = false; @@ -187,41 +187,41 @@ export default class MSSQL extends AbstractDatabase { } } - removeSQL += ');'; - replacements.push('COMMIT TRANSACTION;'); + removeSQL += ");"; + replacements.push("COMMIT TRANSACTION;"); async.parallel( - [ - (callback) => { - if (!firstReplace) { - request.batch(replacements.join('\n'), (err, results) => { - if (err) { - callback(err); - } - callback(err, results); - }); - } else { - callback(); - } - }, - (callback) => { - if (!firstRemove) { - request.query(removeSQL, callback); - } else { - callback(); - } - }, - ], - (err, results) => { - if (err) { - callback(err); + [ + (callback) => { + if (!firstReplace) { + request.batch(replacements.join("\n"), (err, results) => { + if (err) { + callback(err); + } + callback(err, results); + }); + } else { + callback(); + } + }, + (callback) => { + if (!firstRemove) { + request.query(removeSQL, callback); + } else { + callback(); } - callback(err, results); }, + ], + (err, results) => { + if (err) { + callback(err); + } + callback(err, results); + }, ); } - close(callback: (err?:Error)=>{}) { + close(callback: (err?: Error) => {}) { this.db && this.db.close(callback); } -}; +} diff --git a/databases/mysql_db.ts b/databases/mysql_db.ts index 71891ef4..07b6c8d7 100644 --- a/databases/mysql_db.ts +++ b/databases/mysql_db.ts @@ -14,24 +14,24 @@ * limitations under the License. */ -import AbstractDatabase, {type Settings} from '../lib/AbstractDatabase'; -import util from 'util'; -import type {BulkObject} from './cassandra_db'; -import {createPool} from 'mysql2'; -import type {ConnectionConfig, Pool, QueryError} from 'mysql2'; +import AbstractDatabase, { type Settings } from "../lib/AbstractDatabase"; +import util from "util"; +import type { BulkObject } from "./cassandra_db"; +import { createPool } from "mysql2"; +import type { ConnectionConfig, Pool, QueryError } from "mysql2"; export default class extends AbstractDatabase { public readonly _mysqlSettings: Settings; - public _pool: Pool|null; - constructor(settings:Settings) { + public _pool: Pool | null; + constructor(settings: Settings) { super(settings); // logger is set by the framework after construction this._mysqlSettings = { - charset: 'utf8mb4', // temp hack needs a proper fix.. + charset: "utf8mb4", // temp hack needs a proper fix.. ...settings, }; this.settings = { - engine: 'InnoDB', + engine: "InnoDB", // Limit the query size to avoid timeouts or other failures. bulkLimit: 100, json: true, @@ -40,182 +40,204 @@ export default class extends AbstractDatabase { this._pool = null; // Initialized in init(); } - get isAsync() { return true; } + get isAsync() { + return true; + } - async _query(options: any):Promise { + async _query(options: any): Promise { try { return await new Promise((resolve, reject) => { - options = {timeout: this.settings.queryTimeout, ...options}; + options = { timeout: this.settings.queryTimeout, ...options }; // mysql2 3.20+ tightened the query() overloads so the // (options, callback) signature is no longer matched directly; // pool.query(options, cb) still works at runtime but the types // expect (sql, values, cb) or (sql, cb). Cast to any to call // the runtime-correct (options, cb) form. - this._pool && (this._pool as any).query(options, (err:QueryError|null, ...args:string[]) => err != null ? reject(err) : resolve(args) - ); + this._pool && + (this._pool as any).query(options, (err: QueryError | null, ...args: string[]) => + err != null ? reject(err) : resolve(args), + ); }); - } catch (err:any) { - this.logger.error(`${err.fatal ? 'Fatal ' : ''}MySQL error: ${err.stack || err}`); + } catch (err: any) { + this.logger.error(`${err.fatal ? "Fatal " : ""}MySQL error: ${err.stack || err}`); throw err; } } async init() { - if("speeds" in this._mysqlSettings) { - delete this._mysqlSettings.speeds + if ("speeds" in this._mysqlSettings) { + delete this._mysqlSettings.speeds; } if ("filename" in this._mysqlSettings) { - delete this._mysqlSettings.filename + delete this._mysqlSettings.filename; } this._pool = createPool(this._mysqlSettings as ConnectionConfig); - const {database, charset} = this._mysqlSettings; + const { database, charset } = this._mysqlSettings; - const sqlCreate = `${'CREATE TABLE IF NOT EXISTS `store` ( ' + - '`key` VARCHAR( 100 ) NOT NULL COLLATE utf8mb4_bin, ' + - '`value` LONGTEXT COLLATE utf8mb4_bin NOT NULL , ' + - 'PRIMARY KEY ( `key` ) ' + - ') ENGINE='}${this.settings.engine} CHARSET=utf8mb4 COLLATE=utf8mb4_bin;`; + const sqlCreate = `${ + "CREATE TABLE IF NOT EXISTS `store` ( " + + "`key` VARCHAR( 100 ) NOT NULL COLLATE utf8mb4_bin, " + + "`value` LONGTEXT COLLATE utf8mb4_bin NOT NULL , " + + "PRIMARY KEY ( `key` ) " + + ") ENGINE=" + }${this.settings.engine} CHARSET=utf8mb4 COLLATE=utf8mb4_bin;`; - const sqlAlter = 'ALTER TABLE store MODIFY `key` VARCHAR(100) COLLATE utf8mb4_bin;'; + const sqlAlter = "ALTER TABLE store MODIFY `key` VARCHAR(100) COLLATE utf8mb4_bin;"; - await this._query({sql: sqlCreate}); + await this._query({ sql: sqlCreate }); // Checks for Database charset et al const dbCharSet = - 'SELECT DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME ' + - `FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = '${database}'`; - let [result] = await this._query({sql: dbCharSet}); + "SELECT DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME " + + `FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = '${database}'`; + let [result] = await this._query({ sql: dbCharSet }); result = JSON.parse(JSON.stringify(result)); if (result[0].DEFAULT_CHARACTER_SET_NAME !== charset) { - this.logger.error(`Database is not configured with charset ${charset} -- ` + - 'This may lead to crashes when certain characters are pasted in pads'); + this.logger.error( + `Database is not configured with charset ${charset} -- ` + + "This may lead to crashes when certain characters are pasted in pads", + ); this.logger.warn(result[0], charset); } if (result[0].DEFAULT_COLLATION_NAME.indexOf(charset) === -1) { this.logger.error( - `Database is not configured with collation name that includes ${charset} -- ` + - 'This may lead to crashes when certain characters are pasted in pads'); + `Database is not configured with collation name that includes ${charset} -- ` + + "This may lead to crashes when certain characters are pasted in pads", + ); this.logger.warn(result[0], charset, result[0].DEFAULT_COLLATION_NAME); } const tableCharSet = - 'SELECT CCSA.character_set_name AS character_set_name ' + - 'FROM information_schema.`TABLES` ' + - 'T,information_schema.`COLLATION_CHARACTER_SET_APPLICABILITY` CCSA ' + - 'WHERE CCSA.collation_name = T.table_collation ' + - `AND T.table_schema = '${database}' ` + - "AND T.table_name = 'store'"; - [result] = await this._query({sql: tableCharSet}); + "SELECT CCSA.character_set_name AS character_set_name " + + "FROM information_schema.`TABLES` " + + "T,information_schema.`COLLATION_CHARACTER_SET_APPLICABILITY` CCSA " + + "WHERE CCSA.collation_name = T.table_collation " + + `AND T.table_schema = '${database}' ` + + "AND T.table_name = 'store'"; + [result] = await this._query({ sql: tableCharSet }); if (!result[0]) { - this.logger.warn('Data has no character_set_name value -- ' + - 'This may lead to crashes when certain characters are pasted in pads'); + this.logger.warn( + "Data has no character_set_name value -- " + + "This may lead to crashes when certain characters are pasted in pads", + ); } - if (result[0] && (result[0].character_set_name !== charset)) { - this.logger.error(`table is not configured with charset ${charset} -- ` + - 'This may lead to crashes when certain characters are pasted in pads'); + if (result[0] && result[0].character_set_name !== charset) { + this.logger.error( + `table is not configured with charset ${charset} -- ` + + "This may lead to crashes when certain characters are pasted in pads", + ); this.logger.warn(result[0], charset); } // check migration level, alter if not migrated - const level = await this.get('MYSQL_MIGRATION_LEVEL'); + const level = await this.get("MYSQL_MIGRATION_LEVEL"); - if (level !== '1') { - await this._query({sql: sqlAlter}); - await this.set('MYSQL_MIGRATION_LEVEL', '1'); + if (level !== "1") { + await this._query({ sql: sqlAlter }); + await this.set("MYSQL_MIGRATION_LEVEL", "1"); } } - async get(key:string) { + async get(key: string) { const [results] = await this._query({ - sql: 'SELECT `value` FROM `store` WHERE `key` = ? AND BINARY `key` = ?', + sql: "SELECT `value` FROM `store` WHERE `key` = ? AND BINARY `key` = ?", values: [key, key], }); return results.length === 1 ? results[0].value : null; } - async findKeys(key:string, notKey:string) { - let query = 'SELECT `key` FROM `store` WHERE `key` LIKE ?'; + async findKeys(key: string, notKey: string) { + let query = "SELECT `key` FROM `store` WHERE `key` LIKE ?"; const params = []; // desired keys are key, e.g. pad:% - key = key.replace(/\*/g, '%'); + key = key.replace(/\*/g, "%"); params.push(key); if (notKey != null) { // not desired keys are notKey, e.g. %:%:% - notKey = notKey.replace(/\*/g, '%'); - query += ' AND `key` NOT LIKE ?'; + notKey = notKey.replace(/\*/g, "%"); + query += " AND `key` NOT LIKE ?"; params.push(notKey); } - const [results] = await this._query({sql: query, values: params}); - return results.map((val:{key:string}) => val.key); + const [results] = await this._query({ sql: query, values: params }); + return results.map((val: { key: string }) => val.key); } async findKeysPaged( key: string, notKey: string | null | undefined, - options: {limit: number; after?: string}, + options: { limit: number; after?: string }, ) { if (!options || !Number.isInteger(options.limit) || options.limit <= 0) { - throw new Error('findKeysPaged requires a positive integer limit'); + throw new Error("findKeysPaged requires a positive integer limit"); } - let query = 'SELECT `key` FROM `store` WHERE `key` LIKE ?'; - const params: (string | number)[] = [key.replace(/\*/g, '%')]; + let query = "SELECT `key` FROM `store` WHERE `key` LIKE ?"; + const params: (string | number)[] = [key.replace(/\*/g, "%")]; if (notKey != null) { - query += ' AND `key` NOT LIKE ?'; - params.push(notKey.replace(/\*/g, '%')); + query += " AND `key` NOT LIKE ?"; + params.push(notKey.replace(/\*/g, "%")); } if (options.after != null) { // BINARY forces a byte-wise comparison so paging is deterministic even // when the column collation is case-insensitive (e.g. utf8mb4_general_ci). - query += ' AND BINARY `key` > ?'; + query += " AND BINARY `key` > ?"; params.push(options.after); } - query += ' ORDER BY BINARY `key` ASC LIMIT ?'; + query += " ORDER BY BINARY `key` ASC LIMIT ?"; params.push(options.limit); - const [results] = await this._query({sql: query, values: params}); - return results.map((val: {key: string}) => val.key); + const [results] = await this._query({ sql: query, values: params }); + return results.map((val: { key: string }) => val.key); } - async set(key:string, value:string) { - if (key.length > 100) throw new Error('Your Key can only be 100 chars'); - await this._query({sql: 'REPLACE INTO `store` VALUES (?,?)', values: [key, value]}); + async set(key: string, value: string) { + if (key.length > 100) throw new Error("Your Key can only be 100 chars"); + await this._query({ sql: "REPLACE INTO `store` VALUES (?,?)", values: [key, value] }); } - async remove(key:string) { + async remove(key: string) { await this._query({ - sql: 'DELETE FROM `store` WHERE `key` = ? AND BINARY `key` = ?', + sql: "DELETE FROM `store` WHERE `key` = ? AND BINARY `key` = ?", values: [key, key], }); } - async doBulk(bulk:BulkObject[]) { + async doBulk(bulk: BulkObject[]) { const replaces = []; const deletes = []; for (const op of bulk) { switch (op.type) { - case 'set': replaces.push([op.key, op.value]); break; - case 'remove': deletes.push(op.key); break; - default: throw new Error(`unknown op type: ${op.type}`); + case "set": + replaces.push([op.key, op.value]); + break; + case "remove": + deletes.push(op.key); + break; + default: + throw new Error(`unknown op type: ${op.type}`); } } await Promise.all([ - replaces.length ? this._query({ - sql: 'REPLACE INTO `store` VALUES ?;', - values: [replaces], - }) : null, - deletes.length ? this._query({ - sql: 'DELETE FROM `store` WHERE `key` IN (?) AND BINARY `key` IN (?);', - values: [deletes, deletes], - }) : null, + replaces.length + ? this._query({ + sql: "REPLACE INTO `store` VALUES ?;", + values: [replaces], + }) + : null, + deletes.length + ? this._query({ + sql: "DELETE FROM `store` WHERE `key` IN (?) AND BINARY `key` IN (?);", + values: [deletes, deletes], + }) + : null, ]); } async close() { await util.promisify(this._pool!.end.bind(this._pool))(); } -}; +} diff --git a/databases/postgres_db.ts b/databases/postgres_db.ts index 5475c5c9..91c9aa2c 100644 --- a/databases/postgres_db.ts +++ b/databases/postgres_db.ts @@ -14,17 +14,17 @@ * limitations under the License. */ -import AbstractDatabase, {type Settings} from '../lib/AbstractDatabase'; -import async from 'async'; -import * as pg from 'pg'; -import type {BulkObject} from './cassandra_db'; +import AbstractDatabase, { type Settings } from "../lib/AbstractDatabase"; +import async from "async"; +import * as pg from "pg"; +import type { BulkObject } from "./cassandra_db"; export default class extends AbstractDatabase { public db: pg.Pool; public upsertStatement: string | null | undefined; - constructor(settings:Settings | string) { + constructor(settings: Settings | string) { super(settings as Settings); - if (typeof settings === 'string') settings = {connectionString: settings}; + if (typeof settings === "string") settings = { connectionString: settings }; this.settings = settings; this.settings.cache = settings.cache || 1000; @@ -38,13 +38,14 @@ export default class extends AbstractDatabase { this.db = new pg.Pool(this.settings as pg.PoolConfig); } - init(callback: (err: Error)=>{}) { + init(callback: (err: Error) => {}) { const testTableExists = "SELECT 1 as exists FROM pg_tables WHERE tablename = 'store'"; - const createTable = 'CREATE TABLE IF NOT EXISTS store (' + - '"key" character varying(100) NOT NULL, ' + - '"value" text NOT NULL, ' + - 'CONSTRAINT store_pkey PRIMARY KEY (key))'; + const createTable = + "CREATE TABLE IF NOT EXISTS store (" + + '"key" character varying(100) NOT NULL, ' + + '"value" text NOT NULL, ' + + "CONSTRAINT store_pkey PRIMARY KEY (key))"; // this variable will be given a value depending on the result of the // feature detection @@ -60,26 +61,26 @@ export default class extends AbstractDatabase { * - calls the callback */ const detectUpsertMethod = (callback: (err?: Error) => {}) => { - const upsertViaFunction = 'SELECT ueberdb_insert_or_update($1,$2)'; + const upsertViaFunction = "SELECT ueberdb_insert_or_update($1,$2)"; const upsertNatively = - 'INSERT INTO store(key, value) VALUES ($1, $2) ' + - 'ON CONFLICT (key) DO UPDATE SET value = excluded.value'; + "INSERT INTO store(key, value) VALUES ($1, $2) " + + "ON CONFLICT (key) DO UPDATE SET value = excluded.value"; const createFunc = - 'CREATE OR REPLACE FUNCTION ueberdb_insert_or_update(character varying, text) ' + - 'RETURNS void AS $$ ' + - 'BEGIN ' + - ' IF EXISTS( SELECT * FROM store WHERE key = $1 ) THEN ' + - ' UPDATE store SET value = $2 WHERE key = $1; ' + - ' ELSE ' + - ' INSERT INTO store(key,value) VALUES( $1, $2 ); ' + - ' END IF; ' + - ' RETURN; ' + - 'END; ' + - '$$ LANGUAGE plpgsql;'; + "CREATE OR REPLACE FUNCTION ueberdb_insert_or_update(character varying, text) " + + "RETURNS void AS $$ " + + "BEGIN " + + " IF EXISTS( SELECT * FROM store WHERE key = $1 ) THEN " + + " UPDATE store SET value = $2 WHERE key = $1; " + + " ELSE " + + " INSERT INTO store(key,value) VALUES( $1, $2 ); " + + " END IF; " + + " RETURN; " + + "END; " + + "$$ LANGUAGE plpgsql;"; const testNativeUpsert = `EXPLAIN ${upsertNatively}`; - this.db.query(testNativeUpsert, ['test-key', 'test-value'], (err) => { + this.db.query(testNativeUpsert, ["test-key", "test-value"], (err) => { if (err) { // the UPSERT statement failed: we will have to emulate it via // an sql function @@ -113,8 +114,8 @@ export default class extends AbstractDatabase { }); } - get(key:string, callback: (err: Error | null, value: any)=>{}) { - this.db.query('SELECT value FROM store WHERE key=$1', [key], (err, results) => { + get(key: string, callback: (err: Error | null, value: any) => {}) { + this.db.query("SELECT value FROM store WHERE key=$1", [key], (err, results) => { let value = null; if (!err && results.rows.length === 1) { @@ -125,21 +126,21 @@ export default class extends AbstractDatabase { }); } - findKeys(key:string, notKey:string, callback: (err: Error | null, value: any)=>{}) { - let query = 'SELECT key FROM store WHERE key LIKE $1'; + findKeys(key: string, notKey: string, callback: (err: Error | null, value: any) => {}) { + let query = "SELECT key FROM store WHERE key LIKE $1"; const params = []; // desired keys are %key:%, e.g. pad:% - key = key.replace(/\*/g, '%'); + key = key.replace(/\*/g, "%"); params.push(key); if (notKey != null) { // not desired keys are notKey:%, e.g. %:%:% - notKey = notKey.replace(/\*/g, '%'); - query += ' AND key NOT LIKE $2'; + notKey = notKey.replace(/\*/g, "%"); + query += " AND key NOT LIKE $2"; params.push(notKey); } this.db.query(query, params, (err, results) => { - const value:string[] = []; + const value: string[] = []; if (!err && results.rows.length > 0) { results.rows.forEach((val) => { @@ -154,18 +155,18 @@ export default class extends AbstractDatabase { findKeysPaged( key: string, notKey: string | null | undefined, - options: {limit: number; after?: string}, + options: { limit: number; after?: string }, callback: (err: Error | null, value: string[]) => void, ) { if (!options || !Number.isInteger(options.limit) || options.limit <= 0) { - return callback(new Error('findKeysPaged requires a positive integer limit'), []); + return callback(new Error("findKeysPaged requires a positive integer limit"), []); } - let query = 'SELECT key FROM store WHERE key LIKE $1'; - const params: (string | number)[] = [key.replace(/\*/g, '%')]; + let query = "SELECT key FROM store WHERE key LIKE $1"; + const params: (string | number)[] = [key.replace(/\*/g, "%")]; let n = 2; if (notKey != null) { query += ` AND key NOT LIKE $${n++}`; - params.push(notKey.replace(/\*/g, '%')); + params.push(notKey.replace(/\*/g, "%")); } if (options.after != null) { query += ` AND key > $${n++}`; @@ -182,31 +183,31 @@ export default class extends AbstractDatabase { }); } - set(key:string, value:string, callback:(err: Error, result: pg.QueryResult) => void) { + set(key: string, value: string, callback: (err: Error, result: pg.QueryResult) => void) { if (key.length > 100) { - const val = '' as any; - callback(Error('Your Key can only be 100 chars'), val); + const val = "" as any; + callback(Error("Your Key can only be 100 chars"), val); } else if (this.upsertStatement != null) { this.db.query(this.upsertStatement, [key, value], callback); } } - remove(key:string, callback:()=>{}) { - this.db.query('DELETE FROM store WHERE key=$1', [key], callback); + remove(key: string, callback: () => {}) { + this.db.query("DELETE FROM store WHERE key=$1", [key], callback); } - doBulk(bulk:BulkObject[], callback:()=>{}) { + doBulk(bulk: BulkObject[], callback: () => {}) { const replaceVALs = []; - let removeSQL = 'DELETE FROM store WHERE key IN ('; + let removeSQL = "DELETE FROM store WHERE key IN ("; const removeVALs: string[] = []; let removeCount = 0; for (const i in bulk) { - if (bulk[i].type === 'set') { + if (bulk[i].type === "set") { replaceVALs.push([bulk[i].key, bulk[i].value]); - } else if (bulk[i].type === 'remove') { - if (removeCount !== 0) removeSQL += ','; + } else if (bulk[i].type === "remove") { + if (removeCount !== 0) removeSQL += ","; removeCount += 1; removeSQL += `$${removeCount}`; @@ -214,26 +215,29 @@ export default class extends AbstractDatabase { } } - removeSQL += ');'; + removeSQL += ");"; if (!this.upsertStatement) { return; } + const functions: any = replaceVALs.map( + (v) => (cb: () => {}) => this.db.query(this.upsertStatement as string, v as string[], cb), + ); - const functions:any = replaceVALs.map((v) => (cb:()=>{}) => this.db.query(this.upsertStatement as string, v as string[], cb)); - - const removeFunction = (callback: ()=>{}) => { + const removeFunction = (callback: () => {}) => { if (!(removeVALs.length < 1)) { this.db.query(removeSQL, removeVALs, callback); - } else { callback(); } + } else { + callback(); + } }; functions.push(removeFunction); async.parallel(functions, callback); } - close(callback:()=>{}) { + close(callback: () => {}) { this.db.end(callback); } -}; +} diff --git a/databases/postgrespool_db.ts b/databases/postgrespool_db.ts index 9b9a2439..6de0268e 100644 --- a/databases/postgrespool_db.ts +++ b/databases/postgrespool_db.ts @@ -1,11 +1,13 @@ -import type {Settings} from '../lib/AbstractDatabase'; +import type { Settings } from "../lib/AbstractDatabase"; -import Postgres_db from './postgres_db' +import Postgres_db from "./postgres_db"; export default class PostgresDB extends Postgres_db { - constructor(settings:Settings) { - console.warn('ueberdb: The postgrespool database driver is deprecated ' + - 'and will be removed in a future version. Use postgres instead.'); + constructor(settings: Settings) { + console.warn( + "ueberdb: The postgrespool database driver is deprecated " + + "and will be removed in a future version. Use postgres instead.", + ); super(settings); } -}; +} diff --git a/databases/redis_db.ts b/databases/redis_db.ts index 62e68b38..e0689385 100644 --- a/databases/redis_db.ts +++ b/databases/redis_db.ts @@ -14,38 +14,40 @@ * limitations under the License. */ -import AbstractDatabase, {type Settings} from '../lib/AbstractDatabase'; -import {createClient} from 'redis'; -import type {RedisClientOptions} from 'redis'; -import type {BulkObject} from './cassandra_db'; +import AbstractDatabase, { type Settings } from "../lib/AbstractDatabase"; +import { createClient } from "redis"; +import type { RedisClientOptions } from "redis"; +import type { BulkObject } from "./cassandra_db"; export default class RedisDB extends AbstractDatabase { - public _client: any - constructor(settings:Settings) { + public _client: any; + constructor(settings: Settings) { super(settings); this._client = null; this.settings = settings || {}; } - get isAsync() { return true; } + get isAsync() { + return true; + } async init() { if (this.settings.url) { - this._client = createClient({url: this.settings.url}); + this._client = createClient({ url: this.settings.url }); } else if (this.settings.host) { - const options:RedisClientOptions = { - socket:{ + const options: RedisClientOptions = { + socket: { host: this.settings.host, port: Number(this.settings.port), - } - } - if (this.settings.password){ + }, + }; + if (this.settings.password) { options.password = this.settings.password; } - if (this.settings.user){ + if (this.settings.user) { options.username = this.settings.user; } - this._client = createClient(options) + this._client = createClient(options); } if (this._client) { await this._client.connect(); @@ -53,27 +55,27 @@ export default class RedisDB extends AbstractDatabase { } } - async get(key:string) { + async get(key: string) { if (this._client == null) return null; return await this._client.get(key); } - async findKeys(key:string, notKey:string) { + async findKeys(key: string, notKey: string) { if (this._client == null) return null; const [_, type] = /^([^:*]+):\*$/.exec(key) || []; - if (type != null && ['*:*:*', `${key}:*`].includes(notKey)) { + if (type != null && ["*:*:*", `${key}:*`].includes(notKey)) { // Performance optimization for a common Etherpad case. return await this._client.sMembers(`ueberDB:keys:${type}`); } - let keys = await this._client.keys(key.replace(/[?[\]\\]/g, '\\$&')); + let keys = await this._client.keys(key.replace(/[?[\]\\]/g, "\\$&")); if (notKey != null) { const regex = this.createFindRegex(key, notKey); - keys = keys.filter((k:string) => regex.test(k)); + keys = keys.filter((k: string) => regex.test(k)); } return keys; } - async set(key:string, value:string) { + async set(key: string, value: string) { if (this._client == null) return null; const matches = /^([^:]+):([^:]+)$/.exec(key); await Promise.all([ @@ -82,7 +84,7 @@ export default class RedisDB extends AbstractDatabase { ]); } - async remove(key:string) { + async remove(key: string) { if (this._client == null) return null; const matches = /^([^:]+):([^:]+)$/.exec(key); await Promise.all([ @@ -95,14 +97,14 @@ export default class RedisDB extends AbstractDatabase { if (this._client == null) return; const multi = this._client.multi(); - for (const {key, type, value} of bulk) { + for (const { key, type, value } of bulk) { const matches = /^([^:]+):([^:]+)$/.exec(key); - if (type === 'set') { + if (type === "set") { if (matches) { multi.sAdd(`ueberDB:keys:${matches[1]}`, matches[0]); } multi.set(key, value as string); - } else if (type === 'remove') { + } else if (type === "remove") { if (matches) { multi.sRem(`ueberDB:keys:${matches[1]}`, matches[0]); } @@ -118,4 +120,4 @@ export default class RedisDB extends AbstractDatabase { await this._client.quit(); this._client = null; } -}; +} diff --git a/databases/rethink_db.ts b/databases/rethink_db.ts index 8f9c0178..10c294ac 100644 --- a/databases/rethink_db.ts +++ b/databases/rethink_db.ts @@ -14,10 +14,10 @@ * limitations under the License. */ -import AbstractDatabase, {type Settings} from '../lib/AbstractDatabase'; -import r from 'rethinkdb'; -import async from 'async'; -import type {BulkObject} from './cassandra_db'; +import AbstractDatabase, { type Settings } from "../lib/AbstractDatabase"; +import r from "rethinkdb"; +import async from "async"; +import type { BulkObject } from "./cassandra_db"; export default class Rethink_db extends AbstractDatabase { public host: string; @@ -25,13 +25,21 @@ export default class Rethink_db extends AbstractDatabase { public port: number | string; public table: string; public connection: r.Connection | null; - constructor(settings:Settings) { + constructor(settings: Settings) { super(settings); if (!settings) settings = {}; - if (!settings.host) { settings.host = 'localhost'; } - if (!settings.port) { settings.port = 28015; } - if (!settings.db) { settings.db = 'test'; } - if (!settings.table) { settings.table = 'test'; } + if (!settings.host) { + settings.host = "localhost"; + } + if (!settings.port) { + settings.port = 28015; + } + if (!settings.db) { + settings.db = "test"; + } + if (!settings.table) { + settings.table = "test"; + } this.host = settings.host; this.db = settings.db; @@ -40,7 +48,7 @@ export default class Rethink_db extends AbstractDatabase { this.connection = null; } - init(callback: (p: any, cursor: any)=>{}) { + init(callback: (p: any, cursor: any) => {}) { // @ts-ignore r.connect(this, (err, conn) => { if (err) throw err; @@ -51,20 +59,24 @@ export default class Rethink_db extends AbstractDatabase { // assuming table does not exists // @ts-ignore r.tableCreate(this.table).run(this.connection, callback); - } else if (callback) { callback(null, cursor); } + } else if (callback) { + callback(null, cursor); + } }); }); } - get(key:string, callback: (err: Error, p: any)=>{}) { + get(key: string, callback: (err: Error, p: any) => {}) { // @ts-ignore - r.table(this.table).get(key).run(this.connection, (err, item) => { - // @ts-ignore - callback(err, (item ? item.content : item)); - }); + r.table(this.table) + .get(key) + .run(this.connection, (err, item) => { + // @ts-ignore + callback(err, item ? item.content : item); + }); } - findKeys(key:string, notKey:string, callback:()=>{}) { + findKeys(key: string, notKey: string, callback: () => {}) { const keys = []; const regex = this.createFindRegex(key, notKey); // @ts-ignore @@ -75,40 +87,47 @@ export default class Rethink_db extends AbstractDatabase { }).run(this.connection, callback); } - set(key:string, value:string, callback:()=>{}) { + set(key: string, value: string, callback: () => {}) { r.table(this.table) - .insert({id: key, content: value}, {conflict: 'replace'}) - .run(this.connection as r.Connection, callback); + .insert({ id: key, content: value }, { conflict: "replace" }) + .run(this.connection as r.Connection, callback); } - doBulk(bulk: BulkObject[], callback: ()=>{}) { + doBulk(bulk: BulkObject[], callback: () => {}) { const _in: any[] = []; const _out: string | string[] | r.Expression = []; for (const i in bulk) { - if (bulk[i].type === 'set') { - _in.push({id: bulk[i].key, content: bulk[i].value}); - } else if (bulk[i].type === 'remove') { + if (bulk[i].type === "set") { + _in.push({ id: bulk[i].key, content: bulk[i].value }); + } else if (bulk[i].type === "remove") { _out.push(bulk[i].key); } } - async.parallel([ - (cb) => { // @ts-ignore - r.table(this.table).insert(_in, {conflict: 'replace'}).run(this.connection, cb); - }, - (cb) => { // @ts-ignore - r.table(this.table).getAll(_out).delete().run(this.connection, cb); - }, - ], callback); + async.parallel( + [ + (cb) => { + // @ts-ignore + r.table(this.table).insert(_in, { conflict: "replace" }).run(this.connection, cb); + }, + (cb) => { + // @ts-ignore + r.table(this.table).getAll(_out).delete().run(this.connection, cb); + }, + ], + callback, + ); } - remove(key:string, callback:()=>{}) { + remove(key: string, callback: () => {}) { // @ts-ignore r.table(this.table).get(key).delete().run(this.connection, callback); } - close(callback:()=>{}) { - if (this.connection) { this.connection.close(callback); } + close(callback: () => {}) { + if (this.connection) { + this.connection.close(callback); + } } -}; +} diff --git a/databases/rusty_db.ts b/databases/rusty_db.ts index e7da4bba..a28b8c8e 100644 --- a/databases/rusty_db.ts +++ b/databases/rusty_db.ts @@ -1,62 +1,61 @@ import AbstractDatabase from "../lib/AbstractDatabase"; -import {KeyValueDB} from 'rusty-store-kv' +import { KeyValueDB } from "rusty-store-kv"; export default class Rusty_db extends AbstractDatabase { - db: any |null| undefined - - constructor(settings: {filename: string}) { - super(settings); - - // set default settings - this.settings.cache = 0; - this.settings.writeInterval = 0; - this.settings.json = false; - } - - get isAsync() { - return true; - } - - findKeys(key: string, notKey?:string) { - return this.db!.findKeys(key, notKey); - } - - get(key: string) { - const val = this.db!.get(key); - if (!val) { - return val - } - try { - return JSON.parse(val) - } catch (e) { - return val - } - } - - async init() { - - this.db = new KeyValueDB(this.settings.filename!); - } - - close() { - this.db?.close() - this.db = null - } - - remove(key: string) { - this.db!.remove(key); - } - - set(key: string, value: string) { - if (typeof value === "object") { - const valStr = JSON.stringify(value) - this.db!.set(key, valStr); - } else { - this.db!.set(key, value.toString()); - } - } - - destroy() { - this.db!.destroy(); - } + db: any | null | undefined; + + constructor(settings: { filename: string }) { + super(settings); + + // set default settings + this.settings.cache = 0; + this.settings.writeInterval = 0; + this.settings.json = false; + } + + get isAsync() { + return true; + } + + findKeys(key: string, notKey?: string) { + return this.db!.findKeys(key, notKey); + } + + get(key: string) { + const val = this.db!.get(key); + if (!val) { + return val; + } + try { + return JSON.parse(val); + } catch (e) { + return val; + } + } + + async init() { + this.db = new KeyValueDB(this.settings.filename!); + } + + close() { + this.db?.close(); + this.db = null; + } + + remove(key: string) { + this.db!.remove(key); + } + + set(key: string, value: string) { + if (typeof value === "object") { + const valStr = JSON.stringify(value); + this.db!.set(key, valStr); + } else { + this.db!.set(key, value.toString()); + } + } + + destroy() { + this.db!.destroy(); + } } diff --git a/databases/sqlite_db.ts b/databases/sqlite_db.ts index 62a43610..4aedf8cf 100644 --- a/databases/sqlite_db.ts +++ b/databases/sqlite_db.ts @@ -1,6 +1,6 @@ -import type {BulkObject} from './cassandra_db'; -import AbstractDatabase, {type Settings} from '../lib/AbstractDatabase'; -import {SQLite} from "rusty-store-kv"; +import type { BulkObject } from "./cassandra_db"; +import AbstractDatabase, { type Settings } from "../lib/AbstractDatabase"; +import { SQLite } from "rusty-store-kv"; /** * 2011 Peter 'Pita' Martischka @@ -19,19 +19,19 @@ import {SQLite} from "rusty-store-kv"; */ export default class SQLiteDB extends AbstractDatabase { - public db: any|null; - constructor(settings:Settings) { + public db: any | null; + constructor(settings: Settings) { super(settings); this.db = null; if (!settings || !settings.filename) { - settings = {filename: ':memory:'}; + settings = { filename: ":memory:" }; } this.settings = settings; // set settings for the dbWrapper - if (settings.filename === ':memory:') { + if (settings.filename === ":memory:") { this.settings.cache = 0; this.settings.writeInterval = 0; this.settings.json = true; @@ -43,51 +43,49 @@ export default class SQLiteDB extends AbstractDatabase { } init(callback: Function) { - this.db = new SQLite(this.settings.filename as string) + this.db = new SQLite(this.settings.filename as string); callback(); } - - get(key:string, callback:Function) { - const res = this.db!.get(key) - callback(null, res ? res : null) + get(key: string, callback: Function) { + const res = this.db!.get(key); + callback(null, res ? res : null); } - findKeys(key:string, notKey:string, callback:Function) { - const res = this.db?.findKeys(key, notKey) + findKeys(key: string, notKey: string, callback: Function) { + const res = this.db?.findKeys(key, notKey); callback(null, res); } - set(key:string, value:string, callback:Function) { - const res = this.db!.set(key, value) - res ? callback(null, null) : callback(null, res) + set(key: string, value: string, callback: Function) { + const res = this.db!.set(key, value); + res ? callback(null, null) : callback(null, res); } - remove(key:string, callback:Function) { - this.db!.remove(key) - callback(null, null) + remove(key: string, callback: Function) { + this.db!.remove(key); + callback(null, null); } - - doBulk(bulk:BulkObject[], callback:Function) { - const convertedBulk = bulk.map(b=>{ + doBulk(bulk: BulkObject[], callback: Function) { + const convertedBulk = bulk.map((b) => { if (b.value === null) { return { key: b.key, - type: b.type - } satisfies BulkObject + type: b.type, + } satisfies BulkObject; } else { - return b + return b; } - }) + }); - this.db!.doBulk(convertedBulk) + this.db!.doBulk(convertedBulk); callback(); } close(callback: Function) { - callback() + callback(); this.db!.close(); } -}; +} diff --git a/databases/surrealdb_db.ts b/databases/surrealdb_db.ts index f8c675f8..3c5ab03d 100644 --- a/databases/surrealdb_db.ts +++ b/databases/surrealdb_db.ts @@ -14,184 +14,186 @@ * limitations under the License. */ -import AbstractDatabase, {type Settings} from '../lib/AbstractDatabase'; -import {Surreal} from 'surrealdb'; -import type {BulkObject} from './cassandra_db'; +import AbstractDatabase, { type Settings } from "../lib/AbstractDatabase"; +import { Surreal } from "surrealdb"; +import type { BulkObject } from "./cassandra_db"; -const DATABASE = 'ueberdb'; -const WILDCARD = '*'; +const DATABASE = "ueberdb"; +const WILDCARD = "*"; type StoreVal = { - key: string; - value: string; + key: string; + value: string; }; -const replaceAt = function(index: number, replacement: string, original: string) { - return original.substring(0, index) + replacement + original.substring(index + replacement.length); +const replaceAt = function (index: number, replacement: string, original: string) { + return ( + original.substring(0, index) + replacement + original.substring(index + replacement.length) + ); }; // surrealdb uses `:` to separate the table name from the record id; an // untreated `:` in a key would create an unintended record id, so we replace // the first `:` with `_` on write and reverse the substitution on read. const escapeKey = (key: string) => { - const index = key.indexOf(':'); - if (index > -1) { - return replaceAt(index, '_', key); - } - return key; + const index = key.indexOf(":"); + if (index > -1) { + return replaceAt(index, "_", key); + } + return key; }; const unescapeKey = (key: string, originalKey: string) => { - const index = originalKey.indexOf(':'); - if (index > -1) { - return replaceAt(index, ':', key); - } - return key; + const index = originalKey.indexOf(":"); + if (index > -1) { + return replaceAt(index, ":", key); + } + return key; }; export default class SurrealDB extends AbstractDatabase { - public _client: Surreal | null; - - constructor(settings: Settings) { - super(settings); - this._client = null; + public _client: Surreal | null; + + constructor(settings: Settings) { + super(settings); + this._client = null; + } + + get isAsync() { + return true; + } + + async init() { + if (this.settings.url) { + this._client = new Surreal(); + await this._client.connect(this.settings.url); + } else if (this.settings.host) { + const port = this.settings.port || 8000; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const opts = (this.settings.clientOptions ?? {}) as Record; + const protocol = (opts.protocol as string | undefined) || "http://"; + const path = (opts.path as string | undefined) || "/rpc"; + const host = this.settings.host; + this._client = new Surreal(); + await this._client.connect(`${protocol}${host}:${port}${path}`); } - - get isAsync() { return true; } - - async init() { - if (this.settings.url) { - this._client = new Surreal(); - await this._client.connect(this.settings.url); - } else if (this.settings.host) { - const port = this.settings.port || 8000; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const opts = (this.settings.clientOptions ?? {}) as Record; - const protocol = (opts.protocol as string | undefined) || 'http://'; - const path = (opts.path as string | undefined) || '/rpc'; - const host = this.settings.host; - this._client = new Surreal(); - await this._client.connect(`${protocol}${host}:${port}${path}`); - } - if (this.settings.user && this.settings.password) { - await this._client!.signin({ - username: this.settings.user!, - password: this.settings.password!, - }); - } - await this._client!.use({namespace: DATABASE, database: DATABASE}); + if (this.settings.user && this.settings.password) { + await this._client!.signin({ + username: this.settings.user!, + password: this.settings.password!, + }); } - - async get(key: string): Promise { - if (this._client == null) return null; - - key = escapeKey(key); - // surrealdb 2.x: query() returns a Query that resolves to an - // array of result sets — one entry per SurrealQL statement. The - // first entry is the rows for our SELECT. - const res = await this._client.query<[StoreVal[]]>( - 'SELECT key, value FROM store WHERE key = $key', - {key}, - ); - const rows = res[0] || []; - if (rows.length === 0) return null; - const row = rows[0]; - if (typeof row === 'string') return row; - return unescapeKey(row.value, key); + await this._client!.use({ namespace: DATABASE, database: DATABASE }); + } + + async get(key: string): Promise { + if (this._client == null) return null; + + key = escapeKey(key); + // surrealdb 2.x: query() returns a Query that resolves to an + // array of result sets — one entry per SurrealQL statement. The + // first entry is the rows for our SELECT. + const res = await this._client.query<[StoreVal[]]>( + "SELECT key, value FROM store WHERE key = $key", + { key }, + ); + const rows = res[0] || []; + if (rows.length === 0) return null; + const row = rows[0]; + if (typeof row === "string") return row; + return unescapeKey(row.value, key); + } + + async findKeys(key: string, notKey: string | null) { + if (this._client == null) return null; + + const queryString = + notKey != null + ? `SELECT key FROM store WHERE ${this.transformWildcard(key, "key")} AND ${this.transformWildcardNegative(notKey, "notKey")}` + : `SELECT key FROM store WHERE ${this.transformWildcard(key, "key")}`; + + const cleanKey = key.replace(WILDCARD, ""); + const cleanNotKey = (notKey || "").replace(WILDCARD, ""); + const bindings: Record = { key: cleanKey }; + if (notKey != null) bindings.notKey = cleanNotKey; + + const res = await this._client.query<[StoreVal[]]>(queryString, bindings); + return this.transformResult(res[0] || [], cleanKey); + } + + transformWildcard(key: string, keyExpr: string) { + if (key.startsWith(WILDCARD) && key.endsWith(WILDCARD)) { + return `${keyExpr} CONTAINS $${keyExpr}`; + } else if (key.startsWith(WILDCARD)) { + return `string::ends_with(${keyExpr}, $${keyExpr})`; + } else if (key.endsWith(WILDCARD)) { + return `string::starts_with(${keyExpr}, $${keyExpr})`; + } else { + return `${keyExpr} = $${keyExpr}`; } - - async findKeys(key: string, notKey: string | null) { - if (this._client == null) return null; - - const queryString = notKey != null - ? `SELECT key FROM store WHERE ${this.transformWildcard(key, 'key')} AND ${this.transformWildcardNegative(notKey, 'notKey')}` - : `SELECT key FROM store WHERE ${this.transformWildcard(key, 'key')}`; - - const cleanKey = key.replace(WILDCARD, ''); - const cleanNotKey = (notKey || '').replace(WILDCARD, ''); - const bindings: Record = {key: cleanKey}; - if (notKey != null) bindings.notKey = cleanNotKey; - - const res = await this._client.query<[StoreVal[]]>(queryString, bindings); - return this.transformResult(res[0] || [], cleanKey); + } + + transformWildcardNegative(key: string, keyExpr: string) { + if (key.startsWith(WILDCARD) && key.endsWith(WILDCARD)) { + return `key CONTAINSNOT $${keyExpr}`; + } else if (key.startsWith(WILDCARD)) { + return `string::ends_with(key, $${keyExpr}) == false`; + } else if (key.endsWith(WILDCARD)) { + return `string::starts_with(key, $${keyExpr}) == false`; + } else { + return `key != $${keyExpr}`; } + } - transformWildcard(key: string, keyExpr: string) { - if (key.startsWith(WILDCARD) && key.endsWith(WILDCARD)) { - return `${keyExpr} CONTAINS $${keyExpr}`; - } else if (key.startsWith(WILDCARD)) { - return `string::ends_with(${keyExpr}, $${keyExpr})`; - } else if (key.endsWith(WILDCARD)) { - return `string::starts_with(${keyExpr}, $${keyExpr})`; - } else { - return `${keyExpr} = $${keyExpr}`; - } + transformResult(rows: StoreVal[] | string, originalKey: string) { + const value: string[] = []; + if (typeof rows === "string") { + value.push(unescapeKey(rows, originalKey)); + return value; } - - transformWildcardNegative(key: string, keyExpr: string) { - if (key.startsWith(WILDCARD) && key.endsWith(WILDCARD)) { - return `key CONTAINSNOT $${keyExpr}`; - } else if (key.startsWith(WILDCARD)) { - return `string::ends_with(key, $${keyExpr}) == false`; - } else if (key.endsWith(WILDCARD)) { - return `string::starts_with(key, $${keyExpr}) == false`; - } else { - return `key != $${keyExpr}`; - } - } - - transformResult(rows: StoreVal[] | string, originalKey: string) { - const value: string[] = []; - if (typeof rows === 'string') { - value.push(unescapeKey(rows, originalKey)); - return value; - } - for (const row of rows) { - value.push(unescapeKey(row.key, originalKey)); - } - return value; - } - - async set(key: string, value: string) { - if (this._client == null) return null; - const exists = await this.get(key); - const escapedKey = escapeKey(key); - if (exists) { - await this._client.query( - 'UPDATE store SET value = $value WHERE key = $key', - {key: escapedKey, value}, - ); - } else { - await this._client.query( - 'INSERT INTO store (key, value) VALUES ($key, $value)', - {key: escapedKey, value}, - ); - } + for (const row of rows) { + value.push(unescapeKey(row.key, originalKey)); } - - async remove(key: string) { - if (this._client == null) return null; - key = escapeKey(key); - return await this._client.query( - 'DELETE FROM store WHERE key = $key', - {key}, - ); + return value; + } + + async set(key: string, value: string) { + if (this._client == null) return null; + const exists = await this.get(key); + const escapedKey = escapeKey(key); + if (exists) { + await this._client.query("UPDATE store SET value = $value WHERE key = $key", { + key: escapedKey, + value, + }); + } else { + await this._client.query("INSERT INTO store (key, value) VALUES ($key, $value)", { + key: escapedKey, + value, + }); } - - async doBulk(bulk: BulkObject[]): Promise { - if (this._client == null) return; - for (const b of bulk) { - if (b.type === 'set') { - await this.set(b.key, b.value!); - } else if (b.type === 'remove') { - await this.remove(b.key); - } - } + } + + async remove(key: string) { + if (this._client == null) return null; + key = escapeKey(key); + return await this._client.query("DELETE FROM store WHERE key = $key", { key }); + } + + async doBulk(bulk: BulkObject[]): Promise { + if (this._client == null) return; + for (const b of bulk) { + if (b.type === "set") { + await this.set(b.key, b.value!); + } else if (b.type === "remove") { + await this.remove(b.key); + } } + } - async close() { - if (this._client == null) return null; - await this._client.close(); - this._client = null; - } + async close() { + if (this._client == null) return null; + await this._client.close(); + this._client = null; + } } diff --git a/docker-compose.yml b/docker-compose.yml index bbc8d18a..7f5c912c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,44 +1,44 @@ # Docker compose setup for testing with multiple databases -version: '3.8' +version: "3.8" services: - couchdb: - image: couchdb - ports: - - "5984:5984" - environment: - COUCHDB_USER: ueberdb - COUCHDB_PASSWORD: ueberdb - elasticsearch: - image: elasticsearch:9.3.3 - ports: - - 9200:9200 - environment: - discovery.type: single-node - mongo: - image: mongo - ports: - - 27017:27017 - environment: - MONGO_INITDB_DATABASE: mydb_test - mysql: - image: mariadb - ports: - - 3306:3306 - environment: - MYSQL_ROOT_PASSWORD: password - MYSQL_USER: ueberdb - MYSQL_PASSWORD: ueberdb - MYSQL_DATABASE: ueberdb - postgres: - image: postgres:14-alpine - ports: - - 5432:5432 - environment: - POSTGRES_USER: ueberdb - POSTGRES_PASSWORD: ueberdb - POSTGRES_DB: ueberdb - POSTGRES_HOST_AUTH_METHOD: "trust" - redis: - image: redis - ports: - - "6379:6379" + couchdb: + image: couchdb + ports: + - "5984:5984" + environment: + COUCHDB_USER: ueberdb + COUCHDB_PASSWORD: ueberdb + elasticsearch: + image: elasticsearch:9.3.3 + ports: + - 9200:9200 + environment: + discovery.type: single-node + mongo: + image: mongo + ports: + - 27017:27017 + environment: + MONGO_INITDB_DATABASE: mydb_test + mysql: + image: mariadb + ports: + - 3306:3306 + environment: + MYSQL_ROOT_PASSWORD: password + MYSQL_USER: ueberdb + MYSQL_PASSWORD: ueberdb + MYSQL_DATABASE: ueberdb + postgres: + image: postgres:14-alpine + ports: + - 5432:5432 + environment: + POSTGRES_USER: ueberdb + POSTGRES_PASSWORD: ueberdb + POSTGRES_DB: ueberdb + POSTGRES_HOST_AUTH_METHOD: "trust" + redis: + image: redis + ports: + - "6379:6379" diff --git a/index.ts b/index.ts index 082e3e33..c1fa8f25 100644 --- a/index.ts +++ b/index.ts @@ -15,35 +15,35 @@ * limitations under the License. */ -import {Database as DatabaseCache, type Metrics} from './lib/CacheAndBufferLayer'; -import {normalizeLogger} from './lib/logging'; -import type {Settings} from './lib/AbstractDatabase'; +import { Database as DatabaseCache, type Metrics } from "./lib/CacheAndBufferLayer"; +import { normalizeLogger } from "./lib/logging"; +import type { Settings } from "./lib/AbstractDatabase"; -export type {Settings} from './lib/AbstractDatabase'; -export type {Metrics, CacheSettings} from './lib/CacheAndBufferLayer'; -export type {Logger} from './lib/logging'; +export type { Settings } from "./lib/AbstractDatabase"; +export type { Metrics, CacheSettings } from "./lib/CacheAndBufferLayer"; +export type { Logger } from "./lib/logging"; // Database drivers are loaded lazily in initDB() so that only the selected // backend's dependencies need to be installed. export type DatabaseType = - | 'cassandra' - | 'couch' - | 'dirty' - | 'dirtygit' - | 'elasticsearch' - | 'memory' - | 'mock' - | 'mongodb' - | 'mssql' - | 'mysql' - | 'postgres' - | 'postgrespool' - | 'redis' - | 'rethink' - | 'rustydb' - | 'sqlite' - | 'surrealdb'; + | "cassandra" + | "couch" + | "dirty" + | "dirtygit" + | "elasticsearch" + | "memory" + | "mock" + | "mongodb" + | "mssql" + | "mysql" + | "postgres" + | "postgrespool" + | "redis" + | "rethink" + | "rustydb" + | "sqlite" + | "surrealdb"; export class Database { public readonly type: DatabaseType; @@ -66,7 +66,7 @@ export class Database { logger: Partial> | null = null, ) { if (!type) { - type = 'sqlite'; + type = "sqlite"; dbSettings = null; wrapperSettings = null; } @@ -88,45 +88,45 @@ export class Database { // eslint-disable-next-line @typescript-eslint/no-explicit-any private async initDB(): Promise { switch (this.type) { - case 'mysql': - return new (await import('./databases/mysql_db')).default(this.dbSettings as Settings); - case 'postgres': - return new (await import('./databases/postgres_db')).default(this.dbSettings as Settings); - case 'sqlite': - return new (await import('./databases/sqlite_db')).default(this.dbSettings as Settings); - case 'rustydb': + case "mysql": + return new (await import("./databases/mysql_db")).default(this.dbSettings as Settings); + case "postgres": + return new (await import("./databases/postgres_db")).default(this.dbSettings as Settings); + case "sqlite": + return new (await import("./databases/sqlite_db")).default(this.dbSettings as Settings); + case "rustydb": // eslint-disable-next-line @typescript-eslint/no-explicit-any - return new (await import('./databases/rusty_db')).default(this.dbSettings as any); - case 'mongodb': - return new (await import('./databases/mongodb_db')).default(this.dbSettings as Settings); - case 'redis': - return new (await import('./databases/redis_db')).default(this.dbSettings as Settings); - case 'cassandra': - return new (await import('./databases/cassandra_db')).default(this.dbSettings as Settings); - case 'dirty': - return new (await import('./databases/dirty_db')).default(this.dbSettings as Settings); - case 'dirtygit': - return new (await import('./databases/dirty_git_db')).default(this.dbSettings as Settings); - case 'elasticsearch': - return new (await import('./databases/elasticsearch_db')).default( + return new (await import("./databases/rusty_db")).default(this.dbSettings as any); + case "mongodb": + return new (await import("./databases/mongodb_db")).default(this.dbSettings as Settings); + case "redis": + return new (await import("./databases/redis_db")).default(this.dbSettings as Settings); + case "cassandra": + return new (await import("./databases/cassandra_db")).default(this.dbSettings as Settings); + case "dirty": + return new (await import("./databases/dirty_db")).default(this.dbSettings as Settings); + case "dirtygit": + return new (await import("./databases/dirty_git_db")).default(this.dbSettings as Settings); + case "elasticsearch": + return new (await import("./databases/elasticsearch_db")).default( this.dbSettings as Settings, ); - case 'memory': - return new (await import('./databases/memory_db')).default(this.dbSettings as Settings); - case 'mock': - return new (await import('./databases/mock_db')).default(this.dbSettings as Settings); - case 'mssql': - return new (await import('./databases/mssql_db')).default(this.dbSettings as Settings); - case 'postgrespool': - return new (await import('./databases/postgrespool_db')).default( + case "memory": + return new (await import("./databases/memory_db")).default(this.dbSettings as Settings); + case "mock": + return new (await import("./databases/mock_db")).default(this.dbSettings as Settings); + case "mssql": + return new (await import("./databases/mssql_db")).default(this.dbSettings as Settings); + case "postgrespool": + return new (await import("./databases/postgrespool_db")).default( this.dbSettings as Settings, ); - case 'rethink': - return new (await import('./databases/rethink_db')).default(this.dbSettings as Settings); - case 'couch': - return new (await import('./databases/couch_db')).default(this.dbSettings as Settings); - case 'surrealdb': - return new (await import('./databases/surrealdb_db')).default(this.dbSettings as Settings); + case "rethink": + return new (await import("./databases/rethink_db")).default(this.dbSettings as Settings); + case "couch": + return new (await import("./databases/couch_db")).default(this.dbSettings as Settings); + case "surrealdb": + return new (await import("./databases/surrealdb_db")).default(this.dbSettings as Settings); default: throw new Error(`Invalid database type: ${this.type as string}`); } @@ -158,7 +158,7 @@ export class Database { async findKeysPaged( key: string, notKey: string | null | undefined, - options: {limit: number; after?: string}, + options: { limit: number; after?: string }, ): Promise { return this.db.findKeysPaged(key, notKey, options); } diff --git a/lib/AbstractDatabase.ts b/lib/AbstractDatabase.ts index cc111dc5..f00d8810 100644 --- a/lib/AbstractDatabase.ts +++ b/lib/AbstractDatabase.ts @@ -1,9 +1,9 @@ -import {normalizeLogger, type Logger} from './logging'; +import { normalizeLogger, type Logger } from "./logging"; const nullLogger = normalizeLogger(null); const simpleGlobToRegExp = (s: string) => - s.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*'); + s.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*"); export type Settings = { data?: unknown; @@ -48,10 +48,10 @@ class AbstractDatabase { constructor(settings: Settings) { if (new.target === AbstractDatabase) { - throw new TypeError('cannot instantiate Abstract Database directly'); + throw new TypeError("cannot instantiate Abstract Database directly"); } - for (const fn of ['init', 'close', 'get', 'findKeys', 'remove', 'set']) { - if (typeof (this as Record)[fn] !== 'function') { + for (const fn of ["init", "close", "get", "findKeys", "remove", "set"]) { + if (typeof (this as Record)[fn] !== "function") { throw new TypeError(`method ${fn} not defined`); } } @@ -73,10 +73,12 @@ class AbstractDatabase { // eslint-disable-next-line @typescript-eslint/no-explicit-any doBulk(..._args: any[]): void | Promise { - throw new Error('the doBulk method must be implemented if write caching is enabled'); + throw new Error("the doBulk method must be implemented if write caching is enabled"); } - get isAsync(): boolean { return false; } + get isAsync(): boolean { + return false; + } } export default AbstractDatabase; diff --git a/lib/logging.ts b/lib/logging.ts index 7cd281dd..bc4ffe76 100644 --- a/lib/logging.ts +++ b/lib/logging.ts @@ -1,5 +1,5 @@ -import {Console} from 'console'; -import {stdout, stderr} from 'process'; +import { Console } from "console"; +import { stdout, stderr } from "process"; export type Logger = { debug(...args: unknown[]): void; @@ -12,25 +12,35 @@ export type Logger = { isErrorEnabled(): boolean; }; -type LogLevel = 'debug' | 'info' | 'warn' | 'error'; +type LogLevel = "debug" | "info" | "warn" | "error"; export class ConsoleLogger extends Console implements Logger { - constructor(opts = {}) { super({stdout, stderr, inspectOptions: {depth: Infinity}, ...opts}); } - isDebugEnabled(): boolean { return false; } - isInfoEnabled(): boolean { return true; } - isWarnEnabled(): boolean { return true; } - isErrorEnabled(): boolean { return true; } + constructor(opts = {}) { + super({ stdout, stderr, inspectOptions: { depth: Infinity }, ...opts }); + } + isDebugEnabled(): boolean { + return false; + } + isInfoEnabled(): boolean { + return true; + } + isWarnEnabled(): boolean { + return true; + } + isErrorEnabled(): boolean { + return true; + } } export const normalizeLogger = (logger: Partial | null): Logger => { - const levels: LogLevel[] = ['debug', 'info', 'warn', 'error']; + const levels: LogLevel[] = ["debug", "info", "warn", "error"]; const normalized = Object.create(logger ?? {}) as Record; for (const level of levels) { const enabledFn = `is${level.charAt(0).toUpperCase()}${level.slice(1)}Enabled`; - if (typeof normalized[level] !== 'function') { + if (typeof normalized[level] !== "function") { normalized[level] = () => {}; normalized[enabledFn] = () => false; - } else if (typeof normalized[enabledFn] !== 'function') { + } else if (typeof normalized[enabledFn] !== "function") { normalized[enabledFn] = () => true; } } diff --git a/package.json b/package.json index ccb38470..608af2fa 100644 --- a/package.json +++ b/package.json @@ -1,16 +1,24 @@ { "name": "ueberdb2", - "description": "Transform every database into a object key value store", - "url": "https://github.com/ether/ueberDB", - "type": "module", "version": "6.1.5", + "description": "Transform every database into a object key value store", "keywords": [ "database", "keyvalue" ], + "homepage": "https://github.com/ether/ueberDB", + "bugs": { + "url": "https://github.com/ether/ueberDB/issues" + }, "author": { "name": "The Etherpad Foundation" }, + "maintainers": [ + { + "name": "John McLear", + "email": "john@mclear.co.uk" + } + ], "contributors": [ { "name": "John McLear" @@ -22,20 +30,65 @@ "name": "Peter Martischka" } ], - "exports": { - ".": { - "import": "./dist/index.js", - "types": "./dist/index.d.ts" - } + "repository": { + "type": "git", + "url": "https://github.com/ether/ueberDB.git" }, - "main": "./dist/index.js", - "types": "./dist/index.d.ts", "files": [ "dist/*.js", "dist/*.d.ts", "dist/databases", "dist/lib" ], + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "scripts": { + "lint": "oxlint", + "lint:fix": "oxlint --fix", + "format": "oxfmt --write .", + "format:check": "oxfmt --check .", + "build": "pnpm run build:js && pnpm run build:types", + "build:js": "pnpm exec rolldown -c rolldown.config.mjs", + "build:types": "pnpm exec tsc --emitDeclarationOnly", + "test": "vitest --test-timeout=120000", + "ts-check": "tsc --noEmit", + "ts-check:watch": "tsc --noEmit --watch" + }, + "devDependencies": { + "@elastic/elasticsearch": "^9.4.2", + "@types/async": "^3.2.25", + "@types/mssql": "^12.3.0", + "@types/node": "^25.9.1", + "@types/pg": "^8.20.0", + "@types/rethinkdb": "^2.3.21", + "@types/wtfnode": "^0.10.0", + "cassandra-driver": "^4.9.0", + "cli-table3": "^0.6.5", + "mongodb": "^7.2.0", + "mssql": "^12.5.4", + "mysql2": "^3.22.4", + "nano": "^11.0.5", + "oxfmt": "^0.52.0", + "oxlint": "^1.67.0", + "pg": "^8.21.0", + "randexp-ts": "^1.0.5", + "redis": "^6.0.0", + "rethinkdb": "^2.4.2", + "rolldown": "^1.0.3", + "semver": "^7.8.1", + "surrealdb": "^2.0.3", + "testcontainers": "^11.14.0", + "typescript": "^6.0.3", + "vitest": "^4.1.7", + "wtfnode": "^0.10.1" + }, "peerDependencies": { "@elastic/elasticsearch": "^9.0.0", "async": "^3.0.0", @@ -96,65 +149,12 @@ "optional": true } }, - "devDependencies": { - "@elastic/elasticsearch": "^9.4.2", - "@types/async": "^3.2.25", - "@types/mssql": "^12.3.0", - "@types/node": "^25.9.1", - "@types/pg": "^8.20.0", - "@types/rethinkdb": "^2.3.21", - "@types/wtfnode": "^0.10.0", - "cassandra-driver": "^4.9.0", - "cli-table3": "^0.6.5", - "mongodb": "^7.2.0", - "mssql": "^12.5.4", - "mysql2": "^3.22.4", - "nano": "^11.0.5", - "oxfmt": "^0.52.0", - "oxlint": "^1.67.0", - "pg": "^8.21.0", - "randexp-ts": "^1.0.5", - "redis": "^6.0.0", - "rethinkdb": "^2.4.2", - "rolldown": "^1.0.3", - "semver": "^7.8.1", - "surrealdb": "^2.0.3", - "testcontainers": "^11.14.0", - "typescript": "^6.0.3", - "vitest": "^4.1.7", - "wtfnode": "^0.10.1" - }, - "repository": { - "type": "git", - "url": "https://github.com/ether/ueberDB.git" - }, - "bugs": { - "url": "https://github.com/ether/ueberDB/issues" - }, - "homepage": "https://github.com/ether/ueberDB", - "scripts": { - "lint": "oxlint", - "lint:fix": "oxlint --fix", - "format": "oxfmt --write .", - "format:check": "oxfmt --check .", - "build": "pnpm run build:js && pnpm run build:types", - "build:js": "pnpm exec rolldown -c rolldown.config.mjs", - "build:types": "pnpm exec tsc --emitDeclarationOnly", - "test": "vitest --test-timeout=120000", - "ts-check": "tsc --noEmit", - "ts-check:watch": "tsc --noEmit --watch" + "engines": { + "node": ">=24.0.0" }, + "url": "https://github.com/ether/ueberDB", "_npmUser": { "name": "johnyma22", "email": "john@mclear.co.uk" - }, - "maintainers": [ - { - "name": "John McLear", - "email": "john@mclear.co.uk" - } - ], - "engines": { - "node": ">=24.0.0" } } diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index f05fc5c7..a2098664 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,5 +1,5 @@ overrides: - uuid@<11.1.1: '>=11.1.1' + uuid@<11.1.1: ">=11.1.1" vite: 8.0.11 allowBuilds: diff --git a/rolldown.config.mjs b/rolldown.config.mjs index 634b4521..9de857bd 100644 --- a/rolldown.config.mjs +++ b/rolldown.config.mjs @@ -1,12 +1,12 @@ -import {defineConfig} from 'rolldown'; -import path from 'node:path'; +import { defineConfig } from "rolldown"; +import path from "node:path"; export default defineConfig({ - input: ['./index.ts'], - external: (id) => !id.startsWith('.') && !path.isAbsolute(id), + input: ["./index.ts"], + external: (id) => !id.startsWith(".") && !path.isAbsolute(id), output: { - dir: './dist', - format: 'esm', - exports: 'named', + dir: "./dist", + format: "esm", + exports: "named", }, }); diff --git a/test/cassandra/test.cassandra.spec.ts b/test/cassandra/test.cassandra.spec.ts index ccf2c582..b863f670 100644 --- a/test/cassandra/test.cassandra.spec.ts +++ b/test/cassandra/test.cassandra.spec.ts @@ -1,25 +1,24 @@ -import {afterAll, beforeAll, describe} from "vitest"; -import {test_db} from "../lib/test_lib"; -import {GenericContainer, PortWithOptionalBinding, StartedTestContainer} from "testcontainers"; +import { afterAll, beforeAll, describe } from "vitest"; +import { test_db } from "../lib/test_lib"; +import { GenericContainer, PortWithOptionalBinding, StartedTestContainer } from "testcontainers"; -describe('cassandra test', ()=>{ - const portMappings: PortWithOptionalBinding[] = [ - { container: 9042, host: 9042 }, - {container: 10000, host: 10000} - ]; - let container: StartedTestContainer +describe("cassandra test", () => { + const portMappings: PortWithOptionalBinding[] = [ + { container: 9042, host: 9042 }, + { container: 10000, host: 10000 }, + ]; + let container: StartedTestContainer; - beforeAll(async () => { - container = await new GenericContainer("scylladb/scylla:2025.3") - .withCommand([" --smp 1"]) - .withExposedPorts(...portMappings) - .start() - }) + beforeAll(async () => { + container = await new GenericContainer("scylladb/scylla:2025.3") + .withCommand([" --smp 1"]) + .withExposedPorts(...portMappings) + .start(); + }); + test_db("cassandra"); - test_db('cassandra') - - afterAll(async () => { - await container.stop() - }) -}) + afterAll(async () => { + await container.stop(); + }); +}); diff --git a/test/couch/test.couch.spec.ts b/test/couch/test.couch.spec.ts index ac7533df..b535770b 100644 --- a/test/couch/test.couch.spec.ts +++ b/test/couch/test.couch.spec.ts @@ -1,48 +1,53 @@ -import {afterAll, beforeAll, describe} from "vitest"; -import {test_db} from "../lib/test_lib"; -import {GenericContainer, PortWithOptionalBinding, StartedTestContainer, Wait} from "testcontainers"; +import { afterAll, beforeAll, describe } from "vitest"; +import { test_db } from "../lib/test_lib"; +import { + GenericContainer, + PortWithOptionalBinding, + StartedTestContainer, + Wait, +} from "testcontainers"; -describe('couch test', () => { - const portMappings: PortWithOptionalBinding[] = [ - { container: 5984, host: 5984 } - ]; - let container: StartedTestContainer | undefined; +describe("couch test", () => { + const portMappings: PortWithOptionalBinding[] = [{ container: 5984, host: 5984 }]; + let container: StartedTestContainer | undefined; - beforeAll(async () => { - // CouchDB 3.5 enables [chttpd_auth_lockout] mode=enforce by default - // (5 failed auth attempts within max_lifetime → 403/401 for the - // rest of that window). On a fresh container with concurrent - // ueberdb test workers we hit the threshold easily, after which - // every request looks like an auth failure ("Account is - // temporarily locked", surfaced by nano as "Name or password is - // incorrect"). Mount a local.d override that switches the - // lockout mode to "warn" so the test matrix's transient blips - // don't lock out the whole run. - container = await new GenericContainer("couchdb:3.5.0") - .withExposedPorts(...portMappings) - .withEnvironment({ - COUCHDB_USER: "ueberdb", - COUCHDB_PASSWORD: "ueberdb", - }) - .withCopyContentToContainer([{ - content: '[chttpd_auth_lockout]\nmode = warn\n', - target: '/opt/couchdb/etc/local.d/disable-lockout.ini', - mode: 0o644, - }]) - .withWaitStrategy(Wait.forHttp('/_up', 5984).forStatusCode(200)) - .withStartupTimeout(120000) - .start(); - }, 180000); + beforeAll(async () => { + // CouchDB 3.5 enables [chttpd_auth_lockout] mode=enforce by default + // (5 failed auth attempts within max_lifetime → 403/401 for the + // rest of that window). On a fresh container with concurrent + // ueberdb test workers we hit the threshold easily, after which + // every request looks like an auth failure ("Account is + // temporarily locked", surfaced by nano as "Name or password is + // incorrect"). Mount a local.d override that switches the + // lockout mode to "warn" so the test matrix's transient blips + // don't lock out the whole run. + container = await new GenericContainer("couchdb:3.5.0") + .withExposedPorts(...portMappings) + .withEnvironment({ + COUCHDB_USER: "ueberdb", + COUCHDB_PASSWORD: "ueberdb", + }) + .withCopyContentToContainer([ + { + content: "[chttpd_auth_lockout]\nmode = warn\n", + target: "/opt/couchdb/etc/local.d/disable-lockout.ini", + mode: 0o644, + }, + ]) + .withWaitStrategy(Wait.forHttp("/_up", 5984).forStatusCode(200)) + .withStartupTimeout(120000) + .start(); + }, 180000); - test_db('couch'); + test_db("couch"); - afterAll(async () => { - if (container != null) { - try { - await container.stop(); - } catch (err) { - console.warn('couch container stop failed:', err); - } - } - }); + afterAll(async () => { + if (container != null) { + try { + await container.stop(); + } catch (err) { + console.warn("couch container stop failed:", err); + } + } + }); }); diff --git a/test/dirty/test.dirty.spec.ts b/test/dirty/test.dirty.spec.ts index 167ea324..c246c905 100644 --- a/test/dirty/test.dirty.spec.ts +++ b/test/dirty/test.dirty.spec.ts @@ -1,6 +1,6 @@ -import {describe} from "vitest"; -import {test_db} from "../lib/test_lib"; +import { describe } from "vitest"; +import { test_db } from "../lib/test_lib"; -describe('dirty test', ()=>{ - test_db('dirty') -}) +describe("dirty test", () => { + test_db("dirty"); +}); diff --git a/test/elasticsearch/test.elasticsearch.spec.ts b/test/elasticsearch/test.elasticsearch.spec.ts index a2ea7ed5..cb6f2af9 100644 --- a/test/elasticsearch/test.elasticsearch.spec.ts +++ b/test/elasticsearch/test.elasticsearch.spec.ts @@ -1,189 +1,199 @@ -import {afterAll, afterEach, beforeAll, beforeEach, describe, it} from "vitest"; -import {test_db} from "../lib/test_lib"; -import {GenericContainer, PortWithOptionalBinding, StartedTestContainer, Wait} from "testcontainers"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, it } from "vitest"; +import { test_db } from "../lib/test_lib"; +import { + GenericContainer, + PortWithOptionalBinding, + StartedTestContainer, + Wait, +} from "testcontainers"; import * as ueberdb from "../../index"; -import {deepEqual, rejects} from "assert"; -import {databases} from "../lib/databases"; -import {ConsoleLogger} from "../../lib/logging"; -import {Client} from "@elastic/elasticsearch"; -const {databases: {elasticsearch: cfg}} = {databases}; -const logger = new class extends ConsoleLogger { - info() { } - isInfoEnabled() { return false; } -}(); -describe('elasticsearch test', ()=>{ - const portMappings: PortWithOptionalBinding[] = [ - { container: 9200, host: 9200 } - ]; - let container: StartedTestContainer | undefined; +import { deepEqual, rejects } from "assert"; +import { databases } from "../lib/databases"; +import { ConsoleLogger } from "../../lib/logging"; +import { Client } from "@elastic/elasticsearch"; +const { + databases: { elasticsearch: cfg }, +} = { databases }; +const logger = new (class extends ConsoleLogger { + info() {} + isInfoEnabled() { + return false; + } +})(); +describe("elasticsearch test", () => { + const portMappings: PortWithOptionalBinding[] = [{ container: 9200, host: 9200 }]; + let container: StartedTestContainer | undefined; - beforeAll(async () => { - // Use a modern Elasticsearch image and seed legacy schema-v1 data - // without mapping types so migration coverage works on current releases. - // - // What we need for reliable CI: - // - constrain heap so the container doesn't OOM on GitHub runners - // - disable xpack.security so we can hit it over plain HTTP - // - use a /_cluster/health wait strategy so testcontainers actually - // waits for ES to be ready instead of returning the moment the - // container is started (which is what caused the original - // "container stopped/paused" failures). - container = await new GenericContainer("elasticsearch:9.3.3") - .withEnvironment({ - "discovery.type": "single-node", - "ES_JAVA_OPTS": "-Xms512m -Xmx512m", - "xpack.security.enabled": "false", - "bootstrap.memory_lock": "false", - "cluster.routing.allocation.disk.threshold_enabled": "false", - }) - .withExposedPorts(...portMappings) - .withWaitStrategy( - Wait.forHttp("/_cluster/health?wait_for_status=yellow&timeout=60s", 9200) - .forStatusCode(200)) - .withStartupTimeout(240000) - .start() - }, 360000) + beforeAll(async () => { + // Use a modern Elasticsearch image and seed legacy schema-v1 data + // without mapping types so migration coverage works on current releases. + // + // What we need for reliable CI: + // - constrain heap so the container doesn't OOM on GitHub runners + // - disable xpack.security so we can hit it over plain HTTP + // - use a /_cluster/health wait strategy so testcontainers actually + // waits for ES to be ready instead of returning the moment the + // container is started (which is what caused the original + // "container stopped/paused" failures). + container = await new GenericContainer("elasticsearch:9.3.3") + .withEnvironment({ + "discovery.type": "single-node", + ES_JAVA_OPTS: "-Xms512m -Xmx512m", + "xpack.security.enabled": "false", + "bootstrap.memory_lock": "false", + "cluster.routing.allocation.disk.threshold_enabled": "false", + }) + .withExposedPorts(...portMappings) + .withWaitStrategy( + Wait.forHttp("/_cluster/health?wait_for_status=yellow&timeout=60s", 9200).forStatusCode( + 200, + ), + ) + .withStartupTimeout(240000) + .start(); + }, 360000); - test_db('elasticsearch') - describe(__filename, function (this: any) { - const {base_index = 'ueberdb_test'} = cfg; - let client: any; - let db: any; - const deleteTestIndices = async () => { - const res = await client.indices.get({index: `${base_index}*`}, {ignore: [404]}); - const indices = Object.keys(res ?? {}); - if (indices.length > 0) { - await client.indices.delete({index: indices}, {ignore: [404]}); + test_db("elasticsearch"); + describe(__filename, function (this: any) { + const { base_index = "ueberdb_test" } = cfg; + let client: any; + let db: any; + const deleteTestIndices = async () => { + const res = await client.indices.get({ index: `${base_index}*` }, { ignore: [404] }); + const indices = Object.keys(res ?? {}); + if (indices.length > 0) { + await client.indices.delete({ index: indices }, { ignore: [404] }); + } + }; + beforeEach(async () => { + client = new Client({ + node: `http://${cfg.host || "127.0.0.1"}:${cfg.port || "9200"}`, + }); + await deleteTestIndices(); + }); + afterEach(async () => { + if (db != null) { + await db.close(); + } + db = null; + if (client != null) { + await deleteTestIndices(); + client.close(); + } + client = null; + }); + describe("migration to schema v2", () => { + describe("no old data", () => { + for (const migrate of [false, true]) { + it(`migration ${migrate ? "en" : "dis"}abled`, async () => { + // @ts-ignore + const settings = { base_index, migrate_to_newer_schema: undefined, ...cfg }; + delete settings.migrate_to_newer_schema; + db = new ueberdb.Database("elasticsearch", settings, {}, logger); + await db.init(); + const indices = []; + const res = await client.indices.get({ index: `${base_index}*` }); + for (const [k, v] of Object.entries(res)) { + indices.push(k); + // @ts-expect-error TS(2571): Object is of type 'unknown'. + indices.push(...Object.keys(v.aliases)); } + deepEqual(indices.sort(), [`${base_index}_s2`, `${base_index}_s2_i0`].sort()); + }); + } + }); + describe("existing data", () => { + // @ts-expect-error TS(2769): No overload matches this call. + const data = new Map([ + ["foo:number", 42], + ["foo:string", "value"], + ["foo:object", { k: "v" }], + ["foo:p:s:number", 42], + ["foo:p:s:string", "value"], + ["foo:p:s:object", { k: "v" }], + ]); + const setOld = async (k: any, v: any) => { + const kp = k.split(":"); + const index = kp.length === 4 ? `${base_index}-${kp[0]}-${kp[2]}` : base_index; + // In modern Elasticsearch, mapping types are gone. Persist legacy-v1 docs with + // a composite _id so migration can reconstruct the original key components. + const id = kp.length === 4 ? `${kp[1]}:${kp[3]}` : `${kp[0]}:${kp[1]}`; + await client.index({ + index, + id, + body: { + // The old elasticsearch driver was inconsistent: doBulk() called JSON.parse() on the + // value from ueberdb before writing, but set() did not. We'll assume that any existing + // data came from set() writes, not doBulk() writes. + val: JSON.stringify(v), + }, + }); + await client.indices.refresh({ index }); }; beforeEach(async () => { - client = new Client({ - node: `http://${cfg.host || '127.0.0.1'}:${cfg.port || '9200'}`, - }); - await deleteTestIndices(); + await Promise.all([...data].map(async ([k, v]) => await setOld(k, v))); }); - afterEach(async () => { - if (db != null) { await db.close(); } - db = null; - if (client != null) { - await deleteTestIndices(); - client.close(); - } - client = null; + it("migration disabled => init error", async () => { + // @ts-ignore + const settings = { base_index, migrate_to_newer_schema: undefined, ...cfg }; + delete settings.migrate_to_newer_schema; + db = new ueberdb.Database("elasticsearch", settings, {}, logger); + await rejects(db.init(), /migrate_to_newer_schema/); }); - describe('migration to schema v2', () => { - describe('no old data', () => { - for (const migrate of [false, true]) { - it(`migration ${migrate ? 'en' : 'dis'}abled`, async () => { - // @ts-ignore - const settings = {base_index, migrate_to_newer_schema: undefined, - ...cfg}; - delete settings.migrate_to_newer_schema; - db = new ueberdb.Database('elasticsearch', settings, {}, logger); - await db.init(); - const indices = []; - const res = await client.indices.get({index: `${base_index}*`}); - for (const [k, v] of Object.entries(res)) { - indices.push(k); - // @ts-expect-error TS(2571): Object is of type 'unknown'. - indices.push(...Object.keys(v.aliases)); - } - deepEqual(indices.sort(), [`${base_index}_s2`, `${base_index}_s2_i0`].sort()); - }); - } - }); - describe('existing data', () => { - // @ts-expect-error TS(2769): No overload matches this call. - const data = new Map([ - ['foo:number', 42], - ['foo:string', 'value'], - ['foo:object', {k: 'v'}], - ['foo:p:s:number', 42], - ['foo:p:s:string', 'value'], - ['foo:p:s:object', {k: 'v'}], - ]); - const setOld = async (k: any, v: any) => { - const kp = k.split(':'); - const index = kp.length === 4 ? `${base_index}-${kp[0]}-${kp[2]}` : base_index; - // In modern Elasticsearch, mapping types are gone. Persist legacy-v1 docs with - // a composite _id so migration can reconstruct the original key components. - const id = kp.length === 4 ? `${kp[1]}:${kp[3]}` : `${kp[0]}:${kp[1]}`; - await client.index({ - index, - id, - body: { - // The old elasticsearch driver was inconsistent: doBulk() called JSON.parse() on the - // value from ueberdb before writing, but set() did not. We'll assume that any existing - // data came from set() writes, not doBulk() writes. - val: JSON.stringify(v), - }, - }); - await client.indices.refresh({index}); - }; - beforeEach(async () => { - await Promise.all([...data].map(async ([k, v]) => await setOld(k, v))); - }); - it('migration disabled => init error', async () => { - // @ts-ignore - const settings = {base_index, migrate_to_newer_schema: undefined, - ...cfg}; - delete settings.migrate_to_newer_schema; - db = new ueberdb.Database('elasticsearch', settings, {}, logger); - await rejects(db.init(), /migrate_to_newer_schema/); - }); - it('migration enabled', async () => { - // @ts-ignore - const settings = {base_index, ...cfg, migrate_to_newer_schema: true}; - db = new ueberdb.Database('elasticsearch', settings, {}, logger); - await db.init(); - await Promise.all([...data].map(async ([k, v]) => { - deepEqual(await db.get(k), v); - })); - }); - it('each attempt uses a new index', async () => { - await setOld('a-x:b:c-x:d', 'v'); // Force a conversion failure. - cfg.base_index = base_index; - const settings = {...cfg, migrate_to_newer_schema: true}; - db = new ueberdb.Database('elasticsearch', settings, {}, logger); - const getIndices = async () => Object.keys((await client.indices.get({index: `${base_index}_s2*`}))); - deepEqual(await getIndices(), []); - await rejects(db.init(), /ambig/); - deepEqual(await getIndices(), [`${base_index}_s2_migrate_attempt_0`]); - await rejects(db.init(), /ambig/); - deepEqual((await getIndices()).sort(), [ - `${base_index}_s2_migrate_attempt_0`, - `${base_index}_s2_migrate_attempt_1`, - ]); - }); - it('final name not created until success', async () => { - }); - describe('ambiguous key', () => { - for (const k of ['a:b:c-x:d', 'a-x:b:c:d', 'a-x:b:c-x:d']) { - it(k, async () => { - await setOld(k, 'v'); - cfg.base_index = base_index; - const settings = {...cfg, migrate_to_newer_schema: true}; - db = new ueberdb.Database('elasticsearch', settings, {}, logger); - await rejects(db.init(), /ambig/); - }); - } - }); + it("migration enabled", async () => { + // @ts-ignore + const settings = { base_index, ...cfg, migrate_to_newer_schema: true }; + db = new ueberdb.Database("elasticsearch", settings, {}, logger); + await db.init(); + await Promise.all( + [...data].map(async ([k, v]) => { + deepEqual(await db.get(k), v); + }), + ); + }); + it("each attempt uses a new index", async () => { + await setOld("a-x:b:c-x:d", "v"); // Force a conversion failure. + cfg.base_index = base_index; + const settings = { ...cfg, migrate_to_newer_schema: true }; + db = new ueberdb.Database("elasticsearch", settings, {}, logger); + const getIndices = async () => + Object.keys(await client.indices.get({ index: `${base_index}_s2*` })); + deepEqual(await getIndices(), []); + await rejects(db.init(), /ambig/); + deepEqual(await getIndices(), [`${base_index}_s2_migrate_attempt_0`]); + await rejects(db.init(), /ambig/); + deepEqual((await getIndices()).sort(), [ + `${base_index}_s2_migrate_attempt_0`, + `${base_index}_s2_migrate_attempt_1`, + ]); + }); + it("final name not created until success", async () => {}); + describe("ambiguous key", () => { + for (const k of ["a:b:c-x:d", "a-x:b:c:d", "a-x:b:c-x:d"]) { + it(k, async () => { + await setOld(k, "v"); + cfg.base_index = base_index; + const settings = { ...cfg, migrate_to_newer_schema: true }; + db = new ueberdb.Database("elasticsearch", settings, {}, logger); + await rejects(db.init(), /ambig/); }); + } }); + }); }); + }); - - afterAll(async () => { - // Defensive: if beforeAll failed mid-flight, container may be undefined. - // Without the guard, afterAll throws "Cannot read properties of - // undefined (reading 'stop')" and masks the real beforeAll error. - if (container != null) { - try { - await container.stop(); - } catch (err) { - // Best-effort cleanup; don't mask test failures. - console.warn("elasticsearch container stop failed:", err); - } - } - }) -}) + afterAll(async () => { + // Defensive: if beforeAll failed mid-flight, container may be undefined. + // Without the guard, afterAll throws "Cannot read properties of + // undefined (reading 'stop')" and masks the real beforeAll error. + if (container != null) { + try { + await container.stop(); + } catch (err) { + // Best-effort cleanup; don't mask test failures. + console.warn("elasticsearch container stop failed:", err); + } + } + }); +}); diff --git a/test/lib/databases.ts b/test/lib/databases.ts index e48c3c68..bc651d41 100644 --- a/test/lib/databases.ts +++ b/test/lib/databases.ts @@ -1,10 +1,10 @@ -import os from 'os'; +import os from "os"; -type DatabaseType ={ - [key:string]:any -} +type DatabaseType = { + [key: string]: any; +}; -export const databases:DatabaseType = { +export const databases: DatabaseType = { memory: {}, dirty: { filename: `${os.tmpdir()}/ueberdb-test.db`, @@ -33,32 +33,32 @@ export const databases:DatabaseType = { }, }, mysql: { - user: 'ueberdb', - host: '127.0.0.1', - password: 'ueberdb', - database: 'ueberdb', - charset: 'utf8mb4', + user: "ueberdb", + host: "127.0.0.1", + password: "ueberdb", + database: "ueberdb", + charset: "utf8mb4", speeds: { findKeysMax: 6, getMax: 1, }, }, postgres: { - user: 'ueberdb', - host: 'localhost', - password: 'ueberdb', - database: 'ueberdb', - charset: 'utf8mb4', + user: "ueberdb", + host: "localhost", + password: "ueberdb", + database: "ueberdb", + charset: "utf8mb4", speeds: { setMax: 6, }, }, redis: { - url: 'redis://localhost/' + url: "redis://localhost/", }, mongodb: { - url: 'mongodb://127.0.0.1:27017', - database: 'mydb_test', + url: "mongodb://127.0.0.1:27017", + database: "mydb_test", speeds: { count: 2000, findKeysMax: 5, @@ -68,44 +68,44 @@ export const databases:DatabaseType = { }, }, couch: { - host: 'localhost', + host: "localhost", port: 5984, - database: 'ueberdb', - user: 'ueberdb', - password: 'ueberdb', + database: "ueberdb", + user: "ueberdb", + password: "ueberdb", speeds: { findKeysMax: 30, }, }, elasticsearch: { - base_index: 'ueberdb_test', + base_index: "ueberdb_test", speeds: { findKeysMax: 30, - }, host: '127.0.0.1', - port: '9200', - + }, + host: "127.0.0.1", + port: "9200", }, surrealdb: { - url: 'http://127.0.0.1:8000/rpc', + url: "http://127.0.0.1:8000/rpc", port: 0, - user: 'root', - password: 'root', + user: "root", + password: "root", speeds: { - // SurrealDB over HTTP/RPC is markedly slower than in-process - // or binary-protocol drivers; relax all per-op thresholds so - // the shared "speed is acceptable" benchmark passes on CI. - setMax: 30, - getMax: 30, - findKeysMax: 60, - removeMax: 30, + // SurrealDB over HTTP/RPC is markedly slower than in-process + // or binary-protocol drivers; relax all per-op thresholds so + // the shared "speed is acceptable" benchmark passes on CI. + setMax: 30, + getMax: 30, + findKeysMax: 60, + removeMax: 30, }, }, cassandra: { - columnFamily: 'test', + columnFamily: "test", clientOptions: { - contactPoints: ['h1', 'h2'], - localDataCenter: 'datacenter1', - keyspace: 'ks1' - } - } + contactPoints: ["h1", "h2"], + localDataCenter: "datacenter1", + keyspace: "ks1", + }, + }, }; diff --git a/test/lib/test_lib.ts b/test/lib/test_lib.ts index 43440b48..aa750086 100644 --- a/test/lib/test_lib.ts +++ b/test/lib/test_lib.ts @@ -1,16 +1,15 @@ -import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it} from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import * as ueberdb from "../../index"; -import {ConsoleLogger} from "../../lib/logging"; +import { ConsoleLogger } from "../../lib/logging"; import Randexp from "randexp-ts"; -import {rejects} from "assert"; +import { rejects } from "assert"; import Clitable from "cli-table3"; -import {databases} from "./databases"; -import {promises} from "fs"; -import {DatabaseType} from "../../index"; -import {existsSync} from "node:fs"; +import { databases } from "./databases"; +import { promises } from "fs"; +import { DatabaseType } from "../../index"; +import { existsSync } from "node:fs"; - -const fs = {promises}.promises; +const fs = { promises }.promises; const maxKeyLength = 100; // Use a URL-safe character set for generated keys. The previous regex @@ -23,399 +22,417 @@ const randomString = (length = maxKeyLength) => new Randexp(new RegExp(`[a-zA-Z0-9.-]{${length}}`)).gen(); export let db: any; -export const test_db = (database: DatabaseType)=>{ - const dbSettings = databases[database]; - let speedTable: any; - beforeAll(async () => { - speedTable = new Clitable({ - head: [ - 'Database', - 'read cache', - 'write buffer', - '#', - 'ms/set', - 'ms/get', - 'ms/findKeys', - 'ms/remove', - 'total ms', - 'total ms/#', - ], - colWidths: [15, 15, 15, 8, 13, 13, 13, 13, 13, 13], - }); +export const test_db = (database: DatabaseType) => { + const dbSettings = databases[database]; + let speedTable: any; + beforeAll(async () => { + speedTable = new Clitable({ + head: [ + "Database", + "read cache", + "write buffer", + "#", + "ms/set", + "ms/get", + "ms/findKeys", + "ms/remove", + "total ms", + "total ms/#", + ], + colWidths: [15, 15, 15, 8, 13, 13, 13, 13, 13, 13], }); - afterAll(async () => { - console.log(speedTable.toString()); - }); - - for (const readCache of [false, true]) { - describe(`${readCache ? '' : 'no '}read cache`, () => { - for (const writeBuffer of [false, true]) { - describe(`${writeBuffer ? '' : 'no '}write buffer`, function (this: any) { - beforeEach(async () => { - if (dbSettings.filename) { - if (existsSync(dbSettings.filename)) { - await fs.unlink(dbSettings.filename).catch((e) => { - console.log(e) - }); - } - } - - let setting = dbSettings.filename - - if (database === 'rustydb') { - - } + }); + afterAll(async () => { + console.log(speedTable.toString()); + }); + for (const readCache of [false, true]) { + describe(`${readCache ? "" : "no "}read cache`, () => { + for (const writeBuffer of [false, true]) { + describe(`${writeBuffer ? "" : "no "}write buffer`, function (this: any) { + beforeEach(async () => { + if (dbSettings.filename) { + if (existsSync(dbSettings.filename)) { + await fs.unlink(dbSettings.filename).catch((e) => { + console.log(e); + }); + } + } - db = new ueberdb.Database(database, dbSettings, { - ...(readCache ? {} : {cache: 0}), - ...(writeBuffer ? {} : {writeInterval: 0}), - }, new ConsoleLogger()); - await db.init(); - }); - afterEach(async () => { - await db.close(); - if (dbSettings.filename) { - if (existsSync(dbSettings.filename)) { - await fs.unlink(dbSettings.filename).catch((e) => { - console.log(e) - }); - } - } - }); - // The couch driver via nano routes trailing-space and adjacent-key - // requests through a code path that returns 401 from CouchDB session - // middleware in a way we have not been able to reproduce locally. - // Skip this entire describe for couch — every other DB still exercises it. - describe.skipIf(database === 'couch')('white space in key is not ignored', () => { - for (const space of [false, true]) { - describe(`key ${space ? 'has' : 'does not have'} a trailing space`, () => { - let input: any; - let key: any; - beforeEach(async () => { - input = {a: 1, b: new Randexp(/[a-zA-Z0-9]+/).gen()}; - key = randomString(maxKeyLength - 1) + (space ? ' ' : ''); - await db.set(key, input); - }); - it('get(key) -> record', async () => { - const output = await db.get(key); - expect(JSON.stringify(output)).toBe(JSON.stringify(input)); - }); - it('get(`${key} `) -> nullish', async () => { - const output = await db.get(`${key} `); - expect(output == null).toBeTruthy(); - }); - if (space) { - it('get(key.slice(0, -1)) -> nullish', async () => { - const output = await db.get(key.slice(0, -1)); - expect(output == null).toBeTruthy(); - }); - } - }); - } - }); - it('get of unknown key -> nullish', async () => { - const key = randomString(); - expect((await db.get(key)) == null).toBeTruthy(); - }); - it('set+get works', async () => { - const input = {a: 1, b: new Randexp(/[a-zA-Z0-9]+/).gen()}; - const key = randomString(); - await db.set(key, input); - const output = await db.get(key); - expect(JSON.stringify(output)).toBe(JSON.stringify(input)); - }); - it('set+get with random key/value works', async () => { - const input = {testLongString: new Randexp(/[a-f0-9]{50000}/).gen()}; - const key = randomString(); - await db.set(key, input); - const output = await db.get(key); - expect(JSON.stringify(output)).toBe(JSON.stringify(input)); - }); - it('findKeys works', async function (context) { - if (database === 'mongodb') { - context.skip() - } // TODO: Fix mongodb. - // TODO setting a key with non ascii chars - const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen(); + let setting = dbSettings.filename; - await db.set(key, true) - await db.set(`${key}a`, true) - await db.set(`nonmatching_${key}`, false) + if (database === "rustydb") { + } - const keys = await db.findKeys(`${key}*`, null); - expect(keys.sort()).toStrictEqual([key, `${key}a`]); - }); - it('findKeys with exclusion works', async function (context) { - if (database === 'mongodb') { - context.skip(); - } // TODO: Fix mongodb. - const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen(); + db = new ueberdb.Database( + database, + dbSettings, + { + ...(readCache ? {} : { cache: 0 }), + ...(writeBuffer ? {} : { writeInterval: 0 }), + }, + new ConsoleLogger(), + ); + await db.init(); + }); + afterEach(async () => { + await db.close(); + if (dbSettings.filename) { + if (existsSync(dbSettings.filename)) { + await fs.unlink(dbSettings.filename).catch((e) => { + console.log(e); + }); + } + } + }); + // The couch driver via nano routes trailing-space and adjacent-key + // requests through a code path that returns 401 from CouchDB session + // middleware in a way we have not been able to reproduce locally. + // Skip this entire describe for couch — every other DB still exercises it. + describe.skipIf(database === "couch")("white space in key is not ignored", () => { + for (const space of [false, true]) { + describe(`key ${space ? "has" : "does not have"} a trailing space`, () => { + let input: any; + let key: any; + beforeEach(async () => { + input = { a: 1, b: new Randexp(/[a-zA-Z0-9]+/).gen() }; + key = randomString(maxKeyLength - 1) + (space ? " " : ""); + await db.set(key, input); + }); + it("get(key) -> record", async () => { + const output = await db.get(key); + expect(JSON.stringify(output)).toBe(JSON.stringify(input)); + }); + it("get(`${key} `) -> nullish", async () => { + const output = await db.get(`${key} `); + expect(output == null).toBeTruthy(); + }); + if (space) { + it("get(key.slice(0, -1)) -> nullish", async () => { + const output = await db.get(key.slice(0, -1)); + expect(output == null).toBeTruthy(); + }); + } + }); + } + }); + it("get of unknown key -> nullish", async () => { + const key = randomString(); + expect((await db.get(key)) == null).toBeTruthy(); + }); + it("set+get works", async () => { + const input = { a: 1, b: new Randexp(/[a-zA-Z0-9]+/).gen() }; + const key = randomString(); + await db.set(key, input); + const output = await db.get(key); + expect(JSON.stringify(output)).toBe(JSON.stringify(input)); + }); + it("set+get with random key/value works", async () => { + const input = { testLongString: new Randexp(/[a-f0-9]{50000}/).gen() }; + const key = randomString(); + await db.set(key, input); + const output = await db.get(key); + expect(JSON.stringify(output)).toBe(JSON.stringify(input)); + }); + it("findKeys works", async function (context) { + if (database === "mongodb") { + context.skip(); + } // TODO: Fix mongodb. + // TODO setting a key with non ascii chars + const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen(); - await db.set(key, true) - await db.set(`${key}a`, true) - await db.set(`${key}b`, false) - await db.set(`${key}b2`, false) - await db.set(`nonmatching_${key}`, false) - const keys = await db.findKeys(`${key}*`, `${key}b*`); - expect(keys.sort()).toStrictEqual([key, `${key}a`]); - }); - it('findKeys with no matches works', async () => { - const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen(); - await db.set(key, true); - const keys = await db.findKeys(`${key}_nomatch_*`, null); - expect(keys).toStrictEqual([]); - }); - it('findKeys with no wildcard works', async () => { - const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen(); - await db.set(key, true); - const keys = await db.findKeys(key, null); - expect(keys).toStrictEqual([key]); - }); + await db.set(key, true); + await db.set(`${key}a`, true); + await db.set(`nonmatching_${key}`, false); - describe('findKeysPaged', () => { - // Per-test prefix keeps these isolated from other findKeys tests above. - const prefix = () => `pg_${new Randexp(/[a-z]{6}/).gen()}`; + const keys = await db.findKeys(`${key}*`, null); + expect(keys.sort()).toStrictEqual([key, `${key}a`]); + }); + it("findKeys with exclusion works", async function (context) { + if (database === "mongodb") { + context.skip(); + } // TODO: Fix mongodb. + const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen(); - it.skipIf(['mongodb', 'surrealdb'].includes(database))('returns empty page when no keys match', async () => { - const p = prefix(); - const keys = await db.findKeysPaged(`${p}_nomatch:*`, null, {limit: 10}); - expect(keys).toStrictEqual([]); - }); + await db.set(key, true); + await db.set(`${key}a`, true); + await db.set(`${key}b`, false); + await db.set(`${key}b2`, false); + await db.set(`nonmatching_${key}`, false); + const keys = await db.findKeys(`${key}*`, `${key}b*`); + expect(keys.sort()).toStrictEqual([key, `${key}a`]); + }); + it("findKeys with no matches works", async () => { + const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen(); + await db.set(key, true); + const keys = await db.findKeys(`${key}_nomatch_*`, null); + expect(keys).toStrictEqual([]); + }); + it("findKeys with no wildcard works", async () => { + const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen(); + await db.set(key, true); + const keys = await db.findKeys(key, null); + expect(keys).toStrictEqual([key]); + }); - it.skipIf(['mongodb', 'surrealdb'].includes(database))('returns all keys in a single page when limit > count', async () => { - const p = prefix(); - await db.set(`${p}:a`, 1); - await db.set(`${p}:b`, 2); - await db.set(`${p}:c`, 3); - const keys = await db.findKeysPaged(`${p}:*`, null, {limit: 10}); - expect(keys.sort()).toStrictEqual([`${p}:a`, `${p}:b`, `${p}:c`]); - }); + describe("findKeysPaged", () => { + // Per-test prefix keeps these isolated from other findKeys tests above. + const prefix = () => `pg_${new Randexp(/[a-z]{6}/).gen()}`; - it.skipIf(['mongodb', 'surrealdb'].includes(database))('honours the limit', async () => { - const p = prefix(); - await db.set(`${p}:a`, 1); - await db.set(`${p}:b`, 2); - await db.set(`${p}:c`, 3); - const keys = await db.findKeysPaged(`${p}:*`, null, {limit: 2}); - expect(keys.length).toBe(2); - }); + it.skipIf(["mongodb", "surrealdb"].includes(database))( + "returns empty page when no keys match", + async () => { + const p = prefix(); + const keys = await db.findKeysPaged(`${p}_nomatch:*`, null, { limit: 10 }); + expect(keys).toStrictEqual([]); + }, + ); - it.skipIf(['mongodb', 'surrealdb'].includes(database))('pages cleanly across the keyspace using after-cursor', async () => { - const p = prefix(); - const expected = ['a', 'b', 'c', 'd', 'e', 'f', 'g'].map((s) => `${p}:${s}`); - for (const k of expected) await db.set(k, 1); - const collected: string[] = []; - let after: string | undefined; - // 7 keys / page size 3 -> 3 pages (3 + 3 + 1) - for (let safety = 0; safety < 10; safety++) { - const page: string[] = await db.findKeysPaged(`${p}:*`, null, { - limit: 3, - ...(after != null ? {after} : {}), - }); - collected.push(...page); - if (page.length < 3) break; - after = page[page.length - 1]; - } - expect(collected.sort()).toStrictEqual([...expected].sort()); - }); + it.skipIf(["mongodb", "surrealdb"].includes(database))( + "returns all keys in a single page when limit > count", + async () => { + const p = prefix(); + await db.set(`${p}:a`, 1); + await db.set(`${p}:b`, 2); + await db.set(`${p}:c`, 3); + const keys = await db.findKeysPaged(`${p}:*`, null, { limit: 10 }); + expect(keys.sort()).toStrictEqual([`${p}:a`, `${p}:b`, `${p}:c`]); + }, + ); - it.skipIf(['mongodb', 'surrealdb'].includes(database))('respects notKey exclusion across pages', async () => { - const p = prefix(); - await db.set(`${p}:keep1`, 1); - await db.set(`${p}:keep2`, 1); - await db.set(`${p}:skip:1`, 0); - await db.set(`${p}:skip:2`, 0); - const collected: string[] = []; - let after: string | undefined; - for (let safety = 0; safety < 10; safety++) { - const page: string[] = await db.findKeysPaged(`${p}:*`, `${p}:skip:*`, { - limit: 10, - ...(after != null ? {after} : {}), - }); - collected.push(...page); - if (page.length < 10) break; - after = page[page.length - 1]; - } - expect(collected.sort()).toStrictEqual([`${p}:keep1`, `${p}:keep2`]); - }); - }); + it.skipIf(["mongodb", "surrealdb"].includes(database))( + "honours the limit", + async () => { + const p = prefix(); + await db.set(`${p}:a`, 1); + await db.set(`${p}:b`, 2); + await db.set(`${p}:c`, 3); + const keys = await db.findKeysPaged(`${p}:*`, null, { limit: 2 }); + expect(keys.length).toBe(2); + }, + ); + it.skipIf(["mongodb", "surrealdb"].includes(database))( + "pages cleanly across the keyspace using after-cursor", + async () => { + const p = prefix(); + const expected = ["a", "b", "c", "d", "e", "f", "g"].map((s) => `${p}:${s}`); + for (const k of expected) await db.set(k, 1); + const collected: string[] = []; + let after: string | undefined; + // 7 keys / page size 3 -> 3 pages (3 + 3 + 1) + for (let safety = 0; safety < 10; safety++) { + const page: string[] = await db.findKeysPaged(`${p}:*`, null, { + limit: 3, + ...(after != null ? { after } : {}), + }); + collected.push(...page); + if (page.length < 3) break; + after = page[page.length - 1]; + } + expect(collected.sort()).toStrictEqual([...expected].sort()); + }, + ); + it.skipIf(["mongodb", "surrealdb"].includes(database))( + "respects notKey exclusion across pages", + async () => { + const p = prefix(); + await db.set(`${p}:keep1`, 1); + await db.set(`${p}:keep2`, 1); + await db.set(`${p}:skip:1`, 0); + await db.set(`${p}:skip:2`, 0); + const collected: string[] = []; + let after: string | undefined; + for (let safety = 0; safety < 10; safety++) { + const page: string[] = await db.findKeysPaged(`${p}:*`, `${p}:skip:*`, { + limit: 10, + ...(after != null ? { after } : {}), + }); + collected.push(...page); + if (page.length < 10) break; + after = page[page.length - 1]; + } + expect(collected.sort()).toStrictEqual([`${p}:keep1`, `${p}:keep2`]); + }, + ); + }); - it('remove works', async () => { - const input = {a: 1, b: new Randexp(/[a-zA-Z0-9]+/).gen()}; - const key = randomString(); - await db.set(key, input); - expect(JSON.stringify(await db.get(key))).toStrictEqual(JSON.stringify(input)); - await db.remove(key); - expect((await db.get(key)) == null).toBeTruthy(); - }); - it('getSub of existing property works', async () => { - await db.set('k', {sub1: {sub2: 'v'}}); - expect(await db.getSub('k', ['sub1', 'sub2'])).toBe('v'); - expect(await db.getSub('k', ['sub1'])).toStrictEqual({sub2: 'v'}); - expect(await db.getSub('k', [])).toStrictEqual({sub1: {sub2: 'v'}}); - }); - it('getSub of missing property returns nullish', async () => { - await db.set('k', {sub1: {}}); - expect((await db.getSub('k', ['sub1', 'sub2'])) == null).toBeTruthy(); - await db.set('k', {}); - expect((await db.getSub('k', ['sub1', 'sub2'])) == null).toBeTruthy(); - expect((await db.getSub('k', ['sub1']))).toBeNull(); - await db.remove('k'); - expect((await db.getSub('k', ['sub1', 'sub2'])) == null).toBeTruthy(); - expect((await db.getSub('k', ['sub1'])) == null).toBeTruthy(); - expect(await db.getSub('k', []) == null).toBeTruthy(); - }); - it('setSub can modify an existing property', async () => { - await db.set('k', {sub1: {sub2: 'v'}}); - await db.setSub('k', ['sub1', 'sub2'], 'v2'); - expect(await db.get('k')).toStrictEqual({sub1: {sub2: 'v2'}}); - await db.setSub('k', ['sub1'], 'v2'); - expect(await db.get('k')).toStrictEqual({sub1: 'v2'}); - await db.setSub('k', [], 'v3'); - expect(await db.get('k')).toStrictEqual('v3'); - }); - it('setSub can add a new property', async () => { - await db.remove('k'); - await db.setSub('k', [], {}); - expect(await db.get('k')).toStrictEqual({}); - await db.setSub('k', ['sub1'], {}); - expect(await db.get('k')).toStrictEqual({sub1: {}}); - await db.setSub('k', ['sub1', 'sub2'], 'v'); - expect(await db.get('k')).toStrictEqual({sub1: {sub2: 'v'}}); - await db.remove('k'); - await db.setSub('k', ['sub1', 'sub2'], 'v'); - expect(await db.get('k')).toStrictEqual({sub1: {sub2: 'v'}}); - }); - it('setSub rejects attempts to set properties on primitives', async () => { - for (const v of ['hello world', 42, true]) { - await db.set('k', v); - await rejects(db.setSub('k', ['sub'], 'x'), { - name: 'TypeError', - message: /property "sub" on non-object/, - }); - expect(await db.get('k')).toBe(v); - } - }); - it('setSub can delete a property', async () => { - await db.set('k', {sub1: {sub2: 'v', sub3: 'v'}, sub4: 'v'}); - await db.setSub('k', ['sub1', 'sub2'], undefined); - expect(await db.get('k')).toStrictEqual({sub1: {sub3: 'v'}, sub4: 'v'}); - await db.setSub('k', ['sub1', 'sub3'], undefined); - expect(await db.get('k')).toStrictEqual({sub1: {}, sub4: 'v'}); - await db.setSub('k', ['sub1'], undefined); - expect(await db.get('k')).toStrictEqual({sub4: 'v'}); - await db.setSub('k', ['sub4'], undefined); - expect(await db.get('k')).toStrictEqual({}); - await db.setSub('k', [], undefined); - expect((await db.get('k')) == null).toBeTruthy(); - }); + it("remove works", async () => { + const input = { a: 1, b: new Randexp(/[a-zA-Z0-9]+/).gen() }; + const key = randomString(); + await db.set(key, input); + expect(JSON.stringify(await db.get(key))).toStrictEqual(JSON.stringify(input)); + await db.remove(key); + expect((await db.get(key)) == null).toBeTruthy(); + }); + it("getSub of existing property works", async () => { + await db.set("k", { sub1: { sub2: "v" } }); + expect(await db.getSub("k", ["sub1", "sub2"])).toBe("v"); + expect(await db.getSub("k", ["sub1"])).toStrictEqual({ sub2: "v" }); + expect(await db.getSub("k", [])).toStrictEqual({ sub1: { sub2: "v" } }); + }); + it("getSub of missing property returns nullish", async () => { + await db.set("k", { sub1: {} }); + expect((await db.getSub("k", ["sub1", "sub2"])) == null).toBeTruthy(); + await db.set("k", {}); + expect((await db.getSub("k", ["sub1", "sub2"])) == null).toBeTruthy(); + expect(await db.getSub("k", ["sub1"])).toBeNull(); + await db.remove("k"); + expect((await db.getSub("k", ["sub1", "sub2"])) == null).toBeTruthy(); + expect((await db.getSub("k", ["sub1"])) == null).toBeTruthy(); + expect((await db.getSub("k", [])) == null).toBeTruthy(); + }); + it("setSub can modify an existing property", async () => { + await db.set("k", { sub1: { sub2: "v" } }); + await db.setSub("k", ["sub1", "sub2"], "v2"); + expect(await db.get("k")).toStrictEqual({ sub1: { sub2: "v2" } }); + await db.setSub("k", ["sub1"], "v2"); + expect(await db.get("k")).toStrictEqual({ sub1: "v2" }); + await db.setSub("k", [], "v3"); + expect(await db.get("k")).toStrictEqual("v3"); + }); + it("setSub can add a new property", async () => { + await db.remove("k"); + await db.setSub("k", [], {}); + expect(await db.get("k")).toStrictEqual({}); + await db.setSub("k", ["sub1"], {}); + expect(await db.get("k")).toStrictEqual({ sub1: {} }); + await db.setSub("k", ["sub1", "sub2"], "v"); + expect(await db.get("k")).toStrictEqual({ sub1: { sub2: "v" } }); + await db.remove("k"); + await db.setSub("k", ["sub1", "sub2"], "v"); + expect(await db.get("k")).toStrictEqual({ sub1: { sub2: "v" } }); + }); + it("setSub rejects attempts to set properties on primitives", async () => { + for (const v of ["hello world", 42, true]) { + await db.set("k", v); + await rejects(db.setSub("k", ["sub"], "x"), { + name: "TypeError", + message: /property "sub" on non-object/, + }); + expect(await db.get("k")).toBe(v); + } + }); + it("setSub can delete a property", async () => { + await db.set("k", { sub1: { sub2: "v", sub3: "v" }, sub4: "v" }); + await db.setSub("k", ["sub1", "sub2"], undefined); + expect(await db.get("k")).toStrictEqual({ sub1: { sub3: "v" }, sub4: "v" }); + await db.setSub("k", ["sub1", "sub3"], undefined); + expect(await db.get("k")).toStrictEqual({ sub1: {}, sub4: "v" }); + await db.setSub("k", ["sub1"], undefined); + expect(await db.get("k")).toStrictEqual({ sub4: "v" }); + await db.setSub("k", ["sub4"], undefined); + expect(await db.get("k")).toStrictEqual({}); + await db.setSub("k", [], undefined); + expect((await db.get("k")) == null).toBeTruthy(); + }); - it('speed is acceptable', async function () { - type TimeSettings = { - remove?: string | number; - findKeys?: number; - get?: number; - set?: number; - start: number, - } + it("speed is acceptable", async function () { + type TimeSettings = { + remove?: string | number; + findKeys?: number; + get?: number; + set?: number; + start: number; + }; - type Speeds = { - speeds: { - count?: number; - setMax?: number; - getMax?: number; - findKeysMax?: number; - removeMax?: number; - } - } + type Speeds = { + speeds: { + count?: number; + setMax?: number; + getMax?: number; + findKeysMax?: number; + removeMax?: number; + }; + }; - const { - speeds: { - count = 1000, - setMax = 3, - getMax = 0.1, - findKeysMax = 3, - removeMax = 1 - } = {} - }: Speeds = dbSettings || {}; - const input = {a: 1, b: new Randexp(/.+/).gen()}; - // TODO setting a key with non ascii chars - const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen(); - // Pre-allocate an array before starting the timer so that time spent growing the - // array doesn't throw off the benchmarks. - const promises = [...Array(count + 1)].map(() => null); - const timers: TimeSettings = {start: Date.now()}; - for (let i = 0; i < count; ++i) { - promises[i] = db.set(key + i, input); - } - promises[count] = db.flush(); - await Promise.all(promises); - timers.set = Date.now(); - for (let i = 0; i < count; ++i) { - promises[i] = db.get(key + i); - } - await Promise.all(promises); - timers.get = Date.now(); - for (let i = 0; i < count; ++i) { - promises[i] = db.findKeys(key + i, null); - } - await Promise.all(promises); - timers.findKeys = Date.now(); - for (let i = 0; i < count; ++i) { - promises[i] = db.remove(key + i); - } - promises[count] = db.flush(); - await Promise.all(promises); - timers.remove = Date.now(); - const timePerOp = { - set: (timers.set - timers.start) / count, - get: (timers.get - timers.set) / count, - findKeys: (timers.findKeys - timers.get) / count, - remove: (timers.remove - timers.findKeys) / count, - }; - speedTable.push([ - database, - readCache ? 'yes' : 'no', - writeBuffer ? 'yes' : 'no', - count, - timePerOp.set, - timePerOp.get, - timePerOp.findKeys, - timePerOp.remove, - timers.remove - timers.start, - (timers.remove - timers.start) / count, - ]); - // Removes the "Acceptable ms/op" column if there is no enforced limit. - const filterColumn = (row: any) => { - if (readCache && writeBuffer) { - return row; - } - row.splice(1, 1); - return row; - }; - const acceptableTable = new Clitable({ - head: filterColumn(['op', 'Acceptable ms/op', 'Actual ms/op']), - colWidths: filterColumn([10, 18, 18]), - }); - acceptableTable.push(...[ - ['set', setMax, timePerOp.set], - ['get', getMax, timePerOp.get], - ['findKeys', findKeysMax, timePerOp.findKeys], - ['remove', removeMax, timePerOp.remove], - ].map(filterColumn)); - console.log(acceptableTable.toString()); - if (readCache && writeBuffer) { - expect(setMax >= timePerOp.set).toBeTruthy(); - expect(getMax >= timePerOp.get).toBeTruthy(); - expect(findKeysMax >= timePerOp.findKeys).toBeTruthy(); - expect(removeMax >= timePerOp.remove).toBeTruthy(); - } - }) - }); - } + const { + speeds: { + count = 1000, + setMax = 3, + getMax = 0.1, + findKeysMax = 3, + removeMax = 1, + } = {}, + }: Speeds = dbSettings || {}; + const input = { a: 1, b: new Randexp(/.+/).gen() }; + // TODO setting a key with non ascii chars + const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen(); + // Pre-allocate an array before starting the timer so that time spent growing the + // array doesn't throw off the benchmarks. + const promises = [...Array(count + 1)].map(() => null); + const timers: TimeSettings = { start: Date.now() }; + for (let i = 0; i < count; ++i) { + promises[i] = db.set(key + i, input); + } + promises[count] = db.flush(); + await Promise.all(promises); + timers.set = Date.now(); + for (let i = 0; i < count; ++i) { + promises[i] = db.get(key + i); + } + await Promise.all(promises); + timers.get = Date.now(); + for (let i = 0; i < count; ++i) { + promises[i] = db.findKeys(key + i, null); + } + await Promise.all(promises); + timers.findKeys = Date.now(); + for (let i = 0; i < count; ++i) { + promises[i] = db.remove(key + i); + } + promises[count] = db.flush(); + await Promise.all(promises); + timers.remove = Date.now(); + const timePerOp = { + set: (timers.set - timers.start) / count, + get: (timers.get - timers.set) / count, + findKeys: (timers.findKeys - timers.get) / count, + remove: (timers.remove - timers.findKeys) / count, + }; + speedTable.push([ + database, + readCache ? "yes" : "no", + writeBuffer ? "yes" : "no", + count, + timePerOp.set, + timePerOp.get, + timePerOp.findKeys, + timePerOp.remove, + timers.remove - timers.start, + (timers.remove - timers.start) / count, + ]); + // Removes the "Acceptable ms/op" column if there is no enforced limit. + const filterColumn = (row: any) => { + if (readCache && writeBuffer) { + return row; + } + row.splice(1, 1); + return row; + }; + const acceptableTable = new Clitable({ + head: filterColumn(["op", "Acceptable ms/op", "Actual ms/op"]), + colWidths: filterColumn([10, 18, 18]), }); - } -} + acceptableTable.push( + ...[ + ["set", setMax, timePerOp.set], + ["get", getMax, timePerOp.get], + ["findKeys", findKeysMax, timePerOp.findKeys], + ["remove", removeMax, timePerOp.remove], + ].map(filterColumn), + ); + console.log(acceptableTable.toString()); + if (readCache && writeBuffer) { + expect(setMax >= timePerOp.set).toBeTruthy(); + expect(getMax >= timePerOp.get).toBeTruthy(); + expect(findKeysMax >= timePerOp.findKeys).toBeTruthy(); + expect(removeMax >= timePerOp.remove).toBeTruthy(); + } + }); + }); + } + }); + } +}; diff --git a/test/memory/test.memory.spec.ts b/test/memory/test.memory.spec.ts index 4d62f6bc..30106859 100644 --- a/test/memory/test.memory.spec.ts +++ b/test/memory/test.memory.spec.ts @@ -1,20 +1,21 @@ -import wtfnode from 'wtfnode'; -import {afterAll, describe} from 'vitest' -import {test_db} from "../lib/test_lib"; - +import wtfnode from "wtfnode"; +import { afterAll, describe } from "vitest"; +import { test_db } from "../lib/test_lib"; // eslint-disable-next-line mocha/no-top-level-hooks afterAll(async () => { // Add a timeout to forcibly exit if something is keeping node from exiting cleanly. // The timeout is unref()ed so that it doesn't prevent node from exiting when done. setTimeout(() => { - console.error('node should have exited by now but something is keeping it open ' + - 'such as an open connection or active timer'); + console.error( + "node should have exited by now but something is keeping it open " + + "such as an open connection or active timer", + ); wtfnode.dump(); process.exit(1); // eslint-disable-line n/no-process-exit }, 5000).unref(); }); -describe('sqlite test', ()=>{ - test_db('memory') -}) +describe("sqlite test", () => { + test_db("memory"); +}); diff --git a/test/memory/test_getSub.spec.ts b/test/memory/test_getSub.spec.ts index f4dd53f8..54d488e7 100644 --- a/test/memory/test_getSub.spec.ts +++ b/test/memory/test_getSub.spec.ts @@ -1,28 +1,28 @@ -import assert$0 from 'assert'; -import * as ueberdb from '../../index'; -import {describe, it, afterEach, beforeEach} from 'vitest' +import assert$0 from "assert"; +import * as ueberdb from "../../index"; +import { describe, it, afterEach, beforeEach } from "vitest"; const assert = assert$0.strict; describe(__filename, () => { - let db: ueberdb.Database|null; + let db: ueberdb.Database | null; beforeEach(async () => { - db = new ueberdb.Database('memory', {}, {}); + db = new ueberdb.Database("memory", {}, {}); await db.init(); - await db.set('k', {s: 'v'}); + await db.set("k", { s: "v" }); }); afterEach(async () => { if (db != null) await db.close(); db = null; }); - it('getSub stops at non-objects', async () => { + it("getSub stops at non-objects", async () => { // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert((await db.getSub('k', ['s', 'length'])) == null); + assert((await db.getSub("k", ["s", "length"])) == null); }); - it('getSub ignores non-own properties', async () => { + it("getSub ignores non-own properties", async () => { // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert((await db.getSub('k', ['toString'])) == null); + assert((await db.getSub("k", ["toString"])) == null); }); - it('getSub ignores __proto__', async () => { + it("getSub ignores __proto__", async () => { // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert((await db.getSub('k', ['__proto__'])) == null); + assert((await db.getSub("k", ["__proto__"])) == null); }); }); diff --git a/test/memory/test_memory.spec.ts b/test/memory/test_memory.spec.ts index ae1145ad..8d17b1db 100644 --- a/test/memory/test_memory.spec.ts +++ b/test/memory/test_memory.spec.ts @@ -1,32 +1,32 @@ -import assert$0 from 'assert'; -import MemoryDB from '../../databases/memory_db'; -import {describe, it} from 'vitest' +import assert$0 from "assert"; +import MemoryDB from "../../databases/memory_db"; +import { describe, it } from "vitest"; const assert = assert$0.strict; describe(__filename, () => { - describe('data option', () => { - it('uses existing records from data option', async () => { - const db = new MemoryDB({data: new Map([['foo', 'bar']])}); + describe("data option", () => { + it("uses existing records from data option", async () => { + const db = new MemoryDB({ data: new Map([["foo", "bar"]]) }); await db.init(); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.equal(await db.get('foo'), 'bar'); + assert.equal(await db.get("foo"), "bar"); }); - it('updates existing map', async () => { + it("updates existing map", async () => { const data = new Map(); - const db = new MemoryDB({data}); + const db = new MemoryDB({ data }); await db.init(); - await db.set('foo', 'bar'); + await db.set("foo", "bar"); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.equal(data.get('foo'), 'bar'); + assert.equal(data.get("foo"), "bar"); }); - it('does not clear map on close', async () => { + it("does not clear map on close", async () => { const data = new Map(); - const db = new MemoryDB({data}); + const db = new MemoryDB({ data }); await db.init(); - await db.set('foo', 'bar'); + await db.set("foo", "bar"); await db.close(); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.equal(data.get('foo'), 'bar'); + assert.equal(data.get("foo"), "bar"); }); }); }); diff --git a/test/memory/test_tojson.spec.ts b/test/memory/test_tojson.spec.ts index bd1dc779..5caf664a 100644 --- a/test/memory/test_tojson.spec.ts +++ b/test/memory/test_tojson.spec.ts @@ -1,35 +1,35 @@ -import assert$0 from 'assert'; -import * as ueberdb from '../../index'; -import {afterAll, describe, it, afterEach, beforeEach, beforeAll, expect} from 'vitest' +import assert$0 from "assert"; +import * as ueberdb from "../../index"; +import { afterAll, describe, it, afterEach, beforeEach, beforeAll, expect } from "vitest"; const assert = assert$0.strict; describe(__filename, () => { let db: any = null; beforeAll(async () => { - db = new ueberdb.Database('memory', {}, {}); + db = new ueberdb.Database("memory", {}, {}); await db.init(); }); afterAll(async () => { await db.close(); }); - it('no .toJSON method', async () => { - await db.set('key', {prop: 'value'}); + it("no .toJSON method", async () => { + await db.set("key", { prop: "value" }); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.deepEqual(await db.get('key'), {prop: 'value'}); + assert.deepEqual(await db.get("key"), { prop: "value" }); }); - it('direct', async () => { - await db.set('key', {toJSON: (arg: any) => `toJSON ${arg}`}); + it("direct", async () => { + await db.set("key", { toJSON: (arg: any) => `toJSON ${arg}` }); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.equal(await db.get('key'), 'toJSON '); + assert.equal(await db.get("key"), "toJSON "); }); - it('object property', async () => { - await db.set('key', {prop: {toJSON: (arg: any) => `toJSON ${arg}`}}); + it("object property", async () => { + await db.set("key", { prop: { toJSON: (arg: any) => `toJSON ${arg}` } }); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.deepEqual(await db.get('key'), {prop: 'toJSON prop'}); + assert.deepEqual(await db.get("key"), { prop: "toJSON prop" }); }); - it('array entry', async () => { - await db.set('key', [{toJSON: (arg: any) => `toJSON ${arg}`}]); + it("array entry", async () => { + await db.set("key", [{ toJSON: (arg: any) => `toJSON ${arg}` }]); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.deepEqual(await db.get('key'), ['toJSON 0']); + assert.deepEqual(await db.get("key"), ["toJSON 0"]); }); }); diff --git a/test/mock/test_bulk.spec.ts b/test/mock/test_bulk.spec.ts index 17c181ec..dc748877 100644 --- a/test/mock/test_bulk.spec.ts +++ b/test/mock/test_bulk.spec.ts @@ -1,30 +1,29 @@ -import {strict} from 'assert'; -import {Database} from '../../index'; -import util from 'util'; -const assert = strict +import { strict } from "assert"; +import { Database } from "../../index"; +import util from "util"; +const assert = strict; const range = (N: any) => [...Array(N).keys()]; -import {describe, it, afterEach, expect} from 'vitest' +import { describe, it, afterEach, expect } from "vitest"; type MockSettings = { mock?: any; -} +}; describe(__filename, () => { let db: any = null; let mock: any = null; const createDb = async (wrapperSettings: any) => { - const settings:MockSettings = {}; - db = new Database('mock', settings, wrapperSettings); + const settings: MockSettings = {}; + db = new Database("mock", settings, wrapperSettings); await db.init(); mock = settings.mock; - mock.once('init', (cb: any) => cb()); - + mock.once("init", (cb: any) => cb()); }; afterEach(async () => { if (mock != null) { mock.removeAllListeners(); - mock.once('close', (cb: any) => cb()); + mock.once("close", (cb: any) => cb()); mock = null; } if (db != null) { @@ -32,39 +31,53 @@ describe(__filename, () => { db = null; } }); - describe('bulkLimit', () => { - const bulkLimits = [0, false, null, undefined, '', 1, 2]; + describe("bulkLimit", () => { + const bulkLimits = [0, false, null, undefined, "", 1, 2]; for (const bulkLimit of bulkLimits) { - it(bulkLimit === undefined ? 'undefined' : JSON.stringify(bulkLimit), async () => { - await createDb({bulkLimit}); + it(bulkLimit === undefined ? "undefined" : JSON.stringify(bulkLimit), async () => { + await createDb({ bulkLimit }); const gotWrites: any = []; - mock.on('set', util.callbackify(async (k: any, v: any) => gotWrites.push(1))); - mock.on('doBulk', util.callbackify(async (ops: any) => gotWrites.push(ops.length))); + mock.on( + "set", + util.callbackify(async (k: any, v: any) => gotWrites.push(1)), + ); + mock.on( + "doBulk", + util.callbackify(async (ops: any) => gotWrites.push(ops.length)), + ); const N = 10; await Promise.all(range(N).map((i) => db.set(`key${i}`, `val${i}`))); - const wantLimit:any = bulkLimit || N; + const wantLimit: any = bulkLimit || N; const wantWrites = range(N / wantLimit).map((i) => wantLimit); expect(gotWrites).toStrictEqual(wantWrites); }); } }); - it('bulk failures are retried individually', async () => { + it("bulk failures are retried individually", async () => { await createDb({}); const gotDoBulkCalls: any = []; - mock.on('doBulk', util.callbackify(async (ops: any) => { - gotDoBulkCalls.push(ops.length); - throw new Error('test'); - })); + mock.on( + "doBulk", + util.callbackify(async (ops: any) => { + gotDoBulkCalls.push(ops.length); + throw new Error("test"); + }), + ); const gotWrites = new Map(); const wantWrites = new Map(); - mock.on('set', util.callbackify(async (k: any, v: any) => gotWrites.set(k, v))); + mock.on( + "set", + util.callbackify(async (k: any, v: any) => gotWrites.set(k, v)), + ); const N = 10; - await Promise.all(range(N).map(async (i) => { - const k = `key${i}`; - const v = `val${i}`; - wantWrites.set(k, JSON.stringify(v)); - await db.set(k, v); - })); + await Promise.all( + range(N).map(async (i) => { + const k = `key${i}`; + const v = `val${i}`; + wantWrites.set(k, JSON.stringify(v)); + await db.set(k, v); + }), + ); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message assert.deepEqual(gotDoBulkCalls, [N]); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message diff --git a/test/mock/test_findKeys.spec.ts b/test/mock/test_findKeys.spec.ts index f0e7bd72..998b7e1a 100644 --- a/test/mock/test_findKeys.spec.ts +++ b/test/mock/test_findKeys.spec.ts @@ -1,30 +1,29 @@ -import assert$0 from 'assert'; -import {ConsoleLogger} from '../../lib/logging'; -import * as ueberdb from '../../index'; -'use strict'; +import assert$0 from "assert"; +import { ConsoleLogger } from "../../lib/logging"; +import * as ueberdb from "../../index"; +("use strict"); const assert = assert$0.strict; const logger = new ConsoleLogger(); -import {afterAll, describe, it, afterEach, beforeEach, beforeAll, expect} from 'vitest' +import { afterAll, describe, it, afterEach, beforeEach, beforeAll, expect } from "vitest"; type MockSettings = { mock?: any; -} +}; describe(__filename, () => { let db: any = null; let mock: any = null; const createDb = async (wrapperSettings = {}) => { - const settings:MockSettings = {}; - db = new ueberdb.Database('mock', settings, {json: false, ...wrapperSettings}, logger); + const settings: MockSettings = {}; + db = new ueberdb.Database("mock", settings, { json: false, ...wrapperSettings }, logger); await db.init(); mock = settings.mock; - mock.once('init', (cb: any) => cb()); - + mock.once("init", (cb: any) => cb()); }; afterEach(async () => { if (mock != null) { mock.removeAllListeners(); - mock.once('close', (cb: any) => cb()); + mock.once("close", (cb: any) => cb()); mock = null; } if (db != null) { @@ -32,16 +31,19 @@ describe(__filename, () => { db = null; } }); - it('cached entries are flushed before calling findKeys', async () => { + it("cached entries are flushed before calling findKeys", async () => { // Trigger a test timeout if flush() completes before the write operation is buffered. - await createDb({writeInterval: 1e9}); + await createDb({ writeInterval: 1e9 }); let called = false; - mock.on('set', (k: any, v: any, cb: any) => { called = true; cb(null); }); + mock.on("set", (k: any, v: any, cb: any) => { + called = true; + cb(null); + }); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - mock.on('findKeys', (k: any, nk: any, cb: any) => { assert(called); cb(null, []); }); - await Promise.all([ - db.set('key', 'value'), - db.findKeys('key', null), - ]); + mock.on("findKeys", (k: any, nk: any, cb: any) => { + assert(called); + cb(null, []); + }); + await Promise.all([db.set("key", "value"), db.findKeys("key", null)]); }); }); diff --git a/test/mock/test_flush.spec.ts b/test/mock/test_flush.spec.ts index 41e40471..c140a5f4 100644 --- a/test/mock/test_flush.spec.ts +++ b/test/mock/test_flush.spec.ts @@ -1,28 +1,26 @@ -import {ConsoleLogger} from '../../lib/logging'; -import * as ueberdb from '../../index'; -import {afterAll, describe, it, afterEach, beforeEach, beforeAll, expect} from 'vitest' +import { ConsoleLogger } from "../../lib/logging"; +import * as ueberdb from "../../index"; +import { afterAll, describe, it, afterEach, beforeEach, beforeAll, expect } from "vitest"; const logger = new ConsoleLogger(); - type MockSettings = { - mock?: any; -} - + mock?: any; +}; describe(__filename, () => { let db: any = null; let mock: any = null; const createDb = async (wrapperSettings = {}) => { - const settings:MockSettings = {}; - db = new ueberdb.Database('mock', settings, {json: false, ...wrapperSettings}, logger); + const settings: MockSettings = {}; + db = new ueberdb.Database("mock", settings, { json: false, ...wrapperSettings }, logger); await db.init(); mock = settings.mock; - mock.once('init', (cb: any) => cb()); + mock.once("init", (cb: any) => cb()); }; afterEach(async () => { if (mock != null) { mock.removeAllListeners(); - mock.once('close', (cb: any) => cb()); + mock.once("close", (cb: any) => cb()); mock = null; } if (db != null) { @@ -30,32 +28,23 @@ describe(__filename, () => { db = null; } }); - it('flush() immediately after set() sees the write operation', async () => { + it("flush() immediately after set() sees the write operation", async () => { // Trigger a test timeout if flush() completes before the write operation is buffered. - await createDb({writeInterval: 1e9}); - mock.on('set', (k: any, v: any, cb: any) => cb()); - await Promise.all([ - db.set('key', 'value'), - db.flush(), - ]); + await createDb({ writeInterval: 1e9 }); + mock.on("set", (k: any, v: any, cb: any) => cb()); + await Promise.all([db.set("key", "value"), db.flush()]); }); - it('flush() immediately after setSub() sees the write operation', async () => { + it("flush() immediately after setSub() sees the write operation", async () => { // Trigger a test timeout if flush() completes before the write operation is buffered. - await createDb({writeInterval: 1e9}); - mock.on('get', (k: any, cb: any) => cb(null, {sub: 'oldvalue'})); - mock.on('set', (k: any, v: any, cb: any) => cb(null)); - await Promise.all([ - db.setSub('key', ['sub'], 'newvalue'), - db.flush(), - ]); + await createDb({ writeInterval: 1e9 }); + mock.on("get", (k: any, cb: any) => cb(null, { sub: "oldvalue" })); + mock.on("set", (k: any, v: any, cb: any) => cb(null)); + await Promise.all([db.setSub("key", ["sub"], "newvalue"), db.flush()]); }); - it('flush() immediately after remove() sees the write operation', async () => { + it("flush() immediately after remove() sees the write operation", async () => { // Trigger a test timeout if flush() completes before the write operation is buffered. - await createDb({writeInterval: 1e9}); - mock.on('remove', (k: any, cb: any) => cb(null)); - await Promise.all([ - db.remove('key'), - db.flush(), - ]); + await createDb({ writeInterval: 1e9 }); + mock.on("remove", (k: any, cb: any) => cb(null)); + await Promise.all([db.remove("key"), db.flush()]); }); }); diff --git a/test/mock/test_lru.spec.ts b/test/mock/test_lru.spec.ts index 855e9fc5..234ba97b 100644 --- a/test/mock/test_lru.spec.ts +++ b/test/mock/test_lru.spec.ts @@ -1,92 +1,92 @@ -import {exportedForTesting} from '../../lib/CacheAndBufferLayer'; -import assert$0 from 'assert'; -import {afterAll, describe, it, afterEach, beforeEach, beforeAll, expect} from 'vitest' -const LRU = {exportedForTesting}.exportedForTesting.LRU; +import { exportedForTesting } from "../../lib/CacheAndBufferLayer"; +import assert$0 from "assert"; +import { afterAll, describe, it, afterEach, beforeEach, beforeAll, expect } from "vitest"; +const LRU = { exportedForTesting }.exportedForTesting.LRU; const assert = assert$0.strict; describe(__filename, () => { - describe('capacity = 0', () => { - it('constructor does not throw', async () => { + describe("capacity = 0", () => { + it("constructor does not throw", async () => { new LRU(0); }); - describe('behavior when empty', () => { - it('get() returns nullish', async () => { + describe("behavior when empty", () => { + it("get() returns nullish", async () => { // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert((new LRU(0)).get('k') == null); + assert(new LRU(0).get("k") == null); }); - it('empty iteration', async () => { + it("empty iteration", async () => { // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.equal([...(new LRU(0))].length, 0); + assert.equal([...new LRU(0)].length, 0); }); - it('evictOld() does not throw', async () => { - (new LRU(0)).evictOld(); + it("evictOld() does not throw", async () => { + new LRU(0).evictOld(); }); }); - describe('single entry with evictable = false', () => { + describe("single entry with evictable = false", () => { let evictable: any, lru: any, key: any, val: any; beforeEach(async () => { evictable = false; lru = new LRU(0, () => evictable); - key = 'k'; - val = 'v'; + key = "k"; + val = "v"; lru.set(key, val); }); - it('get() works', async () => { + it("get() works", async () => { // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message assert.equal(lru.get(key), val); }); - it('iterate works', async () => { + it("iterate works", async () => { // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message assert.deepEqual([...lru], [[key, val]]); }); - it('re-set() works', async () => { - const val2 = 'v2'; + it("re-set() works", async () => { + const val2 = "v2"; lru.set(key, val2); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message assert.equal(lru.get(key), val2); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message assert.deepEqual([...lru], [[key, val2]]); }); - it('evictOld() does not evict', async () => { + it("evictOld() does not evict", async () => { lru.evictOld(); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message assert.deepEqual([...lru], [[key, val]]); }); - it('evictOld() evicts after setting evictable = true', async () => { + it("evictOld() evicts after setting evictable = true", async () => { evictable = true; lru.evictOld(); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message assert.deepEqual([...lru], []); }); }); - describe('set immediately evicts if evictable', () => { - it('explicitly evictable', async () => { + describe("set immediately evicts if evictable", () => { + it("explicitly evictable", async () => { const lru = new LRU(0, () => true); - lru.set('k', 'v'); + lru.set("k", "v"); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert(lru.get('k') == null); + assert(lru.get("k") == null); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message assert.deepEqual([...lru], []); }); - it('is evictable by default', async () => { + it("is evictable by default", async () => { const lru = new LRU(0); - lru.set('k', 'v'); + lru.set("k", "v"); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert(lru.get('k') == null); + assert(lru.get("k") == null); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message assert.deepEqual([...lru], []); }); }); }); - describe('capacity = 2', () => { + describe("capacity = 2", () => { let evictable: any, lru: any; beforeEach(async () => { evictable = () => false; lru = new LRU(2, (k: any, v: any) => evictable(k, v)); }); - it('iterates oldest first', async () => { - lru.set(0, '0'); - lru.set(1, '1'); + it("iterates oldest first", async () => { + lru.set(0, "0"); + lru.set(1, "1"); let i = 0; for (const [k, v] of lru) { // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message @@ -98,53 +98,91 @@ describe(__filename, () => { // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message assert.equal(i, 2); }); - it('get(k) updates recently used', async () => { - lru.set(0, '0'); - lru.set(1, '1'); + it("get(k) updates recently used", async () => { + lru.set(0, "0"); + lru.set(1, "1"); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.equal(lru.get(0), '0'); + assert.equal(lru.get(0), "0"); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.deepEqual([...lru], [[1, '1'], [0, '0']]); + assert.deepEqual( + [...lru], + [ + [1, "1"], + [0, "0"], + ], + ); }); - it('get(k, false) does not update recently used', async () => { - lru.set(0, '0'); - lru.set(1, '1'); + it("get(k, false) does not update recently used", async () => { + lru.set(0, "0"); + lru.set(1, "1"); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.equal(lru.get(0, false), '0'); + assert.equal(lru.get(0, false), "0"); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.deepEqual([...lru], [[0, '0'], [1, '1']]); + assert.deepEqual( + [...lru], + [ + [0, "0"], + [1, "1"], + ], + ); }); - it('re-set() updates recently used', async () => { - lru.set(0, '0'); - lru.set(1, '1'); - lru.set(0, '00'); + it("re-set() updates recently used", async () => { + lru.set(0, "0"); + lru.set(1, "1"); + lru.set(0, "00"); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.deepEqual([...lru], [[1, '1'], [0, '00']]); + assert.deepEqual( + [...lru], + [ + [1, "1"], + [0, "00"], + ], + ); }); - it('evictOld() only evicts evictable entries', async () => { + it("evictOld() only evicts evictable entries", async () => { evictable = () => false; - lru.set(0, '0'); - lru.set(1, '1'); - lru.set(2, '2'); - lru.set(3, '3'); + lru.set(0, "0"); + lru.set(1, "1"); + lru.set(2, "2"); + lru.set(3, "3"); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.deepEqual([...lru], [[0, '0'], [1, '1'], [2, '2'], [3, '3']]); + assert.deepEqual( + [...lru], + [ + [0, "0"], + [1, "1"], + [2, "2"], + [3, "3"], + ], + ); evictable = (k: any) => k >= 2; lru.evictOld(); // The newer entries should be evicted because the older are dirty/writingInProgress. // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.deepEqual([...lru], [[0, '0'], [1, '1']]); + assert.deepEqual( + [...lru], + [ + [0, "0"], + [1, "1"], + ], + ); }); - it('evictOld() does nothing if at or below capacity', async () => { + it("evictOld() does nothing if at or below capacity", async () => { evictable = () => true; - lru.set(0, '0'); + lru.set(0, "0"); lru.evictOld(); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.deepEqual([...lru], [[0, '0']]); - lru.set(1, '1'); + assert.deepEqual([...lru], [[0, "0"]]); + lru.set(1, "1"); lru.evictOld(); // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert.deepEqual([...lru], [[0, '0'], [1, '1']]); + assert.deepEqual( + [...lru], + [ + [0, "0"], + [1, "1"], + ], + ); }); }); }); diff --git a/test/mock/test_setSub.spec.ts b/test/mock/test_setSub.spec.ts index 4579b878..4ceac15a 100644 --- a/test/mock/test_setSub.spec.ts +++ b/test/mock/test_setSub.spec.ts @@ -1,20 +1,20 @@ -import assert$0 from 'assert'; -import * as ueberdb from '../../index'; -import {afterAll, describe, it, afterEach, beforeEach, beforeAll, expect} from 'vitest' +import assert$0 from "assert"; +import * as ueberdb from "../../index"; +import { afterAll, describe, it, afterEach, beforeEach, beforeAll, expect } from "vitest"; const assert = assert$0.strict; describe(__filename, () => { let db: any; beforeEach(async () => { - db = new ueberdb.Database('memory', {}, {}); + db = new ueberdb.Database("memory", {}, {}); await db.init(); }); afterEach(async () => { if (db != null) await db.close(); db = null; }); - it('setSub rejects __proto__', async () => { - await db.set('k', {}); - await assert.rejects(db.setSub('k', ['__proto__'], 'v')); + it("setSub rejects __proto__", async () => { + await db.set("k", {}); + await assert.rejects(db.setSub("k", ["__proto__"], "v")); }); }); diff --git a/test/mongodb/test.spec.ts b/test/mongodb/test.spec.ts index 214e7630..003009ca 100644 --- a/test/mongodb/test.spec.ts +++ b/test/mongodb/test.spec.ts @@ -1,19 +1,15 @@ -import {afterAll, beforeAll, describe} from "vitest"; -import {test_db} from "../lib/test_lib"; -import {GenericContainer, PortWithOptionalBinding, StartedTestContainer} from "testcontainers"; +import { afterAll, beforeAll, describe } from "vitest"; +import { test_db } from "../lib/test_lib"; +import { GenericContainer, PortWithOptionalBinding, StartedTestContainer } from "testcontainers"; -describe('mongo test', async () => { - const portMappings: PortWithOptionalBinding[] = [ - {container: 27017, host: 27017} - ]; - let container: StartedTestContainer +describe("mongo test", async () => { + const portMappings: PortWithOptionalBinding[] = [{ container: 27017, host: 27017 }]; + let container: StartedTestContainer; - container = await new GenericContainer("mongo:latest") - .withExposedPorts(...portMappings) - .start() - test_db('mongodb') + container = await new GenericContainer("mongo:latest").withExposedPorts(...portMappings).start(); + test_db("mongodb"); - afterAll(async () => { - await container.stop() - }) -}, 120000) + afterAll(async () => { + await container.stop(); + }); +}, 120000); diff --git a/test/mysql/test.mysql.spec.ts b/test/mysql/test.mysql.spec.ts index 0670af2c..89381863 100644 --- a/test/mysql/test.mysql.spec.ts +++ b/test/mysql/test.mysql.spec.ts @@ -1,29 +1,28 @@ -import {afterAll, beforeAll, describe} from "vitest"; -import {test_db} from "../lib/test_lib"; -import {GenericContainer, PortWithOptionalBinding, StartedTestContainer} from "testcontainers"; +import { afterAll, beforeAll, describe } from "vitest"; +import { test_db } from "../lib/test_lib"; +import { GenericContainer, PortWithOptionalBinding, StartedTestContainer } from "testcontainers"; -describe('postgres test', ()=>{ - const portMappings: PortWithOptionalBinding[] = [ - { container: 3306, host: 3306 } - ]; - let container: StartedTestContainer +describe("postgres test", () => { + const portMappings: PortWithOptionalBinding[] = [{ container: 3306, host: 3306 }]; + let container: StartedTestContainer; - beforeAll(async () => { - container = await new GenericContainer("mariadb:latest") - .withExposedPorts(...portMappings) - .withEnvironment({ - MYSQL_ROOT_PASSWORD: "password", - MYSQL_USER: "ueberdb", - MYSQL_PASSWORD: "ueberdb", - MYSQL_DATABASE: "ueberdb" - }).start() - }) + beforeAll(async () => { + container = await new GenericContainer("mariadb:latest") + .withExposedPorts(...portMappings) + .withEnvironment({ + MYSQL_ROOT_PASSWORD: "password", + MYSQL_USER: "ueberdb", + MYSQL_PASSWORD: "ueberdb", + MYSQL_DATABASE: "ueberdb", + }) + .start(); + }); - test_db('mysql') + test_db("mysql"); - afterAll(async () => { - if (container){ - await container.stop() - } - }) -}, 120000) + afterAll(async () => { + if (container) { + await container.stop(); + } + }); +}, 120000); diff --git a/test/mysql/test_mysql.spec.ts b/test/mysql/test_mysql.spec.ts index 7da160be..65d6c6ba 100644 --- a/test/mysql/test_mysql.spec.ts +++ b/test/mysql/test_mysql.spec.ts @@ -1,83 +1,86 @@ -import assert$0 from 'assert'; -import {databases} from '../lib/databases'; -import Mysql_db from '../../databases/mysql_db'; -import {describe, it, beforeEach, beforeAll, afterAll} from 'vitest' -import {GenericContainer, PortWithOptionalBinding, StartedTestContainer} from "testcontainers"; +import assert$0 from "assert"; +import { databases } from "../lib/databases"; +import Mysql_db from "../../databases/mysql_db"; +import { describe, it, beforeEach, beforeAll, afterAll } from "vitest"; +import { GenericContainer, PortWithOptionalBinding, StartedTestContainer } from "testcontainers"; const assert = assert$0.strict; -describe(__filename, () => { - let container: StartedTestContainer - const portMappings: PortWithOptionalBinding[] = [ - { container: 3306, host: 3307 } - ]; +describe( + __filename, + () => { + let container: StartedTestContainer; + const portMappings: PortWithOptionalBinding[] = [{ container: 3306, host: 3307 }]; - beforeAll(async () => { - container = await new GenericContainer("mariadb:latest") + beforeAll(async () => { + container = await new GenericContainer("mariadb:latest") .withExposedPorts(...portMappings) .withEnvironment({ MYSQL_ROOT_PASSWORD: "password", MYSQL_USER: "ueberdb", MYSQL_PASSWORD: "ueberdb", - MYSQL_DATABASE: "ueberdb" - }).start() - }) + MYSQL_DATABASE: "ueberdb", + }) + .start(); + }); - beforeEach(async function (this: any) { - if (databases.mysql == null) return this.skip(); - }); - it('connect error is detected during init()', async () => { - // Use an invalid TCP port to force a connection error. - const db = new Mysql_db({...databases.mysql, port: 65536}); - // An error is expected; prevent it from being logged. - db.logger = Object.setPrototypeOf({error() { }}, db.logger); - await assert.rejects(db.init()); - }); - it('query after fatal error works', async () => { - const db = new Mysql_db({...databases.mysql, port: 3307}); - await db.init(); - // An error is expected; prevent it from being logged. - db.logger = Object.setPrototypeOf({error() { }}, db.logger); - // Sleep longer than the timeout to force a fatal error. - await assert.rejects(db._query({sql: 'DO SLEE(1);', timeout: 2000}), undefined); - await assert.doesNotReject(db._query({sql: 'SELECT 1;'})); - await db.close(); - }); - it('query times out', async () => { - const db = new Mysql_db({...databases.mysql, port: 3307}); - await db.init(); - // Timeout error messages are expected; prevent them from being logged. - db.logger = Object.setPrototypeOf({error() { }}, db.logger); - db.settings.queryTimeout = 100; - await assert.doesNotReject(db._query({sql: 'DO SLEEP(0.090);'})); - await assert.rejects(db._query({sql: 'DO SLEEP(0.110);'})); - await db.close(); - }); - it('queries run concurrently and are queued when pool is busy', async () => { - const connectionLimit = 10; - const db = new Mysql_db({...databases.mysql, connectionLimit, port: 3307}); - await db.init(); - // Set the query duration high enough to avoid flakiness on slow machines but low enough to keep - // the overall test duration short. - const queryDuration = 100; - db.settings.queryTimeout = queryDuration + 100; - const enqueueQuery = () => db._query({sql: `DO SLEEP(${queryDuration / 1000});`}); - // Reduce test flakiness by using slow queries to warm up the pool's connections. - await Promise.all([...Array(connectionLimit)].map(enqueueQuery)); - // Time how long it takes to run just under 2 * connectionLimit queries. - const nQueries = 2 * connectionLimit - 1; - const start = Date.now(); - await Promise.all([...Array(nQueries)].map(enqueueQuery)); - const duration = Date.now() - start; - const wantDurationLower = Math.ceil(nQueries / connectionLimit) * queryDuration; - // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert(duration >= wantDurationLower, `took ${duration}ms, want >= ${wantDurationLower}ms`); - const wantDurationUpper = wantDurationLower + queryDuration; - // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message - assert(duration < wantDurationUpper, `took ${duration}ms, want < ${wantDurationUpper}ms`); - await db.close(); - }); + beforeEach(async function (this: any) { + if (databases.mysql == null) return this.skip(); + }); + it("connect error is detected during init()", async () => { + // Use an invalid TCP port to force a connection error. + const db = new Mysql_db({ ...databases.mysql, port: 65536 }); + // An error is expected; prevent it from being logged. + db.logger = Object.setPrototypeOf({ error() {} }, db.logger); + await assert.rejects(db.init()); + }); + it("query after fatal error works", async () => { + const db = new Mysql_db({ ...databases.mysql, port: 3307 }); + await db.init(); + // An error is expected; prevent it from being logged. + db.logger = Object.setPrototypeOf({ error() {} }, db.logger); + // Sleep longer than the timeout to force a fatal error. + await assert.rejects(db._query({ sql: "DO SLEE(1);", timeout: 2000 }), undefined); + await assert.doesNotReject(db._query({ sql: "SELECT 1;" })); + await db.close(); + }); + it("query times out", async () => { + const db = new Mysql_db({ ...databases.mysql, port: 3307 }); + await db.init(); + // Timeout error messages are expected; prevent them from being logged. + db.logger = Object.setPrototypeOf({ error() {} }, db.logger); + db.settings.queryTimeout = 100; + await assert.doesNotReject(db._query({ sql: "DO SLEEP(0.090);" })); + await assert.rejects(db._query({ sql: "DO SLEEP(0.110);" })); + await db.close(); + }); + it("queries run concurrently and are queued when pool is busy", async () => { + const connectionLimit = 10; + const db = new Mysql_db({ ...databases.mysql, connectionLimit, port: 3307 }); + await db.init(); + // Set the query duration high enough to avoid flakiness on slow machines but low enough to keep + // the overall test duration short. + const queryDuration = 100; + db.settings.queryTimeout = queryDuration + 100; + const enqueueQuery = () => db._query({ sql: `DO SLEEP(${queryDuration / 1000});` }); + // Reduce test flakiness by using slow queries to warm up the pool's connections. + await Promise.all([...Array(connectionLimit)].map(enqueueQuery)); + // Time how long it takes to run just under 2 * connectionLimit queries. + const nQueries = 2 * connectionLimit - 1; + const start = Date.now(); + await Promise.all([...Array(nQueries)].map(enqueueQuery)); + const duration = Date.now() - start; + const wantDurationLower = Math.ceil(nQueries / connectionLimit) * queryDuration; + // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message + assert(duration >= wantDurationLower, `took ${duration}ms, want >= ${wantDurationLower}ms`); + const wantDurationUpper = wantDurationLower + queryDuration; + // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message + assert(duration < wantDurationUpper, `took ${duration}ms, want < ${wantDurationUpper}ms`); + await db.close(); + }); - afterAll(async () => { - await container.stop() - }) -}, 120000); + afterAll(async () => { + await container.stop(); + }); + }, + 120000, +); diff --git a/test/postgres/test.postgresql.spec.ts b/test/postgres/test.postgresql.spec.ts index 0342613f..807d5851 100644 --- a/test/postgres/test.postgresql.spec.ts +++ b/test/postgres/test.postgresql.spec.ts @@ -1,72 +1,70 @@ -import {afterAll, beforeAll, describe, it} from "vitest"; -import {db, test_db} from "../lib/test_lib"; -import {GenericContainer, PortWithOptionalBinding, StartedTestContainer} from "testcontainers"; -import {databases} from "../lib/databases"; +import { afterAll, beforeAll, describe, it } from "vitest"; +import { db, test_db } from "../lib/test_lib"; +import { GenericContainer, PortWithOptionalBinding, StartedTestContainer } from "testcontainers"; +import { databases } from "../lib/databases"; import * as ueberdb from "../../index"; -import {equal} from "assert"; +import { equal } from "assert"; -describe('postgres test', async () => { - const portMappings: PortWithOptionalBinding[] = [ - { container: 5432, host: 5432 } - ]; - let container: StartedTestContainer +describe("postgres test", async () => { + const portMappings: PortWithOptionalBinding[] = [{ container: 5432, host: 5432 }]; + let container: StartedTestContainer; - beforeAll(async () => { - container = await new GenericContainer("postgres:alpine3.21") - .withExposedPorts(...portMappings) - .withEnvironment({ - POSTGRES_USER: "ueberdb", - POSTGRES_PASSWORD: "ueberdb", - POSTGRES_DB: "ueberdb" - }).start() - }) + beforeAll(async () => { + container = await new GenericContainer("postgres:alpine3.21") + .withExposedPorts(...portMappings) + .withEnvironment({ + POSTGRES_USER: "ueberdb", + POSTGRES_PASSWORD: "ueberdb", + POSTGRES_DB: "ueberdb", + }) + .start(); + }); + test_db("postgres"); - test_db('postgres') + afterAll(async () => { + db.close(); + await container.stop(); + }); +}, 1200000); +describe("postgres test individual", async () => { + const portMappings: PortWithOptionalBinding[] = [{ container: 5432, host: 5444 }]; + let container: StartedTestContainer; - afterAll(async () => { - db.close() - await container.stop() - }) -}, 1200000) + beforeAll(async () => { + container = await new GenericContainer("postgres:alpine3.21") + .withExposedPorts(...portMappings) + .withHealthCheck({ + test: ["CMD-SHELL", "pg_isready -d postgresql://ueberdb:ueberdb@127.0.0.1/ueberdb"], + interval: 10000, + timeout: 5000, + retries: 5, + }) + .withEnvironment({ + POSTGRES_USER: "ueberdb", + POSTGRES_PASSWORD: "ueberdb", + POSTGRES_DB: "ueberdb", + }) + .start(); + }); -describe('postgres test individual', async () => { - const portMappings: PortWithOptionalBinding[] = [ - { container: 5432, host: 5444 } - ]; - let container: StartedTestContainer + it("connection string instead of settings object", async () => { + const { user, password, host, database } = databases.postgres; - beforeAll(async () => { - container = await new GenericContainer("postgres:alpine3.21") - .withExposedPorts(...portMappings) - .withHealthCheck({ - test: ["CMD-SHELL", "pg_isready -d postgresql://ueberdb:ueberdb@127.0.0.1/ueberdb"], - interval: 10000, - timeout: 5000, - retries: 5 - }) - .withEnvironment({ - POSTGRES_USER: "ueberdb", - POSTGRES_PASSWORD: "ueberdb", - POSTGRES_DB: "ueberdb" - }).start() - }) + console.log(`postgres://${user}:${password}@${host}:5444/${database}`); + const db = new ueberdb.Database( + "postgres", + `postgres://${user}:${password}@${host}:5444/${database}`, + ); + await db.init(); + await db.set("key", "val"); + const val = (await db.get("key")) as string; + equal(val, "val"); + db.close(); + }); - it('connection string instead of settings object', async () => { - const {user, password, host, database} = databases.postgres; - - console.log(`postgres://${user}:${password}@${host}:5444/${database}`); - const db = new ueberdb.Database('postgres', `postgres://${user}:${password}@${host}:5444/${database}`); - await db.init(); - await db.set('key', 'val'); - const val = await db.get('key') as string; - equal(val, 'val'); - db.close() - }); - - afterAll(async () => { - await container.stop() - }) - -}, 120000) + afterAll(async () => { + await container.stop(); + }); +}, 120000); diff --git a/test/redis/test.redis.spec.ts b/test/redis/test.redis.spec.ts index a5ec3406..31870b24 100644 --- a/test/redis/test.redis.spec.ts +++ b/test/redis/test.redis.spec.ts @@ -1,22 +1,19 @@ -import {afterAll, beforeAll, describe} from "vitest"; -import {test_db} from "../lib/test_lib"; -import {GenericContainer, PortWithOptionalBinding, StartedTestContainer} from "testcontainers"; +import { afterAll, beforeAll, describe } from "vitest"; +import { test_db } from "../lib/test_lib"; +import { GenericContainer, PortWithOptionalBinding, StartedTestContainer } from "testcontainers"; -describe('redis test', ()=>{ - const portMappings: PortWithOptionalBinding[] = [ - { container: 6379, host: 6379 } - ]; - let container: StartedTestContainer +describe("redis test", () => { + const portMappings: PortWithOptionalBinding[] = [{ container: 6379, host: 6379 }]; + let container: StartedTestContainer; - beforeAll(async () => { - container = await new GenericContainer("redis:bookworm") - .withExposedPorts(...portMappings) - .start() - }) + beforeAll(async () => { + container = await new GenericContainer("redis:bookworm") + .withExposedPorts(...portMappings) + .start(); + }); - - afterAll(async () => { - await container.stop() - }) - test_db('redis') -}) + afterAll(async () => { + await container.stop(); + }); + test_db("redis"); +}); diff --git a/test/rethinkdb/rethinkdb.spec.ts b/test/rethinkdb/rethinkdb.spec.ts index b42ba4ae..af7bd725 100644 --- a/test/rethinkdb/rethinkdb.spec.ts +++ b/test/rethinkdb/rethinkdb.spec.ts @@ -1,23 +1,20 @@ -import {afterAll, beforeAll, describe} from "vitest"; -import {GenericContainer, PortWithOptionalBinding, StartedTestContainer} from "testcontainers"; -import {test_db} from "../lib/test_lib"; +import { afterAll, beforeAll, describe } from "vitest"; +import { GenericContainer, PortWithOptionalBinding, StartedTestContainer } from "testcontainers"; +import { test_db } from "../lib/test_lib"; -describe('rethinkdb test', ()=>{ - const portMappings: PortWithOptionalBinding[] = [ - { container: 7000, host: 7000 } - ]; - let container: StartedTestContainer +describe("rethinkdb test", () => { + const portMappings: PortWithOptionalBinding[] = [{ container: 7000, host: 7000 }]; + let container: StartedTestContainer; - beforeAll(async () => { - container = await new GenericContainer("rethinkdb:latest") - .withExposedPorts(...portMappings) - .start() - }) + beforeAll(async () => { + container = await new GenericContainer("rethinkdb:latest") + .withExposedPorts(...portMappings) + .start(); + }); + test_db("cassandra"); - test_db('cassandra') - - afterAll(async () => { - await container.stop() - }) -}) \ No newline at end of file + afterAll(async () => { + await container.stop(); + }); +}); diff --git a/test/rusty/test.rusty.spec.ts b/test/rusty/test.rusty.spec.ts index f9299035..f9304b32 100644 --- a/test/rusty/test.rusty.spec.ts +++ b/test/rusty/test.rusty.spec.ts @@ -1,6 +1,6 @@ -import {describe, it} from "vitest"; -import {test_db} from "../lib/test_lib"; +import { describe, it } from "vitest"; +import { test_db } from "../lib/test_lib"; -describe('rusty test', ()=>{ - test_db('rustydb') -}) +describe("rusty test", () => { + test_db("rustydb"); +}); diff --git a/test/rusty/test_rusty.spec.ts b/test/rusty/test_rusty.spec.ts index 09b72649..ee2147af 100644 --- a/test/rusty/test_rusty.spec.ts +++ b/test/rusty/test_rusty.spec.ts @@ -1,125 +1,115 @@ -import {beforeAll, expect, test} from "vitest"; +import { beforeAll, expect, test } from "vitest"; import os from "os"; import Rusty_db from "../../databases/rusty_db"; import Database from "../../index"; +const TEST = `${os.tmpdir()}/ueberdb-test-${new Date().getTime()}.db`; -const TEST = `${os.tmpdir()}/ueberdb-test-${new Date().getTime()}.db` - -let db: Database +let db: Database; beforeAll(async () => { - db = new Database('rustydb', { - filename: TEST - }) - await db.init() -}) - - - - -test('test get', async () => { - db.set('key:test', 'value:test') - let res = await db.get('key:test'); - expect(res).toBe('value:test') - db.remove("key:test") - expect(await db.get('key:test')).toBeNull() -}) - -test('test remove', async () => { - db.set('key:test', 'value:test') - db.remove('key:test') - expect(await db.get('key:test')).toBeNull() -}) - -test('Key value set', async () => { - db.set('key:test', 'value:test') - let res = await db.get('key:test') - expect(res).toBe('value:test') -}) - -test('Key value remove', async () => { - db.set('key:test', 'value:test') - db.remove('key:test') - let res = await db.get('key:test') - expect(res).toBeNull() -}) - - -test('Key value findKeys 2', async () => { - db.set('key:test', 'value:test') - db.set('key:test2', 'value:test2') - db.set('key:123', "value:123") - let res = await db.findKeys('key:test*') - expect(res).toEqual(['key:test', 'key:test2']) -}) - -test('Key value findKeys', async () => { - db.set('key:test', 'value:test') - db.set('key:test2', 'value:test2') - db.set('key:123', "value:123") - let res = await db.findKeys('key:test2*') - expect(res).toEqual(['key:test2']) -}) - -test('Key value findKeys rev', async () => { - db.set('key:2:test', 'value:test') - db.set('key:3:test2', 'value:test2') - db.set('key:4:123', "value:123") - let res = await db.findKeys('key:*:test') - expect(res).toEqual(['key:2:test']) -}) - -test('Key value findKeys none', async () => { - db.set('key:2:test', 'value:test') - db.set('key:3:test2', 'value:test2') - db.set('key:4:123', "value:123") - let res = await db.findKeys('key:*5:test') - expect(res).toEqual([]) -}) - -test('Key value findKeys 45', async () => { - db.set('key:2:test', 'value:test') - db.set('key:3:test2', 'value:test2') - db.set('key:4:123', "value:123") - db.set('key:45:test', "value:123") - let res = await db.findKeys('key:*5:test') - expect(res).toEqual(["key:45:test"]) -}) - - -test('findKeys with exclusion works', async () => { - db.set('key:2:test', 'test') - db.set('key:2:testa', 'true') - db.set('key:2:testb', 'true') - db.set('key:2:testb2', 'true') - db.set('nonmatching_key:2:test', 'true') - const keys = await db.findKeys('key:2:test*', "key:2:testb*") - expect(keys.sort()).toStrictEqual(['key:2:test', 'key:2:testa']) -}) - - -test('findKeys with no matches works', async ()=>{ - - db.set('key:2:test','test') - const keys = await db.findKeys('123', "key:2:testb*") - expect(keys).toStrictEqual([]) -}) - - -test('find keys with no wildcards works', async ()=>{ - db.set('key:2:test','') - db.set('key:2:testa', '') - const keys = await db.findKeys('key:2:testa') - expect(keys).toStrictEqual(['key:2:testa']) -}) - -test('Set string with whitespace', async () => { - db.set('my-custom-key ', 'value') - const val = await db.get('my-custom-key ') - expect(val).toEqual('value') -}) - -test('get without table', async ()=>{ - - db.get('234dsfsdfsdf') -}) + db = new Database("rustydb", { + filename: TEST, + }); + await db.init(); +}); + +test("test get", async () => { + db.set("key:test", "value:test"); + let res = await db.get("key:test"); + expect(res).toBe("value:test"); + db.remove("key:test"); + expect(await db.get("key:test")).toBeNull(); +}); + +test("test remove", async () => { + db.set("key:test", "value:test"); + db.remove("key:test"); + expect(await db.get("key:test")).toBeNull(); +}); + +test("Key value set", async () => { + db.set("key:test", "value:test"); + let res = await db.get("key:test"); + expect(res).toBe("value:test"); +}); + +test("Key value remove", async () => { + db.set("key:test", "value:test"); + db.remove("key:test"); + let res = await db.get("key:test"); + expect(res).toBeNull(); +}); + +test("Key value findKeys 2", async () => { + db.set("key:test", "value:test"); + db.set("key:test2", "value:test2"); + db.set("key:123", "value:123"); + let res = await db.findKeys("key:test*"); + expect(res).toEqual(["key:test", "key:test2"]); +}); + +test("Key value findKeys", async () => { + db.set("key:test", "value:test"); + db.set("key:test2", "value:test2"); + db.set("key:123", "value:123"); + let res = await db.findKeys("key:test2*"); + expect(res).toEqual(["key:test2"]); +}); + +test("Key value findKeys rev", async () => { + db.set("key:2:test", "value:test"); + db.set("key:3:test2", "value:test2"); + db.set("key:4:123", "value:123"); + let res = await db.findKeys("key:*:test"); + expect(res).toEqual(["key:2:test"]); +}); + +test("Key value findKeys none", async () => { + db.set("key:2:test", "value:test"); + db.set("key:3:test2", "value:test2"); + db.set("key:4:123", "value:123"); + let res = await db.findKeys("key:*5:test"); + expect(res).toEqual([]); +}); + +test("Key value findKeys 45", async () => { + db.set("key:2:test", "value:test"); + db.set("key:3:test2", "value:test2"); + db.set("key:4:123", "value:123"); + db.set("key:45:test", "value:123"); + let res = await db.findKeys("key:*5:test"); + expect(res).toEqual(["key:45:test"]); +}); + +test("findKeys with exclusion works", async () => { + db.set("key:2:test", "test"); + db.set("key:2:testa", "true"); + db.set("key:2:testb", "true"); + db.set("key:2:testb2", "true"); + db.set("nonmatching_key:2:test", "true"); + const keys = await db.findKeys("key:2:test*", "key:2:testb*"); + expect(keys.sort()).toStrictEqual(["key:2:test", "key:2:testa"]); +}); + +test("findKeys with no matches works", async () => { + db.set("key:2:test", "test"); + const keys = await db.findKeys("123", "key:2:testb*"); + expect(keys).toStrictEqual([]); +}); + +test("find keys with no wildcards works", async () => { + db.set("key:2:test", ""); + db.set("key:2:testa", ""); + const keys = await db.findKeys("key:2:testa"); + expect(keys).toStrictEqual(["key:2:testa"]); +}); + +test("Set string with whitespace", async () => { + db.set("my-custom-key ", "value"); + const val = await db.get("my-custom-key "); + expect(val).toEqual("value"); +}); + +test("get without table", async () => { + db.get("234dsfsdfsdf"); +}); diff --git a/test/sqlite/test.sqlite.spec.ts b/test/sqlite/test.sqlite.spec.ts index 01a7dd9f..43f5ef74 100644 --- a/test/sqlite/test.sqlite.spec.ts +++ b/test/sqlite/test.sqlite.spec.ts @@ -1,6 +1,6 @@ -import {describe} from "vitest"; -import {test_db} from "../lib/test_lib"; +import { describe } from "vitest"; +import { test_db } from "../lib/test_lib"; -describe('sqlite test', ()=>{ - test_db('sqlite') -}) +describe("sqlite test", () => { + test_db("sqlite"); +}); diff --git a/test/surrealdb/test.surrealdb.spec.ts b/test/surrealdb/test.surrealdb.spec.ts index f83576c5..9092e4a1 100644 --- a/test/surrealdb/test.surrealdb.spec.ts +++ b/test/surrealdb/test.surrealdb.spec.ts @@ -1,42 +1,48 @@ -import {afterAll, beforeAll, describe} from "vitest"; -import {test_db} from "../lib/test_lib"; -import {GenericContainer, PortWithOptionalBinding, StartedTestContainer, Wait} from "testcontainers"; -import {databases} from "../lib/databases"; +import { afterAll, beforeAll, describe } from "vitest"; +import { test_db } from "../lib/test_lib"; +import { + GenericContainer, + PortWithOptionalBinding, + StartedTestContainer, + Wait, +} from "testcontainers"; +import { databases } from "../lib/databases"; -describe('surrealdb test', () => { - const portMappings: PortWithOptionalBinding[] = [ - { container: 8000, host: 8000 }, - ]; - let container: StartedTestContainer | undefined; +describe("surrealdb test", () => { + const portMappings: PortWithOptionalBinding[] = [{ container: 8000, host: 8000 }]; + let container: StartedTestContainer | undefined; - beforeAll(async () => { - // Configure root credentials and start in-memory storage so the - // ueberdb test can sign in and use it. Wait for the HTTP root to - // respond so testcontainers doesn't return before SurrealDB is ready. - // surrealdb client 2.0.3 requires server >= 2.1.0 < 4.0.0 - container = await new GenericContainer("surrealdb/surrealdb:v2.3.10") - .withExposedPorts(...portMappings) - .withCommand([ - "start", - "--user", databases.surrealdb.user || "root", - "--pass", databases.surrealdb.password || "root", - "--bind", "0.0.0.0:8000", - "memory", - ]) - .withWaitStrategy(Wait.forHttp("/health", 8000).forStatusCode(200)) - .withStartupTimeout(120000) - .start(); - }, 180000); + beforeAll(async () => { + // Configure root credentials and start in-memory storage so the + // ueberdb test can sign in and use it. Wait for the HTTP root to + // respond so testcontainers doesn't return before SurrealDB is ready. + // surrealdb client 2.0.3 requires server >= 2.1.0 < 4.0.0 + container = await new GenericContainer("surrealdb/surrealdb:v2.3.10") + .withExposedPorts(...portMappings) + .withCommand([ + "start", + "--user", + databases.surrealdb.user || "root", + "--pass", + databases.surrealdb.password || "root", + "--bind", + "0.0.0.0:8000", + "memory", + ]) + .withWaitStrategy(Wait.forHttp("/health", 8000).forStatusCode(200)) + .withStartupTimeout(120000) + .start(); + }, 180000); - test_db('surrealdb'); + test_db("surrealdb"); - afterAll(async () => { - if (container != null) { - try { - await container.stop(); - } catch (err) { - console.warn("surrealdb container stop failed:", err); - } - } - }); + afterAll(async () => { + if (container != null) { + try { + await container.stop(); + } catch (err) { + console.warn("surrealdb container stop failed:", err); + } + } + }); }); diff --git a/tsconfig.json b/tsconfig.json index dcfc34c1..6ce63b83 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,8 +12,8 @@ // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ /* Language and Environment */ - "target": "esnext", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ - "module": "ESNext", /* Specify what module code is generated. */ + "target": "esnext" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, + "module": "ESNext" /* Specify what module code is generated. */, "moduleResolution": "bundler", // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ // "jsx": "preserve", /* Specify what JSX code is generated. */ @@ -46,8 +46,8 @@ // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ /* Emit */ - "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ - "declarationMap": true, /* Create sourcemaps for d.ts files. */ + "declaration": true /* Generate .d.ts files from TypeScript and JavaScript files in your project. */, + "declarationMap": true /* Create sourcemaps for d.ts files. */, // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ @@ -73,13 +73,13 @@ /* Interop Constraints */ // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ - "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */, // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ - "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ - "verbatimModuleSyntax": true, /* Enforce type-only imports for type-only symbols; ensures correct ESM output. */ + "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, + "verbatimModuleSyntax": true /* Enforce type-only imports for type-only symbols; ensures correct ESM output. */, /* Type Checking */ - "strict": true, /* Enable all strict type-checking options. */ + "strict": true /* Enable all strict type-checking options. */, // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ @@ -101,11 +101,7 @@ /* Completeness */ // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ - "skipLibCheck": true /* Skip type checking all .d.ts files. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ }, - "include": [ - "index.ts", - "lib", - "databases" - ] + "include": ["index.ts", "lib", "databases"] } diff --git a/vitest.config.ts b/vitest.config.ts index f3e39771..5792460d 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,17 +1,17 @@ -import { defineConfig } from 'vitest/config' +import { defineConfig } from "vitest/config"; export default defineConfig({ - test: { - testTimeout: 120000, - hookTimeout: 120000, - // Integration tests against real DB containers are inherently - // flaky on shared CI runners (network blips, slow startup, - // intermittent connection resets, especially the nano + CouchDB - // 3.5 stack which intermittently returns 401 from session - // middleware on the first request after a fresh connection). - // Retry up to 5 times before giving up so transient blips don't - // fail the whole job. The underlying bug still surfaces if the - // test fails consistently. - retry: 5, - } -}) + test: { + testTimeout: 120000, + hookTimeout: 120000, + // Integration tests against real DB containers are inherently + // flaky on shared CI runners (network blips, slow startup, + // intermittent connection resets, especially the nano + CouchDB + // 3.5 stack which intermittently returns 401 from session + // middleware on the first request after a fresh connection). + // Retry up to 5 times before giving up so transient blips don't + // fail the whole job. The underlying bug still surfaces if the + // test fails consistently. + retry: 5, + }, +}); From 72c2431b92da25d49b8f0fde786715645714ad40 Mon Sep 17 00:00:00 2001 From: SamTV12345 <40429738+samtv12345@users.noreply.github.com> Date: Thu, 28 May 2026 18:23:35 +0200 Subject: [PATCH 2/4] feat: removed console.log in test case --- test/postgres/test.postgresql.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/postgres/test.postgresql.spec.ts b/test/postgres/test.postgresql.spec.ts index 807d5851..c180ddf0 100644 --- a/test/postgres/test.postgresql.spec.ts +++ b/test/postgres/test.postgresql.spec.ts @@ -52,7 +52,6 @@ describe("postgres test individual", async () => { it("connection string instead of settings object", async () => { const { user, password, host, database } = databases.postgres; - console.log(`postgres://${user}:${password}@${host}:5444/${database}`); const db = new ueberdb.Database( "postgres", `postgres://${user}:${password}@${host}:5444/${database}`, From f5d4ce252821269ec4d4cd383c3ecf39a649f5d0 Mon Sep 17 00:00:00 2001 From: SamTV12345 <40429738+samtv12345@users.noreply.github.com> Date: Thu, 28 May 2026 18:33:19 +0200 Subject: [PATCH 3/4] feat: removed console.log in test case --- test/postgres/test.postgresql.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/postgres/test.postgresql.spec.ts b/test/postgres/test.postgresql.spec.ts index c180ddf0..f41bb3a7 100644 --- a/test/postgres/test.postgresql.spec.ts +++ b/test/postgres/test.postgresql.spec.ts @@ -1,6 +1,6 @@ import { afterAll, beforeAll, describe, it } from "vitest"; import { db, test_db } from "../lib/test_lib"; -import { GenericContainer, PortWithOptionalBinding, StartedTestContainer } from "testcontainers"; +import { GenericContainer, type PortWithOptionalBinding, type StartedTestContainer } from "testcontainers"; import { databases } from "../lib/databases"; import * as ueberdb from "../../index"; import { equal } from "assert"; From 45904fa0497e8158f409362464a8acb9e34d81b2 Mon Sep 17 00:00:00 2001 From: SamTV12345 <40429738+samtv12345@users.noreply.github.com> Date: Thu, 28 May 2026 22:50:11 +0200 Subject: [PATCH 4/4] fix(rethink_db): non-null assert connection after oxfmt chain split oxfmt reformats get()'s r.table().get().run() into a multi-line chain, which moves the existing // @ts-ignore off the .run(this.connection, ...) line and re-surfaces TS2769 (Connection | null). Assert non-null to keep the reformatted code type-clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- databases/rethink_db.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databases/rethink_db.ts b/databases/rethink_db.ts index 10c294ac..b151084b 100644 --- a/databases/rethink_db.ts +++ b/databases/rethink_db.ts @@ -70,7 +70,7 @@ export default class Rethink_db extends AbstractDatabase { // @ts-ignore r.table(this.table) .get(key) - .run(this.connection, (err, item) => { + .run(this.connection!, (err, item) => { // @ts-ignore callback(err, item ? item.content : item); });